Merge remote-tracking branch 'origin/main' into feat/account-level-proxy
# Conflicts: # src/server/routes/proxy/chat.ts # src/server/routes/proxy/responses.ts
This commit is contained in:
Hureru
@@ -25,6 +25,7 @@ import {
|
||||
getLocalDayRangeUtc,
|
||||
getLocalRangeStartUtc,
|
||||
parseStoredUtcDateTime,
|
||||
type StoredUtcDateTimeInput,
|
||||
toLocalDayKeyFromStoredUtc,
|
||||
} from '../../services/localTimeService.js';
|
||||
import { createRateLimitGuard } from '../../middleware/requestRateLimit.js';
|
||||
@@ -202,7 +203,7 @@ type SiteAvailabilitySiteRow = {
|
||||
|
||||
type SiteAvailabilityLogRow = {
|
||||
siteId: number | null;
|
||||
createdAt: string | null;
|
||||
createdAt: StoredUtcDateTimeInput;
|
||||
status: string | null;
|
||||
latencyMs: number | null;
|
||||
};
|
||||
|
||||
@@ -244,6 +244,75 @@ describe('PUT /api/routes/:id route rebuild', () => {
|
||||
]));
|
||||
});
|
||||
|
||||
it('fills missing sourceModel from source exact routes when loading explicit-group channels', async () => {
|
||||
const sourceA = await seedAccountWithToken('deepseek-chat');
|
||||
const sourceB = await seedAccountWithToken('deepseek-reasoner');
|
||||
|
||||
const exactRouteA = await db.insert(schema.tokenRoutes).values({
|
||||
modelPattern: 'deepseek-chat',
|
||||
enabled: true,
|
||||
}).returning().get();
|
||||
const exactRouteB = await db.insert(schema.tokenRoutes).values({
|
||||
modelPattern: 'deepseek-reasoner',
|
||||
enabled: true,
|
||||
}).returning().get();
|
||||
|
||||
await db.insert(schema.routeChannels).values([
|
||||
{
|
||||
routeId: exactRouteA.id,
|
||||
accountId: sourceA.account.id,
|
||||
tokenId: sourceA.token.id,
|
||||
sourceModel: null,
|
||||
priority: 0,
|
||||
weight: 10,
|
||||
enabled: true,
|
||||
manualOverride: false,
|
||||
},
|
||||
{
|
||||
routeId: exactRouteB.id,
|
||||
accountId: sourceB.account.id,
|
||||
tokenId: sourceB.token.id,
|
||||
sourceModel: null,
|
||||
priority: 1,
|
||||
weight: 8,
|
||||
enabled: true,
|
||||
manualOverride: false,
|
||||
},
|
||||
]).run();
|
||||
|
||||
const createResponse = await app.inject({
|
||||
method: 'POST',
|
||||
url: '/api/routes',
|
||||
payload: {
|
||||
routeMode: 'explicit_group',
|
||||
displayName: 'deepseekv1',
|
||||
sourceRouteIds: [exactRouteA.id, exactRouteB.id],
|
||||
},
|
||||
});
|
||||
|
||||
expect(createResponse.statusCode).toBe(200);
|
||||
const createdRouteId = (createResponse.json() as { id: number }).id;
|
||||
|
||||
const channelsResponse = await app.inject({
|
||||
method: 'GET',
|
||||
url: `/api/routes/${createdRouteId}/channels`,
|
||||
});
|
||||
|
||||
expect(channelsResponse.statusCode).toBe(200);
|
||||
expect(channelsResponse.json()).toEqual(expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
routeId: exactRouteA.id,
|
||||
accountId: sourceA.account.id,
|
||||
sourceModel: 'deepseek-chat',
|
||||
}),
|
||||
expect.objectContaining({
|
||||
routeId: exactRouteB.id,
|
||||
accountId: sourceB.account.id,
|
||||
sourceModel: 'deepseek-reasoner',
|
||||
}),
|
||||
]));
|
||||
});
|
||||
|
||||
it('rejects invalid explicit-group payloads', async () => {
|
||||
const response = await app.inject({
|
||||
method: 'POST',
|
||||
|
||||
@@ -534,7 +534,7 @@ async function fetchChannelsForRouteRows(routes: RouteRow[]): Promise<Map<number
|
||||
const explicitSourceRouteIds = Array.from(new Set(routes
|
||||
.filter((route) => isExplicitGroupRoute(route))
|
||||
.flatMap((route) => route.sourceRouteIds)));
|
||||
const enabledExplicitSourceRouteIds = explicitSourceRouteIds.length > 0
|
||||
const explicitSourceRoutes = explicitSourceRouteIds.length > 0
|
||||
? (await db.select({
|
||||
id: schema.tokenRoutes.id,
|
||||
modelPattern: schema.tokenRoutes.modelPattern,
|
||||
@@ -543,9 +543,10 @@ async function fetchChannelsForRouteRows(routes: RouteRow[]): Promise<Map<number
|
||||
}).from(schema.tokenRoutes)
|
||||
.where(inArray(schema.tokenRoutes.id, explicitSourceRouteIds))
|
||||
.all())
|
||||
.filter((route) => route.enabled && !isExplicitGroupRoute(route) && isExactModelPattern(route.modelPattern))
|
||||
.map((route) => route.id)
|
||||
: [];
|
||||
const enabledExplicitSourceRouteIds = explicitSourceRoutes
|
||||
.filter((route) => route.enabled && !isExplicitGroupRoute(route) && isExactModelPattern(route.modelPattern))
|
||||
.map((route) => route.id);
|
||||
const actualRouteIds = Array.from(new Set([
|
||||
...routes.filter((route) => !isExplicitGroupRoute(route)).map((route) => route.id),
|
||||
...enabledExplicitSourceRouteIds,
|
||||
@@ -554,6 +555,14 @@ async function fetchChannelsForRouteRows(routes: RouteRow[]): Promise<Map<number
|
||||
return new Map(routes.map((route) => [route.id, []]));
|
||||
}
|
||||
|
||||
const actualRouteById = new Map<number, { modelPattern: string; routeMode: string | null }>();
|
||||
for (const route of routes.filter((item) => !isExplicitGroupRoute(item))) {
|
||||
actualRouteById.set(route.id, { modelPattern: route.modelPattern, routeMode: route.routeMode ?? null });
|
||||
}
|
||||
for (const route of explicitSourceRoutes) {
|
||||
actualRouteById.set(route.id, { modelPattern: route.modelPattern, routeMode: route.routeMode ?? null });
|
||||
}
|
||||
|
||||
const channelRows = await db.select().from(schema.routeChannels)
|
||||
.innerJoin(schema.accounts, eq(schema.routeChannels.accountId, schema.accounts.id))
|
||||
.innerJoin(schema.sites, eq(schema.accounts.siteId, schema.sites.id))
|
||||
@@ -565,13 +574,19 @@ async function fetchChannelsForRouteRows(routes: RouteRow[]): Promise<Map<number
|
||||
|
||||
for (const row of channelRows) {
|
||||
const routeId = row.route_channels.routeId;
|
||||
const actualRoute = actualRouteById.get(routeId);
|
||||
const fallbackSourceModel = actualRoute && !isExplicitGroupRoute(actualRoute) && isExactModelPattern(actualRoute.modelPattern)
|
||||
? actualRoute.modelPattern
|
||||
: null;
|
||||
const resolvedSourceModel = (row.route_channels.sourceModel || fallbackSourceModel || '').trim();
|
||||
if (!channelsByActualRouteId.has(routeId)) channelsByActualRouteId.set(routeId, []);
|
||||
channelsByActualRouteId.get(routeId)!.push({
|
||||
...row.route_channels,
|
||||
sourceModel: resolvedSourceModel || null,
|
||||
account: row.accounts,
|
||||
site: row.sites,
|
||||
token: row.account_tokens
|
||||
? {
|
||||
token: row.account_tokens
|
||||
? {
|
||||
id: row.account_tokens.id,
|
||||
name: row.account_tokens.name,
|
||||
accountId: row.account_tokens.accountId,
|
||||
|
||||
@@ -8,12 +8,15 @@ function readSource(relativePath: string): string {
|
||||
describe('proxy route architecture boundaries', () => {
|
||||
it('keeps shared protocol helpers out of chat route', () => {
|
||||
const source = readSource('./chat.ts');
|
||||
const surfaceSource = readSource('../../proxy-core/surfaces/chatSurface.ts');
|
||||
expect(source).toContain("from '../../proxy-core/surfaces/chatSurface.js'");
|
||||
expect(source).not.toContain("from './chatFormats.js'");
|
||||
expect(source).toContain("from '../../transformers/openai/chat/index.js'");
|
||||
expect(source).toContain("from '../../transformers/anthropic/messages/index.js'");
|
||||
expect(surfaceSource).toContain("from '../../transformers/openai/chat/index.js'");
|
||||
expect(surfaceSource).toContain("from '../../transformers/anthropic/messages/index.js'");
|
||||
});
|
||||
it('keeps anthropic-specific stream orchestration out of chat route', () => {
|
||||
const source = readSource('./chat.ts');
|
||||
const surfaceSource = readSource('../../proxy-core/surfaces/chatSurface.ts');
|
||||
expect(source).not.toContain('serializeAnthropicRawSseEvent');
|
||||
expect(source).not.toContain('syncAnthropicRawStreamStateFromEvent');
|
||||
expect(source).not.toContain('isAnthropicRawSseEventName');
|
||||
@@ -28,14 +31,15 @@ describe('proxy route architecture boundaries', () => {
|
||||
expect(source).not.toContain('anthropicMessagesTransformer.consumeSseEventBlock(');
|
||||
expect(source).not.toContain('anthropicMessagesTransformer.serializeUpstreamFinalAsStream(');
|
||||
expect(source).not.toContain('openAiChatTransformer.serializeUpstreamFinalAsStream(');
|
||||
expect(source).toContain('openAiChatTransformer.proxyStream.createSession(');
|
||||
expect(source).toContain('streamSession.consumeUpstreamFinalPayload(');
|
||||
expect(source).toContain('streamSession.run(');
|
||||
expect(surfaceSource).toContain('openAiChatTransformer.proxyStream.createSession(');
|
||||
expect(surfaceSource).toContain('streamSession.consumeUpstreamFinalPayload(');
|
||||
expect(surfaceSource).toContain('streamSession.run(');
|
||||
});
|
||||
|
||||
it('keeps chat endpoint retry and downgrade strategy out of the route', () => {
|
||||
const source = readSource('./chat.ts');
|
||||
expect(source).toContain('downstreamTransformer.compatibility.createEndpointStrategy(');
|
||||
const surfaceSource = readSource('../../proxy-core/surfaces/chatSurface.ts');
|
||||
expect(surfaceSource).toContain('downstreamTransformer.compatibility.createEndpointStrategy(');
|
||||
expect(source).not.toContain('anthropicMessagesTransformer.compatibility.shouldRetryNormalizedBody(');
|
||||
expect(source).not.toContain('buildMinimalJsonHeadersForCompatibility(');
|
||||
expect(source).not.toContain('promoteResponsesCandidateAfterLegacyChatError(');
|
||||
@@ -46,6 +50,8 @@ describe('proxy route architecture boundaries', () => {
|
||||
|
||||
it('keeps responses protocol assembly out of responses route', () => {
|
||||
const source = readSource('./responses.ts');
|
||||
const surfaceSource = readSource('../../proxy-core/surfaces/openAiResponsesSurface.ts');
|
||||
expect(source).toContain("from '../../proxy-core/surfaces/openAiResponsesSurface.js'");
|
||||
expect(source).not.toContain('function toResponsesPayload(');
|
||||
expect(source).not.toContain('function createResponsesStreamState(');
|
||||
expect(source).not.toContain("from '../../transformers/openai/responses/conversion.js'");
|
||||
@@ -57,19 +63,20 @@ describe('proxy route architecture boundaries', () => {
|
||||
expect(source).not.toContain("from './protocolCompat.js'");
|
||||
expect(source).not.toContain('function shouldDowngradeFromChatToMessagesForResponses(');
|
||||
expect(source).not.toContain('function normalizeText(');
|
||||
expect(source).toContain('openAiResponsesTransformer.inbound.toOpenAiBody(');
|
||||
expect(source).toContain('openAiResponsesTransformer.compatibility.createEndpointStrategy(');
|
||||
expect(source).not.toContain('openAiResponsesTransformer.aggregator.createState(');
|
||||
expect(source).not.toContain('openAiResponsesTransformer.aggregator.serialize(');
|
||||
expect(source).not.toContain('openAiResponsesTransformer.aggregator.complete(');
|
||||
expect(source).toContain('openAiResponsesTransformer.proxyStream.createSession(');
|
||||
expect(source).toContain('streamSession.run(');
|
||||
expect(source).toContain('openAiResponsesTransformer.outbound.serializeFinal(');
|
||||
expect(surfaceSource).toContain('openAiResponsesTransformer.inbound.toOpenAiBody(');
|
||||
expect(surfaceSource).toContain('openAiResponsesTransformer.compatibility.createEndpointStrategy(');
|
||||
expect(surfaceSource).not.toContain('openAiResponsesTransformer.aggregator.createState(');
|
||||
expect(surfaceSource).not.toContain('openAiResponsesTransformer.aggregator.serialize(');
|
||||
expect(surfaceSource).not.toContain('openAiResponsesTransformer.aggregator.complete(');
|
||||
expect(surfaceSource).toContain('openAiResponsesTransformer.proxyStream.createSession(');
|
||||
expect(surfaceSource).toContain('streamSession.run(');
|
||||
expect(surfaceSource).toContain('openAiResponsesTransformer.outbound.serializeFinal(');
|
||||
});
|
||||
|
||||
it('keeps responses endpoint retry and downgrade strategy out of the route', () => {
|
||||
const source = readSource('./responses.ts');
|
||||
expect(source).toContain('openAiResponsesTransformer.compatibility.createEndpointStrategy(');
|
||||
const surfaceSource = readSource('../../proxy-core/surfaces/openAiResponsesSurface.ts');
|
||||
expect(surfaceSource).toContain('openAiResponsesTransformer.compatibility.createEndpointStrategy(');
|
||||
expect(source).not.toContain('openAiResponsesTransformer.compatibility.shouldRetry(');
|
||||
expect(source).not.toContain('openAiResponsesTransformer.compatibility.buildRetryBodies(');
|
||||
expect(source).not.toContain('openAiResponsesTransformer.compatibility.buildRetryHeaders(');
|
||||
@@ -87,35 +94,65 @@ describe('proxy route architecture boundaries', () => {
|
||||
|
||||
it('keeps gemini runtime closure in transformer-owned helpers', () => {
|
||||
const source = readSource('./gemini.ts');
|
||||
const surfaceSource = readSource('../../proxy-core/surfaces/geminiSurface.ts');
|
||||
expect(source).toContain("from '../../proxy-core/surfaces/geminiSurface.js'");
|
||||
expect(source).not.toContain('outbound.serializeAggregateResponse(');
|
||||
expect(source).not.toContain('aggregator.apply(');
|
||||
expect(source).not.toContain('stream.serializeAggregateSsePayload(');
|
||||
expect(source).not.toContain('stream.serializeAggregateJsonPayload(');
|
||||
expect(source).not.toContain('stream.applyJsonPayloadToAggregate(');
|
||||
expect(source).not.toContain('stream.parseSsePayloads(');
|
||||
expect(source).toContain('stream.consumeUpstreamSseBuffer(');
|
||||
expect(source).toContain('stream.serializeUpstreamJsonPayload(');
|
||||
expect(surfaceSource).toContain('stream.consumeUpstreamSseBuffer(');
|
||||
expect(surfaceSource).toContain('stream.serializeUpstreamJsonPayload(');
|
||||
});
|
||||
|
||||
it('keeps proxy file persistence out of files route', () => {
|
||||
const source = readSource('./files.ts');
|
||||
expect(source).toContain("from '../../proxy-core/surfaces/filesSurface.js'");
|
||||
expect(source).not.toContain('saveProxyFile(');
|
||||
expect(source).not.toContain('listProxyFilesByOwner(');
|
||||
expect(source).not.toContain('getProxyFileByPublicIdForOwner(');
|
||||
expect(source).not.toContain('getProxyFileContentByPublicIdForOwner(');
|
||||
expect(source).not.toContain('softDeleteProxyFileByPublicIdForOwner(');
|
||||
});
|
||||
|
||||
it('keeps chat stream lifecycle behind transformer-owned facade', () => {
|
||||
const source = readSource('./chat.ts');
|
||||
const surfaceSource = readSource('../../proxy-core/surfaces/chatSurface.ts');
|
||||
expect(source).not.toContain("from '../../transformers/shared/protocolLifecycle.js'");
|
||||
expect(source).not.toContain('createProxyStreamLifecycle');
|
||||
expect(source).not.toContain('let shouldTerminateEarly = false;');
|
||||
expect(source).not.toContain('const consumeSseBuffer = (incoming: string): string => {');
|
||||
expect(source).not.toContain('writeDone();');
|
||||
expect(source).toContain('openAiChatTransformer.proxyStream.createSession(');
|
||||
expect(surfaceSource).toContain('openAiChatTransformer.proxyStream.createSession(');
|
||||
});
|
||||
|
||||
it('keeps responses stream lifecycle behind transformer-owned facade', () => {
|
||||
const source = readSource('./responses.ts');
|
||||
const surfaceSource = readSource('../../proxy-core/surfaces/openAiResponsesSurface.ts');
|
||||
expect(source).not.toContain("from '../../transformers/shared/protocolLifecycle.js'");
|
||||
expect(source).not.toContain('createProxyStreamLifecycle');
|
||||
expect(source).not.toContain('const consumeSseBuffer = (incoming: string): string => {');
|
||||
expect(source).not.toContain('openAiResponsesTransformer.aggregator.complete(');
|
||||
expect(source).not.toContain('reply.raw.end();');
|
||||
expect(source).toContain('reply.hijack();');
|
||||
expect(source).toContain('openAiResponsesTransformer.proxyStream.createSession(');
|
||||
expect(surfaceSource).not.toContain('openAiResponsesTransformer.aggregator.complete(');
|
||||
expect(surfaceSource).toContain('reply.hijack();');
|
||||
expect(surfaceSource).toContain('openAiResponsesTransformer.proxyStream.createSession(');
|
||||
});
|
||||
|
||||
it('keeps canonical transformer contracts imported from the transformer boundary only', () => {
|
||||
const chatSource = readSource('./chat.ts');
|
||||
const responsesSource = readSource('./responses.ts');
|
||||
const geminiSource = readSource('./gemini.ts');
|
||||
|
||||
expect(chatSource).not.toContain("from '../proxy-core/");
|
||||
expect(responsesSource).not.toContain("from '../proxy-core/");
|
||||
expect(geminiSource).not.toContain("from '../proxy-core/");
|
||||
expect(chatSource).not.toContain("from '../../transformers/contracts.js'");
|
||||
expect(responsesSource).not.toContain("from '../../transformers/contracts.js'");
|
||||
expect(geminiSource).not.toContain("from '../../transformers/contracts.js'");
|
||||
expect(chatSource).not.toContain("from '../../transformers/canonical/");
|
||||
expect(responsesSource).not.toContain("from '../../transformers/canonical/");
|
||||
expect(geminiSource).not.toContain("from '../../transformers/canonical/");
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
+7
-1003
File diff suppressed because it is too large
Load Diff
@@ -19,7 +19,7 @@ describe('extractClaudeCodeSessionId', () => {
|
||||
});
|
||||
|
||||
describe('isCodexResponsesSurface', () => {
|
||||
it('detects Codex responses surface from originator and stainless headers', () => {
|
||||
it('detects Codex responses surface from originator, stainless, and turn-state headers', () => {
|
||||
expect(isCodexResponsesSurface({
|
||||
originator: 'codex_cli_rs',
|
||||
})).toBe(true);
|
||||
@@ -27,6 +27,10 @@ describe('isCodexResponsesSurface', () => {
|
||||
expect(isCodexResponsesSurface({
|
||||
'x-stainless-lang': 'typescript',
|
||||
})).toBe(true);
|
||||
|
||||
expect(isCodexResponsesSurface({
|
||||
'x-codex-turn-state': 'turn-state-123',
|
||||
})).toBe(true);
|
||||
});
|
||||
|
||||
it('returns false for generic responses clients', () => {
|
||||
@@ -109,4 +113,21 @@ describe('detectDownstreamClientContext', () => {
|
||||
clientKind: 'generic',
|
||||
});
|
||||
});
|
||||
|
||||
it('recognizes Gemini CLI internal routes as a first-class downstream client kind', () => {
|
||||
expect(detectDownstreamClientContext({
|
||||
downstreamPath: '/v1internal:generateContent',
|
||||
body: {
|
||||
model: 'gpt-4.1',
|
||||
contents: [
|
||||
{
|
||||
role: 'user',
|
||||
parts: [{ text: 'hello' }],
|
||||
},
|
||||
],
|
||||
},
|
||||
})).toEqual({
|
||||
clientKind: 'gemini_cli',
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,4 +1,11 @@
|
||||
export type DownstreamClientKind = 'generic' | 'codex' | 'claude_code';
|
||||
import { extractClaudeCodeSessionId as extractClaudeCodeSessionIdViaProfile } from '../../proxy-core/cliProfiles/claudeCodeProfile.js';
|
||||
import {
|
||||
detectCliProfile,
|
||||
} from '../../proxy-core/cliProfiles/registry.js';
|
||||
import { isCodexResponsesSurface as isCodexResponsesSurfaceViaProfile } from '../../proxy-core/cliProfiles/codexProfile.js';
|
||||
import type { CliProfileId } from '../../proxy-core/cliProfiles/types.js';
|
||||
|
||||
export type DownstreamClientKind = CliProfileId;
|
||||
|
||||
export type DownstreamClientContext = {
|
||||
clientKind: DownstreamClientKind;
|
||||
@@ -6,76 +13,12 @@ export type DownstreamClientContext = {
|
||||
traceHint?: string;
|
||||
};
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return !!value && typeof value === 'object';
|
||||
}
|
||||
|
||||
function headerValueToString(value: unknown): string | null {
|
||||
if (typeof value === 'string') {
|
||||
const trimmed = value.trim();
|
||||
return trimmed || null;
|
||||
}
|
||||
|
||||
if (Array.isArray(value)) {
|
||||
for (const item of value) {
|
||||
if (typeof item !== 'string') continue;
|
||||
const trimmed = item.trim();
|
||||
if (trimmed) return trimmed;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function getHeaderValue(headers: Record<string, unknown> | undefined, targetKey: string): string | null {
|
||||
if (!headers) return null;
|
||||
const normalizedTarget = targetKey.trim().toLowerCase();
|
||||
|
||||
for (const [rawKey, rawValue] of Object.entries(headers)) {
|
||||
if (rawKey.trim().toLowerCase() !== normalizedTarget) continue;
|
||||
return headerValueToString(rawValue);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
export function isCodexResponsesSurface(headers?: Record<string, unknown>): boolean {
|
||||
if (!headers) return false;
|
||||
|
||||
let sawOpenAiBeta = false;
|
||||
let sawStainless = false;
|
||||
|
||||
for (const [rawKey, rawValue] of Object.entries(headers)) {
|
||||
const key = rawKey.trim().toLowerCase();
|
||||
const value = headerValueToString(rawValue);
|
||||
if (!key || !value) continue;
|
||||
|
||||
if (key === 'originator' && value.toLowerCase() === 'codex_cli_rs') {
|
||||
return true;
|
||||
}
|
||||
if (key === 'openai-beta') {
|
||||
sawOpenAiBeta = true;
|
||||
}
|
||||
if (key.startsWith('x-stainless-')) {
|
||||
sawStainless = true;
|
||||
}
|
||||
}
|
||||
|
||||
return sawOpenAiBeta || sawStainless;
|
||||
return isCodexResponsesSurfaceViaProfile(headers);
|
||||
}
|
||||
|
||||
const claudeCodeUserIdPattern = /^user_[0-9a-f]{64}_account__session_[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
||||
|
||||
export function extractClaudeCodeSessionId(userId: string): string | null {
|
||||
const trimmed = userId.trim();
|
||||
if (!claudeCodeUserIdPattern.test(trimmed)) return null;
|
||||
|
||||
const sessionPrefix = '__session_';
|
||||
const sessionIndex = trimmed.lastIndexOf(sessionPrefix);
|
||||
if (sessionIndex === -1) return null;
|
||||
|
||||
const sessionId = trimmed.slice(sessionIndex + sessionPrefix.length).trim();
|
||||
return sessionId || null;
|
||||
return extractClaudeCodeSessionIdViaProfile(userId);
|
||||
}
|
||||
|
||||
export function detectDownstreamClientContext(input: {
|
||||
@@ -83,38 +26,10 @@ export function detectDownstreamClientContext(input: {
|
||||
headers?: Record<string, unknown>;
|
||||
body?: unknown;
|
||||
}): DownstreamClientContext {
|
||||
const normalizedPath = input.downstreamPath.trim().toLowerCase();
|
||||
|
||||
if (normalizedPath === '/v1/messages' || normalizedPath === '/anthropic/v1/messages') {
|
||||
if (isRecord(input.body) && isRecord(input.body.metadata)) {
|
||||
const userId = typeof input.body.metadata.user_id === 'string'
|
||||
? input.body.metadata.user_id.trim()
|
||||
: '';
|
||||
const sessionId = userId ? extractClaudeCodeSessionId(userId) : null;
|
||||
if (sessionId) {
|
||||
return {
|
||||
clientKind: 'claude_code',
|
||||
sessionId,
|
||||
traceHint: sessionId,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
return { clientKind: 'generic' };
|
||||
}
|
||||
|
||||
if (normalizedPath.startsWith('/v1/responses') && isCodexResponsesSurface(input.headers)) {
|
||||
const sessionId = getHeaderValue(input.headers, 'session_id') || getHeaderValue(input.headers, 'session-id');
|
||||
if (sessionId) {
|
||||
return {
|
||||
clientKind: 'codex',
|
||||
sessionId,
|
||||
traceHint: sessionId,
|
||||
};
|
||||
}
|
||||
|
||||
return { clientKind: 'codex' };
|
||||
}
|
||||
|
||||
return { clientKind: 'generic' };
|
||||
const detected = detectCliProfile(input);
|
||||
return {
|
||||
clientKind: detected.id,
|
||||
...(detected.sessionId ? { sessionId: detected.sessionId } : {}),
|
||||
...(detected.traceHint ? { traceHint: detected.traceHint } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,159 +1 @@
|
||||
import { FastifyInstance, FastifyReply, FastifyRequest } from 'fastify';
|
||||
import { getProxyResourceOwner } from '../../middleware/auth.js';
|
||||
import {
|
||||
getProxyFileByPublicIdForOwner,
|
||||
getProxyFileContentByPublicIdForOwner,
|
||||
listProxyFilesByOwner,
|
||||
saveProxyFile,
|
||||
softDeleteProxyFileByPublicIdForOwner,
|
||||
} from '../../services/proxyFileStore.js';
|
||||
import { ensureMultipartBufferParser, parseMultipartFormData } from './multipart.js';
|
||||
|
||||
function invalidRequest(reply: FastifyReply, message: string) {
|
||||
return reply.code(400).send({ error: { message, type: 'invalid_request_error' } });
|
||||
}
|
||||
|
||||
function notFound(reply: FastifyReply, message = 'file not found') {
|
||||
return reply.code(404).send({ error: { message, type: 'not_found_error' } });
|
||||
}
|
||||
|
||||
function toUnixSeconds(sqlDateTime: string | null | undefined): number {
|
||||
if (!sqlDateTime) return Math.floor(Date.now() / 1000);
|
||||
const parsed = Date.parse(sqlDateTime.replace(' ', 'T') + 'Z');
|
||||
if (!Number.isFinite(parsed)) return Math.floor(Date.now() / 1000);
|
||||
return Math.floor(parsed / 1000);
|
||||
}
|
||||
|
||||
function toFileObject(record: {
|
||||
publicId: string;
|
||||
filename: string;
|
||||
mimeType: string;
|
||||
purpose: string | null;
|
||||
byteSize: number;
|
||||
createdAt: string | null;
|
||||
}) {
|
||||
return {
|
||||
id: record.publicId,
|
||||
object: 'file',
|
||||
bytes: record.byteSize,
|
||||
created_at: toUnixSeconds(record.createdAt),
|
||||
filename: record.filename,
|
||||
purpose: record.purpose || 'assistants',
|
||||
mime_type: record.mimeType,
|
||||
};
|
||||
}
|
||||
|
||||
function inferMimeTypeFromFilename(filename: string): string {
|
||||
const normalized = filename.trim().toLowerCase();
|
||||
if (normalized.endsWith('.pdf')) return 'application/pdf';
|
||||
if (normalized.endsWith('.txt')) return 'text/plain';
|
||||
if (normalized.endsWith('.md') || normalized.endsWith('.markdown')) return 'text/markdown';
|
||||
if (normalized.endsWith('.json')) return 'application/json';
|
||||
if (normalized.endsWith('.png')) return 'image/png';
|
||||
if (normalized.endsWith('.jpg') || normalized.endsWith('.jpeg')) return 'image/jpeg';
|
||||
if (normalized.endsWith('.gif')) return 'image/gif';
|
||||
if (normalized.endsWith('.webp')) return 'image/webp';
|
||||
if (normalized.endsWith('.wav')) return 'audio/wav';
|
||||
if (normalized.endsWith('.mp3')) return 'audio/mpeg';
|
||||
return 'application/octet-stream';
|
||||
}
|
||||
|
||||
function isSupportedProxyFileMimeType(mimeType: string): boolean {
|
||||
return mimeType === 'application/pdf'
|
||||
|| mimeType === 'text/plain'
|
||||
|| mimeType === 'text/markdown'
|
||||
|| mimeType === 'application/json'
|
||||
|| mimeType.startsWith('image/')
|
||||
|| mimeType.startsWith('audio/');
|
||||
}
|
||||
|
||||
export async function filesProxyRoute(app: FastifyInstance) {
|
||||
ensureMultipartBufferParser(app);
|
||||
|
||||
app.post('/v1/files', async (request: FastifyRequest, reply: FastifyReply) => {
|
||||
const owner = getProxyResourceOwner(request);
|
||||
if (!owner) {
|
||||
return reply.code(401).send({ error: { message: 'Missing proxy auth context', type: 'authentication_error' } });
|
||||
}
|
||||
|
||||
const formData = await parseMultipartFormData(request);
|
||||
if (!formData) {
|
||||
return invalidRequest(reply, 'multipart/form-data with a file field is required');
|
||||
}
|
||||
|
||||
const fileEntry = formData.get('file');
|
||||
if (!fileEntry || typeof fileEntry !== 'object' || typeof (fileEntry as File).arrayBuffer !== 'function') {
|
||||
return invalidRequest(reply, 'file field is required');
|
||||
}
|
||||
|
||||
const filename = fileEntry.name || 'upload.bin';
|
||||
const mimeType = (fileEntry.type || inferMimeTypeFromFilename(filename)).trim().toLowerCase();
|
||||
if (!isSupportedProxyFileMimeType(mimeType)) {
|
||||
return invalidRequest(reply, `unsupported file mime type: ${mimeType || 'application/octet-stream'}`);
|
||||
}
|
||||
|
||||
const purposeValue = formData.get('purpose');
|
||||
const purpose = typeof purposeValue === 'string' && purposeValue.trim().length > 0
|
||||
? purposeValue.trim()
|
||||
: 'assistants';
|
||||
const buffer = Buffer.from(await fileEntry.arrayBuffer());
|
||||
|
||||
const saved = await saveProxyFile({
|
||||
...owner,
|
||||
filename,
|
||||
mimeType,
|
||||
purpose,
|
||||
contentBase64: buffer.toString('base64'),
|
||||
});
|
||||
return reply.send(toFileObject(saved));
|
||||
});
|
||||
|
||||
app.get('/v1/files', async (request: FastifyRequest, reply: FastifyReply) => {
|
||||
const owner = getProxyResourceOwner(request);
|
||||
if (!owner) {
|
||||
return reply.code(401).send({ error: { message: 'Missing proxy auth context', type: 'authentication_error' } });
|
||||
}
|
||||
const files = await listProxyFilesByOwner(owner);
|
||||
return reply.send({
|
||||
object: 'list',
|
||||
data: files.map((item) => toFileObject(item)),
|
||||
has_more: false,
|
||||
});
|
||||
});
|
||||
|
||||
app.get<{ Params: { fileId: string } }>('/v1/files/:fileId', async (request, reply) => {
|
||||
const owner = getProxyResourceOwner(request);
|
||||
if (!owner) {
|
||||
return reply.code(401).send({ error: { message: 'Missing proxy auth context', type: 'authentication_error' } });
|
||||
}
|
||||
const file = await getProxyFileByPublicIdForOwner(request.params.fileId, owner);
|
||||
if (!file) return notFound(reply);
|
||||
return reply.send(toFileObject(file));
|
||||
});
|
||||
|
||||
app.get<{ Params: { fileId: string } }>('/v1/files/:fileId/content', async (request, reply) => {
|
||||
const owner = getProxyResourceOwner(request);
|
||||
if (!owner) {
|
||||
return reply.code(401).send({ error: { message: 'Missing proxy auth context', type: 'authentication_error' } });
|
||||
}
|
||||
const file = await getProxyFileContentByPublicIdForOwner(request.params.fileId, owner);
|
||||
if (!file) return notFound(reply);
|
||||
reply.type(file.mimeType);
|
||||
reply.header('Content-Disposition', `inline; filename="${encodeURIComponent(file.filename)}"`);
|
||||
return reply.send(file.buffer);
|
||||
});
|
||||
|
||||
app.delete<{ Params: { fileId: string } }>('/v1/files/:fileId', async (request, reply) => {
|
||||
const owner = getProxyResourceOwner(request);
|
||||
if (!owner) {
|
||||
return reply.code(401).send({ error: { message: 'Missing proxy auth context', type: 'authentication_error' } });
|
||||
}
|
||||
const deleted = await softDeleteProxyFileByPublicIdForOwner(request.params.fileId, owner);
|
||||
if (!deleted) return notFound(reply);
|
||||
return reply.send({
|
||||
id: request.params.fileId,
|
||||
object: 'file',
|
||||
deleted: true,
|
||||
});
|
||||
});
|
||||
}
|
||||
export { filesProxyRoute } from '../../proxy-core/surfaces/filesSurface.js';
|
||||
|
||||
@@ -1,843 +1 @@
|
||||
import { FastifyInstance, FastifyReply, FastifyRequest } from 'fastify';
|
||||
import { TextDecoder } from 'node:util';
|
||||
import { fetch } from 'undici';
|
||||
import { and, eq } from 'drizzle-orm';
|
||||
import { db, schema } from '../../db/index.js';
|
||||
import { formatUtcSqlDateTime } from '../../services/localTimeService.js';
|
||||
import { parseProxyUsage } from '../../services/proxyUsageParser.js';
|
||||
import { isModelAllowedByPolicyOrAllowedRoutes } from '../../services/downstreamApiKeyService.js';
|
||||
import { tokenRouter } from '../../services/tokenRouter.js';
|
||||
import { buildOauthProviderHeaders } from '../../services/oauth/service.js';
|
||||
import { getOauthInfoFromExtraConfig } from '../../services/oauth/oauthAccount.js';
|
||||
import { withSiteProxyRequestInit } from '../../services/siteProxy.js';
|
||||
import { refreshModelsAndRebuildRoutes } from '../../services/modelService.js';
|
||||
import { getDownstreamRoutingPolicy } from './downstreamPolicy.js';
|
||||
import { executeEndpointFlow, type BuiltEndpointRequest } from './endpointFlow.js';
|
||||
import { composeProxyLogMessage } from './logPathMeta.js';
|
||||
import {
|
||||
buildUpstreamEndpointRequest,
|
||||
resolveUpstreamEndpointCandidates,
|
||||
} from './upstreamEndpoint.js';
|
||||
import {
|
||||
geminiGenerateContentTransformer,
|
||||
} from '../../transformers/gemini/generate-content/index.js';
|
||||
import { createChatEndpointStrategy } from '../../transformers/shared/chatEndpointStrategy.js';
|
||||
import { normalizeUpstreamFinalResponse } from '../../transformers/shared/normalized.js';
|
||||
import {
|
||||
createGeminiCliStreamReader,
|
||||
unwrapGeminiCliPayload,
|
||||
wrapGeminiCliRequest,
|
||||
} from './geminiCliCompat.js';
|
||||
import { dispatchRuntimeRequest } from './runtimeExecutor.js';
|
||||
|
||||
const MAX_RETRIES = 2;
|
||||
const GEMINI_MODEL_PROBES = [
|
||||
'gemini-2.5-flash',
|
||||
'gemini-2.0-flash',
|
||||
'gemini-1.5-flash',
|
||||
'gemini-pro',
|
||||
];
|
||||
const GEMINI_CLI_STATIC_MODELS = [
|
||||
{ name: 'models/gemini-2.5-pro', displayName: 'Gemini 2.5 Pro' },
|
||||
{ name: 'models/gemini-2.5-flash', displayName: 'Gemini 2.5 Flash' },
|
||||
{ name: 'models/gemini-2.5-flash-lite', displayName: 'Gemini 2.5 Flash Lite' },
|
||||
{ name: 'models/gemini-3-pro-preview', displayName: 'Gemini 3 Pro Preview' },
|
||||
{ name: 'models/gemini-3.1-pro-preview', displayName: 'Gemini 3.1 Pro Preview' },
|
||||
{ name: 'models/gemini-3-flash-preview', displayName: 'Gemini 3 Flash Preview' },
|
||||
{ name: 'models/gemini-3.1-flash-lite-preview', displayName: 'Gemini 3.1 Flash Lite Preview' },
|
||||
];
|
||||
const EMPTY_PROXY_USAGE = {
|
||||
promptTokens: 0,
|
||||
completionTokens: 0,
|
||||
totalTokens: 0,
|
||||
};
|
||||
|
||||
function isGeminiCliPlatform(platform: unknown): boolean {
|
||||
return String(platform || '').trim().toLowerCase() === 'gemini-cli';
|
||||
}
|
||||
|
||||
function isAntigravityPlatform(platform: unknown): boolean {
|
||||
return String(platform || '').trim().toLowerCase() === 'antigravity';
|
||||
}
|
||||
|
||||
function isInternalGeminiPlatform(platform: unknown): boolean {
|
||||
return isGeminiCliPlatform(platform) || isAntigravityPlatform(platform);
|
||||
}
|
||||
|
||||
function buildGeminiCliActionPath(input: {
|
||||
isStreamAction: boolean;
|
||||
isCountTokensAction: boolean;
|
||||
}) {
|
||||
if (input.isCountTokensAction) return '/v1internal:countTokens';
|
||||
if (input.isStreamAction) return '/v1internal:streamGenerateContent?alt=sse';
|
||||
return '/v1internal:generateContent';
|
||||
}
|
||||
|
||||
function isDirectGeminiFamilyPlatform(platform: unknown): boolean {
|
||||
const normalized = String(platform || '').trim().toLowerCase();
|
||||
return normalized === 'gemini' || normalized === 'gemini-cli' || normalized === 'antigravity';
|
||||
}
|
||||
|
||||
function omitGeminiCliModelField(body: unknown): Record<string, unknown> {
|
||||
if (!body || typeof body !== 'object' || Array.isArray(body)) return {};
|
||||
const { model: _model, ...rest } = body as Record<string, unknown>;
|
||||
return rest;
|
||||
}
|
||||
|
||||
async function selectGeminiChannel(request: FastifyRequest) {
|
||||
const policy = getDownstreamRoutingPolicy(request);
|
||||
for (const candidate of GEMINI_MODEL_PROBES) {
|
||||
const selected = await tokenRouter.selectChannel(candidate, policy);
|
||||
if (selected) return selected;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
async function selectNextGeminiProbeChannel(request: FastifyRequest, excludeChannelIds: number[]) {
|
||||
const policy = getDownstreamRoutingPolicy(request);
|
||||
for (const candidate of GEMINI_MODEL_PROBES) {
|
||||
const selected = await tokenRouter.selectNextChannel(candidate, excludeChannelIds, policy);
|
||||
if (selected) return selected;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function resolveDownstreamPath(request: FastifyRequest): string {
|
||||
const rawUrl = request.raw.url || request.url || '';
|
||||
const withoutQuery = rawUrl.split('?')[0] || rawUrl;
|
||||
return withoutQuery || '/v1beta/models';
|
||||
}
|
||||
|
||||
function resolveUpstreamPath(apiVersion: string, modelActionPath: string): string {
|
||||
const normalizedVersion = apiVersion.replace(/^\/+/, '');
|
||||
const normalizedAction = modelActionPath.replace(/^\/+/, '');
|
||||
return `/${normalizedVersion}/${normalizedAction}`;
|
||||
}
|
||||
|
||||
function hasDownstreamModelRestrictions(policy: { supportedModels?: unknown; allowedRouteIds?: unknown }): boolean {
|
||||
const supportedModels = Array.isArray(policy.supportedModels) ? policy.supportedModels : [];
|
||||
const allowedRouteIds = Array.isArray(policy.allowedRouteIds) ? policy.allowedRouteIds : [];
|
||||
return supportedModels.length > 0 || allowedRouteIds.length > 0;
|
||||
}
|
||||
|
||||
function extractGeminiListedModelName(item: unknown): string {
|
||||
if (!item || typeof item !== 'object') return '';
|
||||
const rawName = typeof (item as { name?: unknown }).name === 'string'
|
||||
? (item as { name: string }).name.trim()
|
||||
: '';
|
||||
if (!rawName) return '';
|
||||
return rawName.startsWith('models/') ? rawName.slice('models/'.length) : rawName;
|
||||
}
|
||||
|
||||
async function filterGeminiListedModelsForPolicy(
|
||||
payload: unknown,
|
||||
request: FastifyRequest,
|
||||
): Promise<unknown> {
|
||||
if (!payload || typeof payload !== 'object' || !Array.isArray((payload as { models?: unknown[] }).models)) {
|
||||
return payload;
|
||||
}
|
||||
|
||||
const policy = getDownstreamRoutingPolicy(request);
|
||||
if (!hasDownstreamModelRestrictions(policy)) {
|
||||
return payload;
|
||||
}
|
||||
|
||||
const filteredModels: unknown[] = [];
|
||||
for (const item of (payload as { models: unknown[] }).models) {
|
||||
const modelName = extractGeminiListedModelName(item);
|
||||
if (!modelName) continue;
|
||||
if (!await isModelAllowedByPolicyOrAllowedRoutes(modelName, policy)) continue;
|
||||
const decision = await tokenRouter.explainSelection?.(modelName, [], policy);
|
||||
if (decision && typeof decision.selectedChannelId !== 'number') continue;
|
||||
filteredModels.push(item);
|
||||
}
|
||||
|
||||
return {
|
||||
...(payload as Record<string, unknown>),
|
||||
models: filteredModels,
|
||||
};
|
||||
}
|
||||
|
||||
async function readRouteAwareGeminiModels(request: FastifyRequest): Promise<Array<{ name: string; displayName: string }>> {
|
||||
const policy = getDownstreamRoutingPolicy(request);
|
||||
const rows = await db.select({ modelName: schema.modelAvailability.modelName })
|
||||
.from(schema.modelAvailability)
|
||||
.innerJoin(schema.accounts, eq(schema.modelAvailability.accountId, schema.accounts.id))
|
||||
.innerJoin(schema.sites, eq(schema.accounts.siteId, schema.sites.id))
|
||||
.where(and(
|
||||
eq(schema.modelAvailability.available, true),
|
||||
eq(schema.accounts.status, 'active'),
|
||||
eq(schema.sites.status, 'active'),
|
||||
))
|
||||
.all();
|
||||
const routeAliases = await db.select({ displayName: schema.tokenRoutes.displayName })
|
||||
.from(schema.tokenRoutes)
|
||||
.where(eq(schema.tokenRoutes.enabled, true))
|
||||
.all();
|
||||
const deduped = Array.from(new Set([
|
||||
...rows.map((row) => String(row.modelName || '').trim()).filter(Boolean),
|
||||
...routeAliases.map((row) => String(row.displayName || '').trim()).filter(Boolean),
|
||||
])).sort();
|
||||
|
||||
const allowed: Array<{ name: string; displayName: string }> = [];
|
||||
for (const modelName of deduped) {
|
||||
if (!await isModelAllowedByPolicyOrAllowedRoutes(modelName, policy)) continue;
|
||||
const decision = await tokenRouter.explainSelection?.(modelName, [], policy);
|
||||
if (decision && typeof decision.selectedChannelId !== 'number') continue;
|
||||
allowed.push({
|
||||
name: `models/${modelName}`,
|
||||
displayName: modelName,
|
||||
});
|
||||
}
|
||||
|
||||
return allowed;
|
||||
}
|
||||
|
||||
async function logProxy(
|
||||
selected: any,
|
||||
modelRequested: string,
|
||||
status: string,
|
||||
httpStatus: number,
|
||||
latencyMs: number,
|
||||
errorMessage: string | null,
|
||||
retryCount: number,
|
||||
downstreamPath: string,
|
||||
upstreamPath: string | null,
|
||||
promptTokens = 0,
|
||||
completionTokens = 0,
|
||||
totalTokens = 0,
|
||||
) {
|
||||
try {
|
||||
const createdAt = formatUtcSqlDateTime(new Date());
|
||||
const normalizedErrorMessage = composeProxyLogMessage({
|
||||
downstreamPath,
|
||||
upstreamPath,
|
||||
errorMessage,
|
||||
});
|
||||
await db.insert(schema.proxyLogs).values({
|
||||
routeId: selected.channel.routeId,
|
||||
channelId: selected.channel.id,
|
||||
accountId: selected.account.id,
|
||||
modelRequested,
|
||||
modelActual: selected.actualModel || modelRequested,
|
||||
status,
|
||||
httpStatus,
|
||||
latencyMs,
|
||||
promptTokens,
|
||||
completionTokens,
|
||||
totalTokens,
|
||||
estimatedCost: 0,
|
||||
billingDetails: null,
|
||||
errorMessage: normalizedErrorMessage,
|
||||
retryCount,
|
||||
createdAt,
|
||||
}).run();
|
||||
} catch (error) {
|
||||
console.warn('[proxy/gemini] failed to write proxy log', error);
|
||||
}
|
||||
}
|
||||
|
||||
export async function geminiProxyRoute(app: FastifyInstance) {
|
||||
const listModels = async (request: FastifyRequest, reply: FastifyReply) => {
|
||||
const apiVersion = geminiGenerateContentTransformer.resolveProxyApiVersion(
|
||||
request.params as { geminiApiVersion?: string } | undefined,
|
||||
);
|
||||
const excludeChannelIds: number[] = [];
|
||||
let retryCount = 0;
|
||||
let lastStatus = 503;
|
||||
let lastText = 'No available channels for Gemini models';
|
||||
let lastContentType = 'application/json';
|
||||
|
||||
while (retryCount <= MAX_RETRIES) {
|
||||
const selected = retryCount === 0
|
||||
? await selectGeminiChannel(request)
|
||||
: await selectNextGeminiProbeChannel(request, excludeChannelIds);
|
||||
if (!selected) {
|
||||
return reply.code(lastStatus).type(lastContentType).send(lastText);
|
||||
}
|
||||
|
||||
excludeChannelIds.push(selected.channel.id);
|
||||
|
||||
try {
|
||||
if (!isDirectGeminiFamilyPlatform(selected.site.platform)) {
|
||||
let models = await readRouteAwareGeminiModels(request);
|
||||
if (models.length <= 0) {
|
||||
await refreshModelsAndRebuildRoutes();
|
||||
models = await readRouteAwareGeminiModels(request);
|
||||
}
|
||||
return reply.code(200).send({ models });
|
||||
}
|
||||
|
||||
if (isGeminiCliPlatform(selected.site.platform)) {
|
||||
const filtered = await filterGeminiListedModelsForPolicy(
|
||||
{ models: GEMINI_CLI_STATIC_MODELS },
|
||||
request,
|
||||
);
|
||||
return reply.code(200).send(filtered);
|
||||
}
|
||||
|
||||
const upstream = await fetch(
|
||||
geminiGenerateContentTransformer.resolveModelsUrl(selected.site.url, apiVersion, selected.tokenValue),
|
||||
{ method: 'GET' },
|
||||
);
|
||||
const text = await upstream.text();
|
||||
if (!upstream.ok) {
|
||||
lastStatus = upstream.status;
|
||||
lastText = text;
|
||||
lastContentType = upstream.headers.get('content-type') || 'application/json';
|
||||
await tokenRouter.recordFailure?.(selected.channel.id);
|
||||
if (retryCount < MAX_RETRIES) {
|
||||
retryCount += 1;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const parsed = JSON.parse(text);
|
||||
const filtered = await filterGeminiListedModelsForPolicy(parsed, request);
|
||||
return reply.code(upstream.status).send(filtered);
|
||||
} catch {
|
||||
return reply.code(upstream.status).type(upstream.headers.get('content-type') || 'application/json').send(text);
|
||||
}
|
||||
} catch (error) {
|
||||
await tokenRouter.recordFailure?.(selected.channel.id);
|
||||
lastStatus = 502;
|
||||
lastContentType = 'application/json';
|
||||
lastText = JSON.stringify({
|
||||
error: {
|
||||
message: error instanceof Error ? error.message : 'Gemini upstream request failed',
|
||||
type: 'upstream_error',
|
||||
},
|
||||
});
|
||||
if (retryCount < MAX_RETRIES) {
|
||||
retryCount += 1;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const handleGenerateContent = async (
|
||||
request: FastifyRequest,
|
||||
reply: FastifyReply,
|
||||
options?: {
|
||||
downstreamProtocol?: 'gemini' | 'gemini-cli';
|
||||
action?: 'generateContent' | 'streamGenerateContent' | 'countTokens';
|
||||
},
|
||||
) => {
|
||||
const downstreamProtocol = options?.downstreamProtocol || 'gemini';
|
||||
const isGeminiCliDownstream = downstreamProtocol === 'gemini-cli';
|
||||
const cliRequestedModel = isGeminiCliDownstream
|
||||
? (typeof (request.body as Record<string, unknown> | null | undefined)?.model === 'string'
|
||||
? String((request.body as Record<string, unknown>).model).trim()
|
||||
: '')
|
||||
: '';
|
||||
const parsedPath = isGeminiCliDownstream
|
||||
? {
|
||||
apiVersion: 'v1beta',
|
||||
modelActionPath: `models/${cliRequestedModel}:${options?.action || 'generateContent'}`,
|
||||
isStreamAction: options?.action === 'streamGenerateContent',
|
||||
requestedModel: cliRequestedModel,
|
||||
}
|
||||
: geminiGenerateContentTransformer.parseProxyRequestPath({
|
||||
rawUrl: request.raw.url || request.url || '',
|
||||
params: request.params as { geminiApiVersion?: string } | undefined,
|
||||
});
|
||||
const { apiVersion, modelActionPath, isStreamAction, requestedModel } = parsedPath;
|
||||
const isCountTokensAction = isGeminiCliDownstream
|
||||
? options?.action === 'countTokens'
|
||||
: modelActionPath.endsWith(':countTokens');
|
||||
if (!requestedModel) {
|
||||
return reply.code(400).send({
|
||||
error: { message: 'Gemini model path is required', type: 'invalid_request_error' },
|
||||
});
|
||||
}
|
||||
|
||||
const policy = getDownstreamRoutingPolicy(request);
|
||||
const downstreamPath = resolveDownstreamPath(request);
|
||||
const excludeChannelIds: number[] = [];
|
||||
let retryCount = 0;
|
||||
let lastStatus = 503;
|
||||
let lastText = 'No available channels for this model';
|
||||
let lastContentType = 'application/json';
|
||||
|
||||
while (retryCount <= MAX_RETRIES) {
|
||||
const selected = retryCount === 0
|
||||
? await tokenRouter.selectChannel(requestedModel, policy)
|
||||
: await tokenRouter.selectNextChannel(requestedModel, excludeChannelIds, policy);
|
||||
if (!selected) {
|
||||
return reply.code(lastStatus).type(lastContentType).send(lastText);
|
||||
}
|
||||
|
||||
excludeChannelIds.push(selected.channel.id);
|
||||
|
||||
const actualModel = selected.actualModel || requestedModel;
|
||||
const normalizedBody = geminiGenerateContentTransformer.inbound.normalizeRequest(
|
||||
isGeminiCliDownstream ? omitGeminiCliModelField(request.body) : (request.body || {}),
|
||||
actualModel,
|
||||
);
|
||||
const oauth = getOauthInfoFromExtraConfig(selected.account.extraConfig);
|
||||
const isGeminiCli = isGeminiCliPlatform(selected.site.platform);
|
||||
const isInternalGemini = isInternalGeminiPlatform(selected.site.platform);
|
||||
const isDirectGeminiFamily = isDirectGeminiFamilyPlatform(selected.site.platform);
|
||||
const startTime = Date.now();
|
||||
let upstreamPath = '';
|
||||
|
||||
try {
|
||||
if (isDirectGeminiFamily) {
|
||||
if (isGeminiCli && !oauth?.projectId) {
|
||||
lastStatus = 500;
|
||||
lastContentType = 'application/json';
|
||||
lastText = JSON.stringify({
|
||||
error: {
|
||||
message: 'Gemini CLI OAuth project is missing',
|
||||
type: 'server_error',
|
||||
},
|
||||
});
|
||||
await tokenRouter.recordFailure?.(selected.channel.id);
|
||||
if (retryCount < MAX_RETRIES) {
|
||||
retryCount += 1;
|
||||
continue;
|
||||
}
|
||||
return reply.code(lastStatus).type(lastContentType).send(lastText);
|
||||
}
|
||||
|
||||
const actualModelAction = modelActionPath.replace(
|
||||
/^models\/[^:]+/,
|
||||
`models/${actualModel}`,
|
||||
);
|
||||
upstreamPath = isInternalGemini
|
||||
? buildGeminiCliActionPath({ isStreamAction, isCountTokensAction })
|
||||
: resolveUpstreamPath(apiVersion, actualModelAction);
|
||||
const query = new URLSearchParams(request.query as Record<string, string>).toString();
|
||||
const requestBody = isInternalGemini
|
||||
? (
|
||||
isCountTokensAction
|
||||
? { request: normalizedBody }
|
||||
: wrapGeminiCliRequest({
|
||||
modelName: actualModel,
|
||||
projectId: oauth?.projectId || '',
|
||||
request: normalizedBody as Record<string, unknown>,
|
||||
})
|
||||
)
|
||||
: normalizedBody;
|
||||
const requestHeaders = isInternalGemini
|
||||
? {
|
||||
'Content-Type': 'application/json',
|
||||
...(isStreamAction ? { Accept: 'text/event-stream' } : {}),
|
||||
Authorization: `Bearer ${selected.tokenValue}`,
|
||||
...buildOauthProviderHeaders({
|
||||
extraConfig: typeof selected.account.extraConfig === 'string' ? selected.account.extraConfig : null,
|
||||
downstreamHeaders: request.headers as Record<string, unknown>,
|
||||
}),
|
||||
}
|
||||
: {
|
||||
'Content-Type': 'application/json',
|
||||
};
|
||||
const targetUrl = isInternalGemini
|
||||
? `${selected.site.url}${upstreamPath}`
|
||||
: geminiGenerateContentTransformer.resolveActionUrl(
|
||||
selected.site.url,
|
||||
apiVersion,
|
||||
actualModelAction,
|
||||
selected.tokenValue,
|
||||
query,
|
||||
);
|
||||
const upstream = isInternalGemini
|
||||
? await dispatchRuntimeRequest({
|
||||
siteUrl: selected.site.url,
|
||||
targetUrl,
|
||||
request: {
|
||||
endpoint: 'chat',
|
||||
path: upstreamPath,
|
||||
headers: requestHeaders,
|
||||
body: requestBody as Record<string, unknown>,
|
||||
runtime: {
|
||||
executor: isGeminiCli ? 'gemini-cli' : 'antigravity',
|
||||
modelName: actualModel,
|
||||
stream: isStreamAction,
|
||||
oauthProjectId: oauth?.projectId || null,
|
||||
action: isCountTokensAction
|
||||
? 'countTokens'
|
||||
: (isStreamAction ? 'streamGenerateContent' : 'generateContent'),
|
||||
},
|
||||
},
|
||||
buildInit: async (requestUrl, requestForFetch) => withSiteProxyRequestInit(requestUrl, {
|
||||
method: 'POST',
|
||||
headers: requestForFetch.headers,
|
||||
body: JSON.stringify(requestForFetch.body),
|
||||
}),
|
||||
})
|
||||
: await fetch(targetUrl, {
|
||||
method: 'POST',
|
||||
headers: requestHeaders,
|
||||
body: JSON.stringify(requestBody),
|
||||
});
|
||||
const contentType = upstream.headers.get('content-type') || 'application/json';
|
||||
if (!upstream.ok) {
|
||||
lastStatus = upstream.status;
|
||||
lastContentType = contentType;
|
||||
lastText = await upstream.text();
|
||||
await tokenRouter.recordFailure?.(selected.channel.id);
|
||||
await logProxy(
|
||||
selected,
|
||||
requestedModel,
|
||||
'failed',
|
||||
lastStatus,
|
||||
Date.now() - startTime,
|
||||
lastText,
|
||||
retryCount,
|
||||
downstreamPath,
|
||||
upstreamPath,
|
||||
);
|
||||
if (retryCount < MAX_RETRIES) {
|
||||
retryCount += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
return reply.code(lastStatus).send(JSON.parse(lastText));
|
||||
} catch {
|
||||
return reply.code(lastStatus).type(lastContentType).send(lastText);
|
||||
}
|
||||
}
|
||||
|
||||
if (geminiGenerateContentTransformer.stream.isSseContentType(contentType)) {
|
||||
reply.hijack();
|
||||
reply.raw.statusCode = upstream.status;
|
||||
reply.raw.setHeader('Content-Type', contentType || 'text/event-stream');
|
||||
const upstreamReader = upstream.body?.getReader();
|
||||
const reader = isInternalGemini && !isGeminiCliDownstream && upstreamReader
|
||||
? createGeminiCliStreamReader(upstreamReader)
|
||||
: upstreamReader;
|
||||
if (!reader) {
|
||||
const latency = Date.now() - startTime;
|
||||
await tokenRouter.recordSuccess?.(selected.channel.id, latency, 0);
|
||||
await logProxy(
|
||||
selected,
|
||||
requestedModel,
|
||||
'success',
|
||||
upstream.status,
|
||||
latency,
|
||||
null,
|
||||
retryCount,
|
||||
downstreamPath,
|
||||
upstreamPath,
|
||||
);
|
||||
reply.raw.end();
|
||||
return;
|
||||
}
|
||||
const aggregateState = geminiGenerateContentTransformer.stream.createAggregateState();
|
||||
const decoder = new TextDecoder();
|
||||
let rest = '';
|
||||
try {
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
if (!value) continue;
|
||||
const chunkText = decoder.decode(value, { stream: true });
|
||||
const consumed = geminiGenerateContentTransformer.stream.consumeUpstreamSseBuffer(
|
||||
aggregateState,
|
||||
rest + chunkText,
|
||||
);
|
||||
rest = consumed.rest;
|
||||
for (const line of consumed.lines) {
|
||||
reply.raw.write(line);
|
||||
}
|
||||
}
|
||||
const tail = decoder.decode();
|
||||
if (tail) {
|
||||
const consumed = geminiGenerateContentTransformer.stream.consumeUpstreamSseBuffer(
|
||||
aggregateState,
|
||||
rest + tail,
|
||||
);
|
||||
for (const line of consumed.lines) {
|
||||
reply.raw.write(line);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
reader.releaseLock();
|
||||
reply.raw.end();
|
||||
}
|
||||
const parsedUsage = parseProxyUsage(aggregateState);
|
||||
const latency = Date.now() - startTime;
|
||||
await tokenRouter.recordSuccess?.(selected.channel.id, latency, 0);
|
||||
await logProxy(
|
||||
selected,
|
||||
requestedModel,
|
||||
'success',
|
||||
upstream.status,
|
||||
latency,
|
||||
null,
|
||||
retryCount,
|
||||
downstreamPath,
|
||||
upstreamPath,
|
||||
parsedUsage.promptTokens,
|
||||
parsedUsage.completionTokens,
|
||||
parsedUsage.totalTokens,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const text = await upstream.text();
|
||||
const aggregateState = geminiGenerateContentTransformer.stream.createAggregateState();
|
||||
let parsedUsage = EMPTY_PROXY_USAGE;
|
||||
try {
|
||||
const parsed = JSON.parse(text);
|
||||
const unwrappedPayload = isInternalGemini && !isGeminiCliDownstream
|
||||
? unwrapGeminiCliPayload(parsed)
|
||||
: parsed;
|
||||
const responsePayload = isCountTokensAction
|
||||
? unwrappedPayload
|
||||
: geminiGenerateContentTransformer.stream.serializeUpstreamJsonPayload(
|
||||
aggregateState,
|
||||
unwrappedPayload,
|
||||
isStreamAction,
|
||||
);
|
||||
parsedUsage = parseProxyUsage(aggregateState);
|
||||
const latency = Date.now() - startTime;
|
||||
await tokenRouter.recordSuccess?.(selected.channel.id, latency, 0);
|
||||
await logProxy(
|
||||
selected,
|
||||
requestedModel,
|
||||
'success',
|
||||
upstream.status,
|
||||
latency,
|
||||
null,
|
||||
retryCount,
|
||||
downstreamPath,
|
||||
upstreamPath,
|
||||
parsedUsage.promptTokens,
|
||||
parsedUsage.completionTokens,
|
||||
parsedUsage.totalTokens,
|
||||
);
|
||||
return reply.code(upstream.status).send(
|
||||
isGeminiCliDownstream && !isCountTokensAction
|
||||
? { response: responsePayload }
|
||||
: responsePayload,
|
||||
);
|
||||
} catch {
|
||||
const latency = Date.now() - startTime;
|
||||
await tokenRouter.recordSuccess?.(selected.channel.id, latency, 0);
|
||||
await logProxy(
|
||||
selected,
|
||||
requestedModel,
|
||||
'success',
|
||||
upstream.status,
|
||||
latency,
|
||||
null,
|
||||
retryCount,
|
||||
downstreamPath,
|
||||
upstreamPath,
|
||||
);
|
||||
return reply.code(upstream.status).type(contentType || 'application/json').send(text);
|
||||
}
|
||||
}
|
||||
|
||||
if (isCountTokensAction) {
|
||||
lastStatus = 501;
|
||||
lastContentType = 'application/json';
|
||||
lastText = JSON.stringify({
|
||||
error: {
|
||||
message: 'Gemini countTokens compatibility is not implemented for this upstream',
|
||||
type: 'invalid_request_error',
|
||||
},
|
||||
});
|
||||
return reply.code(lastStatus).type(lastContentType).send(lastText);
|
||||
}
|
||||
|
||||
const openAiBody = geminiGenerateContentTransformer.compatibility.buildOpenAiBodyFromGeminiRequest({
|
||||
body: normalizedBody as Record<string, unknown>,
|
||||
modelName: actualModel,
|
||||
stream: isStreamAction,
|
||||
});
|
||||
const endpointCandidates = await resolveUpstreamEndpointCandidates(
|
||||
{
|
||||
site: selected.site,
|
||||
account: selected.account,
|
||||
},
|
||||
actualModel,
|
||||
'openai',
|
||||
requestedModel,
|
||||
);
|
||||
const buildEndpointRequest = (
|
||||
endpoint: 'chat' | 'messages' | 'responses',
|
||||
requestOptions: { forceNormalizeClaudeBody?: boolean } = {},
|
||||
) => {
|
||||
const endpointRequest = buildUpstreamEndpointRequest({
|
||||
endpoint,
|
||||
modelName: actualModel,
|
||||
stream: isStreamAction,
|
||||
tokenValue: selected.tokenValue,
|
||||
oauthProvider: oauth?.provider,
|
||||
oauthProjectId: oauth?.projectId,
|
||||
sitePlatform: selected.site.platform,
|
||||
siteUrl: selected.site.url,
|
||||
openaiBody: openAiBody,
|
||||
downstreamFormat: 'openai',
|
||||
forceNormalizeClaudeBody: requestOptions.forceNormalizeClaudeBody,
|
||||
downstreamHeaders: request.headers as Record<string, unknown>,
|
||||
providerHeaders: buildOauthProviderHeaders({
|
||||
extraConfig: typeof selected.account.extraConfig === 'string' ? selected.account.extraConfig : null,
|
||||
downstreamHeaders: request.headers as Record<string, unknown>,
|
||||
}),
|
||||
});
|
||||
return {
|
||||
endpoint,
|
||||
path: endpointRequest.path,
|
||||
headers: endpointRequest.headers,
|
||||
body: endpointRequest.body as Record<string, unknown>,
|
||||
runtime: endpointRequest.runtime,
|
||||
};
|
||||
};
|
||||
const dispatchRequest = (compatibilityRequest: BuiltEndpointRequest, targetUrl?: string) => (
|
||||
dispatchRuntimeRequest({
|
||||
siteUrl: selected.site.url,
|
||||
targetUrl,
|
||||
request: compatibilityRequest,
|
||||
buildInit: async (requestUrl, requestForFetch) => withSiteProxyRequestInit(requestUrl, {
|
||||
method: 'POST',
|
||||
headers: requestForFetch.headers,
|
||||
body: JSON.stringify(requestForFetch.body),
|
||||
}),
|
||||
})
|
||||
);
|
||||
const endpointStrategy = createChatEndpointStrategy({
|
||||
downstreamFormat: 'openai',
|
||||
endpointCandidates,
|
||||
modelName: actualModel,
|
||||
requestedModelHint: requestedModel,
|
||||
sitePlatform: selected.site.platform,
|
||||
isStream: isStreamAction,
|
||||
buildRequest: ({ endpoint, forceNormalizeClaudeBody }) => buildEndpointRequest(
|
||||
endpoint,
|
||||
{ forceNormalizeClaudeBody },
|
||||
),
|
||||
dispatchRequest,
|
||||
});
|
||||
const endpointResult = await executeEndpointFlow({
|
||||
siteUrl: selected.site.url,
|
||||
endpointCandidates,
|
||||
buildRequest: (endpoint) => buildEndpointRequest(endpoint),
|
||||
dispatchRequest,
|
||||
tryRecover: endpointStrategy.tryRecover,
|
||||
shouldDowngrade: endpointStrategy.shouldDowngrade,
|
||||
});
|
||||
if (!endpointResult.ok) {
|
||||
lastStatus = endpointResult.status;
|
||||
lastContentType = 'application/json';
|
||||
lastText = JSON.stringify({
|
||||
error: {
|
||||
message: endpointResult.errText,
|
||||
type: 'upstream_error',
|
||||
},
|
||||
});
|
||||
await tokenRouter.recordFailure?.(selected.channel.id);
|
||||
await logProxy(
|
||||
selected,
|
||||
requestedModel,
|
||||
'failed',
|
||||
lastStatus,
|
||||
Date.now() - startTime,
|
||||
endpointResult.errText,
|
||||
retryCount,
|
||||
downstreamPath,
|
||||
null,
|
||||
);
|
||||
if (retryCount < MAX_RETRIES) {
|
||||
retryCount += 1;
|
||||
continue;
|
||||
}
|
||||
return reply.code(lastStatus).type(lastContentType).send(lastText);
|
||||
}
|
||||
|
||||
upstreamPath = endpointResult.upstreamPath;
|
||||
const upstream = endpointResult.upstream;
|
||||
const rawText = await upstream.text();
|
||||
let upstreamData: unknown = rawText;
|
||||
try {
|
||||
upstreamData = JSON.parse(rawText);
|
||||
} catch {}
|
||||
const parsedUsage = parseProxyUsage(upstreamData);
|
||||
const normalizedFinal = normalizeUpstreamFinalResponse(upstreamData, actualModel, rawText);
|
||||
const geminiResponse = geminiGenerateContentTransformer.compatibility.serializeNormalizedFinalToGemini({
|
||||
normalized: normalizedFinal,
|
||||
usage: {
|
||||
promptTokens: parsedUsage.promptTokens,
|
||||
completionTokens: parsedUsage.completionTokens,
|
||||
totalTokens: parsedUsage.totalTokens,
|
||||
},
|
||||
});
|
||||
const latency = Date.now() - startTime;
|
||||
await tokenRouter.recordSuccess?.(selected.channel.id, latency, 0);
|
||||
await logProxy(
|
||||
selected,
|
||||
requestedModel,
|
||||
'success',
|
||||
upstream.status,
|
||||
latency,
|
||||
null,
|
||||
retryCount,
|
||||
downstreamPath,
|
||||
upstreamPath,
|
||||
parsedUsage.promptTokens,
|
||||
parsedUsage.completionTokens,
|
||||
parsedUsage.totalTokens,
|
||||
);
|
||||
return reply.code(upstream.status).send(
|
||||
isGeminiCliDownstream
|
||||
? { response: geminiResponse }
|
||||
: geminiResponse,
|
||||
);
|
||||
} catch (error) {
|
||||
lastStatus = 502;
|
||||
lastContentType = 'application/json';
|
||||
lastText = JSON.stringify({
|
||||
error: {
|
||||
message: error instanceof Error ? error.message : 'Gemini upstream request failed',
|
||||
type: 'upstream_error',
|
||||
},
|
||||
});
|
||||
await tokenRouter.recordFailure?.(selected.channel.id);
|
||||
await logProxy(
|
||||
selected,
|
||||
requestedModel,
|
||||
'failed',
|
||||
0,
|
||||
Date.now() - startTime,
|
||||
error instanceof Error ? error.message : 'Gemini upstream request failed',
|
||||
retryCount,
|
||||
downstreamPath,
|
||||
upstreamPath || null,
|
||||
);
|
||||
if (retryCount < MAX_RETRIES) {
|
||||
retryCount += 1;
|
||||
continue;
|
||||
}
|
||||
return reply.code(lastStatus).type(lastContentType).send(lastText);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const generateContent = async (request: FastifyRequest, reply: FastifyReply) => handleGenerateContent(request, reply);
|
||||
const geminiCliGenerateContent = async (request: FastifyRequest, reply: FastifyReply) => handleGenerateContent(request, reply, {
|
||||
downstreamProtocol: 'gemini-cli',
|
||||
action: 'generateContent',
|
||||
});
|
||||
const geminiCliStreamGenerateContent = async (request: FastifyRequest, reply: FastifyReply) => handleGenerateContent(request, reply, {
|
||||
downstreamProtocol: 'gemini-cli',
|
||||
action: 'streamGenerateContent',
|
||||
});
|
||||
const geminiCliCountTokens = async (request: FastifyRequest, reply: FastifyReply) => handleGenerateContent(request, reply, {
|
||||
downstreamProtocol: 'gemini-cli',
|
||||
action: 'countTokens',
|
||||
});
|
||||
|
||||
app.get('/v1beta/models', listModels);
|
||||
app.get('/gemini/:geminiApiVersion/models', listModels);
|
||||
app.post('/v1beta/models/*', generateContent);
|
||||
app.post('/gemini/:geminiApiVersion/models/*', generateContent);
|
||||
app.post('/v1internal::generateContent', geminiCliGenerateContent);
|
||||
app.post('/v1internal::streamGenerateContent', geminiCliStreamGenerateContent);
|
||||
app.post('/v1internal::countTokens', geminiCliCountTokens);
|
||||
}
|
||||
export { geminiProxyRoute } from '../../proxy-core/surfaces/geminiSurface.js';
|
||||
|
||||
@@ -1,114 +1,4 @@
|
||||
import type { ProxyResourceOwner } from '../../middleware/auth.js';
|
||||
import { getProxyFileByPublicIdForOwner } from '../../services/proxyFileStore.js';
|
||||
import {
|
||||
inferInputFileMimeType,
|
||||
normalizeInputFileBlock,
|
||||
toOpenAiChatFileBlock,
|
||||
toResponsesInputFileBlock,
|
||||
type NormalizedInputFile,
|
||||
} from '../../transformers/shared/inputFile.js';
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return !!value && typeof value === 'object';
|
||||
}
|
||||
|
||||
function isSupportedFileMimeType(mimeType: string): boolean {
|
||||
return mimeType === 'application/pdf'
|
||||
|| mimeType === 'text/plain'
|
||||
|| mimeType === 'text/markdown'
|
||||
|| mimeType === 'application/json'
|
||||
|| mimeType.startsWith('image/')
|
||||
|| mimeType.startsWith('audio/');
|
||||
}
|
||||
|
||||
export class ProxyInputFileResolutionError extends Error {
|
||||
statusCode: number;
|
||||
payload: unknown;
|
||||
|
||||
constructor(statusCode: number, payload: unknown) {
|
||||
super(typeof payload === 'object' && payload && 'error' in (payload as Record<string, unknown>)
|
||||
? String(((payload as { error?: { message?: unknown } }).error?.message) || 'input file resolution failed')
|
||||
: 'input file resolution failed');
|
||||
this.name = 'ProxyInputFileResolutionError';
|
||||
this.statusCode = statusCode;
|
||||
this.payload = payload;
|
||||
}
|
||||
}
|
||||
|
||||
function buildInvalidRequest(message: string): ProxyInputFileResolutionError {
|
||||
return new ProxyInputFileResolutionError(400, {
|
||||
error: {
|
||||
message,
|
||||
type: 'invalid_request_error',
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function buildNotFound(message: string): ProxyInputFileResolutionError {
|
||||
return new ProxyInputFileResolutionError(404, {
|
||||
error: {
|
||||
message,
|
||||
type: 'not_found_error',
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async function resolveInputFileBlock(
|
||||
item: Record<string, unknown>,
|
||||
owner: ProxyResourceOwner,
|
||||
): Promise<Record<string, unknown>> {
|
||||
const normalized = normalizeInputFileBlock(item);
|
||||
if (!normalized) return item;
|
||||
|
||||
let resolved: NormalizedInputFile = { ...normalized };
|
||||
|
||||
if (!resolved.fileData && resolved.fileId?.startsWith('file-metapi-')) {
|
||||
const stored = await getProxyFileByPublicIdForOwner(resolved.fileId, owner);
|
||||
if (!stored) {
|
||||
throw buildNotFound(`file not found: ${resolved.fileId}`);
|
||||
}
|
||||
resolved = {
|
||||
filename: stored.filename,
|
||||
mimeType: stored.mimeType,
|
||||
fileData: stored.contentBase64,
|
||||
};
|
||||
}
|
||||
|
||||
const inferredMimeType = inferInputFileMimeType(resolved);
|
||||
if (resolved.fileData && (!inferredMimeType || !isSupportedFileMimeType(inferredMimeType))) {
|
||||
throw buildInvalidRequest(`unsupported file mime type: ${inferredMimeType || 'application/octet-stream'}`);
|
||||
}
|
||||
if (inferredMimeType) {
|
||||
resolved.mimeType = inferredMimeType;
|
||||
}
|
||||
|
||||
const type = typeof item.type === 'string' ? item.type : '';
|
||||
if (type === 'input_file') {
|
||||
return toResponsesInputFileBlock(resolved);
|
||||
}
|
||||
if (type === 'file') {
|
||||
return toOpenAiChatFileBlock(resolved);
|
||||
}
|
||||
return item;
|
||||
}
|
||||
|
||||
export async function inlineLocalInputFileReferences(
|
||||
value: unknown,
|
||||
owner: ProxyResourceOwner,
|
||||
): Promise<unknown> {
|
||||
if (Array.isArray(value)) {
|
||||
return Promise.all(value.map((item) => inlineLocalInputFileReferences(item, owner)));
|
||||
}
|
||||
|
||||
if (!isRecord(value)) return value;
|
||||
|
||||
const type = typeof value.type === 'string' ? value.type : '';
|
||||
if (type === 'input_file' || type === 'file') {
|
||||
return resolveInputFileBlock(value, owner);
|
||||
}
|
||||
|
||||
const entries = await Promise.all(
|
||||
Object.entries(value).map(async ([key, entryValue]) => [key, await inlineLocalInputFileReferences(entryValue, owner)] as const),
|
||||
);
|
||||
return Object.fromEntries(entries);
|
||||
}
|
||||
export {
|
||||
inlineLocalInputFileReferences,
|
||||
ProxyInputFileResolutionError,
|
||||
} from '../../proxy-core/surfaces/inputFilesSurface.js';
|
||||
|
||||
@@ -1,67 +1,21 @@
|
||||
import { FastifyInstance } from 'fastify';
|
||||
import { listModelsSurface } from '../../proxy-core/surfaces/modelsSurface.js';
|
||||
import { refreshModelsAndRebuildRoutes } from '../../services/modelService.js';
|
||||
import { getDownstreamRoutingPolicy } from './downstreamPolicy.js';
|
||||
import { tokenRouter } from '../../services/tokenRouter.js';
|
||||
import { isModelAllowedByPolicyOrAllowedRoutes } from '../../services/downstreamApiKeyService.js';
|
||||
|
||||
function isSearchPseudoModel(modelName: string): boolean {
|
||||
const normalized = (modelName || '').trim().toLowerCase();
|
||||
if (!normalized) return false;
|
||||
return normalized === '__search' || /^__.+_search$/.test(normalized);
|
||||
}
|
||||
|
||||
export async function modelsProxyRoute(app: FastifyInstance) {
|
||||
app.get('/v1/models', async (request) => {
|
||||
const downstreamPolicy = getDownstreamRoutingPolicy(request);
|
||||
|
||||
const readModels = async () => {
|
||||
const deduped = Array.from(new Set(await tokenRouter.getAvailableModels()))
|
||||
.filter((modelName) => !isSearchPseudoModel(modelName))
|
||||
.sort();
|
||||
const allowed: string[] = [];
|
||||
for (const modelName of deduped) {
|
||||
if (!await isModelAllowedByPolicyOrAllowedRoutes(modelName, downstreamPolicy)) {
|
||||
continue;
|
||||
}
|
||||
const decision = await tokenRouter.explainSelection(modelName, [], downstreamPolicy);
|
||||
if (typeof decision.selectedChannelId === 'number') {
|
||||
allowed.push(modelName);
|
||||
}
|
||||
}
|
||||
return allowed;
|
||||
};
|
||||
|
||||
let models = await readModels();
|
||||
if (models.length === 0) {
|
||||
await refreshModelsAndRebuildRoutes();
|
||||
models = await readModels();
|
||||
}
|
||||
|
||||
const wantsClaudeFormat = typeof request.headers['anthropic-version'] === 'string'
|
||||
|| typeof request.headers['x-api-key'] === 'string';
|
||||
if (wantsClaudeFormat) {
|
||||
const data = models.map((id) => ({
|
||||
id,
|
||||
type: 'model',
|
||||
display_name: id,
|
||||
created_at: new Date().toISOString(),
|
||||
}));
|
||||
return {
|
||||
data,
|
||||
first_id: data[0]?.id || null,
|
||||
last_id: data[data.length - 1]?.id || null,
|
||||
has_more: false,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
object: 'list',
|
||||
data: models.map(id => ({
|
||||
id,
|
||||
object: 'model',
|
||||
created: Math.floor(Date.now() / 1000),
|
||||
owned_by: 'metapi',
|
||||
})),
|
||||
};
|
||||
return listModelsSurface({
|
||||
downstreamPolicy,
|
||||
responseFormat: wantsClaudeFormat ? 'claude' : 'openai',
|
||||
tokenRouter,
|
||||
refreshModelsAndRebuildRoutes,
|
||||
isModelAllowed: isModelAllowedByPolicyOrAllowedRoutes,
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
@@ -467,4 +467,115 @@ describe('responses proxy codex oauth refresh', () => {
|
||||
});
|
||||
expect(String(insertedProxyLogs.at(-1)?.errorMessage || '')).toContain('tool execution failed');
|
||||
});
|
||||
|
||||
it('does not record success when a native responses stream closes before response.completed', async () => {
|
||||
fetchMock.mockResolvedValue(createSseResponse([
|
||||
'event: response.created\n',
|
||||
'data: {"type":"response.created","response":{"id":"resp_codex_truncated","model":"gpt-5.4","created_at":1706000000,"status":"in_progress","output":[]}}\n\n',
|
||||
'event: response.output_item.added\n',
|
||||
'data: {"type":"response.output_item.added","output_index":0,"item":{"id":"msg_codex_truncated","type":"message","role":"assistant","status":"in_progress","content":[]}}\n\n',
|
||||
'event: response.output_text.delta\n',
|
||||
'data: {"type":"response.output_text.delta","output_index":0,"item_id":"msg_codex_truncated","delta":"partial"}\n\n',
|
||||
'data: [DONE]\n\n',
|
||||
]));
|
||||
|
||||
const response = await app.inject({
|
||||
method: 'POST',
|
||||
url: '/v1/responses',
|
||||
payload: {
|
||||
model: 'gpt-5.4',
|
||||
input: 'hello codex',
|
||||
stream: true,
|
||||
},
|
||||
});
|
||||
|
||||
expect(response.statusCode).toBe(200);
|
||||
expect(response.body).toContain('event: response.failed');
|
||||
expect(response.body).not.toContain('event: response.completed');
|
||||
expect(recordSuccessMock).not.toHaveBeenCalled();
|
||||
expect(recordFailureMock).toHaveBeenCalledTimes(1);
|
||||
expect(insertedProxyLogs.at(-1)).toMatchObject({
|
||||
status: 'failed',
|
||||
httpStatus: 200,
|
||||
});
|
||||
expect(String(insertedProxyLogs.at(-1)?.errorMessage || '')).toContain('stream closed before response.completed');
|
||||
});
|
||||
|
||||
it('does not retry or mark failure after converting a non-stream upstream payload into SSE when post-stream usage accounting fails', async () => {
|
||||
resolveProxyUsageWithSelfLogFallbackMock.mockRejectedValueOnce(new Error('usage accounting failed'));
|
||||
fetchMock.mockResolvedValue(new Response(JSON.stringify({
|
||||
id: 'resp_nonstream_final',
|
||||
object: 'response',
|
||||
model: 'gpt-5.4',
|
||||
status: 'completed',
|
||||
output: [
|
||||
{
|
||||
id: 'msg_nonstream_final',
|
||||
type: 'message',
|
||||
role: 'assistant',
|
||||
status: 'completed',
|
||||
content: [
|
||||
{
|
||||
type: 'output_text',
|
||||
text: 'pong',
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
output_text: 'pong',
|
||||
usage: { input_tokens: 3, output_tokens: 1, total_tokens: 4 },
|
||||
}), {
|
||||
status: 200,
|
||||
headers: { 'content-type': 'application/json' },
|
||||
}));
|
||||
|
||||
const response = await app.inject({
|
||||
method: 'POST',
|
||||
url: '/v1/responses',
|
||||
payload: {
|
||||
model: 'gpt-5.4',
|
||||
input: 'hello codex',
|
||||
stream: true,
|
||||
},
|
||||
});
|
||||
|
||||
expect(response.statusCode).toBe(200);
|
||||
expect(response.body).toContain('event: response.completed');
|
||||
expect(response.body).toContain('"output_text":"pong"');
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||
expect(recordFailureMock).not.toHaveBeenCalled();
|
||||
expect(recordSuccessMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('does not retry or mark failure after streaming SSE success when post-stream usage accounting fails', async () => {
|
||||
resolveProxyUsageWithSelfLogFallbackMock.mockRejectedValueOnce(new Error('usage accounting failed'));
|
||||
fetchMock.mockResolvedValue(createSseResponse([
|
||||
'event: response.created\n',
|
||||
'data: {"type":"response.created","response":{"id":"resp_codex_stream_ok","model":"gpt-5.4","created_at":1706000000,"status":"in_progress","output":[]}}\n\n',
|
||||
'event: response.output_item.added\n',
|
||||
'data: {"type":"response.output_item.added","output_index":0,"item":{"id":"msg_codex_stream_ok","type":"message","role":"assistant","status":"in_progress","content":[]}}\n\n',
|
||||
'event: response.output_text.delta\n',
|
||||
'data: {"type":"response.output_text.delta","output_index":0,"item_id":"msg_codex_stream_ok","delta":"pong"}\n\n',
|
||||
'event: response.completed\n',
|
||||
'data: {"type":"response.completed","response":{"id":"resp_codex_stream_ok","model":"gpt-5.4","status":"completed","usage":{"input_tokens":3,"output_tokens":1,"total_tokens":4}}}\n\n',
|
||||
'data: [DONE]\n\n',
|
||||
]));
|
||||
|
||||
const response = await app.inject({
|
||||
method: 'POST',
|
||||
url: '/v1/responses',
|
||||
payload: {
|
||||
model: 'gpt-5.4',
|
||||
input: 'hello codex',
|
||||
stream: true,
|
||||
},
|
||||
});
|
||||
|
||||
expect(response.statusCode).toBe(200);
|
||||
expect(response.body).toContain('event: response.completed');
|
||||
expect(response.body).toContain('"id":"resp_codex_stream_ok"');
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||
expect(recordFailureMock).not.toHaveBeenCalled();
|
||||
expect(recordSuccessMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -182,4 +182,113 @@ describe('responses proxy compact upstream routing', () => {
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
it('preserves native response.compaction payloads when the upstream compact surface closes via SSE', async () => {
|
||||
fetchMock.mockResolvedValue(new Response([
|
||||
'event: response.output_item.added',
|
||||
`data: ${JSON.stringify({
|
||||
type: 'response.output_item.added',
|
||||
output_index: 0,
|
||||
item: {
|
||||
id: 'rs_123',
|
||||
type: 'compaction',
|
||||
encrypted_content: 'enc-compact-payload',
|
||||
},
|
||||
})}`,
|
||||
'',
|
||||
'event: response.completed',
|
||||
`data: ${JSON.stringify({
|
||||
type: 'response.completed',
|
||||
response: {
|
||||
id: 'cmp_123',
|
||||
object: 'response.compaction',
|
||||
created_at: 1700000000,
|
||||
output: [
|
||||
{
|
||||
id: 'rs_123',
|
||||
type: 'compaction',
|
||||
encrypted_content: 'enc-compact-payload',
|
||||
},
|
||||
],
|
||||
usage: {
|
||||
input_tokens: 1234,
|
||||
output_tokens: 321,
|
||||
total_tokens: 1555,
|
||||
},
|
||||
},
|
||||
})}`,
|
||||
'',
|
||||
'data: [DONE]',
|
||||
'',
|
||||
].join('\n'), {
|
||||
status: 200,
|
||||
headers: { 'content-type': 'text/event-stream' },
|
||||
}));
|
||||
|
||||
const response = await app.inject({
|
||||
method: 'POST',
|
||||
url: '/v1/responses/compact',
|
||||
payload: {
|
||||
model: 'gpt-5.2',
|
||||
input: 'hello',
|
||||
},
|
||||
});
|
||||
|
||||
expect(response.statusCode).toBe(200);
|
||||
expect(response.json()).toEqual({
|
||||
id: 'cmp_123',
|
||||
object: 'response.compaction',
|
||||
created_at: 1700000000,
|
||||
output: [
|
||||
{
|
||||
id: 'rs_123',
|
||||
type: 'compaction',
|
||||
encrypted_content: 'enc-compact-payload',
|
||||
},
|
||||
],
|
||||
usage: {
|
||||
input_tokens: 1234,
|
||||
output_tokens: 321,
|
||||
total_tokens: 1555,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('collects final payloads when non-stream compact upstreams still respond with SSE final payloads directly', async () => {
|
||||
fetchMock.mockResolvedValue(new Response([
|
||||
'event: response.completed',
|
||||
'data: {"id":"cmp_sse_123","object":"response.compaction","input_tokens":12,"output_tokens":3,"total_tokens":15,"output":[{"id":"rs_123","type":"compaction","encrypted_content":"enc-from-sse"}]}',
|
||||
'',
|
||||
'data: [DONE]',
|
||||
'',
|
||||
].join('\n'), {
|
||||
status: 200,
|
||||
headers: { 'content-type': 'text/event-stream; charset=utf-8' },
|
||||
}));
|
||||
|
||||
const response = await app.inject({
|
||||
method: 'POST',
|
||||
url: '/v1/responses/compact',
|
||||
payload: {
|
||||
model: 'gpt-5.2',
|
||||
input: 'hello',
|
||||
},
|
||||
});
|
||||
|
||||
expect(response.statusCode).toBe(200);
|
||||
expect(response.json()).toEqual({
|
||||
id: 'cmp_sse_123',
|
||||
object: 'response.compaction',
|
||||
input_tokens: 12,
|
||||
output_tokens: 3,
|
||||
total_tokens: 15,
|
||||
output: [
|
||||
{
|
||||
id: 'rs_123',
|
||||
type: 'compaction',
|
||||
encrypted_content: 'enc-from-sse',
|
||||
},
|
||||
],
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,814 +1,12 @@
|
||||
import { FastifyInstance, FastifyReply, FastifyRequest } from 'fastify';
|
||||
import { db, hasProxyLogDownstreamApiKeyIdColumn, schema } from '../../db/index.js';
|
||||
import { tokenRouter } from '../../services/tokenRouter.js';
|
||||
import { refreshModelsAndRebuildRoutes } from '../../services/modelService.js';
|
||||
import { reportProxyAllFailed, reportTokenExpired } from '../../services/alertService.js';
|
||||
import { isTokenExpiredError } from '../../services/alertRules.js';
|
||||
import { shouldRetryProxyRequest } from '../../services/proxyRetryPolicy.js';
|
||||
import { resolveProxyUsageWithSelfLogFallback } from '../../services/proxyUsageFallbackService.js';
|
||||
import { mergeProxyUsage, parseProxyUsage } from '../../services/proxyUsageParser.js';
|
||||
import { resolveChannelProxyUrl, withSiteRecordProxyRequestInit } from '../../services/siteProxy.js';
|
||||
import { getProxyUrlFromExtraConfig } from '../../services/accountExtraConfig.js';
|
||||
import { openAiResponsesTransformer } from '../../transformers/openai/responses/index.js';
|
||||
import {
|
||||
buildUpstreamEndpointRequest,
|
||||
resolveUpstreamEndpointCandidates,
|
||||
} from './upstreamEndpoint.js';
|
||||
import { ensureModelAllowedForDownstreamKey, getDownstreamRoutingPolicy, recordDownstreamCostUsage } from './downstreamPolicy.js';
|
||||
import { composeProxyLogMessage } from './logPathMeta.js';
|
||||
import { executeEndpointFlow, withUpstreamPath, type BuiltEndpointRequest } from './endpointFlow.js';
|
||||
import { detectProxyFailure } from './proxyFailureJudge.js';
|
||||
import { formatUtcSqlDateTime } from '../../services/localTimeService.js';
|
||||
import { resolveProxyLogBilling } from './proxyBilling.js';
|
||||
import { getProxyAuthContext, getProxyResourceOwner } from '../../middleware/auth.js';
|
||||
import { detectDownstreamClientContext, isCodexResponsesSurface, type DownstreamClientContext } from './downstreamClientContext.js';
|
||||
import { dispatchRuntimeRequest } from './runtimeExecutor.js';
|
||||
import { normalizeInputFileBlock } from '../../transformers/shared/inputFile.js';
|
||||
import {
|
||||
ProxyInputFileResolutionError,
|
||||
hasNonImageFileInputInOpenAiBody,
|
||||
resolveResponsesBodyInputFiles,
|
||||
} from '../../services/proxyInputFileResolver.js';
|
||||
import {
|
||||
buildOauthProviderHeaders,
|
||||
} from '../../services/oauth/service.js';
|
||||
import { getOauthInfoFromExtraConfig } from '../../services/oauth/oauthAccount.js';
|
||||
import { recordOauthQuotaResetHint } from '../../services/oauth/quota.js';
|
||||
import { refreshOauthAccessTokenSingleflight } from '../../services/oauth/refreshSingleflight.js';
|
||||
import { collectResponsesFinalPayloadFromSse } from './responsesSseFinal.js';
|
||||
import {
|
||||
createGeminiCliStreamReader,
|
||||
unwrapGeminiCliPayload,
|
||||
} from './geminiCliCompat.js';
|
||||
import type { FastifyInstance, FastifyReply, FastifyRequest } from 'fastify';
|
||||
import { handleOpenAiResponsesSurfaceRequest } from '../../proxy-core/surfaces/openAiResponsesSurface.js';
|
||||
import { ensureResponsesWebsocketTransport } from './responsesWebsocket.js';
|
||||
|
||||
const MAX_RETRIES = 2;
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return !!value && typeof value === 'object';
|
||||
}
|
||||
|
||||
function normalizeIncludeList(value: unknown): string[] {
|
||||
if (typeof value === 'string') {
|
||||
const trimmed = value.trim();
|
||||
return trimmed ? [trimmed] : [];
|
||||
}
|
||||
if (!Array.isArray(value)) return [];
|
||||
return value
|
||||
.map((item) => (typeof item === 'string' ? item.trim() : ''))
|
||||
.filter((item) => item.length > 0);
|
||||
}
|
||||
|
||||
function hasExplicitInclude(body: Record<string, unknown>): boolean {
|
||||
return Object.prototype.hasOwnProperty.call(body, 'include');
|
||||
}
|
||||
|
||||
function hasResponsesReasoningRequest(value: unknown): boolean {
|
||||
if (!isRecord(value)) return false;
|
||||
const relevantKeys = ['effort', 'budget_tokens', 'budgetTokens', 'max_tokens', 'maxTokens', 'summary'];
|
||||
return relevantKeys.some((key) => {
|
||||
const entry = value[key];
|
||||
if (typeof entry === 'string') return entry.trim().length > 0;
|
||||
return entry !== undefined && entry !== null;
|
||||
});
|
||||
}
|
||||
|
||||
function carriesResponsesReasoningContinuity(value: unknown): boolean {
|
||||
if (Array.isArray(value)) {
|
||||
return value.some((item) => carriesResponsesReasoningContinuity(item));
|
||||
}
|
||||
if (!isRecord(value)) return false;
|
||||
|
||||
const type = typeof value.type === 'string' ? value.type.trim().toLowerCase() : '';
|
||||
if (type === 'reasoning') {
|
||||
if (typeof value.encrypted_content === 'string' && value.encrypted_content.trim()) {
|
||||
return true;
|
||||
}
|
||||
if (Array.isArray(value.summary) && value.summary.length > 0) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
if (typeof value.reasoning_signature === 'string' && value.reasoning_signature.trim()) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return carriesResponsesReasoningContinuity(value.input)
|
||||
|| carriesResponsesReasoningContinuity(value.content);
|
||||
}
|
||||
|
||||
function wantsNativeResponsesReasoning(body: unknown): boolean {
|
||||
if (!isRecord(body)) return false;
|
||||
const include = normalizeIncludeList(body.include);
|
||||
if (include.some((item) => item.toLowerCase() === 'reasoning.encrypted_content')) {
|
||||
return true;
|
||||
}
|
||||
if (carriesResponsesReasoningContinuity(body.input)) {
|
||||
return true;
|
||||
}
|
||||
if (hasExplicitInclude(body)) {
|
||||
return false;
|
||||
}
|
||||
return hasResponsesReasoningRequest(body.reasoning);
|
||||
}
|
||||
|
||||
function carriesResponsesFileUrlInput(value: unknown): boolean {
|
||||
if (Array.isArray(value)) {
|
||||
return value.some((item) => carriesResponsesFileUrlInput(item));
|
||||
}
|
||||
if (!isRecord(value)) return false;
|
||||
|
||||
const normalizedFile = normalizeInputFileBlock(value);
|
||||
if (normalizedFile?.fileUrl) return true;
|
||||
|
||||
return Object.values(value).some((entry) => carriesResponsesFileUrlInput(entry));
|
||||
}
|
||||
|
||||
type UsageSummary = ReturnType<typeof parseProxyUsage>;
|
||||
|
||||
function deriveCodexExplicitSessionId(body: Record<string, unknown>): string | null {
|
||||
const promptCacheKey = typeof body.prompt_cache_key === 'string'
|
||||
? body.prompt_cache_key.trim()
|
||||
: '';
|
||||
return promptCacheKey || null;
|
||||
}
|
||||
|
||||
export async function responsesProxyRoute(app: FastifyInstance) {
|
||||
ensureResponsesWebsocketTransport(app);
|
||||
const handleResponsesRequest = async (
|
||||
request: FastifyRequest,
|
||||
reply: FastifyReply,
|
||||
downstreamPath: string,
|
||||
) => {
|
||||
const body = request.body as Record<string, unknown>;
|
||||
const clientContext = detectDownstreamClientContext({
|
||||
downstreamPath,
|
||||
headers: request.headers as Record<string, unknown>,
|
||||
body,
|
||||
});
|
||||
const defaultEncryptedReasoningInclude = isCodexResponsesSurface(
|
||||
request.headers as Record<string, unknown>,
|
||||
);
|
||||
const parsedRequestEnvelope = openAiResponsesTransformer.transformRequest(body, {
|
||||
defaultEncryptedReasoningInclude,
|
||||
});
|
||||
if (parsedRequestEnvelope.error) {
|
||||
return reply.code(parsedRequestEnvelope.error.statusCode).send(parsedRequestEnvelope.error.payload);
|
||||
}
|
||||
const requestEnvelope = parsedRequestEnvelope.value!;
|
||||
const requestedModel = requestEnvelope.model;
|
||||
const isStream = requestEnvelope.stream;
|
||||
const isCompactRequest = downstreamPath === '/v1/responses/compact';
|
||||
if (isCompactRequest && isStream) {
|
||||
return reply.code(400).send({
|
||||
error: {
|
||||
message: 'stream is not supported on /v1/responses/compact',
|
||||
type: 'invalid_request_error',
|
||||
},
|
||||
});
|
||||
}
|
||||
if (!await ensureModelAllowedForDownstreamKey(request, reply, requestedModel)) return;
|
||||
const downstreamPolicy = getDownstreamRoutingPolicy(request);
|
||||
const downstreamApiKeyId = getProxyAuthContext(request)?.keyId ?? null;
|
||||
const logDownstreamApiKeyId = downstreamApiKeyId !== null
|
||||
&& await hasProxyLogDownstreamApiKeyIdColumn();
|
||||
const excludeChannelIds: number[] = [];
|
||||
let retryCount = 0;
|
||||
|
||||
while (retryCount <= MAX_RETRIES) {
|
||||
let selected = retryCount === 0
|
||||
? await tokenRouter.selectChannel(requestedModel, downstreamPolicy)
|
||||
: await tokenRouter.selectNextChannel(requestedModel, excludeChannelIds, downstreamPolicy);
|
||||
|
||||
if (!selected && retryCount === 0) {
|
||||
await refreshModelsAndRebuildRoutes();
|
||||
selected = await tokenRouter.selectChannel(requestedModel, downstreamPolicy);
|
||||
}
|
||||
|
||||
if (!selected) {
|
||||
await reportProxyAllFailed({
|
||||
model: requestedModel,
|
||||
reason: 'No available channels after retries',
|
||||
});
|
||||
return reply.code(503).send({
|
||||
error: { message: 'No available channels for this model', type: 'server_error' },
|
||||
});
|
||||
}
|
||||
|
||||
excludeChannelIds.push(selected.channel.id);
|
||||
|
||||
const modelName = selected.actualModel || requestedModel;
|
||||
const oauth = getOauthInfoFromExtraConfig(selected.account.extraConfig);
|
||||
const isCodexSite = String(selected.site.platform || '').trim().toLowerCase() === 'codex';
|
||||
const owner = getProxyResourceOwner(request);
|
||||
let normalizedResponsesBody: Record<string, unknown> = {
|
||||
...requestEnvelope.parsed.normalizedBody,
|
||||
model: modelName,
|
||||
stream: isStream,
|
||||
};
|
||||
if (body.generate === false) {
|
||||
normalizedResponsesBody.generate = false;
|
||||
}
|
||||
if (owner) {
|
||||
try {
|
||||
normalizedResponsesBody = await resolveResponsesBodyInputFiles(normalizedResponsesBody, owner);
|
||||
} catch (error) {
|
||||
if (error instanceof ProxyInputFileResolutionError) {
|
||||
return reply.code(error.statusCode).send(error.payload);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
const openAiBody = openAiResponsesTransformer.inbound.toOpenAiBody(
|
||||
normalizedResponsesBody,
|
||||
modelName,
|
||||
isStream,
|
||||
{ defaultEncryptedReasoningInclude },
|
||||
);
|
||||
const hasNonImageFileInput = hasNonImageFileInputInOpenAiBody(openAiBody);
|
||||
const prefersNativeResponsesReasoning = wantsNativeResponsesReasoning(normalizedResponsesBody);
|
||||
const requiresNativeResponsesFileUrl = carriesResponsesFileUrlInput(normalizedResponsesBody.input);
|
||||
if (requiresNativeResponsesFileUrl && String(selected.site.platform || '').trim().toLowerCase() === 'claude') {
|
||||
return reply.code(400).send({
|
||||
error: {
|
||||
message: 'Responses input_file.file_url requires an upstream /v1/responses endpoint; current site only supports /v1/messages.',
|
||||
type: 'invalid_request_error',
|
||||
},
|
||||
});
|
||||
}
|
||||
const endpointCandidates = await resolveUpstreamEndpointCandidates(
|
||||
{
|
||||
site: selected.site,
|
||||
account: selected.account,
|
||||
},
|
||||
modelName,
|
||||
'responses',
|
||||
requestedModel,
|
||||
{
|
||||
hasNonImageFileInput,
|
||||
wantsNativeResponsesReasoning: prefersNativeResponsesReasoning,
|
||||
},
|
||||
);
|
||||
if (requiresNativeResponsesFileUrl) {
|
||||
endpointCandidates.splice(0, endpointCandidates.length, 'responses');
|
||||
}
|
||||
if (endpointCandidates.length === 0) {
|
||||
endpointCandidates.push('responses', 'chat', 'messages');
|
||||
}
|
||||
const buildProviderHeaders = () => (
|
||||
buildOauthProviderHeaders({
|
||||
extraConfig: typeof selected.account.extraConfig === 'string' ? selected.account.extraConfig : null,
|
||||
downstreamHeaders: request.headers as Record<string, unknown>,
|
||||
})
|
||||
);
|
||||
const buildEndpointRequest = (endpoint: 'chat' | 'messages' | 'responses') => {
|
||||
const upstreamStream = isStream || (isCodexSite && endpoint === 'responses');
|
||||
const codexExplicitSessionId = deriveCodexExplicitSessionId(normalizedResponsesBody);
|
||||
const endpointRequest = buildUpstreamEndpointRequest({
|
||||
endpoint,
|
||||
modelName,
|
||||
stream: upstreamStream,
|
||||
tokenValue: selected.tokenValue,
|
||||
oauthProvider: oauth?.provider,
|
||||
oauthProjectId: oauth?.projectId,
|
||||
sitePlatform: selected.site.platform,
|
||||
siteUrl: selected.site.url,
|
||||
openaiBody: openAiBody,
|
||||
downstreamFormat: 'responses',
|
||||
responsesOriginalBody: normalizedResponsesBody,
|
||||
downstreamHeaders: request.headers as Record<string, unknown>,
|
||||
providerHeaders: buildProviderHeaders(),
|
||||
codexExplicitSessionId,
|
||||
});
|
||||
const upstreamPath = (
|
||||
isCompactRequest && endpoint === 'responses'
|
||||
? `${endpointRequest.path}/compact`
|
||||
: endpointRequest.path
|
||||
);
|
||||
return {
|
||||
endpoint,
|
||||
path: upstreamPath,
|
||||
headers: endpointRequest.headers,
|
||||
body: endpointRequest.body as Record<string, unknown>,
|
||||
runtime: endpointRequest.runtime,
|
||||
};
|
||||
};
|
||||
const dispatchRequest = (compatibilityRequest: BuiltEndpointRequest, targetUrl?: string) => (
|
||||
dispatchRuntimeRequest({
|
||||
siteUrl: selected.site.url,
|
||||
targetUrl,
|
||||
request: compatibilityRequest,
|
||||
buildInit: (_requestUrl, requestForFetch) => withSiteRecordProxyRequestInit(selected.site, {
|
||||
method: 'POST',
|
||||
headers: requestForFetch.headers,
|
||||
body: JSON.stringify(requestForFetch.body),
|
||||
}),
|
||||
})
|
||||
);
|
||||
const endpointStrategy = openAiResponsesTransformer.compatibility.createEndpointStrategy({
|
||||
isStream: isStream || isCodexSite,
|
||||
requiresNativeResponsesFileUrl,
|
||||
dispatchRequest,
|
||||
});
|
||||
const tryRecover = async (ctx: Parameters<NonNullable<typeof endpointStrategy.tryRecover>>[0]) => {
|
||||
if (ctx.response.status === 401 && oauth) {
|
||||
try {
|
||||
const refreshed = await refreshOauthAccessTokenSingleflight(selected.account.id);
|
||||
selected.tokenValue = refreshed.accessToken;
|
||||
selected.account = {
|
||||
...selected.account,
|
||||
accessToken: refreshed.accessToken,
|
||||
extraConfig: refreshed.extraConfig ?? selected.account.extraConfig,
|
||||
};
|
||||
const refreshedRequest = buildEndpointRequest(ctx.request.endpoint);
|
||||
const refreshedTargetUrl = `${selected.site.url}${refreshedRequest.path}`;
|
||||
const refreshedResponse = await dispatchRequest(refreshedRequest, refreshedTargetUrl);
|
||||
if (refreshedResponse.ok) {
|
||||
return {
|
||||
upstream: refreshedResponse,
|
||||
upstreamPath: refreshedRequest.path,
|
||||
};
|
||||
}
|
||||
ctx.request = refreshedRequest;
|
||||
ctx.response = refreshedResponse;
|
||||
ctx.rawErrText = await refreshedResponse.text().catch(() => 'unknown error');
|
||||
} catch {
|
||||
return endpointStrategy.tryRecover(ctx);
|
||||
}
|
||||
}
|
||||
return endpointStrategy.tryRecover(ctx);
|
||||
};
|
||||
|
||||
const startTime = Date.now();
|
||||
|
||||
try {
|
||||
const endpointResult = await executeEndpointFlow({
|
||||
siteUrl: selected.site.url,
|
||||
proxyUrl: resolveChannelProxyUrl(selected.site, selected.account.extraConfig),
|
||||
endpointCandidates,
|
||||
buildRequest: (endpoint) => buildEndpointRequest(endpoint),
|
||||
dispatchRequest,
|
||||
tryRecover,
|
||||
shouldDowngrade: endpointStrategy.shouldDowngrade,
|
||||
onDowngrade: (ctx) => {
|
||||
logProxy(
|
||||
selected,
|
||||
requestedModel,
|
||||
'failed',
|
||||
ctx.response.status,
|
||||
Date.now() - startTime,
|
||||
ctx.errText,
|
||||
retryCount,
|
||||
downstreamPath,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
null,
|
||||
null,
|
||||
clientContext,
|
||||
logDownstreamApiKeyId ? downstreamApiKeyId : null,
|
||||
);
|
||||
},
|
||||
});
|
||||
|
||||
if (!endpointResult.ok) {
|
||||
const status = endpointResult.status || 502;
|
||||
const errText = endpointResult.errText || 'unknown error';
|
||||
const rawErrText = endpointResult.rawErrText || errText;
|
||||
tokenRouter.recordFailure(selected.channel.id);
|
||||
logProxy(
|
||||
selected,
|
||||
requestedModel,
|
||||
'failed',
|
||||
status,
|
||||
Date.now() - startTime,
|
||||
errText,
|
||||
retryCount,
|
||||
downstreamPath,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
null,
|
||||
null,
|
||||
clientContext,
|
||||
logDownstreamApiKeyId ? downstreamApiKeyId : null,
|
||||
);
|
||||
await recordOauthQuotaResetHint({
|
||||
accountId: selected.account.id,
|
||||
statusCode: status,
|
||||
errorText: rawErrText,
|
||||
});
|
||||
|
||||
if (isTokenExpiredError({ status, message: errText })) {
|
||||
await reportTokenExpired({
|
||||
accountId: selected.account.id,
|
||||
username: selected.account.username,
|
||||
siteName: selected.site.name,
|
||||
detail: `HTTP ${status}`,
|
||||
});
|
||||
}
|
||||
|
||||
if (shouldRetryProxyRequest(status, errText) && retryCount < MAX_RETRIES) {
|
||||
retryCount += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
await reportProxyAllFailed({
|
||||
model: requestedModel,
|
||||
reason: `upstream returned HTTP ${status}`,
|
||||
});
|
||||
return reply.code(status).send({ error: { message: errText, type: 'upstream_error' } });
|
||||
}
|
||||
|
||||
const upstream = endpointResult.upstream;
|
||||
const successfulUpstreamPath = endpointResult.upstreamPath;
|
||||
|
||||
if (isStream) {
|
||||
const upstreamContentType = (upstream.headers.get('content-type') || '').toLowerCase();
|
||||
const startSseResponse = () => {
|
||||
reply.hijack();
|
||||
reply.raw.statusCode = 200;
|
||||
reply.raw.setHeader('Content-Type', 'text/event-stream; charset=utf-8');
|
||||
reply.raw.setHeader('Cache-Control', 'no-cache, no-transform');
|
||||
reply.raw.setHeader('Connection', 'keep-alive');
|
||||
reply.raw.setHeader('X-Accel-Buffering', 'no');
|
||||
};
|
||||
|
||||
if (!upstreamContentType.includes('text/event-stream')) {
|
||||
const rawText = await upstream.text();
|
||||
let upstreamData: unknown = rawText;
|
||||
try {
|
||||
upstreamData = JSON.parse(rawText);
|
||||
} catch {
|
||||
upstreamData = rawText;
|
||||
}
|
||||
if (String(selected.site.platform || '').trim().toLowerCase() === 'gemini-cli') {
|
||||
upstreamData = unwrapGeminiCliPayload(upstreamData);
|
||||
}
|
||||
|
||||
const parsedUsage = parseProxyUsage(upstreamData);
|
||||
const latency = Date.now() - startTime;
|
||||
const failure = detectProxyFailure({ rawText, usage: parsedUsage });
|
||||
if (failure) {
|
||||
tokenRouter.recordFailure(selected.channel.id);
|
||||
logProxy(
|
||||
selected,
|
||||
requestedModel,
|
||||
'failed',
|
||||
failure.status,
|
||||
latency,
|
||||
failure.reason,
|
||||
retryCount,
|
||||
downstreamPath,
|
||||
parsedUsage.promptTokens,
|
||||
parsedUsage.completionTokens,
|
||||
parsedUsage.totalTokens,
|
||||
0,
|
||||
null,
|
||||
successfulUpstreamPath,
|
||||
clientContext,
|
||||
logDownstreamApiKeyId ? downstreamApiKeyId : null,
|
||||
);
|
||||
|
||||
if (shouldRetryProxyRequest(failure.status, failure.reason) && retryCount < MAX_RETRIES) {
|
||||
retryCount += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
await reportProxyAllFailed({
|
||||
model: requestedModel,
|
||||
reason: failure.reason,
|
||||
});
|
||||
return reply.code(failure.status).send({ error: { message: failure.reason, type: 'upstream_error' } });
|
||||
}
|
||||
|
||||
const normalized = openAiResponsesTransformer.transformFinalResponse(
|
||||
upstreamData,
|
||||
modelName,
|
||||
rawText,
|
||||
);
|
||||
const downstreamData = openAiResponsesTransformer.outbound.serializeFinal({
|
||||
upstreamPayload: upstreamData,
|
||||
normalized,
|
||||
usage: parsedUsage,
|
||||
serializationMode: 'response',
|
||||
});
|
||||
const resolvedUsage = await resolveProxyUsageWithSelfLogFallback({
|
||||
site: selected.site,
|
||||
account: selected.account,
|
||||
tokenValue: selected.tokenValue,
|
||||
tokenName: selected.tokenName,
|
||||
modelName: selected.actualModel || requestedModel,
|
||||
requestStartedAtMs: startTime,
|
||||
requestEndedAtMs: startTime + latency,
|
||||
localLatencyMs: latency,
|
||||
usage: {
|
||||
promptTokens: parsedUsage.promptTokens,
|
||||
completionTokens: parsedUsage.completionTokens,
|
||||
totalTokens: parsedUsage.totalTokens,
|
||||
},
|
||||
});
|
||||
const { estimatedCost, billingDetails } = await resolveProxyLogBilling({
|
||||
site: selected.site,
|
||||
account: selected.account,
|
||||
modelName: selected.actualModel || requestedModel,
|
||||
parsedUsage,
|
||||
resolvedUsage,
|
||||
});
|
||||
|
||||
tokenRouter.recordSuccess(selected.channel.id, latency, estimatedCost);
|
||||
recordDownstreamCostUsage(request, estimatedCost);
|
||||
logProxy(
|
||||
selected, requestedModel, 'success', 200, latency, null, retryCount, downstreamPath,
|
||||
resolvedUsage.promptTokens, resolvedUsage.completionTokens, resolvedUsage.totalTokens, estimatedCost, billingDetails,
|
||||
successfulUpstreamPath,
|
||||
clientContext,
|
||||
logDownstreamApiKeyId ? downstreamApiKeyId : null,
|
||||
);
|
||||
|
||||
const streamPayload = isRecord(downstreamData) ? downstreamData : {
|
||||
id: `resp_${Date.now()}`,
|
||||
object: 'response',
|
||||
status: 'completed',
|
||||
model: modelName,
|
||||
output: [],
|
||||
output_text: '',
|
||||
usage: {
|
||||
input_tokens: parsedUsage.promptTokens,
|
||||
output_tokens: parsedUsage.completionTokens,
|
||||
total_tokens: parsedUsage.totalTokens,
|
||||
},
|
||||
};
|
||||
const createdPayload = {
|
||||
...streamPayload,
|
||||
status: 'in_progress',
|
||||
output: [],
|
||||
output_text: '',
|
||||
};
|
||||
|
||||
startSseResponse();
|
||||
reply.raw.write(`event: response.created\ndata: ${JSON.stringify({ type: 'response.created', response: createdPayload })}\n\n`);
|
||||
reply.raw.write(`event: response.completed\ndata: ${JSON.stringify({ type: 'response.completed', response: streamPayload })}\n\n`);
|
||||
reply.raw.write('data: [DONE]\n\n');
|
||||
reply.raw.end();
|
||||
return;
|
||||
}
|
||||
|
||||
startSseResponse();
|
||||
|
||||
const upstreamReader = upstream.body?.getReader();
|
||||
const baseReader = String(selected.site.platform || '').trim().toLowerCase() === 'gemini-cli' && upstreamReader
|
||||
? createGeminiCliStreamReader(upstreamReader)
|
||||
: upstreamReader;
|
||||
let parsedUsage: UsageSummary = {
|
||||
promptTokens: 0,
|
||||
completionTokens: 0,
|
||||
totalTokens: 0,
|
||||
cacheReadTokens: 0,
|
||||
cacheCreationTokens: 0,
|
||||
promptTokensIncludeCache: null,
|
||||
};
|
||||
let rawText = '';
|
||||
const decoder = new TextDecoder();
|
||||
const reader = baseReader
|
||||
? {
|
||||
async read() {
|
||||
const result = await baseReader.read();
|
||||
if (result.value) {
|
||||
rawText += decoder.decode(result.value, { stream: true });
|
||||
}
|
||||
return result;
|
||||
},
|
||||
async cancel(reason?: unknown) {
|
||||
return baseReader.cancel(reason);
|
||||
},
|
||||
releaseLock() {
|
||||
return baseReader.releaseLock();
|
||||
},
|
||||
}
|
||||
: baseReader;
|
||||
const writeLines = (lines: string[]) => {
|
||||
for (const line of lines) reply.raw.write(line);
|
||||
};
|
||||
const streamSession = openAiResponsesTransformer.proxyStream.createSession({
|
||||
modelName,
|
||||
successfulUpstreamPath,
|
||||
strictTerminalEvents: Object.entries(request.headers as Record<string, unknown>)
|
||||
.some(([rawKey, rawValue]) => rawKey.trim().toLowerCase() === 'x-metapi-responses-websocket-transport'
|
||||
&& String(rawValue).trim() === '1'),
|
||||
getUsage: () => parsedUsage,
|
||||
onParsedPayload: (payload) => {
|
||||
if (payload && typeof payload === 'object') {
|
||||
parsedUsage = mergeProxyUsage(parsedUsage, parseProxyUsage(payload));
|
||||
}
|
||||
},
|
||||
writeLines,
|
||||
writeRaw: (chunk) => {
|
||||
reply.raw.write(chunk);
|
||||
},
|
||||
});
|
||||
const streamResult = await streamSession.run(reader, reply.raw);
|
||||
rawText += decoder.decode();
|
||||
|
||||
const latency = Date.now() - startTime;
|
||||
if (streamResult.status === 'failed') {
|
||||
tokenRouter.recordFailure(selected.channel.id);
|
||||
logProxy(
|
||||
selected,
|
||||
requestedModel,
|
||||
'failed',
|
||||
200,
|
||||
latency,
|
||||
streamResult.errorMessage,
|
||||
retryCount,
|
||||
downstreamPath,
|
||||
parsedUsage.promptTokens,
|
||||
parsedUsage.completionTokens,
|
||||
parsedUsage.totalTokens,
|
||||
0,
|
||||
null,
|
||||
successfulUpstreamPath,
|
||||
clientContext,
|
||||
logDownstreamApiKeyId ? downstreamApiKeyId : null,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// Once SSE has been hijacked and bytes may already be on the wire, we
|
||||
// must not attempt to convert stream failures into a fresh HTTP error
|
||||
// response or retry on another channel. Responses stream failures are
|
||||
// handled in-band by the proxy stream session.
|
||||
|
||||
const resolvedUsage = await resolveProxyUsageWithSelfLogFallback({
|
||||
site: selected.site,
|
||||
account: selected.account,
|
||||
tokenValue: selected.tokenValue,
|
||||
tokenName: selected.tokenName,
|
||||
modelName: selected.actualModel || requestedModel,
|
||||
requestStartedAtMs: startTime,
|
||||
requestEndedAtMs: startTime + latency,
|
||||
localLatencyMs: latency,
|
||||
usage: {
|
||||
promptTokens: parsedUsage.promptTokens,
|
||||
completionTokens: parsedUsage.completionTokens,
|
||||
totalTokens: parsedUsage.totalTokens,
|
||||
},
|
||||
});
|
||||
const { estimatedCost, billingDetails } = await resolveProxyLogBilling({
|
||||
site: selected.site,
|
||||
account: selected.account,
|
||||
modelName: selected.actualModel || requestedModel,
|
||||
parsedUsage,
|
||||
resolvedUsage,
|
||||
});
|
||||
tokenRouter.recordSuccess(selected.channel.id, latency, estimatedCost);
|
||||
recordDownstreamCostUsage(request, estimatedCost);
|
||||
logProxy(
|
||||
selected, requestedModel, 'success', 200, latency, null, retryCount, downstreamPath,
|
||||
resolvedUsage.promptTokens, resolvedUsage.completionTokens, resolvedUsage.totalTokens, estimatedCost, billingDetails,
|
||||
successfulUpstreamPath,
|
||||
clientContext,
|
||||
logDownstreamApiKeyId ? downstreamApiKeyId : null,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const upstreamContentType = (upstream.headers.get('content-type') || '').toLowerCase();
|
||||
let rawText = '';
|
||||
let upstreamData: unknown;
|
||||
if (upstreamContentType.includes('text/event-stream') && successfulUpstreamPath.endsWith('/responses')) {
|
||||
const collected = await collectResponsesFinalPayloadFromSse(upstream, modelName);
|
||||
rawText = collected.rawText;
|
||||
upstreamData = collected.payload;
|
||||
} else {
|
||||
rawText = await upstream.text();
|
||||
upstreamData = rawText;
|
||||
try {
|
||||
upstreamData = JSON.parse(rawText);
|
||||
} catch {
|
||||
upstreamData = rawText;
|
||||
}
|
||||
}
|
||||
if (String(selected.site.platform || '').trim().toLowerCase() === 'gemini-cli') {
|
||||
upstreamData = unwrapGeminiCliPayload(upstreamData);
|
||||
}
|
||||
const latency = Date.now() - startTime;
|
||||
const parsedUsage = parseProxyUsage(upstreamData);
|
||||
const failure = detectProxyFailure({ rawText, usage: parsedUsage });
|
||||
if (failure) {
|
||||
tokenRouter.recordFailure(selected.channel.id);
|
||||
logProxy(
|
||||
selected,
|
||||
requestedModel,
|
||||
'failed',
|
||||
failure.status,
|
||||
latency,
|
||||
failure.reason,
|
||||
retryCount,
|
||||
downstreamPath,
|
||||
parsedUsage.promptTokens,
|
||||
parsedUsage.completionTokens,
|
||||
parsedUsage.totalTokens,
|
||||
0,
|
||||
null,
|
||||
successfulUpstreamPath,
|
||||
clientContext,
|
||||
logDownstreamApiKeyId ? downstreamApiKeyId : null,
|
||||
);
|
||||
if (shouldRetryProxyRequest(failure.status, failure.reason) && retryCount < MAX_RETRIES) {
|
||||
retryCount += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
await reportProxyAllFailed({
|
||||
model: requestedModel,
|
||||
reason: failure.reason,
|
||||
});
|
||||
return reply.code(failure.status).send({ error: { message: failure.reason, type: 'upstream_error' } });
|
||||
}
|
||||
const normalized = openAiResponsesTransformer.transformFinalResponse(
|
||||
upstreamData,
|
||||
modelName,
|
||||
rawText,
|
||||
);
|
||||
const downstreamData = openAiResponsesTransformer.outbound.serializeFinal({
|
||||
upstreamPayload: upstreamData,
|
||||
normalized,
|
||||
usage: parsedUsage,
|
||||
serializationMode: isCompactRequest ? 'compact' : 'response',
|
||||
});
|
||||
const resolvedUsage = await resolveProxyUsageWithSelfLogFallback({
|
||||
site: selected.site,
|
||||
account: selected.account,
|
||||
tokenValue: selected.tokenValue,
|
||||
tokenName: selected.tokenName,
|
||||
modelName: selected.actualModel || requestedModel,
|
||||
requestStartedAtMs: startTime,
|
||||
requestEndedAtMs: startTime + latency,
|
||||
localLatencyMs: latency,
|
||||
usage: {
|
||||
promptTokens: parsedUsage.promptTokens,
|
||||
completionTokens: parsedUsage.completionTokens,
|
||||
totalTokens: parsedUsage.totalTokens,
|
||||
},
|
||||
});
|
||||
const { estimatedCost, billingDetails } = await resolveProxyLogBilling({
|
||||
site: selected.site,
|
||||
account: selected.account,
|
||||
modelName: selected.actualModel || requestedModel,
|
||||
parsedUsage,
|
||||
resolvedUsage,
|
||||
});
|
||||
|
||||
tokenRouter.recordSuccess(selected.channel.id, latency, estimatedCost);
|
||||
recordDownstreamCostUsage(request, estimatedCost);
|
||||
logProxy(
|
||||
selected, requestedModel, 'success', 200, latency, null, retryCount, downstreamPath,
|
||||
resolvedUsage.promptTokens, resolvedUsage.completionTokens, resolvedUsage.totalTokens, estimatedCost, billingDetails,
|
||||
successfulUpstreamPath,
|
||||
clientContext,
|
||||
logDownstreamApiKeyId ? downstreamApiKeyId : null,
|
||||
);
|
||||
return reply.send(downstreamData);
|
||||
} catch (err: any) {
|
||||
tokenRouter.recordFailure(selected.channel.id);
|
||||
logProxy(
|
||||
selected,
|
||||
requestedModel,
|
||||
'failed',
|
||||
0,
|
||||
Date.now() - startTime,
|
||||
err.message,
|
||||
retryCount,
|
||||
downstreamPath,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
null,
|
||||
null,
|
||||
clientContext,
|
||||
logDownstreamApiKeyId ? downstreamApiKeyId : null,
|
||||
);
|
||||
if (retryCount < MAX_RETRIES) {
|
||||
retryCount += 1;
|
||||
continue;
|
||||
}
|
||||
await reportProxyAllFailed({
|
||||
model: requestedModel,
|
||||
reason: err.message || 'network failure',
|
||||
});
|
||||
return reply.code(502).send({
|
||||
error: { message: `Upstream error: ${err.message}`, type: 'upstream_error' },
|
||||
});
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
app.post('/v1/responses', async (request: FastifyRequest, reply: FastifyReply) =>
|
||||
handleResponsesRequest(request, reply, '/v1/responses'));
|
||||
handleOpenAiResponsesSurfaceRequest(request, reply, '/v1/responses'));
|
||||
app.get('/v1/responses', async (_request: FastifyRequest, reply: FastifyReply) =>
|
||||
reply.code(426).send({
|
||||
error: {
|
||||
@@ -817,59 +15,5 @@ export async function responsesProxyRoute(app: FastifyInstance) {
|
||||
},
|
||||
}));
|
||||
app.post('/v1/responses/compact', async (request: FastifyRequest, reply: FastifyReply) =>
|
||||
handleResponsesRequest(request, reply, '/v1/responses/compact'));
|
||||
}
|
||||
|
||||
async function logProxy(
|
||||
selected: any,
|
||||
modelRequested: string,
|
||||
status: string,
|
||||
httpStatus: number,
|
||||
latencyMs: number,
|
||||
errorMessage: string | null,
|
||||
retryCount: number,
|
||||
downstreamPath: string,
|
||||
promptTokens = 0,
|
||||
completionTokens = 0,
|
||||
totalTokens = 0,
|
||||
estimatedCost = 0,
|
||||
billingDetails: unknown = null,
|
||||
upstreamPath: string | null = null,
|
||||
clientContext: DownstreamClientContext | null = null,
|
||||
downstreamApiKeyId: number | null = null,
|
||||
) {
|
||||
try {
|
||||
const createdAt = formatUtcSqlDateTime(new Date());
|
||||
const normalizedErrorMessage = composeProxyLogMessage({
|
||||
clientKind: clientContext?.clientKind && clientContext.clientKind !== 'generic'
|
||||
? clientContext.clientKind
|
||||
: null,
|
||||
sessionId: clientContext?.sessionId || null,
|
||||
traceHint: clientContext?.traceHint || null,
|
||||
downstreamPath,
|
||||
upstreamPath,
|
||||
errorMessage,
|
||||
});
|
||||
await db.insert(schema.proxyLogs).values({
|
||||
routeId: selected.channel.routeId,
|
||||
channelId: selected.channel.id,
|
||||
accountId: selected.account.id,
|
||||
...(downstreamApiKeyId !== null ? { downstreamApiKeyId } : {}),
|
||||
modelRequested,
|
||||
modelActual: selected.actualModel,
|
||||
status,
|
||||
httpStatus,
|
||||
latencyMs,
|
||||
promptTokens,
|
||||
completionTokens,
|
||||
totalTokens,
|
||||
estimatedCost,
|
||||
billingDetails: billingDetails ? JSON.stringify(billingDetails) : null,
|
||||
errorMessage: normalizedErrorMessage,
|
||||
retryCount,
|
||||
createdAt,
|
||||
}).run();
|
||||
} catch (error) {
|
||||
console.warn('[proxy/responses] failed to write proxy log', error);
|
||||
}
|
||||
handleOpenAiResponsesSurfaceRequest(request, reply, '/v1/responses/compact'));
|
||||
}
|
||||
|
||||
@@ -526,7 +526,7 @@ describe('responses websocket transport', () => {
|
||||
|
||||
const socket = new WebSocket(`${baseUrl}/v1/responses`);
|
||||
await waitForSocketOpen(socket);
|
||||
const messagesPromise = waitForSocketMessages(socket, 3, 400);
|
||||
const messagesPromise = waitForSocketMessages(socket, 6, 400);
|
||||
|
||||
socket.send(JSON.stringify({
|
||||
type: 'response.create',
|
||||
@@ -546,9 +546,12 @@ describe('responses websocket transport', () => {
|
||||
expect(messages.map((message) => message.type)).toEqual([
|
||||
'response.created',
|
||||
'response.output_text.delta',
|
||||
'error',
|
||||
'response.output_text.done',
|
||||
'response.content_part.done',
|
||||
'response.output_item.done',
|
||||
'response.failed',
|
||||
]);
|
||||
expect(messages[2]?.status).toBe(408);
|
||||
expect(messages[2]?.error?.message).toContain('stream closed before response.completed');
|
||||
expect(messages[5]?.response?.status).toBe('failed');
|
||||
expect(messages[5]?.error?.message).toContain('stream closed before response.completed');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
import { collectResponsesFinalPayloadFromSse } from './responsesSseFinal.js';
|
||||
|
||||
describe('collectResponsesFinalPayloadFromSse', () => {
|
||||
it('treats event:error payloads as upstream failures', async () => {
|
||||
const upstream = {
|
||||
async text() {
|
||||
return [
|
||||
'event: error',
|
||||
'data: {"error":{"message":"quota exceeded"},"type":"error"}',
|
||||
'',
|
||||
'data: [DONE]',
|
||||
'',
|
||||
].join('\n');
|
||||
},
|
||||
};
|
||||
|
||||
await expect(collectResponsesFinalPayloadFromSse(upstream, 'gpt-5.4'))
|
||||
.rejects
|
||||
.toThrow('quota exceeded');
|
||||
});
|
||||
});
|
||||
@@ -24,6 +24,14 @@ function getResponsesFailureMessage(payload: Record<string, unknown>): string {
|
||||
return 'upstream stream failed';
|
||||
}
|
||||
|
||||
function hasCompleteFinalResponsesPayload(payload: Record<string, unknown>): boolean {
|
||||
return (
|
||||
payload.object === 'response.compaction'
|
||||
|| Array.isArray(payload.output)
|
||||
|| Object.prototype.hasOwnProperty.call(payload, 'output_text')
|
||||
);
|
||||
}
|
||||
|
||||
export async function collectResponsesFinalPayloadFromSse(
|
||||
upstream: { text(): Promise<string> },
|
||||
modelName: string,
|
||||
@@ -42,18 +50,36 @@ export async function collectResponsesFinalPayloadFromSse(
|
||||
};
|
||||
let completedPayload: Record<string, unknown> | null = null;
|
||||
|
||||
const captureCompletedPayload = (lines: string[]) => {
|
||||
const captureCompletedPayloadFromEvent = (
|
||||
eventType: string,
|
||||
payload: Record<string, unknown>,
|
||||
) => {
|
||||
if (completedPayload) return;
|
||||
if (eventType === 'response.failed' || eventType === 'response.incomplete' || eventType === 'error') {
|
||||
throw new Error(getResponsesFailureMessage(payload));
|
||||
}
|
||||
if (eventType !== 'response.completed') {
|
||||
return;
|
||||
}
|
||||
if (isRecord(payload.response) && hasCompleteFinalResponsesPayload(payload.response)) {
|
||||
completedPayload = payload.response;
|
||||
return;
|
||||
}
|
||||
if (hasCompleteFinalResponsesPayload(payload)) {
|
||||
completedPayload = payload;
|
||||
}
|
||||
};
|
||||
|
||||
const captureCompletedPayloadFromLines = (lines: string[]) => {
|
||||
if (completedPayload) return;
|
||||
const parsed = openAiResponsesTransformer.pullSseEvents(lines.join(''));
|
||||
for (const event of parsed.events) {
|
||||
if (event.data === '[DONE]') continue;
|
||||
const payload = parseResponsesSsePayload(event.data);
|
||||
if (!payload) continue;
|
||||
if (payload.type === 'response.failed') {
|
||||
throw new Error(getResponsesFailureMessage(payload));
|
||||
}
|
||||
if (payload.type === 'response.completed' && isRecord(payload.response)) {
|
||||
completedPayload = payload.response;
|
||||
const payloadType = typeof payload.type === 'string' ? payload.type : '';
|
||||
captureCompletedPayloadFromEvent(payloadType || event.event, payload);
|
||||
if (completedPayload) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
@@ -64,13 +90,15 @@ export async function collectResponsesFinalPayloadFromSse(
|
||||
const payload = parseResponsesSsePayload(event.data);
|
||||
if (!payload) continue;
|
||||
|
||||
if (payload.type === 'response.failed') {
|
||||
throw new Error(getResponsesFailureMessage(payload));
|
||||
}
|
||||
|
||||
const payloadType = typeof payload.type === 'string' ? payload.type : '';
|
||||
const eventType = payloadType || event.event;
|
||||
usage = mergeProxyUsage(usage, parseProxyUsage(payload));
|
||||
captureCompletedPayloadFromEvent(eventType, payload);
|
||||
if (completedPayload) {
|
||||
continue;
|
||||
}
|
||||
const normalizedEvent = openAiResponsesTransformer.transformStreamEvent(payload, streamContext, modelName);
|
||||
captureCompletedPayload(openAiResponsesTransformer.aggregator.serialize({
|
||||
captureCompletedPayloadFromLines(openAiResponsesTransformer.aggregator.serialize({
|
||||
state: aggregateState,
|
||||
streamContext,
|
||||
event: normalizedEvent,
|
||||
@@ -79,7 +107,7 @@ export async function collectResponsesFinalPayloadFromSse(
|
||||
}
|
||||
|
||||
if (!completedPayload) {
|
||||
captureCompletedPayload(openAiResponsesTransformer.aggregator.complete(
|
||||
captureCompletedPayloadFromLines(openAiResponsesTransformer.aggregator.complete(
|
||||
aggregateState,
|
||||
streamContext,
|
||||
usage,
|
||||
|
||||
@@ -2,9 +2,13 @@ import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { fetch } from 'undici';
|
||||
import type { BuiltEndpointRequest } from './endpointFlow.js';
|
||||
|
||||
vi.mock('undici', () => ({
|
||||
fetch: vi.fn(),
|
||||
}));
|
||||
vi.mock('undici', async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import('undici')>();
|
||||
return {
|
||||
...actual,
|
||||
fetch: vi.fn(),
|
||||
};
|
||||
});
|
||||
|
||||
const fetchMock = vi.mocked(fetch);
|
||||
|
||||
|
||||
@@ -1,346 +1,24 @@
|
||||
import { createHash, randomUUID } from 'node:crypto';
|
||||
import { fetch, type RequestInit as UndiciRequestInit } from 'undici';
|
||||
import type { BuiltEndpointRequest } from './endpointFlow.js';
|
||||
import { buildUpstreamUrl } from './upstreamUrl.js';
|
||||
|
||||
const ANTIGRAVITY_RUNTIME_BASE_URLS = [
|
||||
'https://daily-cloudcode-pa.googleapis.com',
|
||||
'https://daily-cloudcode-pa.sandbox.googleapis.com',
|
||||
] as const;
|
||||
|
||||
type RuntimeDispatchInput = {
|
||||
siteUrl: string;
|
||||
request: BuiltEndpointRequest;
|
||||
targetUrl?: string;
|
||||
buildInit: (requestUrl: string, request: BuiltEndpointRequest) => Promise<UndiciRequestInit> | UndiciRequestInit;
|
||||
};
|
||||
|
||||
function asTrimmedString(value: unknown): string {
|
||||
return typeof value === 'string' ? value.trim() : '';
|
||||
}
|
||||
|
||||
function withRequestBody(
|
||||
request: BuiltEndpointRequest,
|
||||
body: Record<string, unknown>,
|
||||
headers?: Record<string, string>,
|
||||
): BuiltEndpointRequest {
|
||||
return {
|
||||
...request,
|
||||
headers: headers ? { ...headers } : { ...request.headers },
|
||||
body,
|
||||
};
|
||||
}
|
||||
|
||||
async function performFetch(
|
||||
input: RuntimeDispatchInput,
|
||||
request: BuiltEndpointRequest,
|
||||
requestUrl = input.targetUrl || buildUpstreamUrl(input.siteUrl, request.path),
|
||||
): Promise<Awaited<ReturnType<typeof fetch>>> {
|
||||
const init = await input.buildInit(requestUrl, request);
|
||||
return fetch(requestUrl, init);
|
||||
}
|
||||
|
||||
async function materializeErrorResponse(
|
||||
response: Awaited<ReturnType<typeof fetch>>,
|
||||
): Promise<Awaited<ReturnType<typeof fetch>>> {
|
||||
if (response.ok) return response;
|
||||
const text = await response.text().catch(() => '');
|
||||
return new Response(text, {
|
||||
status: response.status,
|
||||
headers: response.headers,
|
||||
}) as unknown as Awaited<ReturnType<typeof fetch>>;
|
||||
}
|
||||
|
||||
function parseGeminiCliPreviewFallbackOrder(modelName: string): string[] {
|
||||
switch (modelName.trim()) {
|
||||
case 'gemini-2.5-pro':
|
||||
return [];
|
||||
case 'gemini-2.5-flash':
|
||||
return [];
|
||||
case 'gemini-2.5-flash-lite':
|
||||
return [];
|
||||
default:
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
function replaceGeminiCliModelInUserAgent(userAgent: string | undefined, modelName: string): string | undefined {
|
||||
const raw = asTrimmedString(userAgent);
|
||||
if (!raw) return undefined;
|
||||
return raw.replace(/^GeminiCLI\/([^/]+)\/[^ ]+ /i, `GeminiCLI/$1/${modelName} `);
|
||||
}
|
||||
|
||||
function buildGeminiCliAttemptRequest(request: BuiltEndpointRequest, modelName: string): BuiltEndpointRequest {
|
||||
const body = structuredClone(request.body);
|
||||
const action = request.runtime?.action;
|
||||
if (action === 'countTokens') {
|
||||
delete body.model;
|
||||
delete body.project;
|
||||
} else {
|
||||
body.model = modelName;
|
||||
}
|
||||
const headers = { ...request.headers };
|
||||
const nextUserAgent = replaceGeminiCliModelInUserAgent(
|
||||
headers['User-Agent'] || headers['user-agent'],
|
||||
modelName,
|
||||
);
|
||||
if (nextUserAgent) {
|
||||
headers['User-Agent'] = nextUserAgent;
|
||||
delete headers['user-agent'];
|
||||
}
|
||||
return withRequestBody(request, body, headers);
|
||||
}
|
||||
|
||||
function antigravityRequestType(modelName: string): 'image_gen' | 'agent' {
|
||||
return modelName.includes('image') ? 'image_gen' : 'agent';
|
||||
}
|
||||
|
||||
function generateAntigravityProjectId(): string {
|
||||
const adjectives = ['useful', 'bright', 'swift', 'calm', 'bold'];
|
||||
const nouns = ['signal', 'river', 'rocket', 'forest', 'bridge'];
|
||||
const adjective = adjectives[Math.floor(Math.random() * adjectives.length)] || 'useful';
|
||||
const noun = nouns[Math.floor(Math.random() * nouns.length)] || 'signal';
|
||||
const suffix = Math.floor(100000 + Math.random() * 900000);
|
||||
return `${adjective}-${noun}-${suffix}`;
|
||||
}
|
||||
|
||||
function extractFirstUserText(value: unknown): string {
|
||||
if (!Array.isArray(value)) return '';
|
||||
for (const content of value) {
|
||||
if (!content || typeof content !== 'object' || Array.isArray(content)) continue;
|
||||
const record = content as Record<string, unknown>;
|
||||
if (asTrimmedString(record.role) !== 'user') continue;
|
||||
const parts = Array.isArray(record.parts) ? record.parts : [];
|
||||
for (const part of parts) {
|
||||
if (!part || typeof part !== 'object' || Array.isArray(part)) continue;
|
||||
const text = asTrimmedString((part as Record<string, unknown>).text);
|
||||
if (text) return text;
|
||||
}
|
||||
}
|
||||
return '';
|
||||
}
|
||||
|
||||
function generateStableAntigravitySessionId(payload: Record<string, unknown>): string {
|
||||
const firstUserText = extractFirstUserText(
|
||||
payload.request && typeof payload.request === 'object' && !Array.isArray(payload.request)
|
||||
? (payload.request as Record<string, unknown>).contents
|
||||
: undefined,
|
||||
);
|
||||
if (!firstUserText) {
|
||||
return `-${BigInt(`0x${randomUUID().replace(/-/g, '').slice(0, 16)}`).toString()}`;
|
||||
}
|
||||
const digest = createHash('sha256').update(firstUserText).digest('hex').slice(0, 16);
|
||||
const bigint = BigInt(`0x${digest}`) & BigInt('0x7fffffffffffffff');
|
||||
return `-${bigint.toString()}`;
|
||||
}
|
||||
|
||||
function renameParametersJsonSchema(value: unknown): unknown {
|
||||
if (Array.isArray(value)) {
|
||||
return value.map((item) => renameParametersJsonSchema(item));
|
||||
}
|
||||
if (!value || typeof value !== 'object') return value;
|
||||
|
||||
const input = value as Record<string, unknown>;
|
||||
const output: Record<string, unknown> = {};
|
||||
for (const [key, entry] of Object.entries(input)) {
|
||||
const nextKey = key === 'parametersJsonSchema' ? 'parameters' : key;
|
||||
output[nextKey] = renameParametersJsonSchema(entry);
|
||||
}
|
||||
return output;
|
||||
}
|
||||
|
||||
function deleteNestedMaxOutputTokens(payload: Record<string, unknown>): void {
|
||||
const request = payload.request;
|
||||
if (!request || typeof request !== 'object' || Array.isArray(request)) return;
|
||||
const generationConfig = (request as Record<string, unknown>).generationConfig;
|
||||
if (!generationConfig || typeof generationConfig !== 'object' || Array.isArray(generationConfig)) return;
|
||||
delete (generationConfig as Record<string, unknown>).maxOutputTokens;
|
||||
}
|
||||
|
||||
function buildAntigravityRuntimeBody(
|
||||
originalBody: Record<string, unknown>,
|
||||
modelName: string,
|
||||
action?: NonNullable<BuiltEndpointRequest['runtime']>['action'],
|
||||
): Record<string, unknown> {
|
||||
const payload = renameParametersJsonSchema(structuredClone(originalBody)) as Record<string, unknown>;
|
||||
if (action === 'countTokens') {
|
||||
return payload;
|
||||
}
|
||||
const requestType = antigravityRequestType(modelName);
|
||||
const projectId = asTrimmedString(payload.project) || generateAntigravityProjectId();
|
||||
|
||||
payload.model = modelName;
|
||||
payload.project = projectId;
|
||||
payload.userAgent = 'antigravity';
|
||||
payload.requestType = requestType;
|
||||
payload.requestId = requestType === 'image_gen'
|
||||
? `image_gen/${Date.now()}/${randomUUID()}/12`
|
||||
: `agent-${randomUUID()}`;
|
||||
|
||||
const request = payload.request;
|
||||
if (request && typeof request === 'object' && !Array.isArray(request)) {
|
||||
delete (request as Record<string, unknown>).safetySettings;
|
||||
if (requestType !== 'image_gen') {
|
||||
(request as Record<string, unknown>).sessionId = generateStableAntigravitySessionId(payload);
|
||||
}
|
||||
if (modelName.includes('claude')) {
|
||||
const toolConfig = (
|
||||
(request as Record<string, unknown>).toolConfig
|
||||
&& typeof (request as Record<string, unknown>).toolConfig === 'object'
|
||||
&& !Array.isArray((request as Record<string, unknown>).toolConfig)
|
||||
)
|
||||
? (request as Record<string, unknown>).toolConfig as Record<string, unknown>
|
||||
: (((request as Record<string, unknown>).toolConfig = {}) as Record<string, unknown>);
|
||||
toolConfig.functionCallingConfig = { mode: 'VALIDATED' };
|
||||
} else {
|
||||
deleteNestedMaxOutputTokens(payload);
|
||||
}
|
||||
}
|
||||
|
||||
return payload;
|
||||
}
|
||||
|
||||
function antigravityShouldRetryNoCapacity(
|
||||
status: number,
|
||||
responseText: string,
|
||||
): boolean {
|
||||
if (status !== 503) return false;
|
||||
return responseText.toLowerCase().includes('no capacity available');
|
||||
}
|
||||
|
||||
function antigravityNoCapacityRetryDelay(attempt: number): number {
|
||||
const delay = Math.min((attempt + 1) * 250, 2000);
|
||||
return delay;
|
||||
}
|
||||
|
||||
async function waitMs(ms: number): Promise<void> {
|
||||
if (ms <= 0) return;
|
||||
await new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function dispatchGeminiCliRuntimeRequest(
|
||||
input: RuntimeDispatchInput,
|
||||
): Promise<Awaited<ReturnType<typeof fetch>>> {
|
||||
const baseModel = asTrimmedString(input.request.runtime?.modelName) || asTrimmedString(input.request.body.model);
|
||||
const models = [baseModel, ...parseGeminiCliPreviewFallbackOrder(baseModel)].filter(Boolean);
|
||||
let lastResponse: Awaited<ReturnType<typeof fetch>> | null = null;
|
||||
|
||||
for (const modelName of models.length > 0 ? models : [baseModel || 'unknown']) {
|
||||
const attemptRequest = buildGeminiCliAttemptRequest(input.request, modelName);
|
||||
const response = await performFetch(input, attemptRequest);
|
||||
if (response.ok) return response;
|
||||
|
||||
if (response.status === 429) {
|
||||
lastResponse = await materializeErrorResponse(response);
|
||||
continue;
|
||||
}
|
||||
|
||||
return materializeErrorResponse(response);
|
||||
}
|
||||
|
||||
return lastResponse || performFetch(input, input.request);
|
||||
}
|
||||
|
||||
async function dispatchCodexRuntimeRequest(
|
||||
input: RuntimeDispatchInput,
|
||||
): Promise<Awaited<ReturnType<typeof fetch>>> {
|
||||
return performFetch(input, input.request);
|
||||
}
|
||||
|
||||
async function dispatchClaudeRuntimeRequest(
|
||||
input: RuntimeDispatchInput,
|
||||
): Promise<Awaited<ReturnType<typeof fetch>>> {
|
||||
return performFetch(input, input.request);
|
||||
}
|
||||
|
||||
async function dispatchAntigravityRuntimeRequest(
|
||||
input: RuntimeDispatchInput,
|
||||
): Promise<Awaited<ReturnType<typeof fetch>>> {
|
||||
const modelName = asTrimmedString(input.request.runtime?.modelName) || asTrimmedString(input.request.body.model);
|
||||
const runtimeBody = buildAntigravityRuntimeBody(
|
||||
input.request.body,
|
||||
modelName,
|
||||
input.request.runtime?.action,
|
||||
);
|
||||
const baseAttempts = 3;
|
||||
let lastResponse: Awaited<ReturnType<typeof fetch>> | null = null;
|
||||
|
||||
attemptLoop:
|
||||
for (let attempt = 0; attempt < baseAttempts; attempt += 1) {
|
||||
for (const baseUrl of ANTIGRAVITY_RUNTIME_BASE_URLS) {
|
||||
const requestUrl = `${baseUrl}${input.request.path}`;
|
||||
const minimalHeaders: Record<string, string> = {
|
||||
Authorization: input.request.headers.Authorization || input.request.headers.authorization || '',
|
||||
'Content-Type': 'application/json',
|
||||
Accept: input.request.runtime?.stream ? 'text/event-stream' : 'application/json',
|
||||
'User-Agent': 'antigravity/1.19.6 darwin/arm64',
|
||||
};
|
||||
let response: Awaited<ReturnType<typeof fetch>>;
|
||||
try {
|
||||
response = await performFetch(
|
||||
input,
|
||||
withRequestBody(input.request, runtimeBody, minimalHeaders),
|
||||
requestUrl,
|
||||
);
|
||||
} catch (error) {
|
||||
if (baseUrl !== ANTIGRAVITY_RUNTIME_BASE_URLS[ANTIGRAVITY_RUNTIME_BASE_URLS.length - 1]) {
|
||||
continue;
|
||||
}
|
||||
if (attempt + 1 < baseAttempts) {
|
||||
continue attemptLoop;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
if (response.ok) return response;
|
||||
|
||||
const errorResponse = await materializeErrorResponse(response);
|
||||
const errorText = await errorResponse.text().catch(() => '');
|
||||
lastResponse = new Response(errorText, {
|
||||
status: errorResponse.status,
|
||||
headers: errorResponse.headers,
|
||||
}) as unknown as Awaited<ReturnType<typeof fetch>>;
|
||||
|
||||
if (errorResponse.status === 429) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (antigravityShouldRetryNoCapacity(errorResponse.status, errorText)) {
|
||||
if (baseUrl !== ANTIGRAVITY_RUNTIME_BASE_URLS[ANTIGRAVITY_RUNTIME_BASE_URLS.length - 1]) {
|
||||
continue;
|
||||
}
|
||||
if (attempt + 1 < baseAttempts) {
|
||||
await waitMs(antigravityNoCapacityRetryDelay(attempt));
|
||||
continue attemptLoop;
|
||||
}
|
||||
}
|
||||
|
||||
return lastResponse;
|
||||
}
|
||||
}
|
||||
|
||||
return lastResponse || performFetch(input, withRequestBody(input.request, runtimeBody, {
|
||||
Authorization: input.request.headers.Authorization || input.request.headers.authorization || '',
|
||||
'Content-Type': 'application/json',
|
||||
Accept: input.request.runtime?.stream ? 'text/event-stream' : 'application/json',
|
||||
'User-Agent': 'antigravity/1.19.6 darwin/arm64',
|
||||
}));
|
||||
}
|
||||
import { antigravityExecutor } from '../../proxy-core/executors/antigravityExecutor.js';
|
||||
import { claudeExecutor } from '../../proxy-core/executors/claudeExecutor.js';
|
||||
import { codexExecutor } from '../../proxy-core/executors/codexExecutor.js';
|
||||
import { geminiCliExecutor } from '../../proxy-core/executors/geminiCliExecutor.js';
|
||||
import type { RuntimeDispatchInput, RuntimeResponse } from '../../proxy-core/executors/types.js';
|
||||
|
||||
export async function dispatchRuntimeRequest(
|
||||
input: RuntimeDispatchInput,
|
||||
): Promise<Awaited<ReturnType<typeof fetch>>> {
|
||||
): Promise<RuntimeResponse> {
|
||||
const executor = input.request.runtime?.executor || 'default';
|
||||
if (executor === 'codex') {
|
||||
return dispatchCodexRuntimeRequest(input);
|
||||
return codexExecutor.dispatch(input);
|
||||
}
|
||||
if (executor === 'claude') {
|
||||
return dispatchClaudeRuntimeRequest(input);
|
||||
return claudeExecutor.dispatch(input);
|
||||
}
|
||||
if (executor === 'gemini-cli') {
|
||||
return dispatchGeminiCliRuntimeRequest(input);
|
||||
return geminiCliExecutor.dispatch(input);
|
||||
}
|
||||
if (executor === 'antigravity') {
|
||||
return dispatchAntigravityRuntimeRequest(input);
|
||||
return antigravityExecutor.dispatch(input);
|
||||
}
|
||||
return performFetch(input, input.request);
|
||||
return codexExecutor.dispatch(input);
|
||||
}
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { createHash, randomUUID } from 'node:crypto';
|
||||
import { resolveProviderProfile } from '../../proxy-core/providers/registry.js';
|
||||
import { fetchModelPricingCatalog } from '../../services/modelPricingService.js';
|
||||
import type { DownstreamFormat } from '../../transformers/shared/normalized.js';
|
||||
import {
|
||||
@@ -11,7 +12,6 @@ import {
|
||||
} from '../../transformers/anthropic/messages/conversion.js';
|
||||
import {
|
||||
buildGeminiGenerateContentRequestFromOpenAi,
|
||||
wrapGeminiCliRequest,
|
||||
} from './geminiCliCompat.js';
|
||||
export {
|
||||
buildMinimalJsonHeadersForCompatibility,
|
||||
@@ -790,6 +790,7 @@ export function buildUpstreamEndpointRequest(input: {
|
||||
};
|
||||
} {
|
||||
const sitePlatform = normalizePlatformName(input.sitePlatform);
|
||||
const providerProfile = resolveProviderProfile(sitePlatform);
|
||||
const isClaudeUpstream = sitePlatform === 'claude';
|
||||
const isGeminiUpstream = sitePlatform === 'gemini';
|
||||
const isGeminiCliUpstream = sitePlatform === 'gemini-cli';
|
||||
@@ -886,10 +887,6 @@ export function buildUpstreamEndpointRequest(input: {
|
||||
};
|
||||
|
||||
if (isInternalGeminiUpstream) {
|
||||
const projectId = asTrimmedString(input.oauthProjectId);
|
||||
if (isGeminiCliUpstream && !projectId) {
|
||||
throw new Error('gemini-cli oauth project id missing');
|
||||
}
|
||||
const instructions = (
|
||||
input.downstreamFormat === 'responses'
|
||||
&& typeof input.responsesOriginalBody?.instructions === 'string'
|
||||
@@ -901,27 +898,22 @@ export function buildUpstreamEndpointRequest(input: {
|
||||
modelName: input.modelName,
|
||||
instructions,
|
||||
});
|
||||
const headers = isGeminiCliUpstream
|
||||
? buildGeminiCliRuntimeHeaders({
|
||||
baseHeaders: commonHeaders,
|
||||
providerHeaders: input.providerHeaders,
|
||||
modelName: input.modelName,
|
||||
stream: input.stream,
|
||||
})
|
||||
: buildAntigravityRuntimeHeaders({
|
||||
baseHeaders: commonHeaders,
|
||||
stream: input.stream,
|
||||
});
|
||||
return {
|
||||
path: resolveEndpointPath(input.endpoint),
|
||||
headers,
|
||||
body: wrapGeminiCliRequest({
|
||||
modelName: input.modelName,
|
||||
projectId: projectId || '',
|
||||
request: geminiRequest,
|
||||
}) as Record<string, unknown>,
|
||||
runtime,
|
||||
};
|
||||
if (!providerProfile) {
|
||||
throw new Error(`missing provider profile for platform: ${sitePlatform}`);
|
||||
}
|
||||
return providerProfile.prepareRequest({
|
||||
endpoint: input.endpoint,
|
||||
modelName: input.modelName,
|
||||
stream: input.stream,
|
||||
tokenValue: input.tokenValue,
|
||||
oauthProvider: input.oauthProvider,
|
||||
oauthProjectId: input.oauthProjectId,
|
||||
sitePlatform,
|
||||
baseHeaders: commonHeaders,
|
||||
providerHeaders: input.providerHeaders,
|
||||
body: geminiRequest,
|
||||
action: input.stream ? 'streamGenerateContent' : 'generateContent',
|
||||
});
|
||||
}
|
||||
|
||||
if (input.endpoint === 'messages') {
|
||||
@@ -961,6 +953,21 @@ export function buildUpstreamEndpointRequest(input: {
|
||||
convertOpenAiBodyToAnthropicMessagesBody(openaiBody, input.modelName, input.stream),
|
||||
);
|
||||
|
||||
if (providerProfile?.id === 'claude') {
|
||||
return providerProfile.prepareRequest({
|
||||
endpoint: 'messages',
|
||||
modelName: input.modelName,
|
||||
stream: input.stream,
|
||||
tokenValue: input.tokenValue,
|
||||
oauthProvider: input.oauthProvider,
|
||||
oauthProjectId: input.oauthProjectId,
|
||||
sitePlatform,
|
||||
baseHeaders: commonHeaders,
|
||||
claudeHeaders,
|
||||
body: sanitizedBody,
|
||||
});
|
||||
}
|
||||
|
||||
const headers = buildClaudeRuntimeHeaders({
|
||||
baseHeaders: commonHeaders,
|
||||
claudeHeaders,
|
||||
@@ -1009,6 +1016,26 @@ export function buildUpstreamEndpointRequest(input: {
|
||||
sitePlatform,
|
||||
);
|
||||
|
||||
if (providerProfile?.id === 'codex') {
|
||||
return providerProfile.prepareRequest({
|
||||
endpoint: 'responses',
|
||||
modelName: input.modelName,
|
||||
stream: input.stream,
|
||||
tokenValue: input.tokenValue,
|
||||
oauthProvider: input.oauthProvider,
|
||||
oauthProjectId: input.oauthProjectId,
|
||||
sitePlatform,
|
||||
baseHeaders: {
|
||||
...commonHeaders,
|
||||
...responsesHeaders,
|
||||
},
|
||||
providerHeaders: input.providerHeaders,
|
||||
codexSessionCacheKey: input.codexSessionCacheKey,
|
||||
codexExplicitSessionId: input.codexExplicitSessionId,
|
||||
body,
|
||||
});
|
||||
}
|
||||
|
||||
const headers = sitePlatform === 'codex'
|
||||
? buildCodexRuntimeHeaders({
|
||||
baseHeaders: {
|
||||
@@ -1078,9 +1105,7 @@ export function buildClaudeCountTokensUpstreamRequest(input: {
|
||||
};
|
||||
} {
|
||||
const sitePlatform = normalizePlatformName(input.sitePlatform);
|
||||
const isClaudeOauthUpstream = sitePlatform === 'claude' && input.oauthProvider === 'claude';
|
||||
const claudeHeaders = extractClaudePassthroughHeaders(input.downstreamHeaders);
|
||||
const anthropicVersion = claudeHeaders['anthropic-version'] || '2023-06-01';
|
||||
const { body: bodyWithoutBetas, betas } = extractClaudeBetasFromBody({
|
||||
...input.claudeBody,
|
||||
model: input.modelName,
|
||||
@@ -1089,23 +1114,33 @@ export function buildClaudeCountTokensUpstreamRequest(input: {
|
||||
delete sanitizedBody.max_tokens;
|
||||
delete sanitizedBody.maxTokens;
|
||||
delete sanitizedBody.stream;
|
||||
const providerProfile = resolveProviderProfile(sitePlatform);
|
||||
if (providerProfile?.id !== 'claude') {
|
||||
throw new Error(`missing claude provider profile for platform: ${sitePlatform || 'unknown'}`);
|
||||
}
|
||||
|
||||
const headers = buildClaudeRuntimeHeaders({
|
||||
const prepared = providerProfile.prepareRequest({
|
||||
endpoint: 'messages',
|
||||
modelName: input.modelName,
|
||||
stream: false,
|
||||
tokenValue: input.tokenValue,
|
||||
oauthProvider: input.oauthProvider,
|
||||
sitePlatform,
|
||||
baseHeaders: {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
claudeHeaders,
|
||||
anthropicVersion,
|
||||
stream: false,
|
||||
isClaudeOauthUpstream,
|
||||
tokenValue: input.tokenValue,
|
||||
extraBetas: betas,
|
||||
claudeHeaders: {
|
||||
...claudeHeaders,
|
||||
...(betas.length > 0 ? { 'anthropic-beta': betas.join(',') } : {}),
|
||||
},
|
||||
body: sanitizedBody,
|
||||
action: 'countTokens',
|
||||
});
|
||||
|
||||
return {
|
||||
path: '/v1/messages/count_tokens?beta=true',
|
||||
headers,
|
||||
body: sanitizedBody,
|
||||
path: prepared.path,
|
||||
headers: prepared.headers,
|
||||
body: prepared.body,
|
||||
runtime: {
|
||||
executor: 'claude',
|
||||
modelName: input.modelName,
|
||||
|
||||
Reference in New Issue
Block a user