boos/BoosAPI
2
0
Fork 2
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'main' into feat/account-level-proxy

Browse Source 
# Conflicts:
#	src/server/routes/api/accountTokens.ts
#	src/server/routes/proxy/chat.ts
#	src/server/routes/proxy/completions.ts
#	src/server/routes/proxy/embeddings.ts
#	src/server/routes/proxy/endpointFlow.ts
#	src/server/routes/proxy/images.ts
#	src/server/routes/proxy/responses.ts
#	src/server/services/accountExtraConfig.test.ts
#	src/server/services/modelService.ts
This commit is contained in:
Hureru
2026-03-18 23:42:21 +08:00
parent f36e867e1d a71ffb3641
commit 803ba84cfd
199 changed files with 27415 additions and 1901 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/server/routes/proxy/chat.codex-oauth.test.ts
+248
View File
@@ -0,0 +1,248 @@
import Fastify, { type FastifyInstance } from 'fastify';
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
const fetchMock = vi.fn();
const selectChannelMock = vi.fn();
const selectNextChannelMock = vi.fn();
const recordSuccessMock = vi.fn();
const recordFailureMock = vi.fn();
const refreshModelsAndRebuildRoutesMock = vi.fn();
const reportProxyAllFailedMock = vi.fn();
const reportTokenExpiredMock = vi.fn();
const resolveProxyUsageWithSelfLogFallbackMock = vi.fn(async ({ usage }: any) => ({
...usage,
estimatedCostFromQuota: 0,
recoveredFromSelfLog: false,
}));
const dbInsertMock = vi.fn((_arg?: any) => ({
values: () => ({
run: () => undefined,
}),
}));
vi.mock('undici', () => ({
fetch: (...args: unknown[]) => fetchMock(...args),
}));
vi.mock('../../services/tokenRouter.js', () => ({
tokenRouter: {
selectChannel: (...args: unknown[]) => selectChannelMock(...args),
selectNextChannel: (...args: unknown[]) => selectNextChannelMock(...args),
recordSuccess: (...args: unknown[]) => recordSuccessMock(...args),
recordFailure: (...args: unknown[]) => recordFailureMock(...args),
},
}));
vi.mock('../../services/modelService.js', () => ({
refreshModelsAndRebuildRoutes: (...args: unknown[]) => refreshModelsAndRebuildRoutesMock(...args),
}));
vi.mock('../../services/alertService.js', () => ({
reportProxyAllFailed: (...args: unknown[]) => reportProxyAllFailedMock(...args),
reportTokenExpired: (...args: unknown[]) => reportTokenExpiredMock(...args),
}));
vi.mock('../../services/alertRules.js', () => ({
isTokenExpiredError: ({ status }: { status?: number }) => status === 401,
}));
vi.mock('../../services/modelPricingService.js', () => ({
estimateProxyCost: async () => 0,
buildProxyBillingDetails: async () => null,
fetchModelPricingCatalog: async () => null,
}));
vi.mock('../../services/proxyRetryPolicy.js', () => ({
shouldRetryProxyRequest: () => false,
}));
vi.mock('../../services/proxyUsageFallbackService.js', () => ({
resolveProxyUsageWithSelfLogFallback: (arg: any) => resolveProxyUsageWithSelfLogFallbackMock(arg),
}));
vi.mock('../../db/index.js', () => ({
db: {
insert: (arg: any) => dbInsertMock(arg),
},
hasProxyLogDownstreamApiKeyIdColumn: async () => false,
schema: {
proxyLogs: {},
},
}));
describe('chat proxy codex oauth compatibility', () => {
let app: FastifyInstance;
const createSseResponse = (chunks: string[], status = 200) => {
const encoder = new TextEncoder();
return new Response(new ReadableStream<Uint8Array>({
start(controller) {
for (const chunk of chunks) {
controller.enqueue(encoder.encode(chunk));
}
controller.close();
},
}), {
status,
headers: { 'content-type': 'text/event-stream; charset=utf-8' },
});
};
beforeAll(async () => {
const { chatProxyRoute, claudeMessagesProxyRoute } = await import('./chat.js');
app = Fastify();
await app.register(chatProxyRoute);
await app.register(claudeMessagesProxyRoute);
});
beforeEach(() => {
fetchMock.mockReset();
selectChannelMock.mockReset();
selectNextChannelMock.mockReset();
recordSuccessMock.mockReset();
recordFailureMock.mockReset();
refreshModelsAndRebuildRoutesMock.mockReset();
reportProxyAllFailedMock.mockReset();
reportTokenExpiredMock.mockReset();
resolveProxyUsageWithSelfLogFallbackMock.mockClear();
dbInsertMock.mockClear();
selectChannelMock.mockReturnValue({
channel: { id: 11, routeId: 22 },
site: { name: 'codex-site', url: 'https://chatgpt.com/backend-api/codex', platform: 'codex' },
account: {
id: 33,
username: 'codex-user@example.com',
extraConfig: JSON.stringify({
credentialMode: 'session',
oauth: {
provider: 'codex',
accountId: 'chatgpt-account-123',
email: 'codex-user@example.com',
planType: 'team',
},
}),
},
tokenName: 'default',
tokenValue: 'oauth-access-token',
actualModel: 'gpt-5.4',
});
selectNextChannelMock.mockReturnValue(null);
});
afterAll(async () => {
await app.close();
});
it('strips codex-unsupported responses fields and aggregates the SSE result for /v1/messages callers', async () => {
fetchMock.mockResolvedValue(createSseResponse([
'event: response.created\n',
'data: {"type":"response.created","response":{"id":"resp_codex_claude","model":"gpt-5.4","created_at":1706000000,"status":"in_progress","output":[]}}\n\n',
'event: response.output_item.added\n',
'data: {"type":"response.output_item.added","output_index":0,"item":{"id":"msg_codex_claude","type":"message","role":"assistant","status":"in_progress","content":[]}}\n\n',
'event: response.output_text.delta\n',
'data: {"type":"response.output_text.delta","output_index":0,"item_id":"msg_codex_claude","delta":"pong from codex"}\n\n',
'event: response.completed\n',
'data: {"type":"response.completed","response":{"id":"resp_codex_claude","model":"gpt-5.4","status":"completed","usage":{"input_tokens":9,"output_tokens":3,"total_tokens":12}}}\n\n',
'data: [DONE]\n\n',
]));
const response = await app.inject({
method: 'POST',
url: '/v1/messages',
payload: {
model: 'gpt-5.4',
max_tokens: 256,
messages: [{ role: 'user', content: 'hello codex' }],
},
});
expect(response.statusCode).toBe(200);
expect(fetchMock).toHaveBeenCalledTimes(1);
const [, options] = fetchMock.mock.calls[0] as [string, any];
const forwardedBody = JSON.parse(options.body);
expect(forwardedBody.stream).toBe(true);
expect(forwardedBody.instructions).toBe('');
expect(forwardedBody.store).toBe(false);
expect(forwardedBody.parallel_tool_calls).toBe(true);
expect(forwardedBody.include).toEqual(['reasoning.encrypted_content']);
expect(forwardedBody.max_output_tokens).toBeUndefined();
expect(forwardedBody.max_completion_tokens).toBeUndefined();
const body = response.json();
expect(body.type).toBe('message');
expect(body.role).toBe('assistant');
expect(body.model).toBe('gpt-5.4');
expect(body.content?.[0]?.type).toBe('text');
expect(body.content?.[0]?.text).toContain('pong from codex');
});
it('translates codex SSE into Claude messages stream events instead of leaking raw response events', async () => {
fetchMock.mockResolvedValue(createSseResponse([
'event: response.created\n',
'data: {"type":"response.created","response":{"id":"resp_codex_claude_stream","model":"gpt-5.4","created_at":1706000000,"status":"in_progress","output":[]}}\n\n',
'event: response.output_item.added\n',
'data: {"type":"response.output_item.added","output_index":0,"item":{"id":"msg_codex_claude_stream","type":"message","role":"assistant","status":"in_progress","content":[]}}\n\n',
'event: response.content_part.added\n',
'data: {"type":"response.content_part.added","output_index":0,"content_index":0,"item_id":"msg_codex_claude_stream","part":{"type":"output_text","text":""}}\n\n',
'event: response.output_text.delta\n',
'data: {"type":"response.output_text.delta","output_index":0,"item_id":"msg_codex_claude_stream","content_index":0,"delta":"pong from codex"}\n\n',
'event: response.content_part.done\n',
'data: {"type":"response.content_part.done","output_index":0,"content_index":0,"item_id":"msg_codex_claude_stream","part":{"type":"output_text","text":"pong from codex"}}\n\n',
'event: response.completed\n',
'data: {"type":"response.completed","response":{"id":"resp_codex_claude_stream","model":"gpt-5.4","status":"completed","usage":{"input_tokens":9,"output_tokens":3,"total_tokens":12}}}\n\n',
'data: [DONE]\n\n',
]));
const response = await app.inject({
method: 'POST',
url: '/v1/messages',
payload: {
model: 'gpt-5.4',
stream: true,
max_tokens: 256,
messages: [{ role: 'user', content: 'hello codex' }],
},
});
expect(response.statusCode).toBe(200);
expect(response.headers['content-type']).toContain('text/event-stream');
expect(response.body).toContain('event: message_start');
expect(response.body).toContain('event: content_block_delta');
expect(response.body).toContain('pong from codex');
expect(response.body).not.toContain('event: response.created');
expect(response.body).not.toContain('"type":"response.output_text.delta"');
});
it('translates codex SSE into OpenAI chat completion chunks instead of leaking raw response events', async () => {
fetchMock.mockResolvedValue(createSseResponse([
'event: response.created\n',
'data: {"type":"response.created","response":{"id":"resp_codex_openai_stream","model":"gpt-5.4","created_at":1706000000,"status":"in_progress","output":[]}}\n\n',
'event: response.output_item.added\n',
'data: {"type":"response.output_item.added","output_index":0,"item":{"id":"msg_codex_openai_stream","type":"message","role":"assistant","status":"in_progress","content":[]}}\n\n',
'event: response.output_text.delta\n',
'data: {"type":"response.output_text.delta","output_index":0,"item_id":"msg_codex_openai_stream","delta":"pong from codex"}\n\n',
'event: response.completed\n',
'data: {"type":"response.completed","response":{"id":"resp_codex_openai_stream","model":"gpt-5.4","status":"completed","usage":{"input_tokens":9,"output_tokens":3,"total_tokens":12}}}\n\n',
'data: [DONE]\n\n',
]));
const response = await app.inject({
method: 'POST',
url: '/v1/chat/completions',
payload: {
model: 'gpt-5.4',
stream: true,
messages: [{ role: 'user', content: 'hello codex' }],
},
});
expect(response.statusCode).toBe(200);
expect(response.headers['content-type']).toContain('text/event-stream');
expect(response.body).toContain('"object":"chat.completion.chunk"');
expect(response.body).toContain('pong from codex');
expect(response.body).not.toContain('event: response.created');
expect(response.body).not.toContain('"type":"response.output_text.delta"');
});
});
src/server/routes/proxy/chat.count-tokens.test.ts
+157
View File
@@ -0,0 +1,157 @@
import Fastify, { type FastifyInstance } from 'fastify';
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
const fetchMock = vi.fn();
const selectChannelMock = vi.fn();
const selectNextChannelMock = vi.fn();
const recordSuccessMock = vi.fn();
const recordFailureMock = vi.fn();
const refreshModelsAndRebuildRoutesMock = vi.fn();
const reportProxyAllFailedMock = vi.fn();
const reportTokenExpiredMock = vi.fn();
const resolveProxyUsageWithSelfLogFallbackMock = vi.fn(async ({ usage }: any) => ({
...usage,
estimatedCostFromQuota: 0,
recoveredFromSelfLog: false,
}));
const dbInsertMock = vi.fn((_arg?: any) => ({
values: () => ({
run: () => undefined,
}),
}));
vi.mock('undici', () => ({
fetch: (...args: unknown[]) => fetchMock(...args),
}));
vi.mock('../../services/tokenRouter.js', () => ({
tokenRouter: {
selectChannel: (...args: unknown[]) => selectChannelMock(...args),
selectNextChannel: (...args: unknown[]) => selectNextChannelMock(...args),
recordSuccess: (...args: unknown[]) => recordSuccessMock(...args),
recordFailure: (...args: unknown[]) => recordFailureMock(...args),
},
}));
vi.mock('../../services/modelService.js', () => ({
refreshModelsAndRebuildRoutes: (...args: unknown[]) => refreshModelsAndRebuildRoutesMock(...args),
}));
vi.mock('../../services/alertService.js', () => ({
reportProxyAllFailed: (...args: unknown[]) => reportProxyAllFailedMock(...args),
reportTokenExpired: (...args: unknown[]) => reportTokenExpiredMock(...args),
}));
vi.mock('../../services/alertRules.js', () => ({
isTokenExpiredError: () => false,
}));
vi.mock('../../services/modelPricingService.js', () => ({
estimateProxyCost: async () => 0,
buildProxyBillingDetails: async () => null,
fetchModelPricingCatalog: async () => null,
}));
vi.mock('../../services/proxyRetryPolicy.js', () => ({
shouldRetryProxyRequest: () => false,
}));
vi.mock('../../services/proxyUsageFallbackService.js', () => ({
resolveProxyUsageWithSelfLogFallback: (arg: any) => resolveProxyUsageWithSelfLogFallbackMock(arg),
}));
vi.mock('../../services/oauth/quota.js', () => ({
recordOauthQuotaResetHint: async () => undefined,
}));
vi.mock('../../db/index.js', () => ({
db: {
insert: (arg: any) => dbInsertMock(arg),
},
hasProxyLogDownstreamApiKeyIdColumn: async () => false,
schema: {
proxyLogs: {},
},
}));
describe('claude count_tokens proxy route', () => {
let app: FastifyInstance;
beforeAll(async () => {
const { claudeMessagesProxyRoute } = await import('./chat.js');
app = Fastify();
await app.register(claudeMessagesProxyRoute);
});
beforeEach(() => {
fetchMock.mockReset();
selectChannelMock.mockReset();
selectNextChannelMock.mockReset();
recordSuccessMock.mockReset();
recordFailureMock.mockReset();
refreshModelsAndRebuildRoutesMock.mockReset();
reportProxyAllFailedMock.mockReset();
reportTokenExpiredMock.mockReset();
resolveProxyUsageWithSelfLogFallbackMock.mockClear();
dbInsertMock.mockClear();
selectChannelMock.mockReturnValue({
channel: { id: 11, routeId: 22 },
site: { name: 'claude-site', url: 'https://api.anthropic.com', platform: 'claude' },
account: { id: 33, username: 'claude-user@example.com' },
tokenName: 'default',
tokenValue: 'sk-claude',
actualModel: 'claude-opus-4-6',
});
selectNextChannelMock.mockReturnValue(null);
});
afterAll(async () => {
await app.close();
});
it('forwards /v1/messages/count_tokens to the claude count_tokens upstream path', async () => {
fetchMock.mockResolvedValue(new Response(JSON.stringify({
input_tokens: 42,
}), {
status: 200,
headers: { 'content-type': 'application/json' },
}));
const response = await app.inject({
method: 'POST',
url: '/v1/messages/count_tokens',
payload: {
model: 'claude-opus-4-6',
tools: [
{ name: 'lookup', input_schema: { type: 'object' } },
],
messages: [
{
role: 'user',
content: [{ type: 'text', text: 'count these tokens' }],
},
],
},
});
expect(response.statusCode).toBe(200);
expect(response.json()).toEqual({ input_tokens: 42 });
expect(fetchMock).toHaveBeenCalledTimes(1);
const [targetUrl, options] = fetchMock.mock.calls[0] as [string, any];
expect(targetUrl).toBe('https://api.anthropic.com/v1/messages/count_tokens?beta=true');
expect(options.headers['anthropic-version']).toBe('2023-06-01');
expect(options.headers['anthropic-beta']).toContain('claude-code-20250219');
expect(options.headers['Accept-Encoding']).toBe('gzip, deflate, br, zstd');
const forwardedBody = JSON.parse(String(options.body));
expect(forwardedBody.model).toBe('claude-opus-4-6');
expect(forwardedBody.messages).toEqual([
{
role: 'user',
content: [{ type: 'text', text: 'count these tokens', cache_control: { type: 'ephemeral' } }],
},
]);
});
});
src/server/routes/proxy/chat.stream.test.ts
+19 -2
View File
@@ -1384,6 +1384,15 @@ describe('chat proxy stream behavior', () => {
status: 400,
headers: { 'content-type': 'application/json' },
}))
.mockResolvedValueOnce(new Response(JSON.stringify({
error: {
message: 'request validation failed',
type: 'invalid_request_error',
},
}), {
status: 400,
headers: { 'content-type': 'application/json' },
}))
.mockResolvedValueOnce(new Response(JSON.stringify({
id: 'resp_retry_minimal_headers',
object: 'response',
@@ -1413,21 +1422,27 @@ describe('chat proxy stream behavior', () => {
const body = response.json();
expect(body.output_text).toContain('ok after minimal headers retry');
expect(fetchMock).toHaveBeenCalledTimes(3);
expect(fetchMock).toHaveBeenCalledTimes(4);
const [, firstOptions] = fetchMock.mock.calls[0] as [string, any];
const [, secondOptions] = fetchMock.mock.calls[1] as [string, any];
const [, thirdOptions] = fetchMock.mock.calls[2] as [string, any];
const [, fourthOptions] = fetchMock.mock.calls[3] as [string, any];
expect(firstOptions.headers['openai-beta']).toBe('responses-2025-03-11');
expect(secondOptions.headers['openai-beta']).toBe('responses-2025-03-11');
expect(thirdOptions.headers['openai-beta']).toBeUndefined();
expect(thirdOptions.headers['openai-beta']).toBe('responses-2025-03-11');
expect(fourthOptions.headers['openai-beta']).toBeUndefined();
const firstBody = JSON.parse(firstOptions.body);
const secondBody = JSON.parse(secondOptions.body);
const thirdBody = JSON.parse(thirdOptions.body);
const fourthBody = JSON.parse(fourthOptions.body);
expect(firstBody.user).toBe('user-123');
expect(secondBody.user).toBeUndefined();
expect(thirdBody.user).toBeUndefined();
expect(secondBody.include).toEqual(['reasoning.encrypted_content']);
expect(thirdBody.include).toBeUndefined();
expect(fourthBody.user).toBeUndefined();
});
it('returns concise Cloudflare host error on /v1/responses 502 html failures', async () => {
@@ -3193,6 +3208,8 @@ describe('chat proxy stream behavior', () => {
expect(response.statusCode).toBe(200);
expect(response.body).toContain('"finish_reason":"error"');
expect(response.body).toContain('[DONE]');
expect(recordSuccessMock).not.toHaveBeenCalled();
expect(recordFailureMock).toHaveBeenCalledTimes(1);
});
it('preserves non-stream function_call output when /v1/chat/completions falls back to /v1/responses', async () => {
src/server/routes/proxy/chat.ts
+441 -32
View File
@@ -1,7 +1,6 @@
import { FastifyInstance, FastifyRequest, FastifyReply } from 'fastify';
import { tokenRouter } from '../../services/tokenRouter.js';
import { db, schema } from '../../db/index.js';
import { fetch } from 'undici';
import { db, hasProxyLogDownstreamApiKeyIdColumn, schema } from '../../db/index.js';
import { refreshModelsAndRebuildRoutes } from '../../services/modelService.js';
import { reportProxyAllFailed, reportTokenExpired } from '../../services/alertService.js';
import { isTokenExpiredError } from '../../services/alertRules.js';
@@ -12,6 +11,7 @@ import { resolveChannelProxyUrl, withSiteRecordProxyRequestInit } from '../../se
import { getProxyUrlFromExtraConfig } from '../../services/accountExtraConfig.js';
import { type DownstreamFormat } from '../../transformers/shared/normalized.js';
import {
buildClaudeCountTokensUpstreamRequest,
buildUpstreamEndpointRequest,
resolveUpstreamEndpointCandidates,
} from './upstreamEndpoint.js';
@@ -21,21 +21,41 @@ import {
recordDownstreamCostUsage,
} from './downstreamPolicy.js';
import { composeProxyLogMessage } from './logPathMeta.js';
import { executeEndpointFlow } from './endpointFlow.js';
import { executeEndpointFlow, type BuiltEndpointRequest } from './endpointFlow.js';
import { formatUtcSqlDateTime } from '../../services/localTimeService.js';
import { resolveProxyLogBilling } from './proxyBilling.js';
import { openAiChatTransformer } from '../../transformers/openai/chat/index.js';
import { anthropicMessagesTransformer } from '../../transformers/anthropic/messages/index.js';
import { detectDownstreamClientContext, type DownstreamClientContext } from './downstreamClientContext.js';
import { getProxyResourceOwner } from '../../middleware/auth.js';
import { getProxyAuthContext, getProxyResourceOwner } from '../../middleware/auth.js';
import {
ProxyInputFileResolutionError,
hasNonImageFileInputInOpenAiBody,
resolveOpenAiBodyInputFiles,
} from '../../services/proxyInputFileResolver.js';
import {
buildOauthProviderHeaders,
} from '../../services/oauth/service.js';
import { getOauthInfoFromExtraConfig } from '../../services/oauth/oauthAccount.js';
import { recordOauthQuotaResetHint } from '../../services/oauth/quota.js';
import { refreshOauthAccessTokenSingleflight } from '../../services/oauth/refreshSingleflight.js';
import { collectResponsesFinalPayloadFromSse } from './responsesSseFinal.js';
import {
createGeminiCliStreamReader,
unwrapGeminiCliPayload,
} from './geminiCliCompat.js';
import { dispatchRuntimeRequest } from './runtimeExecutor.js';
const MAX_RETRIES = 2;
function isRecord(value: unknown): value is Record<string, unknown> {
return !!value && typeof value === 'object' && !Array.isArray(value);
}
function asTrimmedString(value: unknown): string {
return typeof value === 'string' ? value.trim() : '';
}
export async function chatProxyRoute(app: FastifyInstance) {
app.post('/v1/chat/completions', async (request: FastifyRequest, reply: FastifyReply) =>
handleChatProxyRequest(request, reply, 'openai'));
@@ -44,6 +64,8 @@ export async function chatProxyRoute(app: FastifyInstance) {
export async function claudeMessagesProxyRoute(app: FastifyInstance) {
app.post('/v1/messages', async (request: FastifyRequest, reply: FastifyReply) =>
handleChatProxyRequest(request, reply, 'claude'));
app.post('/v1/messages/count_tokens', async (request: FastifyRequest, reply: FastifyReply) =>
handleClaudeCountTokensRequest(request, reply));
}
async function handleChatProxyRequest(
@@ -87,6 +109,15 @@ async function handleChatProxyRequest(
}
}
const hasNonImageFileInput = hasNonImageFileInputInOpenAiBody(resolvedOpenAiBody);
const codexSessionCacheKey = deriveCodexSessionCacheKey({
downstreamFormat,
body: downstreamFormat === 'claude' ? claudeOriginalBody : request.body,
requestedModel,
proxyToken: getProxyAuthContext(request)?.token || null,
});
const downstreamApiKeyId = getProxyAuthContext(request)?.keyId ?? null;
const logDownstreamApiKeyId = downstreamApiKeyId !== null
&& await hasProxyLogDownstreamApiKeyIdColumn();
const excludeChannelIds: number[] = [];
let retryCount = 0;
@@ -114,6 +145,8 @@ async function handleChatProxyRequest(
excludeChannelIds.push(selected.channel.id);
const modelName = selected.actualModel || requestedModel;
const oauth = getOauthInfoFromExtraConfig(selected.account.extraConfig);
const isCodexSite = String(selected.site.platform || '').trim().toLowerCase() === 'codex';
const endpointCandidates = [
...await resolveUpstreamEndpointCandidates(
{
@@ -128,15 +161,24 @@ async function handleChatProxyRequest(
},
),
];
const buildProviderHeaders = () => (
buildOauthProviderHeaders({
extraConfig: typeof selected.account.extraConfig === 'string' ? selected.account.extraConfig : null,
downstreamHeaders: request.headers as Record<string, unknown>,
})
);
const buildEndpointRequest = (
endpoint: 'chat' | 'messages' | 'responses',
options: { forceNormalizeClaudeBody?: boolean } = {},
) => {
const upstreamStream = isStream || (isCodexSite && endpoint === 'responses');
const endpointRequest = buildUpstreamEndpointRequest({
endpoint,
modelName,
stream: isStream,
stream: upstreamStream,
tokenValue: selected.tokenValue,
oauthProvider: oauth?.provider,
oauthProjectId: oauth?.projectId,
sitePlatform: selected.site.platform,
siteUrl: selected.site.url,
openaiBody: resolvedOpenAiBody,
@@ -144,34 +186,73 @@ async function handleChatProxyRequest(
claudeOriginalBody,
forceNormalizeClaudeBody: options.forceNormalizeClaudeBody,
downstreamHeaders: request.headers as Record<string, unknown>,
providerHeaders: buildProviderHeaders(),
codexSessionCacheKey,
});
return {
endpoint,
path: endpointRequest.path,
headers: endpointRequest.headers,
body: endpointRequest.body as Record<string, unknown>,
runtime: endpointRequest.runtime,
};
};
const dispatchRequest = (
compatibilityRequest: BuiltEndpointRequest,
targetUrl?: string,
) => (
dispatchRuntimeRequest({
siteUrl: selected.site.url,
targetUrl,
request: compatibilityRequest,
buildInit: (_requestUrl, requestForFetch) => withSiteRecordProxyRequestInit(selected.site, {
method: 'POST',
headers: requestForFetch.headers,
body: JSON.stringify(requestForFetch.body),
}),
})
);
const endpointStrategy = downstreamTransformer.compatibility.createEndpointStrategy({
downstreamFormat,
endpointCandidates,
modelName,
requestedModelHint: requestedModel,
sitePlatform: selected.site.platform,
isStream,
isStream: isStream || isCodexSite,
buildRequest: ({ endpoint, forceNormalizeClaudeBody }) => buildEndpointRequest(
endpoint,
{ forceNormalizeClaudeBody },
),
dispatchRequest: (compatibilityRequest, targetUrl) => fetch(
targetUrl ?? `${selected.site.url}${compatibilityRequest.path}`,
withSiteRecordProxyRequestInit(selected.site, {
method: 'POST',
headers: compatibilityRequest.headers,
body: JSON.stringify(compatibilityRequest.body),
}, getProxyUrlFromExtraConfig(selected.account.extraConfig)),
),
dispatchRequest,
});
const tryRecover = async (ctx: Parameters<NonNullable<typeof endpointStrategy.tryRecover>>[0]) => {
if (ctx.response.status === 401 && oauth) {
try {
const refreshed = await refreshOauthAccessTokenSingleflight(selected.account.id);
selected.tokenValue = refreshed.accessToken;
selected.account = {
...selected.account,
accessToken: refreshed.accessToken,
extraConfig: refreshed.extraConfig ?? selected.account.extraConfig,
};
const refreshedRequest = buildEndpointRequest(ctx.request.endpoint);
const refreshedTargetUrl = `${selected.site.url}${refreshedRequest.path}`;
const refreshedResponse = await dispatchRequest(refreshedRequest, refreshedTargetUrl);
if (refreshedResponse.ok) {
return {
upstream: refreshedResponse,
upstreamPath: refreshedRequest.path,
};
}
ctx.request = refreshedRequest;
ctx.response = refreshedResponse;
ctx.rawErrText = await refreshedResponse.text().catch(() => 'unknown error');
} catch {
return endpointStrategy.tryRecover(ctx);
}
}
return endpointStrategy.tryRecover(ctx);
};
let startTime = Date.now();
try {
@@ -180,7 +261,8 @@ async function handleChatProxyRequest(
proxyUrl: resolveChannelProxyUrl(selected.site, selected.account.extraConfig),
endpointCandidates,
buildRequest: (endpoint) => buildEndpointRequest(endpoint),
tryRecover: endpointStrategy.tryRecover,
dispatchRequest,
tryRecover,
shouldDowngrade: endpointStrategy.shouldDowngrade,
onDowngrade: (ctx) => {
logProxy(
@@ -192,20 +274,22 @@ async function handleChatProxyRequest(
ctx.errText,
retryCount,
downstreamPath,
0,
0,
0,
0,
null,
null,
clientContext,
);
},
0,
0,
0,
0,
null,
null,
clientContext,
logDownstreamApiKeyId ? downstreamApiKeyId : null,
);
},
});
if (!endpointResult.ok) {
const status = endpointResult.status || 502;
const errText = endpointResult.errText || 'unknown error';
const rawErrText = endpointResult.rawErrText || errText;
tokenRouter.recordFailure(selected.channel.id);
logProxy(
selected,
@@ -223,7 +307,13 @@ async function handleChatProxyRequest(
null,
null,
clientContext,
logDownstreamApiKeyId ? downstreamApiKeyId : null,
);
await recordOauthQuotaResetHint({
accountId: selected.account.id,
statusCode: status,
errorText: rawErrText,
});
if (isTokenExpiredError({ status, message: errText })) {
await reportTokenExpired({
@@ -277,6 +367,7 @@ async function handleChatProxyRequest(
const streamSession = openAiChatTransformer.proxyStream.createSession({
downstreamFormat,
modelName,
successfulUpstreamPath,
onParsedPayload: (payload) => {
if (payload && typeof payload === 'object') {
parsedUsage = mergeProxyUsage(parsedUsage, parseProxyUsage(payload));
@@ -297,9 +388,35 @@ async function handleChatProxyRequest(
} catch {
fallbackData = fallbackText;
}
streamSession.consumeUpstreamFinalPayload(fallbackData, fallbackText, reply.raw);
if (String(selected.site.platform || '').trim().toLowerCase() === 'gemini-cli') {
fallbackData = unwrapGeminiCliPayload(fallbackData);
}
const streamResult = streamSession.consumeUpstreamFinalPayload(fallbackData, fallbackText, reply.raw);
const latency = Date.now() - startTime;
if (streamResult.status === 'failed') {
tokenRouter.recordFailure(selected.channel.id);
logProxy(
selected,
requestedModel,
'failed',
200,
latency,
streamResult.errorMessage,
retryCount,
downstreamPath,
parsedUsage.promptTokens,
parsedUsage.completionTokens,
parsedUsage.totalTokens,
0,
null,
successfulUpstreamPath,
clientContext,
logDownstreamApiKeyId ? downstreamApiKeyId : null,
);
return;
}
const resolvedUsage = await resolveProxyUsageWithSelfLogFallback({
site: selected.site,
account: selected.account,
@@ -341,14 +458,42 @@ async function handleChatProxyRequest(
estimatedCost,
billingDetails,
successfulUpstreamPath,
clientContext,
logDownstreamApiKeyId ? downstreamApiKeyId : null,
);
return;
}
const reader = upstream.body?.getReader();
await streamSession.run(reader, reply.raw);
const upstreamReader = upstream.body?.getReader();
const reader = String(selected.site.platform || '').trim().toLowerCase() === 'gemini-cli' && upstreamReader
? createGeminiCliStreamReader(upstreamReader)
: upstreamReader;
const streamResult = await streamSession.run(reader, reply.raw);
const latency = Date.now() - startTime;
if (streamResult.status === 'failed') {
tokenRouter.recordFailure(selected.channel.id);
logProxy(
selected,
requestedModel,
'failed',
200,
latency,
streamResult.errorMessage,
retryCount,
downstreamPath,
parsedUsage.promptTokens,
parsedUsage.completionTokens,
parsedUsage.totalTokens,
0,
null,
successfulUpstreamPath,
clientContext,
logDownstreamApiKeyId ? downstreamApiKeyId : null,
);
return;
}
const resolvedUsage = await resolveProxyUsageWithSelfLogFallback({
site: selected.site,
account: selected.account,
@@ -390,16 +535,30 @@ async function handleChatProxyRequest(
estimatedCost,
billingDetails,
successfulUpstreamPath,
clientContext,
logDownstreamApiKeyId ? downstreamApiKeyId : null,
);
return;
}
const rawText = await upstream.text();
let upstreamData: unknown = rawText;
try {
upstreamData = JSON.parse(rawText);
} catch {
const upstreamContentType = (upstream.headers.get('content-type') || '').toLowerCase();
let rawText = '';
let upstreamData: unknown;
if (upstreamContentType.includes('text/event-stream') && successfulUpstreamPath.endsWith('/responses')) {
const collected = await collectResponsesFinalPayloadFromSse(upstream, modelName);
rawText = collected.rawText;
upstreamData = collected.payload;
} else {
rawText = await upstream.text();
upstreamData = rawText;
try {
upstreamData = JSON.parse(rawText);
} catch {
upstreamData = rawText;
}
}
if (String(selected.site.platform || '').trim().toLowerCase() === 'gemini-cli') {
upstreamData = unwrapGeminiCliPayload(upstreamData);
}
const latency = Date.now() - startTime;
@@ -448,6 +607,8 @@ async function handleChatProxyRequest(
estimatedCost,
billingDetails,
successfulUpstreamPath,
clientContext,
logDownstreamApiKeyId ? downstreamApiKeyId : null,
);
return reply.send(downstreamResponse);
@@ -469,6 +630,7 @@ async function handleChatProxyRequest(
null,
null,
clientContext,
logDownstreamApiKeyId ? downstreamApiKeyId : null,
);
if (retryCount < MAX_RETRIES) {
@@ -491,6 +653,251 @@ async function handleChatProxyRequest(
}
}
function deriveCodexSessionCacheKey(input: {
downstreamFormat: DownstreamFormat | 'responses';
body: unknown;
requestedModel: string;
proxyToken: string | null;
}): string | null {
if (isRecord(input.body)) {
if (input.downstreamFormat === 'claude' && isRecord(input.body.metadata)) {
const userId = asTrimmedString(input.body.metadata.user_id);
if (userId) return `${input.requestedModel}:claude:${userId}`;
}
const promptCacheKey = asTrimmedString(input.body.prompt_cache_key);
if (promptCacheKey) return `${input.requestedModel}:responses:${promptCacheKey}`;
}
const proxyToken = asTrimmedString(input.proxyToken);
if (proxyToken) {
return `${input.requestedModel}:proxy:${proxyToken}`;
}
return null;
}
async function handleClaudeCountTokensRequest(
request: FastifyRequest,
reply: FastifyReply,
) {
const rawBody = isRecord(request.body) ? { ...request.body } : null;
if (!rawBody) {
return reply.code(400).send({
error: {
message: 'Request body must be a JSON object',
type: 'invalid_request_error',
},
});
}
const requestedModel = asTrimmedString(rawBody.model);
if (!requestedModel) {
return reply.code(400).send({
error: {
message: 'model is required',
type: 'invalid_request_error',
},
});
}
if (!await ensureModelAllowedForDownstreamKey(request, reply, requestedModel)) return;
const downstreamPath = '/v1/messages/count_tokens';
const clientContext = detectDownstreamClientContext({
downstreamPath,
headers: request.headers as Record<string, unknown>,
body: rawBody,
});
const downstreamPolicy = getDownstreamRoutingPolicy(request);
const downstreamApiKeyId = getProxyAuthContext(request)?.keyId ?? null;
const logDownstreamApiKeyId = downstreamApiKeyId !== null
&& await hasProxyLogDownstreamApiKeyIdColumn();
const excludeChannelIds: number[] = [];
let retryCount = 0;
while (retryCount <= MAX_RETRIES) {
let selected = retryCount === 0
? await tokenRouter.selectChannel(requestedModel, downstreamPolicy)
: await tokenRouter.selectNextChannel(requestedModel, excludeChannelIds, downstreamPolicy);
if (!selected && retryCount === 0) {
await refreshModelsAndRebuildRoutes();
selected = await tokenRouter.selectChannel(requestedModel, downstreamPolicy);
}
if (!selected) {
await reportProxyAllFailed({
model: requestedModel,
reason: 'No available channels after retries',
});
return reply.code(503).send({
error: { message: 'No available channels for this model', type: 'server_error' },
});
}
excludeChannelIds.push(selected.channel.id);
if (String(selected.site.platform || '').trim().toLowerCase() !== 'claude') {
if (retryCount < MAX_RETRIES) {
retryCount += 1;
continue;
}
return reply.code(501).send({
error: {
message: 'Claude count_tokens compatibility is not implemented for this upstream',
type: 'invalid_request_error',
},
});
}
const modelName = selected.actualModel || requestedModel;
const oauth = getOauthInfoFromExtraConfig(selected.account.extraConfig);
const startTime = Date.now();
const buildRequest = () => {
const upstreamRequest = buildClaudeCountTokensUpstreamRequest({
modelName,
tokenValue: selected.tokenValue,
oauthProvider: oauth?.provider,
sitePlatform: selected.site.platform,
claudeBody: rawBody,
downstreamHeaders: request.headers as Record<string, unknown>,
});
return {
endpoint: 'messages' as const,
path: upstreamRequest.path,
headers: upstreamRequest.headers,
body: upstreamRequest.body,
runtime: upstreamRequest.runtime,
};
};
try {
let upstreamRequest = buildRequest();
let upstream = await dispatchRuntimeRequest({
siteUrl: selected.site.url,
request: upstreamRequest,
buildInit: (_requestUrl, requestForFetch) => withSiteRecordProxyRequestInit(selected.site, {
method: 'POST',
headers: requestForFetch.headers,
body: JSON.stringify(requestForFetch.body),
}),
});
if (upstream.status === 401 && oauth) {
try {
const refreshed = await refreshOauthAccessTokenSingleflight(selected.account.id);
selected.tokenValue = refreshed.accessToken;
selected.account = {
...selected.account,
accessToken: refreshed.accessToken,
extraConfig: refreshed.extraConfig ?? selected.account.extraConfig,
};
upstreamRequest = buildRequest();
upstream = await dispatchRuntimeRequest({
siteUrl: selected.site.url,
request: upstreamRequest,
buildInit: (_requestUrl, requestForFetch) => withSiteRecordProxyRequestInit(selected.site, {
method: 'POST',
headers: requestForFetch.headers,
body: JSON.stringify(requestForFetch.body),
}),
});
} catch {
// Fall through to the regular upstream error handling below.
}
}
const latency = Date.now() - startTime;
const contentType = upstream.headers.get('content-type') || 'application/json';
const text = await upstream.text();
let payload: unknown = text;
try {
payload = JSON.parse(text);
} catch {
payload = text;
}
if (!upstream.ok) {
tokenRouter.recordFailure(selected.channel.id);
logProxy(
selected,
requestedModel,
'failed',
upstream.status,
latency,
typeof payload === 'string' ? payload : JSON.stringify(payload),
retryCount,
downstreamPath,
0,
0,
0,
0,
null,
upstreamRequest.path,
clientContext,
logDownstreamApiKeyId ? downstreamApiKeyId : null,
);
if (shouldRetryProxyRequest(upstream.status, typeof payload === 'string' ? payload : text) && retryCount < MAX_RETRIES) {
retryCount += 1;
continue;
}
return reply.code(upstream.status).type(contentType).send(payload);
}
tokenRouter.recordSuccess(selected.channel.id, latency, 0);
recordDownstreamCostUsage(request, 0);
logProxy(
selected,
requestedModel,
'success',
upstream.status,
latency,
null,
retryCount,
downstreamPath,
0,
0,
0,
0,
null,
upstreamRequest.path,
clientContext,
logDownstreamApiKeyId ? downstreamApiKeyId : null,
);
return reply.code(upstream.status).type(contentType).send(payload);
} catch (error: any) {
tokenRouter.recordFailure(selected.channel.id);
logProxy(
selected,
requestedModel,
'failed',
0,
Date.now() - startTime,
error?.message || 'network error',
retryCount,
downstreamPath,
0,
0,
0,
0,
null,
null,
clientContext,
logDownstreamApiKeyId ? downstreamApiKeyId : null,
);
if (retryCount < MAX_RETRIES) {
retryCount += 1;
continue;
}
return reply.code(502).send({
error: {
message: `Upstream error: ${error?.message || 'network failure'}`,
type: 'upstream_error',
},
});
}
}
}
async function logProxy(
selected: any,
modelRequested: string,
@@ -507,6 +914,7 @@ async function logProxy(
billingDetails: unknown = null,
upstreamPath: string | null = null,
clientContext: DownstreamClientContext | null = null,
downstreamApiKeyId: number | null = null,
) {
try {
const createdAt = formatUtcSqlDateTime(new Date());
@@ -524,6 +932,7 @@ async function logProxy(
routeId: selected.channel.routeId,
channelId: selected.channel.id,
accountId: selected.account.id,
...(downstreamApiKeyId !== null ? { downstreamApiKeyId } : {}),
modelRequested,
modelActual: selected.actualModel,
status,
src/server/routes/proxy/completions.ts
+11 -5
View File
@@ -1,6 +1,6 @@
import { FastifyInstance, FastifyReply, FastifyRequest } from 'fastify';
import { fetch } from 'undici';
import { db, schema } from '../../db/index.js';
import { db, hasProxyLogDownstreamApiKeyIdColumn, schema } from '../../db/index.js';
import { tokenRouter } from '../../services/tokenRouter.js';
import { refreshModelsAndRebuildRoutes } from '../../services/modelService.js';
import { reportProxyAllFailed, reportTokenExpired } from '../../services/alertService.js';
@@ -14,6 +14,7 @@ import { getProxyUrlFromExtraConfig } from '../../services/accountExtraConfig.js
import { composeProxyLogMessage } from './logPathMeta.js';
import { formatUtcSqlDateTime } from '../../services/localTimeService.js';
import { resolveProxyLogBilling } from './proxyBilling.js';
import { getProxyAuthContext } from '../../middleware/auth.js';
import { buildUpstreamUrl } from './upstreamUrl.js';
const MAX_RETRIES = 2;
@@ -27,6 +28,9 @@ export async function completionsProxyRoute(app: FastifyInstance) {
}
if (!await ensureModelAllowedForDownstreamKey(request, reply, requestedModel)) return;
const downstreamPolicy = getDownstreamRoutingPolicy(request);
const downstreamApiKeyId = getProxyAuthContext(request)?.keyId ?? null;
const logDownstreamApiKeyId = downstreamApiKeyId !== null
&& await hasProxyLogDownstreamApiKeyIdColumn();
const isStream = body.stream === true;
const excludeChannelIds: number[] = [];
@@ -71,7 +75,7 @@ export async function completionsProxyRoute(app: FastifyInstance) {
if (!upstream.ok) {
const errText = await upstream.text().catch(() => 'unknown error');
tokenRouter.recordFailure(selected.channel.id);
logProxy(selected, requestedModel, 'failed', upstream.status, Date.now() - startTime, errText, retryCount);
logProxy(selected, requestedModel, 'failed', upstream.status, Date.now() - startTime, errText, retryCount, logDownstreamApiKeyId ? downstreamApiKeyId : null);
if (isTokenExpiredError({ status: upstream.status, message: errText })) {
await reportTokenExpired({
@@ -172,7 +176,7 @@ export async function completionsProxyRoute(app: FastifyInstance) {
tokenRouter.recordSuccess(selected.channel.id, latency, estimatedCost);
recordDownstreamCostUsage(request, estimatedCost);
logProxy(
selected, requestedModel, 'success', 200, latency, null, retryCount,
selected, requestedModel, 'success', 200, latency, null, retryCount, logDownstreamApiKeyId ? downstreamApiKeyId : null,
resolvedUsage.promptTokens, resolvedUsage.completionTokens, resolvedUsage.totalTokens, estimatedCost, billingDetails,
);
return;
@@ -207,13 +211,13 @@ export async function completionsProxyRoute(app: FastifyInstance) {
tokenRouter.recordSuccess(selected.channel.id, latency, estimatedCost);
recordDownstreamCostUsage(request, estimatedCost);
logProxy(
selected, requestedModel, 'success', 200, latency, null, retryCount,
selected, requestedModel, 'success', 200, latency, null, retryCount, logDownstreamApiKeyId ? downstreamApiKeyId : null,
resolvedUsage.promptTokens, resolvedUsage.completionTokens, resolvedUsage.totalTokens, estimatedCost, billingDetails,
);
return reply.send(data);
} catch (err: any) {
tokenRouter.recordFailure(selected.channel.id);
logProxy(selected, requestedModel, 'failed', 0, Date.now() - startTime, err.message, retryCount);
logProxy(selected, requestedModel, 'failed', 0, Date.now() - startTime, err.message, retryCount, logDownstreamApiKeyId ? downstreamApiKeyId : null);
if (retryCount < MAX_RETRIES) {
retryCount++;
continue;
@@ -238,6 +242,7 @@ async function logProxy(
latencyMs: number,
errorMessage: string | null,
retryCount: number,
downstreamApiKeyId: number | null = null,
promptTokens = 0,
completionTokens = 0,
totalTokens = 0,
@@ -254,6 +259,7 @@ async function logProxy(
routeId: selected.channel.routeId,
channelId: selected.channel.id,
accountId: selected.account.id,
...(downstreamApiKeyId !== null ? { downstreamApiKeyId } : {}),
modelRequested,
modelActual: selected.actualModel,
status,
src/server/routes/proxy/embeddings.ts
+10 -31
View File
@@ -1,6 +1,6 @@
import { FastifyInstance, FastifyReply, FastifyRequest } from 'fastify';
import { fetch } from 'undici';
import { db, schema } from '../../db/index.js';
import { db, hasProxyLogDownstreamApiKeyIdColumn, schema } from '../../db/index.js';
import { tokenRouter } from '../../services/tokenRouter.js';
import { refreshModelsAndRebuildRoutes } from '../../services/modelService.js';
import { reportProxyAllFailed, reportTokenExpired } from '../../services/alertService.js';
@@ -14,6 +14,7 @@ import { getProxyUrlFromExtraConfig } from '../../services/accountExtraConfig.js
import { composeProxyLogMessage } from './logPathMeta.js';
import { formatUtcSqlDateTime } from '../../services/localTimeService.js';
import { resolveProxyLogBilling } from './proxyBilling.js';
import { getProxyAuthContext } from '../../middleware/auth.js';
import { buildUpstreamUrl } from './upstreamUrl.js';
const MAX_RETRIES = 2;
@@ -27,6 +28,9 @@ export async function embeddingsProxyRoute(app: FastifyInstance) {
}
if (!await ensureModelAllowedForDownstreamKey(request, reply, requestedModel)) return;
const downstreamPolicy = getDownstreamRoutingPolicy(request);
const downstreamApiKeyId = getProxyAuthContext(request)?.keyId ?? null;
const logDownstreamApiKeyId = downstreamApiKeyId !== null
&& await hasProxyLogDownstreamApiKeyIdColumn();
const excludeChannelIds: number[] = [];
let retryCount = 0;
@@ -68,7 +72,7 @@ export async function embeddingsProxyRoute(app: FastifyInstance) {
const text = await upstream.text();
if (!upstream.ok) {
tokenRouter.recordFailure(selected.channel.id);
logProxy(selected, requestedModel, 'failed', upstream.status, Date.now() - startTime, text, retryCount);
logProxy(selected, requestedModel, 'failed', upstream.status, Date.now() - startTime, text, retryCount, logDownstreamApiKeyId ? downstreamApiKeyId : null);
if (isTokenExpiredError({ status: upstream.status, message: text })) {
await reportTokenExpired({
@@ -91,43 +95,16 @@ export async function embeddingsProxyRoute(app: FastifyInstance) {
return reply.code(upstream.status).send({ error: { message: text, type: 'upstream_error' } });
}
let data: any = {};
try { data = JSON.parse(text); } catch { data = {}; }
const latency = Date.now() - startTime;
const parsedUsage = parseProxyUsage(data);
const resolvedUsage = await resolveProxyUsageWithSelfLogFallback({
site: selected.site,
account: selected.account,
tokenValue: selected.tokenValue,
tokenName: selected.tokenName,
modelName: selected.actualModel || requestedModel,
requestStartedAtMs: startTime,
requestEndedAtMs: startTime + latency,
localLatencyMs: latency,
usage: {
promptTokens: parsedUsage.promptTokens,
completionTokens: parsedUsage.completionTokens,
totalTokens: parsedUsage.totalTokens,
},
});
const { estimatedCost, billingDetails } = await resolveProxyLogBilling({
site: selected.site,
account: selected.account,
modelName: selected.actualModel || requestedModel,
parsedUsage,
resolvedUsage,
});
tokenRouter.recordSuccess(selected.channel.id, latency, estimatedCost);
recordDownstreamCostUsage(request, estimatedCost);
logProxy(
selected, requestedModel, 'success', upstream.status, latency, null, retryCount,
selected, requestedModel, 'success', upstream.status, latency, null, retryCount, logDownstreamApiKeyId ? downstreamApiKeyId : null,
resolvedUsage.promptTokens, resolvedUsage.completionTokens, resolvedUsage.totalTokens, estimatedCost, billingDetails,
);
return reply.code(upstream.status).send(data);
} catch (err: any) {
tokenRouter.recordFailure(selected.channel.id);
logProxy(selected, requestedModel, 'failed', 0, Date.now() - startTime, err.message, retryCount);
logProxy(selected, requestedModel, 'failed', 0, Date.now() - startTime, err.message, retryCount, logDownstreamApiKeyId ? downstreamApiKeyId : null);
if (retryCount < MAX_RETRIES) {
retryCount++;
continue;
@@ -150,6 +127,7 @@ async function logProxy(
latencyMs: number,
errorMessage: string | null,
retryCount: number,
downstreamApiKeyId: number | null = null,
promptTokens = 0,
completionTokens = 0,
totalTokens = 0,
@@ -166,6 +144,7 @@ async function logProxy(
routeId: selected.channel.routeId,
channelId: selected.channel.id,
accountId: selected.account.id,
...(downstreamApiKeyId !== null ? { downstreamApiKeyId } : {}),
modelRequested,
modelActual: selected.actualModel,
status,
src/server/routes/proxy/endpointFlow.test.ts
+34
View File
@@ -58,6 +58,24 @@ describe('executeEndpointFlow', () => {
expect(fetchMock).toHaveBeenCalledTimes(1);
});
it('uses the injected dispatchRequest hook instead of the default fetch path', async () => {
const dispatchRequest = vi.fn(async () => toUndiciResponse(new Response(JSON.stringify({ ok: true }), {
status: 200,
headers: { 'content-type': 'application/json' },
})));
const result = await executeEndpointFlow({
siteUrl: 'https://example.com',
endpointCandidates: ['responses'],
buildRequest: () => requestFor('/v1/responses'),
dispatchRequest,
});
expect(result.ok).toBe(true);
expect(dispatchRequest).toHaveBeenCalledTimes(1);
expect(fetchMock).not.toHaveBeenCalled();
});
it('avoids duplicated /v1 when base url already ends with /v1', async () => {
fetchMock.mockResolvedValueOnce(toUndiciResponse(new Response(JSON.stringify({ ok: true }), {
status: 200,
@@ -167,6 +185,22 @@ describe('executeEndpointFlow', () => {
expect(fetchMock).toHaveBeenCalledTimes(1);
});
it('uses proxyUrl for the default fetch path when no dispatch hook is provided', async () => {
fetchMock.mockResolvedValueOnce(toUndiciResponse(new Response(JSON.stringify({ ok: true }), {
status: 200,
headers: { 'content-type': 'application/json' },
})));
await executeEndpointFlow({
siteUrl: 'https://example.com',
proxyUrl: 'https://proxy.internal/base',
endpointCandidates: ['responses'],
buildRequest: () => requestFor('/v1/responses'),
});
expect(fetchMock.mock.calls[0]?.[0]).toBe('https://proxy.internal/base/v1/responses');
});
it('returns normalized final error when all endpoints fail', async () => {
fetchMock.mockResolvedValueOnce(toUndiciResponse(new Response(JSON.stringify({
error: { message: 'upstream_error', type: 'upstream_error' },
src/server/routes/proxy/endpointFlow.ts
+29 -9
View File
@@ -1,5 +1,5 @@
import { fetch } from 'undici';
import { withExplicitProxyRequestInit, withSiteProxyRequestInit } from '../../services/siteProxy.js';
import { withSiteProxyRequestInit } from '../../services/siteProxy.js';
import { summarizeUpstreamError } from './upstreamError.js';
import type { UpstreamEndpoint } from './upstreamEndpoint.js';
import { buildUpstreamUrl } from './upstreamUrl.js';
@@ -9,6 +9,13 @@ export type BuiltEndpointRequest = {
path: string;
headers: Record<string, string>;
body: Record<string, unknown>;
runtime?: {
executor: 'default' | 'codex' | 'gemini-cli' | 'antigravity' | 'claude';
modelName?: string;
stream?: boolean;
oauthProjectId?: string | null;
action?: 'generateContent' | 'streamGenerateContent' | 'countTokens';
};
};
export type EndpointAttemptContext = {
@@ -35,6 +42,7 @@ export type EndpointFlowResult =
ok: false;
status: number;
errText: string;
rawErrText?: string;
};
export function withUpstreamPath(path: string, message: string): string {
@@ -46,11 +54,21 @@ type ExecuteEndpointFlowInput = {
proxyUrl?: string | null;
endpointCandidates: UpstreamEndpoint[];
buildRequest: (endpoint: UpstreamEndpoint, endpointIndex: number) => BuiltEndpointRequest;
dispatchRequest?: (
request: BuiltEndpointRequest,
targetUrl: string,
) => Promise<Awaited<ReturnType<typeof fetch>>>;
tryRecover?: (ctx: EndpointAttemptContext) => Promise<EndpointRecoverResult>;
shouldDowngrade?: (ctx: EndpointAttemptContext) => boolean;
onDowngrade?: (ctx: EndpointAttemptContext & { errText: string }) => void | Promise<void>;
};
function buildAbsoluteUrl(base: string, path: string): string {
const normalizedBase = base.replace(/\/+$/, '');
const normalizedPath = path.replace(/^\/+/, '');
return normalizedPath ? `${normalizedBase}/${normalizedPath}` : normalizedBase;
}
export async function executeEndpointFlow(input: ExecuteEndpointFlowInput): Promise<EndpointFlowResult> {
const endpointCount = input.endpointCandidates.length;
if (endpointCount <= 0) {
@@ -63,19 +81,19 @@ export async function executeEndpointFlow(input: ExecuteEndpointFlowInput): Prom
let finalStatus = 0;
let finalErrText = 'unknown error';
let finalRawErrText: string | undefined;
for (let endpointIndex = 0; endpointIndex < endpointCount; endpointIndex += 1) {
const endpoint = input.endpointCandidates[endpointIndex] as UpstreamEndpoint;
const request = input.buildRequest(endpoint, endpointIndex);
const targetUrl = buildUpstreamUrl(input.siteUrl, request.path);
const defaultTarget = buildUpstreamUrl(input.siteUrl, request.path);
const targetUrl = input.proxyUrl
? buildAbsoluteUrl(input.proxyUrl, request.path)
: defaultTarget;
let response = await fetch(targetUrl, input.proxyUrl
? withExplicitProxyRequestInit(input.proxyUrl, {
method: 'POST',
headers: request.headers,
body: JSON.stringify(request.body),
})
: await withSiteProxyRequestInit(targetUrl, {
let response = input.dispatchRequest
? await input.dispatchRequest(request, targetUrl)
: await fetch(targetUrl, await withSiteProxyRequestInit(targetUrl, {
method: 'POST',
headers: request.headers,
body: JSON.stringify(request.body),
@@ -130,6 +148,7 @@ export async function executeEndpointFlow(input: ExecuteEndpointFlowInput): Prom
finalStatus = response.status;
finalErrText = errText;
finalRawErrText = rawErrText;
break;
}
@@ -137,5 +156,6 @@ export async function executeEndpointFlow(input: ExecuteEndpointFlowInput): Prom
ok: false,
status: finalStatus || 502,
errText: finalErrText || 'unknown error',
rawErrText: finalRawErrText,
};
}
src/server/routes/proxy/gemini.test.ts
+882 -65
View File
File diff suppressed because it is too large Load Diff
src/server/routes/proxy/gemini.ts
+685 -94
View File
@@ -1,11 +1,34 @@
import { FastifyInstance, FastifyReply, FastifyRequest } from 'fastify';
import { TextDecoder } from 'node:util';
import { fetch } from 'undici';
import { and, eq } from 'drizzle-orm';
import { db, schema } from '../../db/index.js';
import { formatUtcSqlDateTime } from '../../services/localTimeService.js';
import { parseProxyUsage } from '../../services/proxyUsageParser.js';
import { isModelAllowedByPolicyOrAllowedRoutes } from '../../services/downstreamApiKeyService.js';
import { tokenRouter } from '../../services/tokenRouter.js';
import { buildOauthProviderHeaders } from '../../services/oauth/service.js';
import { getOauthInfoFromExtraConfig } from '../../services/oauth/oauthAccount.js';
import { withSiteProxyRequestInit } from '../../services/siteProxy.js';
import { refreshModelsAndRebuildRoutes } from '../../services/modelService.js';
import { getDownstreamRoutingPolicy } from './downstreamPolicy.js';
import { executeEndpointFlow, type BuiltEndpointRequest } from './endpointFlow.js';
import { composeProxyLogMessage } from './logPathMeta.js';
import {
buildUpstreamEndpointRequest,
resolveUpstreamEndpointCandidates,
} from './upstreamEndpoint.js';
import {
geminiGenerateContentTransformer,
} from '../../transformers/gemini/generate-content/index.js';
import { createChatEndpointStrategy } from '../../transformers/shared/chatEndpointStrategy.js';
import { normalizeUpstreamFinalResponse } from '../../transformers/shared/normalized.js';
import {
createGeminiCliStreamReader,
unwrapGeminiCliPayload,
wrapGeminiCliRequest,
} from './geminiCliCompat.js';
import { dispatchRuntimeRequest } from './runtimeExecutor.js';
const MAX_RETRIES = 2;
const GEMINI_MODEL_PROBES = [
@@ -14,6 +37,52 @@ const GEMINI_MODEL_PROBES = [
'gemini-1.5-flash',
'gemini-pro',
];
const GEMINI_CLI_STATIC_MODELS = [
{ name: 'models/gemini-2.5-pro', displayName: 'Gemini 2.5 Pro' },
{ name: 'models/gemini-2.5-flash', displayName: 'Gemini 2.5 Flash' },
{ name: 'models/gemini-2.5-flash-lite', displayName: 'Gemini 2.5 Flash Lite' },
{ name: 'models/gemini-3-pro-preview', displayName: 'Gemini 3 Pro Preview' },
{ name: 'models/gemini-3.1-pro-preview', displayName: 'Gemini 3.1 Pro Preview' },
{ name: 'models/gemini-3-flash-preview', displayName: 'Gemini 3 Flash Preview' },
{ name: 'models/gemini-3.1-flash-lite-preview', displayName: 'Gemini 3.1 Flash Lite Preview' },
];
const EMPTY_PROXY_USAGE = {
promptTokens: 0,
completionTokens: 0,
totalTokens: 0,
};
function isGeminiCliPlatform(platform: unknown): boolean {
return String(platform || '').trim().toLowerCase() === 'gemini-cli';
}
function isAntigravityPlatform(platform: unknown): boolean {
return String(platform || '').trim().toLowerCase() === 'antigravity';
}
function isInternalGeminiPlatform(platform: unknown): boolean {
return isGeminiCliPlatform(platform) || isAntigravityPlatform(platform);
}
function buildGeminiCliActionPath(input: {
isStreamAction: boolean;
isCountTokensAction: boolean;
}) {
if (input.isCountTokensAction) return '/v1internal:countTokens';
if (input.isStreamAction) return '/v1internal:streamGenerateContent?alt=sse';
return '/v1internal:generateContent';
}
function isDirectGeminiFamilyPlatform(platform: unknown): boolean {
const normalized = String(platform || '').trim().toLowerCase();
return normalized === 'gemini' || normalized === 'gemini-cli' || normalized === 'antigravity';
}
function omitGeminiCliModelField(body: unknown): Record<string, unknown> {
if (!body || typeof body !== 'object' || Array.isArray(body)) return {};
const { model: _model, ...rest } = body as Record<string, unknown>;
return rest;
}
async function selectGeminiChannel(request: FastifyRequest) {
const policy = getDownstreamRoutingPolicy(request);
@@ -33,6 +102,141 @@ async function selectNextGeminiProbeChannel(request: FastifyRequest, excludeChan
return null;
}
function resolveDownstreamPath(request: FastifyRequest): string {
const rawUrl = request.raw.url || request.url || '';
const withoutQuery = rawUrl.split('?')[0] || rawUrl;
return withoutQuery || '/v1beta/models';
}
function resolveUpstreamPath(apiVersion: string, modelActionPath: string): string {
const normalizedVersion = apiVersion.replace(/^\/+/, '');
const normalizedAction = modelActionPath.replace(/^\/+/, '');
return `/${normalizedVersion}/${normalizedAction}`;
}
function hasDownstreamModelRestrictions(policy: { supportedModels?: unknown; allowedRouteIds?: unknown }): boolean {
const supportedModels = Array.isArray(policy.supportedModels) ? policy.supportedModels : [];
const allowedRouteIds = Array.isArray(policy.allowedRouteIds) ? policy.allowedRouteIds : [];
return supportedModels.length > 0 || allowedRouteIds.length > 0;
}
function extractGeminiListedModelName(item: unknown): string {
if (!item || typeof item !== 'object') return '';
const rawName = typeof (item as { name?: unknown }).name === 'string'
? (item as { name: string }).name.trim()
: '';
if (!rawName) return '';
return rawName.startsWith('models/') ? rawName.slice('models/'.length) : rawName;
}
async function filterGeminiListedModelsForPolicy(
payload: unknown,
request: FastifyRequest,
): Promise<unknown> {
if (!payload || typeof payload !== 'object' || !Array.isArray((payload as { models?: unknown[] }).models)) {
return payload;
}
const policy = getDownstreamRoutingPolicy(request);
if (!hasDownstreamModelRestrictions(policy)) {
return payload;
}
const filteredModels: unknown[] = [];
for (const item of (payload as { models: unknown[] }).models) {
const modelName = extractGeminiListedModelName(item);
if (!modelName) continue;
if (!await isModelAllowedByPolicyOrAllowedRoutes(modelName, policy)) continue;
const decision = await tokenRouter.explainSelection?.(modelName, [], policy);
if (decision && typeof decision.selectedChannelId !== 'number') continue;
filteredModels.push(item);
}
return {
...(payload as Record<string, unknown>),
models: filteredModels,
};
}
async function readRouteAwareGeminiModels(request: FastifyRequest): Promise<Array<{ name: string; displayName: string }>> {
const policy = getDownstreamRoutingPolicy(request);
const rows = await db.select({ modelName: schema.modelAvailability.modelName })
.from(schema.modelAvailability)
.innerJoin(schema.accounts, eq(schema.modelAvailability.accountId, schema.accounts.id))
.innerJoin(schema.sites, eq(schema.accounts.siteId, schema.sites.id))
.where(and(
eq(schema.modelAvailability.available, true),
eq(schema.accounts.status, 'active'),
eq(schema.sites.status, 'active'),
))
.all();
const routeAliases = await db.select({ displayName: schema.tokenRoutes.displayName })
.from(schema.tokenRoutes)
.where(eq(schema.tokenRoutes.enabled, true))
.all();
const deduped = Array.from(new Set([
...rows.map((row) => String(row.modelName || '').trim()).filter(Boolean),
...routeAliases.map((row) => String(row.displayName || '').trim()).filter(Boolean),
])).sort();
const allowed: Array<{ name: string; displayName: string }> = [];
for (const modelName of deduped) {
if (!await isModelAllowedByPolicyOrAllowedRoutes(modelName, policy)) continue;
const decision = await tokenRouter.explainSelection?.(modelName, [], policy);
if (decision && typeof decision.selectedChannelId !== 'number') continue;
allowed.push({
name: `models/${modelName}`,
displayName: modelName,
});
}
return allowed;
}
async function logProxy(
selected: any,
modelRequested: string,
status: string,
httpStatus: number,
latencyMs: number,
errorMessage: string | null,
retryCount: number,
downstreamPath: string,
upstreamPath: string | null,
promptTokens = 0,
completionTokens = 0,
totalTokens = 0,
) {
try {
const createdAt = formatUtcSqlDateTime(new Date());
const normalizedErrorMessage = composeProxyLogMessage({
downstreamPath,
upstreamPath,
errorMessage,
});
await db.insert(schema.proxyLogs).values({
routeId: selected.channel.routeId,
channelId: selected.channel.id,
accountId: selected.account.id,
modelRequested,
modelActual: selected.actualModel || modelRequested,
status,
httpStatus,
latencyMs,
promptTokens,
completionTokens,
totalTokens,
estimatedCost: 0,
billingDetails: null,
errorMessage: normalizedErrorMessage,
retryCount,
createdAt,
}).run();
} catch (error) {
console.warn('[proxy/gemini] failed to write proxy log', error);
}
}
export async function geminiProxyRoute(app: FastifyInstance) {
const listModels = async (request: FastifyRequest, reply: FastifyReply) => {
const apiVersion = geminiGenerateContentTransformer.resolveProxyApiVersion(
@@ -55,6 +259,23 @@ export async function geminiProxyRoute(app: FastifyInstance) {
excludeChannelIds.push(selected.channel.id);
try {
if (!isDirectGeminiFamilyPlatform(selected.site.platform)) {
let models = await readRouteAwareGeminiModels(request);
if (models.length <= 0) {
await refreshModelsAndRebuildRoutes();
models = await readRouteAwareGeminiModels(request);
}
return reply.code(200).send({ models });
}
if (isGeminiCliPlatform(selected.site.platform)) {
const filtered = await filterGeminiListedModelsForPolicy(
{ models: GEMINI_CLI_STATIC_MODELS },
request,
);
return reply.code(200).send(filtered);
}
const upstream = await fetch(
geminiGenerateContentTransformer.resolveModelsUrl(selected.site.url, apiVersion, selected.tokenValue),
{ method: 'GET' },
@@ -72,7 +293,9 @@ export async function geminiProxyRoute(app: FastifyInstance) {
}
try {
return reply.code(upstream.status).send(JSON.parse(text));
const parsed = JSON.parse(text);
const filtered = await filterGeminiListedModelsForPolicy(parsed, request);
return reply.code(upstream.status).send(filtered);
} catch {
return reply.code(upstream.status).type(upstream.headers.get('content-type') || 'application/json').send(text);
}
@@ -94,12 +317,36 @@ export async function geminiProxyRoute(app: FastifyInstance) {
}
};
const generateContent = async (request: FastifyRequest, reply: FastifyReply) => {
const parsedPath = geminiGenerateContentTransformer.parseProxyRequestPath({
rawUrl: request.raw.url || request.url || '',
params: request.params as { geminiApiVersion?: string } | undefined,
});
const handleGenerateContent = async (
request: FastifyRequest,
reply: FastifyReply,
options?: {
downstreamProtocol?: 'gemini' | 'gemini-cli';
action?: 'generateContent' | 'streamGenerateContent' | 'countTokens';
},
) => {
const downstreamProtocol = options?.downstreamProtocol || 'gemini';
const isGeminiCliDownstream = downstreamProtocol === 'gemini-cli';
const cliRequestedModel = isGeminiCliDownstream
? (typeof (request.body as Record<string, unknown> | null | undefined)?.model === 'string'
? String((request.body as Record<string, unknown>).model).trim()
: '')
: '';
const parsedPath = isGeminiCliDownstream
? {
apiVersion: 'v1beta',
modelActionPath: `models/${cliRequestedModel}:${options?.action || 'generateContent'}`,
isStreamAction: options?.action === 'streamGenerateContent',
requestedModel: cliRequestedModel,
}
: geminiGenerateContentTransformer.parseProxyRequestPath({
rawUrl: request.raw.url || request.url || '',
params: request.params as { geminiApiVersion?: string } | undefined,
});
const { apiVersion, modelActionPath, isStreamAction, requestedModel } = parsedPath;
const isCountTokensAction = isGeminiCliDownstream
? options?.action === 'countTokens'
: modelActionPath.endsWith(':countTokens');
if (!requestedModel) {
return reply.code(400).send({
error: { message: 'Gemini model path is required', type: 'invalid_request_error' },
@@ -107,6 +354,7 @@ export async function geminiProxyRoute(app: FastifyInstance) {
}
const policy = getDownstreamRoutingPolicy(request);
const downstreamPath = resolveDownstreamPath(request);
const excludeChannelIds: number[] = [];
let retryCount = 0;
let lastStatus = 503;
@@ -123,109 +371,424 @@ export async function geminiProxyRoute(app: FastifyInstance) {
excludeChannelIds.push(selected.channel.id);
const body = geminiGenerateContentTransformer.inbound.normalizeRequest(
request.body || {},
selected.actualModel || requestedModel,
const actualModel = selected.actualModel || requestedModel;
const normalizedBody = geminiGenerateContentTransformer.inbound.normalizeRequest(
isGeminiCliDownstream ? omitGeminiCliModelField(request.body) : (request.body || {}),
actualModel,
);
const oauth = getOauthInfoFromExtraConfig(selected.account.extraConfig);
const isGeminiCli = isGeminiCliPlatform(selected.site.platform);
const isInternalGemini = isInternalGeminiPlatform(selected.site.platform);
const isDirectGeminiFamily = isDirectGeminiFamilyPlatform(selected.site.platform);
const startTime = Date.now();
let upstreamPath = '';
const actualModelAction = modelActionPath.replace(
/^models\/[^:]+/,
`models/${selected.actualModel || requestedModel}`,
);
const query = new URLSearchParams(request.query as Record<string, string>).toString();
try {
const upstream = await fetch(
geminiGenerateContentTransformer.resolveActionUrl(
selected.site.url,
apiVersion,
actualModelAction,
selected.tokenValue,
query,
),
{
method: 'POST',
headers: {
if (isDirectGeminiFamily) {
if (isGeminiCli && !oauth?.projectId) {
lastStatus = 500;
lastContentType = 'application/json';
lastText = JSON.stringify({
error: {
message: 'Gemini CLI OAuth project is missing',
type: 'server_error',
},
});
await tokenRouter.recordFailure?.(selected.channel.id);
if (retryCount < MAX_RETRIES) {
retryCount += 1;
continue;
}
return reply.code(lastStatus).type(lastContentType).send(lastText);
}
const actualModelAction = modelActionPath.replace(
/^models\/[^:]+/,
`models/${actualModel}`,
);
upstreamPath = isInternalGemini
? buildGeminiCliActionPath({ isStreamAction, isCountTokensAction })
: resolveUpstreamPath(apiVersion, actualModelAction);
const query = new URLSearchParams(request.query as Record<string, string>).toString();
const requestBody = isInternalGemini
? (
isCountTokensAction
? { request: normalizedBody }
: wrapGeminiCliRequest({
modelName: actualModel,
projectId: oauth?.projectId || '',
request: normalizedBody as Record<string, unknown>,
})
)
: normalizedBody;
const requestHeaders = isInternalGemini
? {
'Content-Type': 'application/json',
...(isStreamAction ? { Accept: 'text/event-stream' } : {}),
Authorization: `Bearer ${selected.tokenValue}`,
...buildOauthProviderHeaders({
extraConfig: typeof selected.account.extraConfig === 'string' ? selected.account.extraConfig : null,
downstreamHeaders: request.headers as Record<string, unknown>,
}),
}
: {
'Content-Type': 'application/json',
};
const targetUrl = isInternalGemini
? `${selected.site.url}${upstreamPath}`
: geminiGenerateContentTransformer.resolveActionUrl(
selected.site.url,
apiVersion,
actualModelAction,
selected.tokenValue,
query,
);
const upstream = isInternalGemini
? await dispatchRuntimeRequest({
siteUrl: selected.site.url,
targetUrl,
request: {
endpoint: 'chat',
path: upstreamPath,
headers: requestHeaders,
body: requestBody as Record<string, unknown>,
runtime: {
executor: isGeminiCli ? 'gemini-cli' : 'antigravity',
modelName: actualModel,
stream: isStreamAction,
oauthProjectId: oauth?.projectId || null,
action: isCountTokensAction
? 'countTokens'
: (isStreamAction ? 'streamGenerateContent' : 'generateContent'),
},
},
buildInit: async (requestUrl, requestForFetch) => withSiteProxyRequestInit(requestUrl, {
method: 'POST',
headers: requestForFetch.headers,
body: JSON.stringify(requestForFetch.body),
}),
})
: await fetch(targetUrl, {
method: 'POST',
headers: requestHeaders,
body: JSON.stringify(requestBody),
});
const contentType = upstream.headers.get('content-type') || 'application/json';
if (!upstream.ok) {
lastStatus = upstream.status;
lastContentType = contentType;
lastText = await upstream.text();
await tokenRouter.recordFailure?.(selected.channel.id);
await logProxy(
selected,
requestedModel,
'failed',
lastStatus,
Date.now() - startTime,
lastText,
retryCount,
downstreamPath,
upstreamPath,
);
if (retryCount < MAX_RETRIES) {
retryCount += 1;
continue;
}
try {
return reply.code(lastStatus).send(JSON.parse(lastText));
} catch {
return reply.code(lastStatus).type(lastContentType).send(lastText);
}
}
if (geminiGenerateContentTransformer.stream.isSseContentType(contentType)) {
reply.hijack();
reply.raw.statusCode = upstream.status;
reply.raw.setHeader('Content-Type', contentType || 'text/event-stream');
const upstreamReader = upstream.body?.getReader();
const reader = isInternalGemini && !isGeminiCliDownstream && upstreamReader
? createGeminiCliStreamReader(upstreamReader)
: upstreamReader;
if (!reader) {
const latency = Date.now() - startTime;
await tokenRouter.recordSuccess?.(selected.channel.id, latency, 0);
await logProxy(
selected,
requestedModel,
'success',
upstream.status,
latency,
null,
retryCount,
downstreamPath,
upstreamPath,
);
reply.raw.end();
return;
}
const aggregateState = geminiGenerateContentTransformer.stream.createAggregateState();
const decoder = new TextDecoder();
let rest = '';
try {
while (true) {
const { done, value } = await reader.read();
if (done) break;
if (!value) continue;
const chunkText = decoder.decode(value, { stream: true });
const consumed = geminiGenerateContentTransformer.stream.consumeUpstreamSseBuffer(
aggregateState,
rest + chunkText,
);
rest = consumed.rest;
for (const line of consumed.lines) {
reply.raw.write(line);
}
}
const tail = decoder.decode();
if (tail) {
const consumed = geminiGenerateContentTransformer.stream.consumeUpstreamSseBuffer(
aggregateState,
rest + tail,
);
for (const line of consumed.lines) {
reply.raw.write(line);
}
}
} finally {
reader.releaseLock();
reply.raw.end();
}
const parsedUsage = parseProxyUsage(aggregateState);
const latency = Date.now() - startTime;
await tokenRouter.recordSuccess?.(selected.channel.id, latency, 0);
await logProxy(
selected,
requestedModel,
'success',
upstream.status,
latency,
null,
retryCount,
downstreamPath,
upstreamPath,
parsedUsage.promptTokens,
parsedUsage.completionTokens,
parsedUsage.totalTokens,
);
return;
}
const text = await upstream.text();
const aggregateState = geminiGenerateContentTransformer.stream.createAggregateState();
let parsedUsage = EMPTY_PROXY_USAGE;
try {
const parsed = JSON.parse(text);
const unwrappedPayload = isInternalGemini && !isGeminiCliDownstream
? unwrapGeminiCliPayload(parsed)
: parsed;
const responsePayload = isCountTokensAction
? unwrappedPayload
: geminiGenerateContentTransformer.stream.serializeUpstreamJsonPayload(
aggregateState,
unwrappedPayload,
isStreamAction,
);
parsedUsage = parseProxyUsage(aggregateState);
const latency = Date.now() - startTime;
await tokenRouter.recordSuccess?.(selected.channel.id, latency, 0);
await logProxy(
selected,
requestedModel,
'success',
upstream.status,
latency,
null,
retryCount,
downstreamPath,
upstreamPath,
parsedUsage.promptTokens,
parsedUsage.completionTokens,
parsedUsage.totalTokens,
);
return reply.code(upstream.status).send(
isGeminiCliDownstream && !isCountTokensAction
? { response: responsePayload }
: responsePayload,
);
} catch {
const latency = Date.now() - startTime;
await tokenRouter.recordSuccess?.(selected.channel.id, latency, 0);
await logProxy(
selected,
requestedModel,
'success',
upstream.status,
latency,
null,
retryCount,
downstreamPath,
upstreamPath,
);
return reply.code(upstream.status).type(contentType || 'application/json').send(text);
}
}
if (isCountTokensAction) {
lastStatus = 501;
lastContentType = 'application/json';
lastText = JSON.stringify({
error: {
message: 'Gemini countTokens compatibility is not implemented for this upstream',
type: 'invalid_request_error',
},
body: JSON.stringify(body),
});
return reply.code(lastStatus).type(lastContentType).send(lastText);
}
const openAiBody = geminiGenerateContentTransformer.compatibility.buildOpenAiBodyFromGeminiRequest({
body: normalizedBody as Record<string, unknown>,
modelName: actualModel,
stream: isStreamAction,
});
const endpointCandidates = await resolveUpstreamEndpointCandidates(
{
site: selected.site,
account: selected.account,
},
actualModel,
'openai',
requestedModel,
);
const contentType = upstream.headers.get('content-type') || 'application/json';
if (!upstream.ok) {
lastStatus = upstream.status;
lastContentType = contentType;
lastText = await upstream.text();
const buildEndpointRequest = (
endpoint: 'chat' | 'messages' | 'responses',
requestOptions: { forceNormalizeClaudeBody?: boolean } = {},
) => {
const endpointRequest = buildUpstreamEndpointRequest({
endpoint,
modelName: actualModel,
stream: isStreamAction,
tokenValue: selected.tokenValue,
oauthProvider: oauth?.provider,
oauthProjectId: oauth?.projectId,
sitePlatform: selected.site.platform,
siteUrl: selected.site.url,
openaiBody: openAiBody,
downstreamFormat: 'openai',
forceNormalizeClaudeBody: requestOptions.forceNormalizeClaudeBody,
downstreamHeaders: request.headers as Record<string, unknown>,
providerHeaders: buildOauthProviderHeaders({
extraConfig: typeof selected.account.extraConfig === 'string' ? selected.account.extraConfig : null,
downstreamHeaders: request.headers as Record<string, unknown>,
}),
});
return {
endpoint,
path: endpointRequest.path,
headers: endpointRequest.headers,
body: endpointRequest.body as Record<string, unknown>,
runtime: endpointRequest.runtime,
};
};
const dispatchRequest = (compatibilityRequest: BuiltEndpointRequest, targetUrl?: string) => (
dispatchRuntimeRequest({
siteUrl: selected.site.url,
targetUrl,
request: compatibilityRequest,
buildInit: async (requestUrl, requestForFetch) => withSiteProxyRequestInit(requestUrl, {
method: 'POST',
headers: requestForFetch.headers,
body: JSON.stringify(requestForFetch.body),
}),
})
);
const endpointStrategy = createChatEndpointStrategy({
downstreamFormat: 'openai',
endpointCandidates,
modelName: actualModel,
requestedModelHint: requestedModel,
sitePlatform: selected.site.platform,
isStream: isStreamAction,
buildRequest: ({ endpoint, forceNormalizeClaudeBody }) => buildEndpointRequest(
endpoint,
{ forceNormalizeClaudeBody },
),
dispatchRequest,
});
const endpointResult = await executeEndpointFlow({
siteUrl: selected.site.url,
endpointCandidates,
buildRequest: (endpoint) => buildEndpointRequest(endpoint),
dispatchRequest,
tryRecover: endpointStrategy.tryRecover,
shouldDowngrade: endpointStrategy.shouldDowngrade,
});
if (!endpointResult.ok) {
lastStatus = endpointResult.status;
lastContentType = 'application/json';
lastText = JSON.stringify({
error: {
message: endpointResult.errText,
type: 'upstream_error',
},
});
await tokenRouter.recordFailure?.(selected.channel.id);
await logProxy(
selected,
requestedModel,
'failed',
lastStatus,
Date.now() - startTime,
endpointResult.errText,
retryCount,
downstreamPath,
null,
);
if (retryCount < MAX_RETRIES) {
retryCount += 1;
continue;
}
try {
return reply.code(lastStatus).send(JSON.parse(lastText));
} catch {
return reply.code(lastStatus).type(lastContentType).send(lastText);
}
return reply.code(lastStatus).type(lastContentType).send(lastText);
}
if (geminiGenerateContentTransformer.stream.isSseContentType(contentType)) {
reply.hijack();
reply.raw.statusCode = upstream.status;
reply.raw.setHeader('Content-Type', contentType || 'text/event-stream');
const reader = upstream.body?.getReader();
if (!reader) {
reply.raw.end();
return;
}
const aggregateState = geminiGenerateContentTransformer.stream.createAggregateState();
const decoder = new TextDecoder();
let rest = '';
try {
while (true) {
const { done, value } = await reader.read();
if (done) break;
if (!value) continue;
const chunkText = decoder.decode(value, { stream: true });
const consumed = geminiGenerateContentTransformer.stream.consumeUpstreamSseBuffer(
aggregateState,
rest + chunkText,
);
rest = consumed.rest;
for (const line of consumed.lines) {
reply.raw.write(line);
}
}
const tail = decoder.decode();
if (tail) {
const consumed = geminiGenerateContentTransformer.stream.consumeUpstreamSseBuffer(
aggregateState,
rest + tail,
);
for (const line of consumed.lines) {
reply.raw.write(line);
}
}
} finally {
reader.releaseLock();
reply.raw.end();
}
return;
}
const text = await upstream.text();
upstreamPath = endpointResult.upstreamPath;
const upstream = endpointResult.upstream;
const rawText = await upstream.text();
let upstreamData: unknown = rawText;
try {
const parsed = JSON.parse(text);
const aggregateState = geminiGenerateContentTransformer.stream.createAggregateState();
return reply.code(upstream.status).send(
geminiGenerateContentTransformer.stream.serializeUpstreamJsonPayload(
aggregateState,
parsed,
isStreamAction,
),
);
} catch {
return reply.code(upstream.status).type(contentType || 'application/json').send(text);
}
upstreamData = JSON.parse(rawText);
} catch {}
const parsedUsage = parseProxyUsage(upstreamData);
const normalizedFinal = normalizeUpstreamFinalResponse(upstreamData, actualModel, rawText);
const geminiResponse = geminiGenerateContentTransformer.compatibility.serializeNormalizedFinalToGemini({
normalized: normalizedFinal,
usage: {
promptTokens: parsedUsage.promptTokens,
completionTokens: parsedUsage.completionTokens,
totalTokens: parsedUsage.totalTokens,
},
});
const latency = Date.now() - startTime;
await tokenRouter.recordSuccess?.(selected.channel.id, latency, 0);
await logProxy(
selected,
requestedModel,
'success',
upstream.status,
latency,
null,
retryCount,
downstreamPath,
upstreamPath,
parsedUsage.promptTokens,
parsedUsage.completionTokens,
parsedUsage.totalTokens,
);
return reply.code(upstream.status).send(
isGeminiCliDownstream
? { response: geminiResponse }
: geminiResponse,
);
} catch (error) {
lastStatus = 502;
lastContentType = 'application/json';
@@ -236,6 +799,17 @@ export async function geminiProxyRoute(app: FastifyInstance) {
},
});
await tokenRouter.recordFailure?.(selected.channel.id);
await logProxy(
selected,
requestedModel,
'failed',
0,
Date.now() - startTime,
error instanceof Error ? error.message : 'Gemini upstream request failed',
retryCount,
downstreamPath,
upstreamPath || null,
);
if (retryCount < MAX_RETRIES) {
retryCount += 1;
continue;
@@ -245,8 +819,25 @@ export async function geminiProxyRoute(app: FastifyInstance) {
}
};
const generateContent = async (request: FastifyRequest, reply: FastifyReply) => handleGenerateContent(request, reply);
const geminiCliGenerateContent = async (request: FastifyRequest, reply: FastifyReply) => handleGenerateContent(request, reply, {
downstreamProtocol: 'gemini-cli',
action: 'generateContent',
});
const geminiCliStreamGenerateContent = async (request: FastifyRequest, reply: FastifyReply) => handleGenerateContent(request, reply, {
downstreamProtocol: 'gemini-cli',
action: 'streamGenerateContent',
});
const geminiCliCountTokens = async (request: FastifyRequest, reply: FastifyReply) => handleGenerateContent(request, reply, {
downstreamProtocol: 'gemini-cli',
action: 'countTokens',
});
app.get('/v1beta/models', listModels);
app.get('/gemini/:geminiApiVersion/models', listModels);
app.post('/v1beta/models/*', generateContent);
app.post('/gemini/:geminiApiVersion/models/*', generateContent);
app.post('/v1internal::generateContent', geminiCliGenerateContent);
app.post('/v1internal::streamGenerateContent', geminiCliStreamGenerateContent);
app.post('/v1internal::countTokens', geminiCliCountTokens);
}
src/server/routes/proxy/geminiCliCompat.thoughtSignature.test.ts
+196
View File
@@ -0,0 +1,196 @@
import { describe, it, expect } from 'vitest';
import { buildGeminiGenerateContentRequestFromOpenAi } from './geminiCliCompat.js';
describe('thoughtSignature injection in OpenAI→Gemini conversion', () => {
it('injects thoughtSignature from provider_specific_fields into functionCall parts', () => {
const result = buildGeminiGenerateContentRequestFromOpenAi({
body: {
model: 'gemini-3-flash-preview',
messages: [
{ role: 'user', content: 'What is the weather?' },
{
role: 'assistant',
content: 'Let me check.',
tool_calls: [
{
id: 'call_123',
type: 'function',
function: { name: 'get_weather', arguments: '{"city":"Tokyo"}' },
provider_specific_fields: { thought_signature: 'real_sig_abc' },
},
],
},
{ role: 'tool', tool_call_id: 'call_123', content: '{"temp":"22C"}' },
],
},
modelName: 'gemini-3-flash-preview',
}) as Record<string, unknown>;
const contents = result.contents as Array<Record<string, unknown>>;
// Find the model message with functionCall
const modelMsgs = contents.filter((c) => c.role === 'model');
const fcParts = modelMsgs
.flatMap((m) => (m.parts as Array<Record<string, unknown>>))
.filter((p) => 'functionCall' in p);
expect(fcParts.length).toBe(1);
expect(fcParts[0].thoughtSignature).toBe('real_sig_abc');
});
it('splits text and signed functionCall parts into separate model messages', () => {
const result = buildGeminiGenerateContentRequestFromOpenAi({
body: {
model: 'gemini-3-flash-preview',
messages: [
{ role: 'user', content: 'Read the file.' },
{
role: 'assistant',
content: 'I will read it.',
tool_calls: [
{
id: 'call_456',
type: 'function',
function: { name: 'Read', arguments: '{"path":"/tmp/x"}' },
provider_specific_fields: { thought_signature: 'sig_split_test' },
},
],
},
{ role: 'tool', tool_call_id: 'call_456', content: 'file content here' },
],
},
modelName: 'gemini-3-flash-preview',
}) as Record<string, unknown>;
const contents = result.contents as Array<Record<string, unknown>>;
const modelMsgs = contents.filter((c) => c.role === 'model');
// Should be 2 model messages: text (unsigned) + functionCall (signed)
expect(modelMsgs.length).toBe(2);
const firstParts = modelMsgs[0].parts as Array<Record<string, unknown>>;
const secondParts = modelMsgs[1].parts as Array<Record<string, unknown>>;
// First: text only, no thoughtSignature
expect(firstParts.every((p) => 'text' in p)).toBe(true);
expect(firstParts.every((p) => !('thoughtSignature' in p))).toBe(true);
// Second: functionCall with thoughtSignature
expect(secondParts.every((p) => 'functionCall' in p)).toBe(true);
expect(secondParts[0].thoughtSignature).toBe('sig_split_test');
});
it('injects dummy sentinel when thinking enabled but no signature available', () => {
const result = buildGeminiGenerateContentRequestFromOpenAi({
body: {
model: 'gemini-3-flash-preview',
reasoning_effort: 'high',
messages: [
{ role: 'user', content: 'Do something.' },
{
role: 'assistant',
tool_calls: [
{
id: 'call_no_sig',
type: 'function',
function: { name: 'Bash', arguments: '{"command":"ls"}' },
// No provider_specific_fields — signature missing
},
],
},
{ role: 'tool', tool_call_id: 'call_no_sig', content: 'file1\nfile2' },
],
},
modelName: 'gemini-3-flash-preview',
}) as Record<string, unknown>;
const contents = result.contents as Array<Record<string, unknown>>;
const modelMsgs = contents.filter((c) => c.role === 'model');
const fcParts = modelMsgs
.flatMap((m) => (m.parts as Array<Record<string, unknown>>))
.filter((p) => 'functionCall' in p);
expect(fcParts.length).toBe(1);
// Should have a dummy sentinel, not be missing
expect(typeof fcParts[0].thoughtSignature).toBe('string');
expect((fcParts[0].thoughtSignature as string).length).toBeGreaterThan(0);
});
it('does not inject thoughtSignature when thinking is not enabled', () => {
const result = buildGeminiGenerateContentRequestFromOpenAi({
body: {
model: 'gemini-2.5-flash',
messages: [
{ role: 'user', content: 'Hello' },
{
role: 'assistant',
tool_calls: [
{
id: 'call_no_think',
type: 'function',
function: { name: 'Read', arguments: '{"path":"/x"}' },
},
],
},
{ role: 'tool', tool_call_id: 'call_no_think', content: 'data' },
],
},
modelName: 'gemini-2.5-flash',
}) as Record<string, unknown>;
const contents = result.contents as Array<Record<string, unknown>>;
const modelMsgs = contents.filter((c) => c.role === 'model');
const fcParts = modelMsgs
.flatMap((m) => (m.parts as Array<Record<string, unknown>>))
.filter((p) => 'functionCall' in p);
expect(fcParts.length).toBe(1);
// No thinking → no signature injected
expect(fcParts[0].thoughtSignature).toBeUndefined();
});
it('preserves functionResponse count matching functionCall count', () => {
const result = buildGeminiGenerateContentRequestFromOpenAi({
body: {
model: 'gemini-3-flash-preview',
messages: [
{ role: 'user', content: 'Read two files.' },
{
role: 'assistant',
tool_calls: [
{
id: 'call_a',
type: 'function',
function: { name: 'Read', arguments: '{"path":"/a"}' },
provider_specific_fields: { thought_signature: 'sig_a' },
},
{
id: 'call_b',
type: 'function',
function: { name: 'Read', arguments: '{"path":"/b"}' },
provider_specific_fields: { thought_signature: 'sig_b' },
},
],
},
{ role: 'tool', tool_call_id: 'call_a', content: 'content a' },
{ role: 'tool', tool_call_id: 'call_b', content: 'content b' },
],
},
modelName: 'gemini-3-flash-preview',
}) as Record<string, unknown>;
const contents = result.contents as Array<Record<string, unknown>>;
// Count functionCall and functionResponse parts
let fcCount = 0;
let frCount = 0;
for (const content of contents) {
for (const part of (content.parts as Array<Record<string, unknown>>)) {
if ('functionCall' in part) fcCount++;
if ('functionResponse' in part) frCount++;
}
}
expect(fcCount).toBe(2);
expect(frCount).toBe(2);
});
});
src/server/routes/proxy/geminiCliCompat.ts
+396
View File
@@ -0,0 +1,396 @@
import { TextDecoder, TextEncoder } from 'node:util';
import { resolveGeminiThinkingConfigFromRequest } from '../../transformers/gemini/generate-content/convert.js';
// Dummy sentinel used when no real thoughtSignature is available but thinking
// mode is enabled. Gemini accepts any base64 string and won't reject this.
const DUMMY_THOUGHT_SIGNATURE = 'c2tpcF90aG91Z2h0X3NpZ25hdHVyZV92YWxpZGF0b3I=';
function isRecord(value: unknown): value is Record<string, unknown> {
return !!value && typeof value === 'object' && !Array.isArray(value);
}
function asTrimmedString(value: unknown): string {
return typeof value === 'string' ? value.trim() : '';
}
function parseDataUrl(url: string): { mimeType: string; data: string } | null {
if (!url.startsWith('data:')) return null;
const [, rest] = url.split('data:', 2);
const [meta, data] = rest.split(',', 2);
if (!meta || !data) return null;
const [mimeType] = meta.split(';', 1);
return {
mimeType: mimeType || 'application/octet-stream',
data,
};
}
function normalizeFunctionResponseResult(value: unknown): unknown {
if (typeof value !== 'string') return value;
const trimmed = value.trim();
if (!trimmed) return value;
try {
return JSON.parse(trimmed);
} catch {
return value;
}
}
function convertContentToGeminiParts(content: unknown): Array<Record<string, unknown>> {
if (typeof content === 'string') {
const trimmed = content.trim();
return trimmed ? [{ text: trimmed }] : [];
}
if (isRecord(content)) {
if (typeof content.text === 'string') {
const trimmed = content.text.trim();
return trimmed ? [{ text: trimmed }] : [];
}
return [];
}
if (!Array.isArray(content)) return [];
const parts: Array<Record<string, unknown>> = [];
for (const item of content) {
if (!isRecord(item)) continue;
const type = asTrimmedString(item.type).toLowerCase();
if (type === 'text') {
const text = asTrimmedString(item.text);
if (text) parts.push({ text });
continue;
}
if (type === 'image_url') {
const imageUrl = asTrimmedString(item.image_url && isRecord(item.image_url) ? item.image_url.url : item.url);
const parsed = imageUrl ? parseDataUrl(imageUrl) : null;
if (parsed) {
parts.push({
inlineData: {
mime_type: parsed.mimeType,
data: parsed.data,
},
});
}
continue;
}
if (type === 'input_audio') {
const data = asTrimmedString(item.data);
if (data) {
parts.push({
inlineData: {
mime_type: 'audio/wav',
data,
},
});
}
}
}
return parts;
}
function buildGeminiTools(tools: unknown): Array<Record<string, unknown>> | undefined {
if (!Array.isArray(tools)) return undefined;
const declarations = tools
.filter((item) => isRecord(item))
.flatMap((item) => {
if (asTrimmedString(item.type) !== 'function' || !isRecord(item.function)) return [];
const fn = item.function as Record<string, unknown>;
const name = asTrimmedString(fn.name);
if (!name) return [];
return [{
name,
...(asTrimmedString(fn.description) ? { description: asTrimmedString(fn.description) } : {}),
parametersJsonSchema: isRecord(fn.parameters) ? fn.parameters : { type: 'object', properties: {} },
}];
});
if (declarations.length <= 0) return undefined;
return [{ functionDeclarations: declarations }];
}
function buildGeminiToolConfig(toolChoice: unknown): Record<string, unknown> | undefined {
if (typeof toolChoice === 'string') {
const normalized = toolChoice.trim().toLowerCase();
if (normalized === 'none') {
return { functionCallingConfig: { mode: 'NONE' } };
}
if (normalized === 'required') {
return { functionCallingConfig: { mode: 'ANY' } };
}
return { functionCallingConfig: { mode: 'AUTO' } };
}
return undefined;
}
export function buildGeminiGenerateContentRequestFromOpenAi(input: {
body: Record<string, unknown>;
modelName: string;
instructions?: string;
}) {
const request: Record<string, unknown> = {
contents: [],
};
const messages = Array.isArray(input.body.messages) ? input.body.messages : [];
const toolNameById = new Map<string, string>();
for (const message of messages) {
if (!isRecord(message) || asTrimmedString(message.role) !== 'assistant') continue;
const toolCalls = Array.isArray(message.tool_calls) ? message.tool_calls : [];
for (const toolCall of toolCalls) {
if (!isRecord(toolCall) || !isRecord(toolCall.function)) continue;
const id = asTrimmedString(toolCall.id);
const name = asTrimmedString(toolCall.function.name);
if (id && name) {
toolNameById.set(id, name);
}
}
}
// Detect if thinking mode is enabled (needed for dummy signature fallback)
const hasThinkingEnabled = !!resolveGeminiThinkingConfigFromRequest(input.modelName, input.body);
// Collect thoughtSignatures from assistant tool_calls for injection.
// OpenAI format stores signatures in provider_specific_fields or encoded in tool call IDs.
const thoughtSignatureById = new Map<string, string>();
for (const message of messages) {
if (!isRecord(message) || asTrimmedString(message.role) !== 'assistant') continue;
const toolCalls = Array.isArray(message.tool_calls) ? message.tool_calls : [];
for (const toolCall of toolCalls) {
if (!isRecord(toolCall)) continue;
const id = asTrimmedString(toolCall.id);
if (!id) continue;
// Check provider_specific_fields first
const providerFields = isRecord(toolCall.provider_specific_fields) ? toolCall.provider_specific_fields : null;
if (providerFields && typeof providerFields.thought_signature === 'string') {
thoughtSignatureById.set(id, providerFields.thought_signature);
}
}
}
const systemParts: Array<Record<string, unknown>> = [];
if (typeof input.instructions === 'string' && input.instructions.trim()) {
systemParts.push({ text: input.instructions.trim() });
}
for (const message of messages) {
if (!isRecord(message)) continue;
const role = asTrimmedString(message.role).toLowerCase();
if (role === 'system' || role === 'developer') {
systemParts.push(...convertContentToGeminiParts(message.content));
continue;
}
if (role === 'tool') {
const toolCallId = asTrimmedString(message.tool_call_id);
const name = toolNameById.get(toolCallId) || 'unknown';
const result = normalizeFunctionResponseResult(message.content);
request.contents = [
...(Array.isArray(request.contents) ? request.contents : []),
{
role: 'user',
parts: [{
functionResponse: {
name,
response: {
result,
},
},
}],
},
];
continue;
}
const textParts = convertContentToGeminiParts(message.content);
const fcParts: Array<Record<string, unknown>> = [];
const toolCalls = Array.isArray(message.tool_calls) ? message.tool_calls : [];
for (const toolCall of toolCalls) {
if (!isRecord(toolCall) || !isRecord(toolCall.function)) continue;
const name = asTrimmedString(toolCall.function.name);
if (!name) continue;
const rawArguments = toolCall.function.arguments;
let args: unknown = {};
if (typeof rawArguments === 'string' && rawArguments.trim()) {
try {
args = JSON.parse(rawArguments);
} catch {
args = { raw: rawArguments };
}
} else if (isRecord(rawArguments)) {
args = rawArguments;
}
const fcPart: Record<string, unknown> = {
functionCall: { name, args },
};
// Inject thoughtSignature if available from provider_specific_fields.
// If no signature is found, use a dummy sentinel so Gemini doesn't reject
// the request outright. Gemini 3+ requires thoughtSignature on all
// functionCall parts when thinking is enabled.
// Ref: https://ai.google.dev/gemini-api/docs/thought-signatures
const id = asTrimmedString(toolCall.id);
const signature = thoughtSignatureById.get(id);
if (signature) {
fcPart.thoughtSignature = signature;
} else if (hasThinkingEnabled) {
fcPart.thoughtSignature = DUMMY_THOUGHT_SIGNATURE;
}
fcParts.push(fcPart);
}
const geminiRole = role === 'assistant' ? 'model' : 'user';
// Gemini requires that parts WITH thoughtSignature and parts WITHOUT
// are not mixed in the same message. Split if needed.
// Ref: https://ai.google.dev/gemini-api/docs/thought-signatures
const hasSigned = fcParts.some((p) => 'thoughtSignature' in p);
if (hasSigned && textParts.length > 0 && fcParts.length > 0) {
// Emit text parts first, then functionCall parts in a separate message
request.contents = [
...(Array.isArray(request.contents) ? request.contents : []),
{ role: geminiRole, parts: textParts },
{ role: geminiRole, parts: fcParts },
];
} else {
const allParts = [...textParts, ...fcParts];
if (allParts.length <= 0) continue;
request.contents = [
...(Array.isArray(request.contents) ? request.contents : []),
{ role: geminiRole, parts: allParts },
];
}
}
if (systemParts.length > 0) {
request.systemInstruction = {
role: 'user',
parts: systemParts,
};
}
const generationConfig: Record<string, unknown> = {};
const maxOutputTokens = Number(
input.body.max_output_tokens
?? input.body.max_completion_tokens
?? input.body.max_tokens
?? 0,
);
if (Number.isFinite(maxOutputTokens) && maxOutputTokens > 0) {
generationConfig.maxOutputTokens = Math.trunc(maxOutputTokens);
}
const temperature = Number(input.body.temperature);
if (Number.isFinite(temperature)) generationConfig.temperature = temperature;
const topP = Number(input.body.top_p);
if (Number.isFinite(topP)) generationConfig.topP = topP;
const topK = Number(input.body.top_k);
if (Number.isFinite(topK)) generationConfig.topK = topK;
if (Array.isArray(input.body.stop) && input.body.stop.length > 0) {
generationConfig.stopSequences = input.body.stop.filter((item): item is string => typeof item === 'string' && item.trim().length > 0);
}
const thinkingConfig = resolveGeminiThinkingConfigFromRequest(input.modelName, input.body);
if (thinkingConfig) {
generationConfig.thinkingConfig = thinkingConfig;
}
if (Object.keys(generationConfig).length > 0) {
request.generationConfig = generationConfig;
}
const geminiTools = buildGeminiTools(input.body.tools);
if (geminiTools) {
request.tools = geminiTools;
}
const toolConfig = buildGeminiToolConfig(input.body.tool_choice);
if (toolConfig) {
request.toolConfig = toolConfig;
}
return request;
}
export function wrapGeminiCliRequest(input: {
modelName: string;
projectId: string;
request: Record<string, unknown>;
}) {
const { model, ...requestPayload } = input.request;
return {
project: input.projectId,
model: input.modelName,
request: requestPayload,
};
}
export function unwrapGeminiCliPayload<T>(payload: T): unknown {
if (!isRecord(payload)) return payload;
if (payload.response !== undefined) {
return payload.response;
}
return payload;
}
function rewriteGeminiCliSseEventBlock(block: string): string {
const lines = block.split(/\r?\n/g);
return lines.map((line) => {
if (!line.startsWith('data:')) return line;
const data = line.slice(5).trim();
if (!data || data === '[DONE]') return line;
try {
const parsed = JSON.parse(data);
return `data: ${JSON.stringify(unwrapGeminiCliPayload(parsed))}`;
} catch {
return line;
}
}).join('\n');
}
export function createGeminiCliStreamReader(reader: {
read(): Promise<{ done: boolean; value?: Uint8Array }>;
cancel(reason?: unknown): Promise<unknown>;
releaseLock(): void;
}) {
const decoder = new TextDecoder();
const encoder = new TextEncoder();
const outputQueue: Uint8Array[] = [];
let buffer = '';
let done = false;
async function fillQueue() {
while (outputQueue.length <= 0 && !done) {
const result = await reader.read();
if (result.done) {
done = true;
const tail = decoder.decode();
if (tail) buffer += tail;
if (buffer.trim()) {
outputQueue.push(encoder.encode(`${rewriteGeminiCliSseEventBlock(buffer)}\n\n`));
buffer = '';
}
break;
}
if (!result.value) continue;
buffer += decoder.decode(result.value, { stream: true });
let separatorIndex = buffer.indexOf('\n\n');
while (separatorIndex >= 0) {
const block = buffer.slice(0, separatorIndex);
buffer = buffer.slice(separatorIndex + 2);
outputQueue.push(encoder.encode(`${rewriteGeminiCliSseEventBlock(block)}\n\n`));
separatorIndex = buffer.indexOf('\n\n');
}
}
}
return {
async read() {
await fillQueue();
if (outputQueue.length > 0) {
return { done: false, value: outputQueue.shift() };
}
return { done: true, value: undefined };
},
cancel(reason?: unknown) {
return reader.cancel(reason);
},
releaseLock() {
reader.releaseLock();
},
};
}
src/server/routes/proxy/images.ts
+18 -26
View File
@@ -1,6 +1,6 @@
import { FastifyInstance, FastifyReply, FastifyRequest } from 'fastify';
import { fetch } from 'undici';
import { db, schema } from '../../db/index.js';
import { db, hasProxyLogDownstreamApiKeyIdColumn, schema } from '../../db/index.js';
import { tokenRouter } from '../../services/tokenRouter.js';
import { refreshModelsAndRebuildRoutes } from '../../services/modelService.js';
import { reportProxyAllFailed, reportTokenExpired } from '../../services/alertService.js';
@@ -13,6 +13,7 @@ import { getProxyUrlFromExtraConfig } from '../../services/accountExtraConfig.js
import { composeProxyLogMessage } from './logPathMeta.js';
import { formatUtcSqlDateTime } from '../../services/localTimeService.js';
import { cloneFormDataWithOverrides, ensureMultipartBufferParser, parseMultipartFormData } from './multipart.js';
import { getProxyAuthContext } from '../../middleware/auth.js';
import { buildUpstreamUrl } from './upstreamUrl.js';
const MAX_RETRIES = 2;
@@ -25,6 +26,9 @@ export async function imagesProxyRoute(app: FastifyInstance) {
const requestedModel = body?.model || 'gpt-image-1';
if (!await ensureModelAllowedForDownstreamKey(request, reply, requestedModel)) return;
const downstreamPolicy = getDownstreamRoutingPolicy(request);
const downstreamApiKeyId = getProxyAuthContext(request)?.keyId ?? null;
const logDownstreamApiKeyId = downstreamApiKeyId !== null
&& await hasProxyLogDownstreamApiKeyIdColumn();
const excludeChannelIds: number[] = [];
let retryCount = 0;
@@ -67,7 +71,7 @@ export async function imagesProxyRoute(app: FastifyInstance) {
const text = await upstream.text();
if (!upstream.ok) {
tokenRouter.recordFailure(selected.channel.id);
logProxy(selected, requestedModel, 'failed', upstream.status, Date.now() - startTime, text, retryCount);
logProxy(selected, requestedModel, 'failed', upstream.status, Date.now() - startTime, text, retryCount, logDownstreamApiKeyId ? downstreamApiKeyId : null);
if (isTokenExpiredError({ status: upstream.status, message: text })) {
await reportTokenExpired({
accountId: selected.account.id,
@@ -101,27 +105,11 @@ export async function imagesProxyRoute(app: FastifyInstance) {
});
tokenRouter.recordSuccess(selected.channel.id, latency, estimatedCost);
recordDownstreamCostUsage(request, estimatedCost);
logProxy(selected, requestedModel, 'success', upstream.status, latency, null, retryCount, estimatedCost);
logProxy(selected, requestedModel, 'success', upstream.status, latency, null, retryCount, logDownstreamApiKeyId ? downstreamApiKeyId : null, estimatedCost);
return reply.code(upstream.status).send(data);
} catch (err: any) {
tokenRouter.recordFailure(selected.channel.id);
logProxy(selected, requestedModel, 'failed', 0, Date.now() - startTime, err.message, retryCount);
if (retryCount < MAX_RETRIES) {
retryCount++;
continue;
}
await reportProxyAllFailed({
model: requestedModel,
reason: err.message || 'network failure',
});
return reply.code(502).send({
error: { message: `Upstream error: ${err.message}`, type: 'upstream_error' },
});
}
}
});
app.post('/v1/images/edits', async (request: FastifyRequest, reply: FastifyReply) => {
logProxy(selected, requestedModel, 'failed', 0, Date.now() - startTime, err.message, retryCount, logDownstreamApiKeyId ? downstreamApiKeyId : null);, async (request: FastifyRequest, reply: FastifyReply) => {
const multipartForm = await parseMultipartFormData(request);
const jsonBody = (!multipartForm && request.body && typeof request.body === 'object')
? request.body as Record<string, unknown>
@@ -132,6 +120,9 @@ export async function imagesProxyRoute(app: FastifyInstance) {
if (!await ensureModelAllowedForDownstreamKey(request, reply, requestedModel)) return;
const downstreamPolicy = getDownstreamRoutingPolicy(request);
const downstreamApiKeyId = getProxyAuthContext(request)?.keyId ?? null;
const logDownstreamApiKeyId = downstreamApiKeyId !== null
&& await hasProxyLogDownstreamApiKeyIdColumn();
const excludeChannelIds: number[] = [];
let retryCount = 0;
@@ -160,7 +151,6 @@ export async function imagesProxyRoute(app: FastifyInstance) {
const startTime = Date.now();
try {
const accountProxy = getProxyUrlFromExtraConfig(selected.account.extraConfig);
const requestInit = multipartForm
? withSiteRecordProxyRequestInit(selected.site, {
method: 'POST',
@@ -170,7 +160,7 @@ export async function imagesProxyRoute(app: FastifyInstance) {
body: cloneFormDataWithOverrides(multipartForm, {
model: selected.actualModel || requestedModel,
}) as any,
}, accountProxy)
})
: withSiteRecordProxyRequestInit(selected.site, {
method: 'POST',
headers: {
@@ -181,13 +171,13 @@ export async function imagesProxyRoute(app: FastifyInstance) {
...(jsonBody || {}),
model: selected.actualModel || requestedModel,
}),
}, accountProxy);
});
const upstream = await fetch(targetUrl, requestInit);
const text = await upstream.text();
if (!upstream.ok) {
tokenRouter.recordFailure(selected.channel.id);
logProxy(selected, requestedModel, 'failed', upstream.status, Date.now() - startTime, text, retryCount, 0, '/v1/images/edits');
logProxy(selected, requestedModel, 'failed', upstream.status, Date.now() - startTime, text, retryCount, logDownstreamApiKeyId ? downstreamApiKeyId : null, 0, '/v1/images/edits');
if (isTokenExpiredError({ status: upstream.status, message: text })) {
await reportTokenExpired({
accountId: selected.account.id,
@@ -221,11 +211,11 @@ export async function imagesProxyRoute(app: FastifyInstance) {
});
tokenRouter.recordSuccess(selected.channel.id, latency, estimatedCost);
recordDownstreamCostUsage(request, estimatedCost);
logProxy(selected, requestedModel, 'success', upstream.status, latency, null, retryCount, estimatedCost, '/v1/images/edits');
logProxy(selected, requestedModel, 'success', upstream.status, latency, null, retryCount, logDownstreamApiKeyId ? downstreamApiKeyId : null, estimatedCost, '/v1/images/edits');
return reply.code(upstream.status).send(data);
} catch (err: any) {
tokenRouter.recordFailure(selected.channel.id);
logProxy(selected, requestedModel, 'failed', 0, Date.now() - startTime, err.message, retryCount, 0, '/v1/images/edits');
logProxy(selected, requestedModel, 'failed', 0, Date.now() - startTime, err.message, retryCount, logDownstreamApiKeyId ? downstreamApiKeyId : null, 0, '/v1/images/edits');
if (retryCount < MAX_RETRIES) {
retryCount++;
continue;
@@ -259,6 +249,7 @@ async function logProxy(
latencyMs: number,
errorMessage: string | null,
retryCount: number,
downstreamApiKeyId: number | null = null,
estimatedCost = 0,
downstreamPath = '/v1/images/generations',
) {
@@ -272,6 +263,7 @@ async function logProxy(
routeId: selected.channel.routeId,
channelId: selected.channel.id,
accountId: selected.account.id,
...(downstreamApiKeyId !== null ? { downstreamApiKeyId } : {}),
modelRequested,
modelActual: selected.actualModel,
status,
src/server/routes/proxy/models.test.ts
+105
View File
@@ -7,11 +7,13 @@ import { join } from 'node:path';
type DbModule = typeof import('../../db/index.js');
type ProxyRouterModule = typeof import('./router.js');
type TokenRouterModule = typeof import('../../services/tokenRouter.js');
type TokensRoutesModule = typeof import('../api/tokens.js');
describe('/v1/models route', () => {
let db: DbModule['db'];
let schema: DbModule['schema'];
let proxyRoutes: ProxyRouterModule['proxyRoutes'];
let tokensRoutes: TokensRoutesModule['tokensRoutes'];
let invalidateTokenRouterCache: TokenRouterModule['invalidateTokenRouterCache'];
let app: FastifyInstance;
let dataDir = '';
@@ -24,13 +26,16 @@ describe('/v1/models route', () => {
const dbModule = await import('../../db/index.js');
const proxyRouterModule = await import('./router.js');
const tokenRouterModule = await import('../../services/tokenRouter.js');
const tokensRoutesModule = await import('../api/tokens.js');
db = dbModule.db;
schema = dbModule.schema;
proxyRoutes = proxyRouterModule.proxyRoutes;
tokensRoutes = tokensRoutesModule.tokensRoutes;
invalidateTokenRouterCache = tokenRouterModule.invalidateTokenRouterCache;
app = Fastify();
await app.register(tokensRoutes);
await app.register(proxyRoutes);
});
@@ -285,6 +290,106 @@ describe('/v1/models route', () => {
expect(ids).not.toContain('claude-sonnet-4-5');
});
it('returns only explicit-group public name while hiding source exact routes', async () => {
const site = await db.insert(schema.sites).values({
name: 'explicit-group-site',
url: 'https://explicit-group.example.com',
platform: 'openai',
status: 'active',
}).returning().get();
const account = await db.insert(schema.accounts).values({
siteId: site.id,
accessToken: 'explicit-group-access-token',
status: 'active',
}).returning().get();
const token = await db.insert(schema.accountTokens).values({
accountId: account.id,
name: 'default',
token: 'explicit-group-api-token',
enabled: true,
isDefault: true,
}).returning().get();
await db.insert(schema.modelAvailability).values([
{
accountId: account.id,
modelName: 'claude-opus-4-5',
available: true,
},
{
accountId: account.id,
modelName: 'claude-sonnet-4-5',
available: true,
},
]).run();
const sourceRouteA = await db.insert(schema.tokenRoutes).values({
modelPattern: 'claude-opus-4-5',
enabled: true,
}).returning().get();
const sourceRouteB = await db.insert(schema.tokenRoutes).values({
modelPattern: 'claude-sonnet-4-5',
enabled: true,
}).returning().get();
await db.insert(schema.routeChannels).values([
{
routeId: sourceRouteA.id,
accountId: account.id,
tokenId: token.id,
sourceModel: 'claude-opus-4-5',
enabled: true,
},
{
routeId: sourceRouteB.id,
accountId: account.id,
tokenId: token.id,
sourceModel: 'claude-sonnet-4-5',
enabled: true,
},
]).run();
const groupResponse = await app.inject({
method: 'POST',
url: '/api/routes',
payload: {
routeMode: 'explicit_group',
displayName: 'claude-opus-4-6',
sourceRouteIds: [sourceRouteA.id, sourceRouteB.id],
},
});
expect(groupResponse.statusCode).toBe(200);
const groupId = (groupResponse.json() as { id: number }).id;
await db.insert(schema.downstreamApiKeys).values({
name: 'managed-explicit-group-key',
key: 'sk-managed-explicit-group',
enabled: true,
allowedRouteIds: JSON.stringify([groupId]),
}).run();
const response = await app.inject({
method: 'GET',
url: '/v1/models',
headers: {
authorization: 'Bearer sk-managed-explicit-group',
},
});
expect(response.statusCode).toBe(200);
const body = response.json() as {
object: 'list';
data: Array<{ id: string }>;
};
const ids = body.data.map((item) => item.id);
expect(ids).toContain('claude-opus-4-6');
expect(ids).not.toContain('claude-opus-4-5');
expect(ids).not.toContain('claude-sonnet-4-5');
});
it('filters search pseudo models out of /v1/models', async () => {
const site = await db.insert(schema.sites).values({
name: 'search-site',
src/server/routes/proxy/models.ts
+1 -24
View File
@@ -1,6 +1,4 @@
import { FastifyInstance } from 'fastify';
import { db, schema } from '../../db/index.js';
import { and, eq } from 'drizzle-orm';
import { refreshModelsAndRebuildRoutes } from '../../services/modelService.js';
import { getDownstreamRoutingPolicy } from './downstreamPolicy.js';
import { tokenRouter } from '../../services/tokenRouter.js';
@@ -17,28 +15,7 @@ export async function modelsProxyRoute(app: FastifyInstance) {
const downstreamPolicy = getDownstreamRoutingPolicy(request);
const readModels = async () => {
const rows = await db.select({ modelName: schema.modelAvailability.modelName })
.from(schema.modelAvailability)
.innerJoin(schema.accounts, eq(schema.modelAvailability.accountId, schema.accounts.id))
.innerJoin(schema.sites, eq(schema.accounts.siteId, schema.sites.id))
.where(
and(
eq(schema.modelAvailability.available, true),
eq(schema.accounts.status, 'active'),
eq(schema.sites.status, 'active'),
),
)
.all();
const routeAliases = (await db.select({ displayName: schema.tokenRoutes.displayName })
.from(schema.tokenRoutes)
.where(eq(schema.tokenRoutes.enabled, true))
.all())
.map((row) => (row.displayName || '').trim())
.filter((name) => name.length > 0);
const deduped = Array.from(new Set([
...rows.map((r) => r.modelName),
...routeAliases,
]))
const deduped = Array.from(new Set(await tokenRouter.getAvailableModels()))
.filter((modelName) => !isSearchPseudoModel(modelName))
.sort();
const allowed: string[] = [];
src/server/routes/proxy/responses.codex-oauth.test.ts
+470
View File
@@ -0,0 +1,470 @@
import Fastify, { type FastifyInstance } from 'fastify';
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
const fetchMock = vi.fn();
const selectChannelMock = vi.fn();
const selectNextChannelMock = vi.fn();
const recordSuccessMock = vi.fn();
const recordFailureMock = vi.fn();
const refreshModelsAndRebuildRoutesMock = vi.fn();
const reportProxyAllFailedMock = vi.fn();
const reportTokenExpiredMock = vi.fn();
const resolveProxyUsageWithSelfLogFallbackMock = vi.fn(async ({ usage }: any) => ({
...usage,
estimatedCostFromQuota: 0,
recoveredFromSelfLog: false,
}));
const refreshOauthAccessTokenSingleflightMock = vi.fn();
const recordOauthQuotaResetHintMock = vi.fn();
const insertedProxyLogs: Record<string, unknown>[] = [];
const dbInsertMock = vi.fn((_arg?: any) => ({
values: (values: Record<string, unknown>) => {
insertedProxyLogs.push(values);
return {
run: () => undefined,
};
},
}));
vi.mock('undici', () => ({
fetch: (...args: unknown[]) => fetchMock(...args),
}));
vi.mock('../../services/tokenRouter.js', () => ({
tokenRouter: {
selectChannel: (...args: unknown[]) => selectChannelMock(...args),
selectNextChannel: (...args: unknown[]) => selectNextChannelMock(...args),
recordSuccess: (...args: unknown[]) => recordSuccessMock(...args),
recordFailure: (...args: unknown[]) => recordFailureMock(...args),
},
}));
vi.mock('../../services/modelService.js', () => ({
refreshModelsAndRebuildRoutes: (...args: unknown[]) => refreshModelsAndRebuildRoutesMock(...args),
}));
vi.mock('../../services/alertService.js', () => ({
reportProxyAllFailed: (...args: unknown[]) => reportProxyAllFailedMock(...args),
reportTokenExpired: (...args: unknown[]) => reportTokenExpiredMock(...args),
}));
vi.mock('../../services/alertRules.js', () => ({
isTokenExpiredError: ({ status }: { status?: number }) => status === 401,
}));
vi.mock('../../services/modelPricingService.js', () => ({
estimateProxyCost: async () => 0,
buildProxyBillingDetails: async () => null,
fetchModelPricingCatalog: async () => null,
}));
vi.mock('../../services/proxyRetryPolicy.js', () => ({
shouldRetryProxyRequest: () => false,
}));
vi.mock('../../services/proxyUsageFallbackService.js', () => ({
resolveProxyUsageWithSelfLogFallback: (arg: any) => resolveProxyUsageWithSelfLogFallbackMock(arg),
}));
vi.mock('../../services/oauth/refreshSingleflight.js', () => ({
refreshOauthAccessTokenSingleflight: (...args: unknown[]) => refreshOauthAccessTokenSingleflightMock(...args),
}));
vi.mock('../../services/oauth/quota.js', () => ({
recordOauthQuotaResetHint: (...args: unknown[]) => recordOauthQuotaResetHintMock(...args),
}));
vi.mock('../../db/index.js', () => ({
db: {
insert: (arg: any) => dbInsertMock(arg),
},
hasProxyLogDownstreamApiKeyIdColumn: async () => false,
schema: {
proxyLogs: {},
},
}));
describe('responses proxy codex oauth refresh', () => {
let app: FastifyInstance;
const createSseResponse = (chunks: string[], status = 200) => {
const encoder = new TextEncoder();
return new Response(new ReadableStream<Uint8Array>({
start(controller) {
for (const chunk of chunks) {
controller.enqueue(encoder.encode(chunk));
}
controller.close();
},
}), {
status,
headers: { 'content-type': 'text/event-stream; charset=utf-8' },
});
};
beforeAll(async () => {
const { responsesProxyRoute } = await import('./responses.js');
app = Fastify();
await app.register(responsesProxyRoute);
});
beforeEach(() => {
fetchMock.mockReset();
selectChannelMock.mockReset();
selectNextChannelMock.mockReset();
recordSuccessMock.mockReset();
recordFailureMock.mockReset();
refreshModelsAndRebuildRoutesMock.mockReset();
reportProxyAllFailedMock.mockReset();
reportTokenExpiredMock.mockReset();
resolveProxyUsageWithSelfLogFallbackMock.mockClear();
refreshOauthAccessTokenSingleflightMock.mockReset();
recordOauthQuotaResetHintMock.mockReset();
dbInsertMock.mockClear();
insertedProxyLogs.length = 0;
selectChannelMock.mockReturnValue({
channel: { id: 11, routeId: 22 },
site: { name: 'codex-site', url: 'https://chatgpt.com/backend-api/codex', platform: 'codex' },
account: {
id: 33,
username: 'codex-user@example.com',
extraConfig: JSON.stringify({
credentialMode: 'session',
oauth: {
provider: 'codex',
accountId: 'chatgpt-account-123',
email: 'codex-user@example.com',
planType: 'plus',
},
}),
},
tokenName: 'default',
tokenValue: 'expired-access-token',
actualModel: 'gpt-5.2-codex',
});
selectNextChannelMock.mockReturnValue(null);
refreshOauthAccessTokenSingleflightMock.mockResolvedValue({
accessToken: 'fresh-access-token',
accountId: 33,
accountKey: 'chatgpt-account-123',
});
});
afterAll(async () => {
await app.close();
});
it('refreshes codex oauth token and retries the same responses request on 401', async () => {
fetchMock
.mockResolvedValueOnce(new Response(JSON.stringify({
error: { message: 'expired token', type: 'invalid_request_error' },
}), {
status: 401,
headers: { 'content-type': 'application/json' },
}))
.mockResolvedValueOnce(new Response(JSON.stringify({
id: 'resp_codex_refreshed',
object: 'response',
model: 'gpt-5.2-codex',
status: 'completed',
output_text: 'ok after codex token refresh',
usage: { input_tokens: 4, output_tokens: 2, total_tokens: 6 },
}), {
status: 200,
headers: { 'content-type': 'application/json' },
}));
const response = await app.inject({
method: 'POST',
url: '/v1/responses',
headers: {
'user-agent': 'CodexClient/1.0',
},
payload: {
model: 'gpt-5.2-codex',
input: 'hello codex',
},
});
expect(response.statusCode).toBe(200);
expect(refreshOauthAccessTokenSingleflightMock).toHaveBeenCalledWith(33);
expect(fetchMock).toHaveBeenCalledTimes(2);
const [firstUrl, firstOptions] = fetchMock.mock.calls[0] as [string, any];
const [secondUrl, secondOptions] = fetchMock.mock.calls[1] as [string, any];
expect(firstUrl).toBe('https://chatgpt.com/backend-api/codex/responses');
expect(secondUrl).toBe('https://chatgpt.com/backend-api/codex/responses');
expect(firstOptions.headers.Authorization).toBe('Bearer expired-access-token');
expect(secondOptions.headers.Authorization).toBe('Bearer fresh-access-token');
expect(secondOptions.headers.Originator || secondOptions.headers.originator).toBe('codex_cli_rs');
expect(secondOptions.headers['Chatgpt-Account-Id'] || secondOptions.headers['chatgpt-account-id']).toBe('chatgpt-account-123');
expect(secondOptions.headers.Version || secondOptions.headers.version).toBe('0.101.0');
expect(String(secondOptions.headers.Session_id || secondOptions.headers.session_id || '')).toMatch(/^[0-9a-f-]{36}$/i);
expect(secondOptions.headers.Conversation_id || secondOptions.headers.conversation_id).toBeUndefined();
expect(secondOptions.headers['User-Agent'] || secondOptions.headers['user-agent']).toBe('CodexClient/1.0');
expect(secondOptions.headers.Accept || secondOptions.headers.accept).toBe('text/event-stream');
expect(secondOptions.headers.Connection || secondOptions.headers.connection).toBe('Keep-Alive');
expect(response.json()?.output_text).toContain('ok after codex token refresh');
});
it('sends an explicit empty instructions field to codex responses when downstream body has no system prompt', async () => {
fetchMock.mockResolvedValue(new Response(JSON.stringify({
id: 'resp_codex_no_system',
object: 'response',
model: 'gpt-5.2-codex',
status: 'completed',
output_text: 'ok without system prompt',
usage: { input_tokens: 4, output_tokens: 2, total_tokens: 6 },
}), {
status: 200,
headers: { 'content-type': 'application/json' },
}));
const response = await app.inject({
method: 'POST',
url: '/v1/responses',
payload: {
model: 'gpt-5.2-codex',
input: 'hello codex',
},
});
expect(response.statusCode).toBe(200);
expect(fetchMock).toHaveBeenCalledTimes(1);
const [, options] = fetchMock.mock.calls[0] as [string, any];
const forwardedBody = JSON.parse(options.body);
expect(forwardedBody.instructions).toBe('');
expect(forwardedBody.prompt_cache_key).toBeUndefined();
expect(forwardedBody.stream).toBe(true);
expect(forwardedBody.input).toEqual([
{
type: 'message',
role: 'user',
content: [{ type: 'input_text', text: 'hello codex' }],
},
]);
});
it('preserves explicit prompt_cache_key for codex responses requests', async () => {
fetchMock.mockResolvedValue(new Response(JSON.stringify({
id: 'resp_codex_with_cache_key',
object: 'response',
model: 'gpt-5.2-codex',
status: 'completed',
output_text: 'ok with cache key',
usage: { input_tokens: 4, output_tokens: 2, total_tokens: 6 },
}), {
status: 200,
headers: { 'content-type': 'application/json' },
}));
const response = await app.inject({
method: 'POST',
url: '/v1/responses',
payload: {
model: 'gpt-5.2-codex',
prompt_cache_key: 'codex-cache-123',
input: 'hello codex',
},
});
expect(response.statusCode).toBe(200);
expect(fetchMock).toHaveBeenCalledTimes(1);
const [, options] = fetchMock.mock.calls[0] as [string, any];
const forwardedBody = JSON.parse(options.body);
expect(options.headers.Session_id || options.headers.session_id).toBe('codex-cache-123');
expect(options.headers.Conversation_id || options.headers.conversation_id).toBe('codex-cache-123');
expect(forwardedBody.prompt_cache_key).toBe('codex-cache-123');
});
it('strips generic downstream headers before forwarding codex responses upstream', async () => {
fetchMock.mockResolvedValue(new Response(JSON.stringify({
id: 'resp_codex_header_filter',
object: 'response',
model: 'gpt-5.2-codex',
status: 'completed',
output_text: 'ok with filtered headers',
usage: { input_tokens: 4, output_tokens: 2, total_tokens: 6 },
}), {
status: 200,
headers: { 'content-type': 'application/json' },
}));
const response = await app.inject({
method: 'POST',
url: '/v1/responses',
headers: {
'openai-beta': 'responses-2025-03-11',
'x-openai-client-user-agent': '{"client":"openclaw"}',
origin: 'https://openclaw.example',
referer: 'https://openclaw.example/app',
'user-agent': 'OpenClaw/1.0',
version: '0.202.0',
session_id: 'session-from-client',
},
payload: {
model: 'gpt-5.2-codex',
input: 'hello codex',
},
});
expect(response.statusCode).toBe(200);
expect(fetchMock).toHaveBeenCalledTimes(1);
const [, options] = fetchMock.mock.calls[0] as [string, any];
expect(options.headers.Version || options.headers.version).toBe('0.202.0');
expect(options.headers.Session_id || options.headers.session_id).toBe('session-from-client');
expect(options.headers['User-Agent'] || options.headers['user-agent']).toBe('OpenClaw/1.0');
expect(options.headers['openai-beta']).toBeUndefined();
expect(options.headers['x-openai-client-user-agent']).toBeUndefined();
expect(options.headers.origin).toBeUndefined();
expect(options.headers.referer).toBeUndefined();
});
it('records codex usage_limit_reached reset hints on upstream 429 failures', async () => {
fetchMock.mockResolvedValue(new Response(JSON.stringify({
error: {
type: 'usage_limit_reached',
resets_at: 1773800400,
message: 'quota exceeded',
},
}), {
status: 429,
headers: { 'content-type': 'application/json' },
}));
const response = await app.inject({
method: 'POST',
url: '/v1/responses',
payload: {
model: 'gpt-5.2-codex',
input: 'hello codex',
},
});
expect(response.statusCode).toBe(429);
expect(recordOauthQuotaResetHintMock).toHaveBeenCalledWith({
accountId: 33,
statusCode: 429,
errorText: JSON.stringify({
error: {
type: 'usage_limit_reached',
resets_at: 1773800400,
message: 'quota exceeded',
},
}),
});
});
it('forces codex upstream responses requests to stream and aggregates the SSE payload for non-stream downstream callers', async () => {
fetchMock.mockResolvedValue(createSseResponse([
'event: response.created\n',
'data: {"type":"response.created","response":{"id":"resp_codex_stream","model":"gpt-5.4","created_at":1706000000,"status":"in_progress","output":[]}}\n\n',
'event: response.output_item.added\n',
'data: {"type":"response.output_item.added","output_index":0,"item":{"id":"msg_codex_stream","type":"message","role":"assistant","status":"in_progress","content":[]}}\n\n',
'event: response.output_text.delta\n',
'data: {"type":"response.output_text.delta","output_index":0,"item_id":"msg_codex_stream","delta":"pong"}\n\n',
'event: response.completed\n',
'data: {"type":"response.completed","response":{"id":"resp_codex_stream","model":"gpt-5.4","status":"completed","usage":{"input_tokens":3,"output_tokens":1,"total_tokens":4}}}\n\n',
'data: [DONE]\n\n',
]));
const response = await app.inject({
method: 'POST',
url: '/v1/responses',
payload: {
model: 'gpt-5.4',
input: 'hello codex',
},
});
expect(response.statusCode).toBe(200);
expect(fetchMock).toHaveBeenCalledTimes(1);
const [, options] = fetchMock.mock.calls[0] as [string, any];
const forwardedBody = JSON.parse(options.body);
expect(forwardedBody.stream).toBe(true);
expect(forwardedBody.instructions).toBe('');
expect(forwardedBody.store).toBe(false);
expect(response.json()).toMatchObject({
id: 'resp_codex_stream',
model: 'gpt-5.4',
status: 'completed',
output_text: 'pong',
usage: {
input_tokens: 3,
output_tokens: 1,
total_tokens: 4,
},
});
});
it('preserves codex-required instructions and store fields across responses compatibility retries', async () => {
fetchMock.mockResolvedValue(new Response(JSON.stringify({
error: { message: 'upstream_error', type: 'upstream_error' },
}), {
status: 400,
headers: { 'content-type': 'application/json' },
}));
const response = await app.inject({
method: 'POST',
url: '/v1/responses',
payload: {
model: 'gpt-5.2-codex',
input: 'hello codex',
metadata: { trace: 'compatibility-retry' },
},
});
expect(response.statusCode).toBe(400);
expect(fetchMock.mock.calls.length).toBeGreaterThanOrEqual(2);
const [, firstOptions] = fetchMock.mock.calls[0] as [string, any];
const [, secondOptions] = fetchMock.mock.calls[1] as [string, any];
const firstBody = JSON.parse(firstOptions.body);
const secondBody = JSON.parse(secondOptions.body);
expect(firstBody.instructions).toBe('');
expect(firstBody.store).toBe(false);
expect(firstBody.stream).toBe(true);
expect(secondBody.instructions).toBe('');
expect(secondBody.store).toBe(false);
expect(secondBody.stream).toBe(true);
});
it('does not record success when a streaming responses request ends with response.failed', async () => {
fetchMock.mockResolvedValue(createSseResponse([
'event: response.created\n',
'data: {"type":"response.created","response":{"id":"resp_codex_failed","model":"gpt-5.4","created_at":1706000000,"status":"in_progress","output":[]}}\n\n',
'event: response.failed\n',
'data: {"type":"response.failed","response":{"id":"resp_codex_failed","model":"gpt-5.4","status":"failed","error":{"message":"tool execution failed"}}}\n\n',
'data: [DONE]\n\n',
]));
const response = await app.inject({
method: 'POST',
url: '/v1/responses',
payload: {
model: 'gpt-5.4',
input: 'hello codex',
stream: true,
},
});
expect(response.statusCode).toBe(200);
expect(response.body).toContain('response.failed');
expect(response.body).not.toContain('response.completed');
expect(recordSuccessMock).not.toHaveBeenCalled();
expect(recordFailureMock).toHaveBeenCalledTimes(1);
expect(insertedProxyLogs.at(-1)).toMatchObject({
status: 'failed',
httpStatus: 200,
});
expect(String(insertedProxyLogs.at(-1)?.errorMessage || '')).toContain('tool execution failed');
});
});
src/server/routes/proxy/responses.ts
+163 -25
View File
@@ -1,6 +1,5 @@
import { FastifyInstance, FastifyReply, FastifyRequest } from 'fastify';
import { fetch } from 'undici';
import { db, schema } from '../../db/index.js';
import { db, hasProxyLogDownstreamApiKeyIdColumn, schema } from '../../db/index.js';
import { tokenRouter } from '../../services/tokenRouter.js';
import { refreshModelsAndRebuildRoutes } from '../../services/modelService.js';
import { reportProxyAllFailed, reportTokenExpired } from '../../services/alertService.js';
@@ -17,17 +16,30 @@ import {
} from './upstreamEndpoint.js';
import { ensureModelAllowedForDownstreamKey, getDownstreamRoutingPolicy, recordDownstreamCostUsage } from './downstreamPolicy.js';
import { composeProxyLogMessage } from './logPathMeta.js';
import { executeEndpointFlow, withUpstreamPath } from './endpointFlow.js';
import { executeEndpointFlow, withUpstreamPath, type BuiltEndpointRequest } from './endpointFlow.js';
import { formatUtcSqlDateTime } from '../../services/localTimeService.js';
import { resolveProxyLogBilling } from './proxyBilling.js';
import { getProxyResourceOwner } from '../../middleware/auth.js';
import { getProxyAuthContext, getProxyResourceOwner } from '../../middleware/auth.js';
import { detectDownstreamClientContext, isCodexResponsesSurface, type DownstreamClientContext } from './downstreamClientContext.js';
import { dispatchRuntimeRequest } from './runtimeExecutor.js';
import { normalizeInputFileBlock } from '../../transformers/shared/inputFile.js';
import {
ProxyInputFileResolutionError,
hasNonImageFileInputInOpenAiBody,
resolveResponsesBodyInputFiles,
} from '../../services/proxyInputFileResolver.js';
import {
buildOauthProviderHeaders,
} from '../../services/oauth/service.js';
import { getOauthInfoFromExtraConfig } from '../../services/oauth/oauthAccount.js';
import { recordOauthQuotaResetHint } from '../../services/oauth/quota.js';
import { refreshOauthAccessTokenSingleflight } from '../../services/oauth/refreshSingleflight.js';
import { collectResponsesFinalPayloadFromSse } from './responsesSseFinal.js';
import {
createGeminiCliStreamReader,
unwrapGeminiCliPayload,
} from './geminiCliCompat.js';
import { ensureResponsesWebsocketTransport } from './responsesWebsocket.js';
const MAX_RETRIES = 2;
@@ -113,7 +125,15 @@ function carriesResponsesFileUrlInput(value: unknown): boolean {
type UsageSummary = ReturnType<typeof parseProxyUsage>;
function deriveCodexExplicitSessionId(body: Record<string, unknown>): string | null {
const promptCacheKey = typeof body.prompt_cache_key === 'string'
? body.prompt_cache_key.trim()
: '';
return promptCacheKey || null;
}
export async function responsesProxyRoute(app: FastifyInstance) {
ensureResponsesWebsocketTransport(app);
const handleResponsesRequest = async (
request: FastifyRequest,
reply: FastifyReply,
@@ -148,6 +168,9 @@ export async function responsesProxyRoute(app: FastifyInstance) {
}
if (!await ensureModelAllowedForDownstreamKey(request, reply, requestedModel)) return;
const downstreamPolicy = getDownstreamRoutingPolicy(request);
const downstreamApiKeyId = getProxyAuthContext(request)?.keyId ?? null;
const logDownstreamApiKeyId = downstreamApiKeyId !== null
&& await hasProxyLogDownstreamApiKeyIdColumn();
const excludeChannelIds: number[] = [];
let retryCount = 0;
@@ -174,12 +197,17 @@ export async function responsesProxyRoute(app: FastifyInstance) {
excludeChannelIds.push(selected.channel.id);
const modelName = selected.actualModel || requestedModel;
const oauth = getOauthInfoFromExtraConfig(selected.account.extraConfig);
const isCodexSite = String(selected.site.platform || '').trim().toLowerCase() === 'codex';
const owner = getProxyResourceOwner(request);
let normalizedResponsesBody: Record<string, unknown> = {
...requestEnvelope.parsed.normalizedBody,
model: modelName,
stream: isStream,
};
if (body.generate === false) {
normalizedResponsesBody.generate = false;
}
if (owner) {
try {
normalizedResponsesBody = await resolveResponsesBodyInputFiles(normalizedResponsesBody, owner);
@@ -226,18 +254,30 @@ export async function responsesProxyRoute(app: FastifyInstance) {
if (endpointCandidates.length === 0) {
endpointCandidates.push('responses', 'chat', 'messages');
}
const buildProviderHeaders = () => (
buildOauthProviderHeaders({
extraConfig: typeof selected.account.extraConfig === 'string' ? selected.account.extraConfig : null,
downstreamHeaders: request.headers as Record<string, unknown>,
})
);
const buildEndpointRequest = (endpoint: 'chat' | 'messages' | 'responses') => {
const upstreamStream = isStream || (isCodexSite && endpoint === 'responses');
const codexExplicitSessionId = deriveCodexExplicitSessionId(normalizedResponsesBody);
const endpointRequest = buildUpstreamEndpointRequest({
endpoint,
modelName,
stream: isStream,
stream: upstreamStream,
tokenValue: selected.tokenValue,
oauthProvider: oauth?.provider,
oauthProjectId: oauth?.projectId,
sitePlatform: selected.site.platform,
siteUrl: selected.site.url,
openaiBody: openAiBody,
downstreamFormat: 'responses',
responsesOriginalBody: normalizedResponsesBody,
downstreamHeaders: request.headers as Record<string, unknown>,
providerHeaders: buildProviderHeaders(),
codexExplicitSessionId,
});
const upstreamPath = (
isCompactRequest && endpoint === 'responses'
@@ -249,20 +289,54 @@ export async function responsesProxyRoute(app: FastifyInstance) {
path: upstreamPath,
headers: endpointRequest.headers,
body: endpointRequest.body as Record<string, unknown>,
runtime: endpointRequest.runtime,
};
};
const endpointStrategy = openAiResponsesTransformer.compatibility.createEndpointStrategy({
isStream,
requiresNativeResponsesFileUrl,
dispatchRequest: (compatibilityRequest, targetUrl) => fetch(
targetUrl ?? `${selected.site.url}${compatibilityRequest.path}`,
withSiteRecordProxyRequestInit(selected.site, {
const dispatchRequest = (compatibilityRequest: BuiltEndpointRequest, targetUrl?: string) => (
dispatchRuntimeRequest({
siteUrl: selected.site.url,
targetUrl,
request: compatibilityRequest,
buildInit: (_requestUrl, requestForFetch) => withSiteRecordProxyRequestInit(selected.site, {
method: 'POST',
headers: compatibilityRequest.headers,
body: JSON.stringify(compatibilityRequest.body),
}, getProxyUrlFromExtraConfig(selected.account.extraConfig)),
),
headers: requestForFetch.headers,
body: JSON.stringify(requestForFetch.body),
}),
})
);
const endpointStrategy = openAiResponsesTransformer.compatibility.createEndpointStrategy({
isStream: isStream || isCodexSite,
requiresNativeResponsesFileUrl,
dispatchRequest,
});
const tryRecover = async (ctx: Parameters<NonNullable<typeof endpointStrategy.tryRecover>>[0]) => {
if (ctx.response.status === 401 && oauth) {
try {
const refreshed = await refreshOauthAccessTokenSingleflight(selected.account.id);
selected.tokenValue = refreshed.accessToken;
selected.account = {
...selected.account,
accessToken: refreshed.accessToken,
extraConfig: refreshed.extraConfig ?? selected.account.extraConfig,
};
const refreshedRequest = buildEndpointRequest(ctx.request.endpoint);
const refreshedTargetUrl = `${selected.site.url}${refreshedRequest.path}`;
const refreshedResponse = await dispatchRequest(refreshedRequest, refreshedTargetUrl);
if (refreshedResponse.ok) {
return {
upstream: refreshedResponse,
upstreamPath: refreshedRequest.path,
};
}
ctx.request = refreshedRequest;
ctx.response = refreshedResponse;
ctx.rawErrText = await refreshedResponse.text().catch(() => 'unknown error');
} catch {
return endpointStrategy.tryRecover(ctx);
}
}
return endpointStrategy.tryRecover(ctx);
};
const startTime = Date.now();
@@ -272,7 +346,8 @@ export async function responsesProxyRoute(app: FastifyInstance) {
proxyUrl: resolveChannelProxyUrl(selected.site, selected.account.extraConfig),
endpointCandidates,
buildRequest: (endpoint) => buildEndpointRequest(endpoint),
tryRecover: endpointStrategy.tryRecover,
dispatchRequest,
tryRecover,
shouldDowngrade: endpointStrategy.shouldDowngrade,
onDowngrade: (ctx) => {
logProxy(
@@ -291,6 +366,7 @@ export async function responsesProxyRoute(app: FastifyInstance) {
null,
null,
clientContext,
logDownstreamApiKeyId ? downstreamApiKeyId : null,
);
},
});
@@ -298,6 +374,7 @@ export async function responsesProxyRoute(app: FastifyInstance) {
if (!endpointResult.ok) {
const status = endpointResult.status || 502;
const errText = endpointResult.errText || 'unknown error';
const rawErrText = endpointResult.rawErrText || errText;
tokenRouter.recordFailure(selected.channel.id);
logProxy(
selected,
@@ -315,7 +392,13 @@ export async function responsesProxyRoute(app: FastifyInstance) {
null,
null,
clientContext,
logDownstreamApiKeyId ? downstreamApiKeyId : null,
);
await recordOauthQuotaResetHint({
accountId: selected.account.id,
statusCode: status,
errorText: rawErrText,
});
if (isTokenExpiredError({ status, message: errText })) {
await reportTokenExpired({
@@ -338,8 +421,8 @@ export async function responsesProxyRoute(app: FastifyInstance) {
return reply.code(status).send({ error: { message: errText, type: 'upstream_error' } });
}
const upstream = endpointResult.upstream;
const successfulUpstreamPath = endpointResult.upstreamPath;
const upstream = endpointResult.upstream;
const successfulUpstreamPath = endpointResult.upstreamPath;
if (isStream) {
reply.hijack();
@@ -349,7 +432,10 @@ export async function responsesProxyRoute(app: FastifyInstance) {
reply.raw.setHeader('Connection', 'keep-alive');
reply.raw.setHeader('X-Accel-Buffering', 'no');
const reader = upstream.body?.getReader();
const upstreamReader = upstream.body?.getReader();
const reader = String(selected.site.platform || '').trim().toLowerCase() === 'gemini-cli' && upstreamReader
? createGeminiCliStreamReader(upstreamReader)
: upstreamReader;
let parsedUsage: UsageSummary = {
promptTokens: 0,
completionTokens: 0,
@@ -364,6 +450,9 @@ export async function responsesProxyRoute(app: FastifyInstance) {
const streamSession = openAiResponsesTransformer.proxyStream.createSession({
modelName,
successfulUpstreamPath,
strictTerminalEvents: Object.entries(request.headers as Record<string, unknown>)
.some(([rawKey, rawValue]) => rawKey.trim().toLowerCase() === 'x-metapi-responses-websocket-transport'
&& String(rawValue).trim() === '1'),
getUsage: () => parsedUsage,
onParsedPayload: (payload) => {
if (payload && typeof payload === 'object') {
@@ -375,9 +464,32 @@ export async function responsesProxyRoute(app: FastifyInstance) {
reply.raw.write(chunk);
},
});
await streamSession.run(reader, reply.raw);
const streamResult = await streamSession.run(reader, reply.raw);
const latency = Date.now() - startTime;
if (streamResult.status === 'failed') {
tokenRouter.recordFailure(selected.channel.id);
logProxy(
selected,
requestedModel,
'failed',
200,
latency,
streamResult.errorMessage,
retryCount,
downstreamPath,
parsedUsage.promptTokens,
parsedUsage.completionTokens,
parsedUsage.totalTokens,
0,
null,
successfulUpstreamPath,
clientContext,
logDownstreamApiKeyId ? downstreamApiKeyId : null,
);
return;
}
const resolvedUsage = await resolveProxyUsageWithSelfLogFallback({
site: selected.site,
account: selected.account,
@@ -406,16 +518,30 @@ export async function responsesProxyRoute(app: FastifyInstance) {
selected, requestedModel, 'success', 200, latency, null, retryCount, downstreamPath,
resolvedUsage.promptTokens, resolvedUsage.completionTokens, resolvedUsage.totalTokens, estimatedCost, billingDetails,
successfulUpstreamPath,
clientContext,
logDownstreamApiKeyId ? downstreamApiKeyId : null,
);
return;
}
const rawText = await upstream.text();
let upstreamData: unknown = rawText;
try {
upstreamData = JSON.parse(rawText);
} catch {
const upstreamContentType = (upstream.headers.get('content-type') || '').toLowerCase();
let rawText = '';
let upstreamData: unknown;
if (upstreamContentType.includes('text/event-stream') && successfulUpstreamPath.endsWith('/responses')) {
const collected = await collectResponsesFinalPayloadFromSse(upstream, modelName);
rawText = collected.rawText;
upstreamData = collected.payload;
} else {
rawText = await upstream.text();
upstreamData = rawText;
try {
upstreamData = JSON.parse(rawText);
} catch {
upstreamData = rawText;
}
}
if (String(selected.site.platform || '').trim().toLowerCase() === 'gemini-cli') {
upstreamData = unwrapGeminiCliPayload(upstreamData);
}
const latency = Date.now() - startTime;
const parsedUsage = parseProxyUsage(upstreamData);
@@ -459,6 +585,8 @@ export async function responsesProxyRoute(app: FastifyInstance) {
selected, requestedModel, 'success', 200, latency, null, retryCount, downstreamPath,
resolvedUsage.promptTokens, resolvedUsage.completionTokens, resolvedUsage.totalTokens, estimatedCost, billingDetails,
successfulUpstreamPath,
clientContext,
logDownstreamApiKeyId ? downstreamApiKeyId : null,
);
return reply.send(downstreamData);
} catch (err: any) {
@@ -479,6 +607,7 @@ export async function responsesProxyRoute(app: FastifyInstance) {
null,
null,
clientContext,
logDownstreamApiKeyId ? downstreamApiKeyId : null,
);
if (retryCount < MAX_RETRIES) {
retryCount += 1;
@@ -497,6 +626,13 @@ export async function responsesProxyRoute(app: FastifyInstance) {
app.post('/v1/responses', async (request: FastifyRequest, reply: FastifyReply) =>
handleResponsesRequest(request, reply, '/v1/responses'));
app.get('/v1/responses', async (_request: FastifyRequest, reply: FastifyReply) =>
reply.code(426).send({
error: {
message: 'WebSocket upgrade required for GET /v1/responses',
type: 'invalid_request_error',
},
}));
app.post('/v1/responses/compact', async (request: FastifyRequest, reply: FastifyReply) =>
handleResponsesRequest(request, reply, '/v1/responses/compact'));
}
@@ -517,6 +653,7 @@ async function logProxy(
billingDetails: unknown = null,
upstreamPath: string | null = null,
clientContext: DownstreamClientContext | null = null,
downstreamApiKeyId: number | null = null,
) {
try {
const createdAt = formatUtcSqlDateTime(new Date());
@@ -534,6 +671,7 @@ async function logProxy(
routeId: selected.channel.routeId,
channelId: selected.channel.id,
accountId: selected.account.id,
...(downstreamApiKeyId !== null ? { downstreamApiKeyId } : {}),
modelRequested,
modelActual: selected.actualModel,
status,
src/server/routes/proxy/responses.websocket.test.ts
+554
View File
@@ -0,0 +1,554 @@
import Fastify, { type FastifyInstance } from 'fastify';
import { AddressInfo } from 'node:net';
import WebSocket from 'ws';
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
const fetchMock = vi.fn();
const selectChannelMock = vi.fn();
const selectNextChannelMock = vi.fn();
const previewSelectedChannelMock = vi.fn();
const recordSuccessMock = vi.fn();
const recordFailureMock = vi.fn();
const refreshModelsAndRebuildRoutesMock = vi.fn();
const reportProxyAllFailedMock = vi.fn();
const reportTokenExpiredMock = vi.fn();
const resolveProxyUsageWithSelfLogFallbackMock = vi.fn(async ({ usage }: any) => ({
...usage,
estimatedCostFromQuota: 0,
recoveredFromSelfLog: false,
}));
const dbInsertMock = vi.fn((_arg?: any) => ({
values: () => ({
run: () => undefined,
}),
}));
vi.mock('undici', () => ({
fetch: (...args: unknown[]) => fetchMock(...args),
}));
vi.mock('../../services/tokenRouter.js', () => ({
tokenRouter: {
selectChannel: (...args: unknown[]) => selectChannelMock(...args),
selectNextChannel: (...args: unknown[]) => selectNextChannelMock(...args),
previewSelectedChannel: (...args: unknown[]) => previewSelectedChannelMock(...args),
recordSuccess: (...args: unknown[]) => recordSuccessMock(...args),
recordFailure: (...args: unknown[]) => recordFailureMock(...args),
},
}));
vi.mock('../../services/modelService.js', () => ({
refreshModelsAndRebuildRoutes: (...args: unknown[]) => refreshModelsAndRebuildRoutesMock(...args),
}));
vi.mock('../../services/alertService.js', () => ({
reportProxyAllFailed: (...args: unknown[]) => reportProxyAllFailedMock(...args),
reportTokenExpired: (...args: unknown[]) => reportTokenExpiredMock(...args),
}));
vi.mock('../../services/alertRules.js', () => ({
isTokenExpiredError: () => false,
}));
vi.mock('../../services/modelPricingService.js', () => ({
estimateProxyCost: async () => 0,
buildProxyBillingDetails: async () => null,
fetchModelPricingCatalog: async () => null,
}));
vi.mock('../../services/proxyRetryPolicy.js', () => ({
shouldRetryProxyRequest: () => false,
}));
vi.mock('../../services/proxyUsageFallbackService.js', () => ({
resolveProxyUsageWithSelfLogFallback: (arg: any) => resolveProxyUsageWithSelfLogFallbackMock(arg),
}));
vi.mock('../../services/oauth/quota.js', () => ({
recordOauthQuotaResetHint: async () => undefined,
}));
vi.mock('../../db/index.js', () => ({
db: {
insert: (arg: any) => dbInsertMock(arg),
},
hasProxyLogDownstreamApiKeyIdColumn: async () => false,
schema: {
proxyLogs: {},
},
}));
function createSseResponse(chunks: string[], status = 200) {
const encoder = new TextEncoder();
return new Response(new ReadableStream<Uint8Array>({
start(controller) {
for (const chunk of chunks) {
controller.enqueue(encoder.encode(chunk));
}
controller.close();
},
}), {
status,
headers: { 'content-type': 'text/event-stream; charset=utf-8' },
});
}
function createSelectedChannel(options?: {
siteName?: string;
siteUrl?: string;
sitePlatform?: string;
username?: string;
extraConfig?: string;
tokenValue?: string;
actualModel?: string;
}) {
const sitePlatform = options?.sitePlatform ?? 'codex';
const isCodex = sitePlatform === 'codex';
return {
channel: { id: 11, routeId: 22 },
site: {
name: options?.siteName ?? (isCodex ? 'codex-site' : 'openai-site'),
url: options?.siteUrl ?? (isCodex ? 'https://chatgpt.com/backend-api/codex' : 'https://api.openai.com'),
platform: sitePlatform,
},
account: {
id: 33,
username: options?.username ?? (isCodex ? 'codex-user@example.com' : 'openai-user@example.com'),
extraConfig: options?.extraConfig ?? (isCodex
? JSON.stringify({
credentialMode: 'session',
oauth: {
provider: 'codex',
accountId: 'chatgpt-account-123',
email: 'codex-user@example.com',
},
})
: '{}'),
},
tokenName: 'default',
tokenValue: options?.tokenValue ?? (isCodex ? 'oauth-access-token' : 'sk-openai-token'),
actualModel: options?.actualModel ?? (isCodex ? 'gpt-5.4' : 'gpt-4.1'),
};
}
function waitForSocketOpen(socket: WebSocket) {
return new Promise<void>((resolve, reject) => {
socket.once('open', () => resolve());
socket.once('error', reject);
});
}
function waitForSocketUpgrade(socket: WebSocket) {
return new Promise<{ headers: Record<string, string | string[] | undefined> }>((resolve, reject) => {
socket.once('upgrade', (response) => resolve({ headers: response.headers as Record<string, string | string[] | undefined> }));
socket.once('error', reject);
});
}
function waitForSocketMessages(socket: WebSocket, count: number, timeoutMs = 1000) {
return new Promise<any[]>((resolve, reject) => {
const messages: any[] = [];
const timeout = setTimeout(() => {
socket.off('message', onMessage);
socket.off('error', onError);
reject(new Error(`Timed out waiting for ${count} websocket messages`));
}, timeoutMs);
const onMessage = (payload: WebSocket.RawData) => {
messages.push(JSON.parse(String(payload)));
if (messages.length >= count) {
clearTimeout(timeout);
socket.off('message', onMessage);
socket.off('error', onError);
resolve(messages);
}
};
const onError = (error: Error) => {
clearTimeout(timeout);
socket.off('message', onMessage);
reject(error);
};
socket.on('message', onMessage);
socket.once('error', onError);
});
}
describe('responses websocket transport', () => {
let app: FastifyInstance;
let baseUrl: string;
beforeAll(async () => {
const { responsesProxyRoute } = await import('./responses.js');
app = Fastify();
await app.register(responsesProxyRoute);
await app.listen({ port: 0, host: '127.0.0.1' });
const address = app.server.address() as AddressInfo;
baseUrl = `ws://127.0.0.1:${address.port}`;
});
beforeEach(() => {
fetchMock.mockReset();
selectChannelMock.mockReset();
selectNextChannelMock.mockReset();
previewSelectedChannelMock.mockReset();
recordSuccessMock.mockReset();
recordFailureMock.mockReset();
refreshModelsAndRebuildRoutesMock.mockReset();
reportProxyAllFailedMock.mockReset();
reportTokenExpiredMock.mockReset();
resolveProxyUsageWithSelfLogFallbackMock.mockClear();
dbInsertMock.mockClear();
const selectedChannel = createSelectedChannel();
selectChannelMock.mockReturnValue(selectedChannel);
selectNextChannelMock.mockReturnValue(null);
previewSelectedChannelMock.mockResolvedValue(selectedChannel);
});
afterAll(async () => {
await app.close();
});
it('accepts response.create over GET /v1/responses websocket and forwards streamed responses events', async () => {
fetchMock.mockResolvedValue(createSseResponse([
'event: response.created\n',
'data: {"type":"response.created","response":{"id":"resp_ws","model":"gpt-5.4","created_at":1706000000,"status":"in_progress","output":[]}}\n\n',
'event: response.output_item.added\n',
'data: {"type":"response.output_item.added","output_index":0,"item":{"id":"msg_ws","type":"message","role":"assistant","status":"in_progress","content":[]}}\n\n',
'event: response.output_text.delta\n',
'data: {"type":"response.output_text.delta","output_index":0,"item_id":"msg_ws","delta":"pong"}\n\n',
'event: response.completed\n',
'data: {"type":"response.completed","response":{"id":"resp_ws","model":"gpt-5.4","status":"completed","output":[{"id":"msg_ws","type":"message","role":"assistant","status":"completed","content":[{"type":"output_text","text":"pong"}]}],"usage":{"input_tokens":3,"output_tokens":1,"total_tokens":4}}}\n\n',
'data: [DONE]\n\n',
]));
const socket = new WebSocket(`${baseUrl}/v1/responses`);
await waitForSocketOpen(socket);
const messagesPromise = waitForSocketMessages(socket, 4);
socket.send(JSON.stringify({
type: 'response.create',
model: 'gpt-5.4',
input: [
{
type: 'message',
role: 'user',
content: [{ type: 'input_text', text: 'hello websocket' }],
},
],
}));
const messages = await messagesPromise;
socket.close();
expect(messages.map((message) => message.type)).toEqual([
'response.created',
'response.output_item.added',
'response.output_text.delta',
'response.completed',
]);
expect(messages[3]?.response?.output?.[0]?.content?.[0]?.text).toBe('pong');
expect(fetchMock).toHaveBeenCalledTimes(1);
});
it('echoes x-codex-turn-state on websocket upgrade responses', async () => {
const socket = new WebSocket(`${baseUrl}/v1/responses`, {
headers: {
'x-codex-turn-state': 'turn-state-123',
},
});
const [upgrade] = await Promise.all([
waitForSocketUpgrade(socket),
waitForSocketOpen(socket),
]);
socket.close();
expect(upgrade.headers['x-codex-turn-state']).toBe('turn-state-123');
});
it('merges follow-up response.create payloads when the selected upstream does not support incremental mode', async () => {
const selectedChannel = createSelectedChannel({
sitePlatform: 'openai',
actualModel: 'gpt-4.1',
});
selectChannelMock.mockReturnValue(selectedChannel);
previewSelectedChannelMock.mockResolvedValue(selectedChannel);
fetchMock
.mockResolvedValueOnce(createSseResponse([
'event: response.output_item.done\n',
'data: {"type":"response.output_item.done","output_index":0,"item":{"id":"fc_1","type":"function_call","call_id":"call_1"}}\n\n',
'event: response.output_item.done\n',
'data: {"type":"response.output_item.done","output_index":1,"item":{"id":"msg_1","type":"message","role":"assistant","status":"completed","content":[{"type":"output_text","text":"call tool"}]}}\n\n',
'event: response.completed\n',
'data: {"type":"response.completed","response":{"id":"resp_ws_1","model":"gpt-4.1","status":"completed","output":[{"id":"fc_1","type":"function_call","call_id":"call_1"},{"id":"msg_1","type":"message","role":"assistant","status":"completed","content":[{"type":"output_text","text":"call tool"}]}],"usage":{"input_tokens":3,"output_tokens":1,"total_tokens":4}}}\n\n',
'data: [DONE]\n\n',
]))
.mockResolvedValueOnce(createSseResponse([
'event: response.completed\n',
'data: {"type":"response.completed","response":{"id":"resp_ws_2","model":"gpt-4.1","status":"completed","output":[{"id":"msg_2","type":"message","role":"assistant","status":"completed","content":[{"type":"output_text","text":"done"}]}],"usage":{"input_tokens":5,"output_tokens":1,"total_tokens":6}}}\n\n',
'data: [DONE]\n\n',
]));
const socket = new WebSocket(`${baseUrl}/v1/responses`);
await waitForSocketOpen(socket);
const firstResponsePromise = waitForSocketMessages(socket, 1);
socket.send(JSON.stringify({
type: 'response.create',
model: 'gpt-4.1',
instructions: 'be helpful',
input: [
{
id: 'msg_user_1',
type: 'message',
role: 'user',
content: [{ type: 'input_text', text: 'call the tool' }],
},
],
}));
await firstResponsePromise;
const secondResponsePromise = waitForSocketMessages(socket, 1);
socket.send(JSON.stringify({
type: 'response.create',
previous_response_id: 'resp_ws_1',
input: [
{
id: 'tool_out_1',
type: 'function_call_output',
call_id: 'call_1',
output: 'tool result',
},
],
}));
await secondResponsePromise;
socket.close();
expect(fetchMock).toHaveBeenCalledTimes(2);
const [, firstOptions] = fetchMock.mock.calls[0] as [string, RequestInit];
const [, secondOptions] = fetchMock.mock.calls[1] as [string, RequestInit];
const firstBody = JSON.parse(String(firstOptions.body));
const secondBody = JSON.parse(String(secondOptions.body));
expect(firstBody.input).toHaveLength(1);
expect(secondBody.previous_response_id).toBeUndefined();
expect(secondBody.model).toBe('gpt-4.1');
expect(secondBody.instructions).toBe('be helpful');
expect(secondBody.input).toEqual([
{
id: 'msg_user_1',
type: 'message',
role: 'user',
content: [{ type: 'input_text', text: 'call the tool' }],
},
{
id: 'fc_1',
type: 'function_call',
call_id: 'call_1',
status: 'completed',
},
{
id: 'msg_1',
type: 'message',
role: 'assistant',
status: 'completed',
content: [{ type: 'output_text', text: 'call tool' }],
},
{
id: 'tool_out_1',
type: 'function_call_output',
call_id: 'call_1',
output: 'tool result',
},
]);
});
it('preserves incremental response.create payloads with previous_response_id for websocket-capable upstreams', async () => {
fetchMock
.mockResolvedValueOnce(createSseResponse([
'event: response.output_item.done\n',
'data: {"type":"response.output_item.done","output_index":0,"item":{"id":"fc_1","type":"function_call","call_id":"call_1"}}\n\n',
'event: response.output_item.done\n',
'data: {"type":"response.output_item.done","output_index":1,"item":{"id":"msg_1","type":"message","role":"assistant","status":"completed","content":[{"type":"output_text","text":"call tool"}]}}\n\n',
'event: response.completed\n',
'data: {"type":"response.completed","response":{"id":"resp_ws_1","model":"gpt-5.4","status":"completed","output":[{"id":"fc_1","type":"function_call","call_id":"call_1"},{"id":"msg_1","type":"message","role":"assistant","status":"completed","content":[{"type":"output_text","text":"call tool"}]}],"usage":{"input_tokens":3,"output_tokens":1,"total_tokens":4}}}\n\n',
'data: [DONE]\n\n',
]))
.mockResolvedValueOnce(createSseResponse([
'event: response.completed\n',
'data: {"type":"response.completed","response":{"id":"resp_ws_2","model":"gpt-5.4","status":"completed","output":[{"id":"msg_2","type":"message","role":"assistant","status":"completed","content":[{"type":"output_text","text":"done"}]}],"usage":{"input_tokens":5,"output_tokens":1,"total_tokens":6}}}\n\n',
'data: [DONE]\n\n',
]));
const socket = new WebSocket(`${baseUrl}/v1/responses`);
await waitForSocketOpen(socket);
const firstResponsePromise = waitForSocketMessages(socket, 1);
socket.send(JSON.stringify({
type: 'response.create',
model: 'gpt-5.4',
instructions: 'be helpful',
input: [
{
id: 'msg_user_1',
type: 'message',
role: 'user',
content: [{ type: 'input_text', text: 'call the tool' }],
},
],
}));
await firstResponsePromise;
const secondResponsePromise = waitForSocketMessages(socket, 1);
socket.send(JSON.stringify({
type: 'response.create',
previous_response_id: 'resp_ws_1',
input: [
{
id: 'tool_out_1',
type: 'function_call_output',
call_id: 'call_1',
output: 'tool result',
},
],
}));
await secondResponsePromise;
socket.close();
expect(fetchMock).toHaveBeenCalledTimes(2);
const [, secondOptions] = fetchMock.mock.calls[1] as [string, RequestInit];
const secondBody = JSON.parse(String(secondOptions.body));
expect(secondBody.previous_response_id).toBe('resp_ws_1');
expect(secondBody.model).toBe('gpt-5.4');
expect(secondBody.instructions).toBe('be helpful');
expect(secondBody.input).toEqual([
{
id: 'tool_out_1',
type: 'function_call_output',
call_id: 'call_1',
output: 'tool result',
},
]);
});
it('handles generate=false locally only for non-websocket-capable upstreams', async () => {
const selectedChannel = createSelectedChannel({
sitePlatform: 'openai',
actualModel: 'gpt-4.1',
});
selectChannelMock.mockReturnValue(selectedChannel);
previewSelectedChannelMock.mockResolvedValue(selectedChannel);
fetchMock.mockResolvedValueOnce(createSseResponse([
'event: response.completed\n',
'data: {"type":"response.completed","response":{"id":"resp_ws_after_prewarm","model":"gpt-4.1","status":"completed","output":[{"id":"msg_2","type":"message","role":"assistant","status":"completed","content":[{"type":"output_text","text":"done"}]}],"usage":{"input_tokens":1,"output_tokens":1,"total_tokens":2}}}\n\n',
'data: [DONE]\n\n',
]));
const socket = new WebSocket(`${baseUrl}/v1/responses`);
await waitForSocketOpen(socket);
const prewarmMessagesPromise = waitForSocketMessages(socket, 2);
socket.send(JSON.stringify({
type: 'response.create',
model: 'gpt-4.1',
generate: false,
}));
const prewarmMessages = await prewarmMessagesPromise;
expect(prewarmMessages.map((message) => message.type)).toEqual(['response.created', 'response.completed']);
expect(fetchMock).toHaveBeenCalledTimes(0);
const secondResponsePromise = waitForSocketMessages(socket, 1);
socket.send(JSON.stringify({
type: 'response.create',
previous_response_id: prewarmMessages[0]?.response?.id,
input: [
{
id: 'msg_followup_1',
type: 'message',
role: 'user',
content: [{ type: 'input_text', text: 'continue' }],
},
],
}));
await secondResponsePromise;
socket.close();
expect(fetchMock).toHaveBeenCalledTimes(1);
const [, options] = fetchMock.mock.calls[0] as [string, RequestInit];
const forwardedBody = JSON.parse(String(options.body));
expect(forwardedBody.generate).toBeUndefined();
expect(forwardedBody.previous_response_id).toBeUndefined();
expect(forwardedBody.model).toBe('gpt-4.1');
expect(forwardedBody.input).toEqual([
{
id: 'msg_followup_1',
type: 'message',
role: 'user',
content: [{ type: 'input_text', text: 'continue' }],
},
]);
});
it('forwards generate=false upstream for websocket-capable upstreams instead of synthesizing prewarm events', async () => {
fetchMock.mockResolvedValueOnce(createSseResponse([
'event: response.completed\n',
'data: {"type":"response.completed","response":{"id":"resp_capable_generate_false","model":"gpt-5.4","status":"completed","output":[],"usage":{"input_tokens":0,"output_tokens":0,"total_tokens":0}}}\n\n',
'data: [DONE]\n\n',
]));
const socket = new WebSocket(`${baseUrl}/v1/responses`);
await waitForSocketOpen(socket);
const messagesPromise = waitForSocketMessages(socket, 1);
socket.send(JSON.stringify({
type: 'response.create',
model: 'gpt-5.4',
generate: false,
}));
const messages = await messagesPromise;
socket.close();
expect(messages.map((message) => message.type)).toEqual(['response.completed']);
expect(fetchMock).toHaveBeenCalledTimes(1);
const [, options] = fetchMock.mock.calls[0] as [string, RequestInit];
const forwardedBody = JSON.parse(String(options.body));
expect(forwardedBody.generate).toBe(false);
});
it('emits websocket error when the upstream stream closes before a terminal responses event', async () => {
fetchMock.mockResolvedValue(createSseResponse([
'event: response.created\n',
'data: {"type":"response.created","response":{"id":"resp_incomplete","model":"gpt-5.4","created_at":1706000000,"status":"in_progress","output":[]}}\n\n',
'event: response.output_text.delta\n',
'data: {"type":"response.output_text.delta","output_index":0,"item_id":"msg_ws","delta":"partial"}\n\n',
]));
const socket = new WebSocket(`${baseUrl}/v1/responses`);
await waitForSocketOpen(socket);
const messagesPromise = waitForSocketMessages(socket, 3, 400);
socket.send(JSON.stringify({
type: 'response.create',
model: 'gpt-5.4',
input: [
{
type: 'message',
role: 'user',
content: [{ type: 'input_text', text: 'hello websocket' }],
},
],
}));
const messages = await messagesPromise;
socket.close();
expect(messages.map((message) => message.type)).toEqual([
'response.created',
'response.output_text.delta',
'error',
]);
expect(messages[2]?.status).toBe(408);
expect(messages[2]?.error?.message).toContain('stream closed before response.completed');
});
});
src/server/routes/proxy/responsesSseFinal.ts
+97
View File
@@ -0,0 +1,97 @@
import { openAiResponsesTransformer } from '../../transformers/openai/responses/index.js';
import { mergeProxyUsage, parseProxyUsage } from '../../services/proxyUsageParser.js';
function isRecord(value: unknown): value is Record<string, unknown> {
return !!value && typeof value === 'object';
}
function parseResponsesSsePayload(data: string): Record<string, unknown> | null {
try {
const parsed = JSON.parse(data);
return isRecord(parsed) ? parsed : null;
} catch {
return null;
}
}
function getResponsesFailureMessage(payload: Record<string, unknown>): string {
if (isRecord(payload.error) && typeof payload.error.message === 'string' && payload.error.message.trim()) {
return payload.error.message.trim();
}
if (typeof payload.message === 'string' && payload.message.trim()) {
return payload.message.trim();
}
return 'upstream stream failed';
}
export async function collectResponsesFinalPayloadFromSse(
upstream: { text(): Promise<string> },
modelName: string,
): Promise<{ payload: Record<string, unknown>; rawText: string }> {
const rawText = await upstream.text();
const { events } = openAiResponsesTransformer.pullSseEvents(rawText);
const streamContext = openAiResponsesTransformer.createStreamContext(modelName);
const aggregateState = openAiResponsesTransformer.aggregator.createState(modelName);
let usage = {
promptTokens: 0,
completionTokens: 0,
totalTokens: 0,
cacheReadTokens: 0,
cacheCreationTokens: 0,
promptTokensIncludeCache: null as boolean | null,
};
let completedPayload: Record<string, unknown> | null = null;
const captureCompletedPayload = (lines: string[]) => {
if (completedPayload) return;
const parsed = openAiResponsesTransformer.pullSseEvents(lines.join(''));
for (const event of parsed.events) {
if (event.data === '[DONE]') continue;
const payload = parseResponsesSsePayload(event.data);
if (!payload) continue;
if (payload.type === 'response.failed') {
throw new Error(getResponsesFailureMessage(payload));
}
if (payload.type === 'response.completed' && isRecord(payload.response)) {
completedPayload = payload.response;
return;
}
}
};
for (const event of events) {
if (event.data === '[DONE]') continue;
const payload = parseResponsesSsePayload(event.data);
if (!payload) continue;
if (payload.type === 'response.failed') {
throw new Error(getResponsesFailureMessage(payload));
}
usage = mergeProxyUsage(usage, parseProxyUsage(payload));
const normalizedEvent = openAiResponsesTransformer.transformStreamEvent(payload, streamContext, modelName);
captureCompletedPayload(openAiResponsesTransformer.aggregator.serialize({
state: aggregateState,
streamContext,
event: normalizedEvent,
usage,
}));
}
if (!completedPayload) {
captureCompletedPayload(openAiResponsesTransformer.aggregator.complete(
aggregateState,
streamContext,
usage,
));
}
if (completedPayload) {
return {
payload: completedPayload,
rawText,
};
}
throw new Error('stream disconnected before response.completed');
}
src/server/routes/proxy/responsesWebsocket.ts
+401
View File
@@ -0,0 +1,401 @@
import { randomUUID } from 'node:crypto';
import type { FastifyInstance } from 'fastify';
import type { IncomingMessage } from 'node:http';
import { WebSocketServer, type RawData, type WebSocket } from 'ws';
import { tokenRouter } from '../../services/tokenRouter.js';
import { openAiResponsesTransformer } from '../../transformers/openai/responses/index.js';
const installedApps = new WeakSet<FastifyInstance>();
const WS_TURN_STATE_HEADER = 'x-codex-turn-state';
const RESPONSES_WEBSOCKET_MODE_HEADER = 'x-metapi-responses-websocket-mode';
const RESPONSES_WEBSOCKET_TRANSPORT_HEADER = 'x-metapi-responses-websocket-transport';
type NormalizedResponsesWebsocketRequest =
| {
ok: true;
request: Record<string, unknown>;
nextRequestSnapshot: Record<string, unknown>;
}
| {
ok: false;
status: number;
message: string;
};
function isRecord(value: unknown): value is Record<string, unknown> {
return !!value && typeof value === 'object' && !Array.isArray(value);
}
function asTrimmedString(value: unknown): string {
return typeof value === 'string' ? value.trim() : '';
}
function headerValueToTrimmedString(value: unknown): string {
if (typeof value === 'string') return value.trim();
if (Array.isArray(value)) {
for (const item of value) {
if (typeof item !== 'string') continue;
const trimmed = item.trim();
if (trimmed) return trimmed;
}
}
return '';
}
function parseJsonObject(raw: RawData): Record<string, unknown> | null {
try {
const parsed = JSON.parse(String(raw));
return isRecord(parsed) ? parsed : null;
} catch {
return null;
}
}
function cloneJsonObject<T>(value: T): T {
return structuredClone(value);
}
function toResponseInputArray(value: unknown): unknown[] {
return Array.isArray(value) ? cloneJsonObject(value) : [];
}
function normalizeResponsesWebsocketRequest(
parsed: Record<string, unknown>,
lastRequest: Record<string, unknown> | null,
lastResponseOutput: unknown[],
supportsIncrementalInput: boolean,
): NormalizedResponsesWebsocketRequest {
const requestType = asTrimmedString(parsed.type);
if (requestType !== 'response.create' && requestType !== 'response.append') {
return {
ok: false,
status: 400,
message: `unsupported websocket request type: ${requestType || 'unknown'}`,
};
}
if (!lastRequest) {
if (requestType !== 'response.create') {
return {
ok: false,
status: 400,
message: 'websocket request received before response.create',
};
}
const next = cloneJsonObject(parsed);
delete next.type;
if (!supportsIncrementalInput && parsed.generate === false) {
delete next.generate;
}
next.stream = true;
if (!Array.isArray(next.input)) next.input = [];
const modelName = asTrimmedString(next.model);
if (!modelName) {
return {
ok: false,
status: 400,
message: 'missing model in response.create request',
};
}
return {
ok: true,
request: next,
nextRequestSnapshot: cloneJsonObject(next),
};
}
if (!Array.isArray(parsed.input)) {
return {
ok: false,
status: 400,
message: 'websocket request requires array field: input',
};
}
const next = cloneJsonObject(parsed);
delete next.type;
next.stream = true;
if (!('model' in next) && typeof lastRequest.model === 'string') {
next.model = lastRequest.model;
}
if (!('instructions' in next) && lastRequest.instructions !== undefined) {
next.instructions = cloneJsonObject(lastRequest.instructions);
}
if (supportsIncrementalInput && requestType === 'response.create' && asTrimmedString(parsed.previous_response_id)) {
return {
ok: true,
request: next,
nextRequestSnapshot: cloneJsonObject(next),
};
}
const mergedInput = [
...toResponseInputArray(lastRequest.input),
...cloneJsonObject(lastResponseOutput),
...cloneJsonObject(parsed.input),
];
delete next.previous_response_id;
next.input = mergedInput;
return {
ok: true,
request: next,
nextRequestSnapshot: cloneJsonObject(next),
};
}
function shouldHandleResponsesWebsocketPrewarmLocally(
parsed: Record<string, unknown>,
lastRequest: Record<string, unknown> | null,
supportsIncrementalInput: boolean,
): boolean {
if (supportsIncrementalInput || lastRequest) return false;
if (asTrimmedString(parsed.type) !== 'response.create') return false;
return parsed.generate === false;
}
function writeResponsesWebsocketError(
socket: WebSocket,
status: number,
message: string,
errorPayload?: unknown,
) {
socket.send(JSON.stringify({
type: 'error',
status,
error: isRecord(errorPayload) && isRecord(errorPayload.error)
? errorPayload.error
: {
type: status >= 500 ? 'server_error' : 'invalid_request_error',
message,
},
}));
}
function synthesizePrewarmResponsePayloads(request: Record<string, unknown>) {
const responseId = `resp_prewarm_${randomUUID()}`;
const modelName = asTrimmedString(request.model) || 'unknown';
const createdAt = Math.floor(Date.now() / 1000);
return [
{
type: 'response.created',
response: {
id: responseId,
object: 'response',
created_at: createdAt,
status: 'in_progress',
model: modelName,
output: [],
},
},
{
type: 'response.completed',
response: {
id: responseId,
object: 'response',
created_at: createdAt,
status: 'completed',
model: modelName,
output: [],
usage: {
input_tokens: 0,
output_tokens: 0,
total_tokens: 0,
},
},
},
];
}
function collectResponsesOutput(payloads: unknown[]): unknown[] {
const outputByIndex = new Map<number, unknown>();
let completedOutput: unknown[] | null = null;
for (const payload of payloads) {
if (!isRecord(payload)) continue;
const type = asTrimmedString(payload.type);
if ((type === 'response.output_item.added' || type === 'response.output_item.done')
&& Number.isInteger(payload.output_index)
&& payload.item !== undefined) {
outputByIndex.set(Number(payload.output_index), cloneJsonObject(payload.item));
continue;
}
if (type === 'response.completed' && isRecord(payload.response) && Array.isArray(payload.response.output)) {
completedOutput = cloneJsonObject(payload.response.output);
}
}
if (completedOutput) return completedOutput;
return [...outputByIndex.entries()]
.sort((left, right) => left[0] - right[0])
.map(([, value]) => value);
}
function buildInjectHeaders(request: IncomingMessage): Record<string, string | string[]> {
const headers: Record<string, string | string[]> = {};
for (const [rawKey, rawValue] of Object.entries(request.headers)) {
const key = rawKey.toLowerCase();
if (!rawValue) continue;
if (
key === 'host'
|| key === 'connection'
|| key === 'upgrade'
|| key === 'sec-websocket-key'
|| key === 'sec-websocket-version'
|| key === 'sec-websocket-extensions'
|| key === 'sec-websocket-protocol'
) {
continue;
}
headers[rawKey] = rawValue as string | string[];
}
return headers;
}
async function supportsResponsesWebsocketIncrementalInput(
parsed: Record<string, unknown>,
lastRequest: Record<string, unknown> | null,
): Promise<boolean> {
const requestModel = asTrimmedString(parsed.model) || asTrimmedString(lastRequest?.model);
if (!requestModel) return false;
try {
const selected = await tokenRouter.previewSelectedChannel(requestModel);
return asTrimmedString(selected?.site?.platform).toLowerCase() === 'codex';
} catch {
return false;
}
}
async function handleResponsesWebsocketConnection(
app: FastifyInstance,
socket: WebSocket,
request: IncomingMessage,
) {
let lastRequest: Record<string, unknown> | null = null;
let lastResponseOutput: unknown[] = [];
socket.on('message', async (raw) => {
const parsed = parseJsonObject(raw);
if (!parsed) {
writeResponsesWebsocketError(socket, 400, 'Invalid websocket JSON payload');
return;
}
const supportsIncrementalInput = await supportsResponsesWebsocketIncrementalInput(parsed, lastRequest);
const shouldHandleLocalPrewarm = shouldHandleResponsesWebsocketPrewarmLocally(
parsed,
lastRequest,
supportsIncrementalInput,
);
const normalized = normalizeResponsesWebsocketRequest(
parsed,
lastRequest,
lastResponseOutput,
supportsIncrementalInput,
);
if (!normalized.ok) {
writeResponsesWebsocketError(socket, normalized.status, normalized.message);
return;
}
lastRequest = normalized.nextRequestSnapshot;
if (shouldHandleLocalPrewarm) {
lastResponseOutput = [];
for (const payload of synthesizePrewarmResponsePayloads(normalized.request)) {
socket.send(JSON.stringify(payload));
}
return;
}
const response = await app.inject({
method: 'POST',
url: '/v1/responses',
headers: {
...buildInjectHeaders(request),
[RESPONSES_WEBSOCKET_TRANSPORT_HEADER]: '1',
...(supportsIncrementalInput ? { [RESPONSES_WEBSOCKET_MODE_HEADER]: 'incremental' } : {}),
},
payload: normalized.request,
});
if (response.statusCode < 200 || response.statusCode >= 300) {
let payload: unknown = null;
try {
payload = JSON.parse(response.body);
} catch {
payload = null;
}
writeResponsesWebsocketError(
socket,
response.statusCode,
response.statusMessage || 'Upstream error',
payload,
);
return;
}
const contentType = String(response.headers['content-type'] || '').toLowerCase();
if (!contentType.includes('text/event-stream')) {
try {
const payload = JSON.parse(response.body);
socket.send(JSON.stringify(payload));
lastResponseOutput = isRecord(payload?.response) && Array.isArray(payload.response.output)
? cloneJsonObject(payload.response.output)
: [];
} catch {
writeResponsesWebsocketError(socket, 502, 'Unexpected non-JSON websocket proxy response');
}
return;
}
const pulled = openAiResponsesTransformer.pullSseEvents(response.body);
const forwardedPayloads: unknown[] = [];
let sawTerminalPayload = false;
for (const event of pulled.events) {
if (event.data === '[DONE]') continue;
try {
const payload = JSON.parse(event.data);
forwardedPayloads.push(payload);
const type = isRecord(payload) ? asTrimmedString(payload.type) : '';
if (type === 'response.completed' || type === 'response.failed') {
sawTerminalPayload = true;
}
socket.send(JSON.stringify(payload));
} catch {
// Ignore malformed SSE frames; the HTTP route already normalizes them.
}
}
lastResponseOutput = collectResponsesOutput(forwardedPayloads);
if (!sawTerminalPayload) {
writeResponsesWebsocketError(socket, 408, 'stream closed before response.completed');
}
});
}
export function ensureResponsesWebsocketTransport(app: FastifyInstance) {
if (installedApps.has(app)) return;
installedApps.add(app);
const websocketServer = new WebSocketServer({ noServer: true });
websocketServer.on('headers', (headers, request) => {
const turnState = headerValueToTrimmedString(request.headers[WS_TURN_STATE_HEADER]);
if (!turnState) return;
headers.push(`${WS_TURN_STATE_HEADER}: ${turnState}`);
});
app.server.on('upgrade', (request, socket, head) => {
const url = new URL(request.url || '/', 'http://localhost');
if (url.pathname !== '/v1/responses') return;
websocketServer.handleUpgrade(request, socket, head, (client) => {
void handleResponsesWebsocketConnection(app, client, request);
});
});
app.addHook('onClose', async () => {
await new Promise<void>((resolve) => {
websocketServer.close(() => resolve());
});
});
}
src/server/routes/proxy/runtimeExecutor.test.ts
+221
View File
@@ -0,0 +1,221 @@
import { beforeEach, describe, expect, it, vi } from 'vitest';
import { fetch } from 'undici';
import type { BuiltEndpointRequest } from './endpointFlow.js';
vi.mock('undici', () => ({
fetch: vi.fn(),
}));
const fetchMock = vi.mocked(fetch);
describe('dispatchRuntimeRequest', () => {
beforeEach(() => {
fetchMock.mockReset();
});
it('routes antigravity runtime requests through daily then sandbox base urls and rewrites the payload fingerprint', async () => {
const { dispatchRuntimeRequest } = await import('./runtimeExecutor.js');
const request: BuiltEndpointRequest = {
endpoint: 'chat',
path: '/v1internal:generateContent',
headers: {
Authorization: 'Bearer antigravity-token',
'Content-Type': 'application/json',
'User-Agent': 'google-api-nodejs-client/9.15.1',
},
body: {
project: 'project-demo',
model: 'gemini-3-pro-preview',
request: {
contents: [
{
role: 'user',
parts: [{ text: 'hello runtime executor' }],
},
],
},
},
runtime: {
executor: 'antigravity',
modelName: 'gemini-3-pro-preview',
stream: false,
},
};
fetchMock
.mockResolvedValueOnce(new Response(JSON.stringify({
error: { message: 'try fallback base url' },
}), {
status: 429,
headers: { 'content-type': 'application/json' },
}) as unknown as Awaited<ReturnType<typeof fetch>>)
.mockResolvedValueOnce(new Response(JSON.stringify({
response: { responseId: 'ok' },
}), {
status: 200,
headers: { 'content-type': 'application/json' },
}) as unknown as Awaited<ReturnType<typeof fetch>>);
const response = await dispatchRuntimeRequest({
siteUrl: 'https://cloudcode-pa.googleapis.com',
request,
buildInit: (_url, nextRequest) => ({
method: 'POST',
headers: nextRequest.headers,
body: JSON.stringify(nextRequest.body),
}),
});
expect(response.ok).toBe(true);
expect(fetchMock).toHaveBeenCalledTimes(2);
expect(fetchMock.mock.calls[0]?.[0]).toBe('https://daily-cloudcode-pa.googleapis.com/v1internal:generateContent');
expect(fetchMock.mock.calls[1]?.[0]).toBe('https://daily-cloudcode-pa.sandbox.googleapis.com/v1internal:generateContent');
const firstInit = fetchMock.mock.calls[0]?.[1] as RequestInit;
expect(firstInit.headers).toMatchObject({
Authorization: 'Bearer antigravity-token',
'Content-Type': 'application/json',
Accept: 'application/json',
'User-Agent': 'antigravity/1.19.6 darwin/arm64',
});
const upstreamBody = JSON.parse(String(firstInit.body));
expect(upstreamBody).toMatchObject({
project: 'project-demo',
model: 'gemini-3-pro-preview',
userAgent: 'antigravity',
requestType: 'agent',
request: {
sessionId: expect.any(String),
contents: [
{
role: 'user',
parts: [{ text: 'hello runtime executor' }],
},
],
},
});
expect(upstreamBody.requestId).toMatch(/^agent-[0-9a-f-]{36}$/i);
});
it('keeps gemini-cli countTokens payload lean while forcing a model-aware user agent', async () => {
const { dispatchRuntimeRequest } = await import('./runtimeExecutor.js');
const request: BuiltEndpointRequest = {
endpoint: 'chat',
path: '/v1internal:countTokens',
headers: {
Authorization: 'Bearer gemini-cli-token',
'Content-Type': 'application/json',
'User-Agent': 'GeminiCLI/0.31.0/unknown (win32; x64)',
'X-Goog-Api-Client': 'google-genai-sdk/1.41.0 gl-node/v22.19.0',
},
body: {
project: 'project-demo',
model: 'gemini-2.5-pro',
request: {
contents: [
{
role: 'user',
parts: [{ text: 'count these tokens' }],
},
],
},
},
runtime: {
executor: 'gemini-cli',
modelName: 'gemini-2.5-pro',
stream: false,
action: 'countTokens',
},
};
fetchMock.mockResolvedValueOnce(new Response(JSON.stringify({
totalTokens: 12,
}), {
status: 200,
headers: { 'content-type': 'application/json' },
}) as unknown as Awaited<ReturnType<typeof fetch>>);
const response = await dispatchRuntimeRequest({
siteUrl: 'https://cloudcode-pa.googleapis.com',
request,
buildInit: (_url, nextRequest) => ({
method: 'POST',
headers: nextRequest.headers,
body: JSON.stringify(nextRequest.body),
}),
});
expect(response.ok).toBe(true);
const requestInit = fetchMock.mock.calls[0]?.[1] as RequestInit;
expect(requestInit.headers).toMatchObject({
Authorization: 'Bearer gemini-cli-token',
'Content-Type': 'application/json',
'User-Agent': 'GeminiCLI/0.31.0/gemini-2.5-pro (win32; x64)',
'X-Goog-Api-Client': 'google-genai-sdk/1.41.0 gl-node/v22.19.0',
});
expect(JSON.parse(String(requestInit.body))).toEqual({
request: {
contents: [
{
role: 'user',
parts: [{ text: 'count these tokens' }],
},
],
},
});
});
it('retries antigravity runtime requests on transport errors before falling back to the next base url', async () => {
const { dispatchRuntimeRequest } = await import('./runtimeExecutor.js');
const request: BuiltEndpointRequest = {
endpoint: 'chat',
path: '/v1internal:generateContent',
headers: {
Authorization: 'Bearer antigravity-token',
'Content-Type': 'application/json',
},
body: {
project: 'project-demo',
model: 'gemini-3-pro-preview',
request: {
contents: [
{
role: 'user',
parts: [{ text: 'hello runtime executor' }],
},
],
},
},
runtime: {
executor: 'antigravity',
modelName: 'gemini-3-pro-preview',
stream: false,
},
};
fetchMock
.mockRejectedValueOnce(new Error('socket hang up'))
.mockResolvedValueOnce(new Response(JSON.stringify({
response: { responseId: 'ok' },
}), {
status: 200,
headers: { 'content-type': 'application/json' },
}) as unknown as Awaited<ReturnType<typeof fetch>>);
const response = await dispatchRuntimeRequest({
siteUrl: 'https://cloudcode-pa.googleapis.com',
request,
buildInit: (_url, nextRequest) => ({
method: 'POST',
headers: nextRequest.headers,
body: JSON.stringify(nextRequest.body),
}),
});
expect(response.ok).toBe(true);
expect(fetchMock).toHaveBeenCalledTimes(2);
expect(fetchMock.mock.calls[0]?.[0]).toBe('https://daily-cloudcode-pa.googleapis.com/v1internal:generateContent');
expect(fetchMock.mock.calls[1]?.[0]).toBe('https://daily-cloudcode-pa.sandbox.googleapis.com/v1internal:generateContent');
});
});
src/server/routes/proxy/runtimeExecutor.ts
+346
View File
@@ -0,0 +1,346 @@
import { createHash, randomUUID } from 'node:crypto';
import { fetch, type RequestInit as UndiciRequestInit } from 'undici';
import type { BuiltEndpointRequest } from './endpointFlow.js';
import { buildUpstreamUrl } from './upstreamUrl.js';
const ANTIGRAVITY_RUNTIME_BASE_URLS = [
'https://daily-cloudcode-pa.googleapis.com',
'https://daily-cloudcode-pa.sandbox.googleapis.com',
] as const;
type RuntimeDispatchInput = {
siteUrl: string;
request: BuiltEndpointRequest;
targetUrl?: string;
buildInit: (requestUrl: string, request: BuiltEndpointRequest) => Promise<UndiciRequestInit> | UndiciRequestInit;
};
function asTrimmedString(value: unknown): string {
return typeof value === 'string' ? value.trim() : '';
}
function withRequestBody(
request: BuiltEndpointRequest,
body: Record<string, unknown>,
headers?: Record<string, string>,
): BuiltEndpointRequest {
return {
...request,
headers: headers ? { ...headers } : { ...request.headers },
body,
};
}
async function performFetch(
input: RuntimeDispatchInput,
request: BuiltEndpointRequest,
requestUrl = input.targetUrl || buildUpstreamUrl(input.siteUrl, request.path),
): Promise<Awaited<ReturnType<typeof fetch>>> {
const init = await input.buildInit(requestUrl, request);
return fetch(requestUrl, init);
}
async function materializeErrorResponse(
response: Awaited<ReturnType<typeof fetch>>,
): Promise<Awaited<ReturnType<typeof fetch>>> {
if (response.ok) return response;
const text = await response.text().catch(() => '');
return new Response(text, {
status: response.status,
headers: response.headers,
}) as unknown as Awaited<ReturnType<typeof fetch>>;
}
function parseGeminiCliPreviewFallbackOrder(modelName: string): string[] {
switch (modelName.trim()) {
case 'gemini-2.5-pro':
return [];
case 'gemini-2.5-flash':
return [];
case 'gemini-2.5-flash-lite':
return [];
default:
return [];
}
}
function replaceGeminiCliModelInUserAgent(userAgent: string | undefined, modelName: string): string | undefined {
const raw = asTrimmedString(userAgent);
if (!raw) return undefined;
return raw.replace(/^GeminiCLI\/([^/]+)\/[^ ]+ /i, `GeminiCLI/$1/${modelName} `);
}
function buildGeminiCliAttemptRequest(request: BuiltEndpointRequest, modelName: string): BuiltEndpointRequest {
const body = structuredClone(request.body);
const action = request.runtime?.action;
if (action === 'countTokens') {
delete body.model;
delete body.project;
} else {
body.model = modelName;
}
const headers = { ...request.headers };
const nextUserAgent = replaceGeminiCliModelInUserAgent(
headers['User-Agent'] || headers['user-agent'],
modelName,
);
if (nextUserAgent) {
headers['User-Agent'] = nextUserAgent;
delete headers['user-agent'];
}
return withRequestBody(request, body, headers);
}
function antigravityRequestType(modelName: string): 'image_gen' | 'agent' {
return modelName.includes('image') ? 'image_gen' : 'agent';
}
function generateAntigravityProjectId(): string {
const adjectives = ['useful', 'bright', 'swift', 'calm', 'bold'];
const nouns = ['signal', 'river', 'rocket', 'forest', 'bridge'];
const adjective = adjectives[Math.floor(Math.random() * adjectives.length)] || 'useful';
const noun = nouns[Math.floor(Math.random() * nouns.length)] || 'signal';
const suffix = Math.floor(100000 + Math.random() * 900000);
return `${adjective}-${noun}-${suffix}`;
}
function extractFirstUserText(value: unknown): string {
if (!Array.isArray(value)) return '';
for (const content of value) {
if (!content || typeof content !== 'object' || Array.isArray(content)) continue;
const record = content as Record<string, unknown>;
if (asTrimmedString(record.role) !== 'user') continue;
const parts = Array.isArray(record.parts) ? record.parts : [];
for (const part of parts) {
if (!part || typeof part !== 'object' || Array.isArray(part)) continue;
const text = asTrimmedString((part as Record<string, unknown>).text);
if (text) return text;
}
}
return '';
}
function generateStableAntigravitySessionId(payload: Record<string, unknown>): string {
const firstUserText = extractFirstUserText(
payload.request && typeof payload.request === 'object' && !Array.isArray(payload.request)
? (payload.request as Record<string, unknown>).contents
: undefined,
);
if (!firstUserText) {
return `-${BigInt(`0x${randomUUID().replace(/-/g, '').slice(0, 16)}`).toString()}`;
}
const digest = createHash('sha256').update(firstUserText).digest('hex').slice(0, 16);
const bigint = BigInt(`0x${digest}`) & BigInt('0x7fffffffffffffff');
return `-${bigint.toString()}`;
}
function renameParametersJsonSchema(value: unknown): unknown {
if (Array.isArray(value)) {
return value.map((item) => renameParametersJsonSchema(item));
}
if (!value || typeof value !== 'object') return value;
const input = value as Record<string, unknown>;
const output: Record<string, unknown> = {};
for (const [key, entry] of Object.entries(input)) {
const nextKey = key === 'parametersJsonSchema' ? 'parameters' : key;
output[nextKey] = renameParametersJsonSchema(entry);
}
return output;
}
function deleteNestedMaxOutputTokens(payload: Record<string, unknown>): void {
const request = payload.request;
if (!request || typeof request !== 'object' || Array.isArray(request)) return;
const generationConfig = (request as Record<string, unknown>).generationConfig;
if (!generationConfig || typeof generationConfig !== 'object' || Array.isArray(generationConfig)) return;
delete (generationConfig as Record<string, unknown>).maxOutputTokens;
}
function buildAntigravityRuntimeBody(
originalBody: Record<string, unknown>,
modelName: string,
action?: NonNullable<BuiltEndpointRequest['runtime']>['action'],
): Record<string, unknown> {
const payload = renameParametersJsonSchema(structuredClone(originalBody)) as Record<string, unknown>;
if (action === 'countTokens') {
return payload;
}
const requestType = antigravityRequestType(modelName);
const projectId = asTrimmedString(payload.project) || generateAntigravityProjectId();
payload.model = modelName;
payload.project = projectId;
payload.userAgent = 'antigravity';
payload.requestType = requestType;
payload.requestId = requestType === 'image_gen'
? `image_gen/${Date.now()}/${randomUUID()}/12`
: `agent-${randomUUID()}`;
const request = payload.request;
if (request && typeof request === 'object' && !Array.isArray(request)) {
delete (request as Record<string, unknown>).safetySettings;
if (requestType !== 'image_gen') {
(request as Record<string, unknown>).sessionId = generateStableAntigravitySessionId(payload);
}
if (modelName.includes('claude')) {
const toolConfig = (
(request as Record<string, unknown>).toolConfig
&& typeof (request as Record<string, unknown>).toolConfig === 'object'
&& !Array.isArray((request as Record<string, unknown>).toolConfig)
)
? (request as Record<string, unknown>).toolConfig as Record<string, unknown>
: (((request as Record<string, unknown>).toolConfig = {}) as Record<string, unknown>);
toolConfig.functionCallingConfig = { mode: 'VALIDATED' };
} else {
deleteNestedMaxOutputTokens(payload);
}
}
return payload;
}
function antigravityShouldRetryNoCapacity(
status: number,
responseText: string,
): boolean {
if (status !== 503) return false;
return responseText.toLowerCase().includes('no capacity available');
}
function antigravityNoCapacityRetryDelay(attempt: number): number {
const delay = Math.min((attempt + 1) * 250, 2000);
return delay;
}
async function waitMs(ms: number): Promise<void> {
if (ms <= 0) return;
await new Promise((resolve) => setTimeout(resolve, ms));
}
async function dispatchGeminiCliRuntimeRequest(
input: RuntimeDispatchInput,
): Promise<Awaited<ReturnType<typeof fetch>>> {
const baseModel = asTrimmedString(input.request.runtime?.modelName) || asTrimmedString(input.request.body.model);
const models = [baseModel, ...parseGeminiCliPreviewFallbackOrder(baseModel)].filter(Boolean);
let lastResponse: Awaited<ReturnType<typeof fetch>> | null = null;
for (const modelName of models.length > 0 ? models : [baseModel || 'unknown']) {
const attemptRequest = buildGeminiCliAttemptRequest(input.request, modelName);
const response = await performFetch(input, attemptRequest);
if (response.ok) return response;
if (response.status === 429) {
lastResponse = await materializeErrorResponse(response);
continue;
}
return materializeErrorResponse(response);
}
return lastResponse || performFetch(input, input.request);
}
async function dispatchCodexRuntimeRequest(
input: RuntimeDispatchInput,
): Promise<Awaited<ReturnType<typeof fetch>>> {
return performFetch(input, input.request);
}
async function dispatchClaudeRuntimeRequest(
input: RuntimeDispatchInput,
): Promise<Awaited<ReturnType<typeof fetch>>> {
return performFetch(input, input.request);
}
async function dispatchAntigravityRuntimeRequest(
input: RuntimeDispatchInput,
): Promise<Awaited<ReturnType<typeof fetch>>> {
const modelName = asTrimmedString(input.request.runtime?.modelName) || asTrimmedString(input.request.body.model);
const runtimeBody = buildAntigravityRuntimeBody(
input.request.body,
modelName,
input.request.runtime?.action,
);
const baseAttempts = 3;
let lastResponse: Awaited<ReturnType<typeof fetch>> | null = null;
attemptLoop:
for (let attempt = 0; attempt < baseAttempts; attempt += 1) {
for (const baseUrl of ANTIGRAVITY_RUNTIME_BASE_URLS) {
const requestUrl = `${baseUrl}${input.request.path}`;
const minimalHeaders: Record<string, string> = {
Authorization: input.request.headers.Authorization || input.request.headers.authorization || '',
'Content-Type': 'application/json',
Accept: input.request.runtime?.stream ? 'text/event-stream' : 'application/json',
'User-Agent': 'antigravity/1.19.6 darwin/arm64',
};
let response: Awaited<ReturnType<typeof fetch>>;
try {
response = await performFetch(
input,
withRequestBody(input.request, runtimeBody, minimalHeaders),
requestUrl,
);
} catch (error) {
if (baseUrl !== ANTIGRAVITY_RUNTIME_BASE_URLS[ANTIGRAVITY_RUNTIME_BASE_URLS.length - 1]) {
continue;
}
if (attempt + 1 < baseAttempts) {
continue attemptLoop;
}
throw error;
}
if (response.ok) return response;
const errorResponse = await materializeErrorResponse(response);
const errorText = await errorResponse.text().catch(() => '');
lastResponse = new Response(errorText, {
status: errorResponse.status,
headers: errorResponse.headers,
}) as unknown as Awaited<ReturnType<typeof fetch>>;
if (errorResponse.status === 429) {
continue;
}
if (antigravityShouldRetryNoCapacity(errorResponse.status, errorText)) {
if (baseUrl !== ANTIGRAVITY_RUNTIME_BASE_URLS[ANTIGRAVITY_RUNTIME_BASE_URLS.length - 1]) {
continue;
}
if (attempt + 1 < baseAttempts) {
await waitMs(antigravityNoCapacityRetryDelay(attempt));
continue attemptLoop;
}
}
return lastResponse;
}
}
return lastResponse || performFetch(input, withRequestBody(input.request, runtimeBody, {
Authorization: input.request.headers.Authorization || input.request.headers.authorization || '',
'Content-Type': 'application/json',
Accept: input.request.runtime?.stream ? 'text/event-stream' : 'application/json',
'User-Agent': 'antigravity/1.19.6 darwin/arm64',
}));
}
export async function dispatchRuntimeRequest(
input: RuntimeDispatchInput,
): Promise<Awaited<ReturnType<typeof fetch>>> {
const executor = input.request.runtime?.executor || 'default';
if (executor === 'codex') {
return dispatchCodexRuntimeRequest(input);
}
if (executor === 'claude') {
return dispatchClaudeRuntimeRequest(input);
}
if (executor === 'gemini-cli') {
return dispatchGeminiCliRuntimeRequest(input);
}
if (executor === 'antigravity') {
return dispatchAntigravityRuntimeRequest(input);
}
return performFetch(input, input.request);
}
src/server/routes/proxy/search.ts
+10 -4
View File
@@ -1,6 +1,6 @@
import { FastifyInstance, FastifyReply, FastifyRequest } from 'fastify';
import { fetch } from 'undici';
import { db, schema } from '../../db/index.js';
import { db, hasProxyLogDownstreamApiKeyIdColumn, schema } from '../../db/index.js';
import { tokenRouter } from '../../services/tokenRouter.js';
import { refreshModelsAndRebuildRoutes } from '../../services/modelService.js';
import { reportProxyAllFailed, reportTokenExpired } from '../../services/alertService.js';
@@ -11,6 +11,7 @@ import { withSiteRecordProxyRequestInit } from '../../services/siteProxy.js';
import { getProxyUrlFromExtraConfig } from '../../services/accountExtraConfig.js';
import { composeProxyLogMessage } from './logPathMeta.js';
import { formatUtcSqlDateTime } from '../../services/localTimeService.js';
import { getProxyAuthContext } from '../../middleware/auth.js';
import { buildUpstreamUrl } from './upstreamUrl.js';
const MAX_RETRIES = 2;
@@ -53,6 +54,9 @@ export async function searchProxyRoute(app: FastifyInstance) {
if (!await ensureModelAllowedForDownstreamKey(request, reply, requestedModel)) return;
const downstreamPolicy = getDownstreamRoutingPolicy(request);
const downstreamApiKeyId = getProxyAuthContext(request)?.keyId ?? null;
const logDownstreamApiKeyId = downstreamApiKeyId !== null
&& await hasProxyLogDownstreamApiKeyIdColumn();
const excludeChannelIds: number[] = [];
let retryCount = 0;
@@ -98,7 +102,7 @@ export async function searchProxyRoute(app: FastifyInstance) {
const text = await upstream.text();
if (!upstream.ok) {
tokenRouter.recordFailure(selected.channel.id);
logProxy(selected, requestedModel, 'failed', upstream.status, Date.now() - startTime, text, retryCount);
logProxy(selected, requestedModel, 'failed', upstream.status, Date.now() - startTime, text, retryCount, logDownstreamApiKeyId ? downstreamApiKeyId : null);
if (isTokenExpiredError({ status: upstream.status, message: text })) {
await reportTokenExpired({
accountId: selected.account.id,
@@ -124,11 +128,11 @@ export async function searchProxyRoute(app: FastifyInstance) {
const latency = Date.now() - startTime;
tokenRouter.recordSuccess(selected.channel.id, latency, 0);
recordDownstreamCostUsage(request, 0);
logProxy(selected, requestedModel, 'success', upstream.status, latency, null, retryCount);
logProxy(selected, requestedModel, 'success', upstream.status, latency, null, retryCount, logDownstreamApiKeyId ? downstreamApiKeyId : null);
return reply.code(upstream.status).send(data);
} catch (error: any) {
tokenRouter.recordFailure(selected.channel.id);
logProxy(selected, requestedModel, 'failed', 0, Date.now() - startTime, error?.message || 'network error', retryCount);
logProxy(selected, requestedModel, 'failed', 0, Date.now() - startTime, error?.message || 'network error', retryCount, logDownstreamApiKeyId ? downstreamApiKeyId : null);
if (retryCount < MAX_RETRIES) {
retryCount += 1;
continue;
@@ -153,6 +157,7 @@ async function logProxy(
latencyMs: number,
errorMessage: string | null,
retryCount: number,
downstreamApiKeyId: number | null = null,
) {
try {
const createdAt = formatUtcSqlDateTime(new Date());
@@ -160,6 +165,7 @@ async function logProxy(
routeId: selected.channel.routeId,
channelId: selected.channel.id,
accountId: selected.account.id,
...(downstreamApiKeyId !== null ? { downstreamApiKeyId } : {}),
modelRequested,
modelActual: selected.actualModel,
status,
src/server/routes/proxy/upstreamEndpoint.test.ts
+343
View File
@@ -151,6 +151,16 @@ describe('resolveUpstreamEndpointCandidates', () => {
'claude',
);
expect(claudeOrder).toEqual(['messages']);
const codexOrder = await resolveUpstreamEndpointCandidates(
{
...baseContext,
site: { ...baseContext.site, platform: 'codex', url: 'https://chatgpt.com/backend-api/codex' },
},
'gpt-5.2-codex',
'openai',
);
expect(codexOrder).toEqual(['responses']);
});
it('prefers document-capable endpoints when downstream content contains non-image files', async () => {
@@ -319,6 +329,339 @@ describe('buildUpstreamEndpointRequest', () => {
]);
});
it('builds codex responses requests against backend-api path and preserves oauth provider headers', () => {
const request = buildUpstreamEndpointRequest({
endpoint: 'responses',
modelName: 'gpt-5.2-codex',
stream: false,
tokenValue: 'oauth-access-token',
sitePlatform: 'codex',
siteUrl: 'https://chatgpt.com/backend-api/codex',
openaiBody: {
model: 'gpt-5.2-codex',
messages: [{ role: 'user', content: 'hello codex' }],
temperature: 0.2,
top_p: 0.9,
user: 'drop-me',
service_tier: 'auto',
},
downstreamFormat: 'openai',
providerHeaders: {
Originator: 'codex_cli_rs',
'Chatgpt-Account-Id': 'chatgpt-account-123',
},
codexSessionCacheKey: 'gpt-5.2-codex:proxy:test-key',
} as any);
expect(request.path).toBe('/responses');
expect(request.headers.Authorization).toBe('Bearer oauth-access-token');
expect(request.headers.Originator).toBe('codex_cli_rs');
expect(request.headers['Chatgpt-Account-Id']).toBe('chatgpt-account-123');
expect(request.headers.Version).toBe('0.101.0');
expect(request.headers.Session_id).toMatch(/^[0-9a-f-]{36}$/i);
expect(request.headers.Conversation_id).toBe(request.headers.Session_id);
expect(request.headers['User-Agent']).toBe('codex_cli_rs/0.101.0 (Mac OS 26.0.1; arm64) Apple_Terminal/464');
expect(request.headers.Accept).toBe('text/event-stream');
expect(request.headers.Connection).toBe('Keep-Alive');
expect(request.body.instructions).toBe('');
expect(request.body.prompt_cache_key).toBe(request.headers.Session_id);
expect(request.body.stream).toBe(true);
expect(request.body.store).toBe(false);
expect(request.body.parallel_tool_calls).toBe(true);
expect(request.body.include).toEqual(['reasoning.encrypted_content']);
expect(request.body.max_output_tokens).toBeUndefined();
expect(request.body.temperature).toBeUndefined();
expect(request.body.top_p).toBeUndefined();
expect(request.body.user).toBeUndefined();
expect(request.body.service_tier).toBeUndefined();
});
it('reuses a stable codex session id when the same downstream continuity key is provided', () => {
const firstRequest = buildUpstreamEndpointRequest({
endpoint: 'responses',
modelName: 'gpt-5.4',
stream: false,
tokenValue: 'oauth-access-token',
sitePlatform: 'codex',
siteUrl: 'https://chatgpt.com/backend-api/codex',
openaiBody: {
model: 'gpt-5.4',
messages: [{ role: 'user', content: 'hello codex' }],
},
downstreamFormat: 'openai',
providerHeaders: {
Originator: 'codex_cli_rs',
},
codexSessionCacheKey: 'gpt-5.4:user-123',
} as any);
const secondRequest = buildUpstreamEndpointRequest({
endpoint: 'responses',
modelName: 'gpt-5.4',
stream: false,
tokenValue: 'oauth-access-token',
sitePlatform: 'codex',
siteUrl: 'https://chatgpt.com/backend-api/codex',
openaiBody: {
model: 'gpt-5.4',
messages: [{ role: 'user', content: 'hello again codex' }],
},
downstreamFormat: 'openai',
providerHeaders: {
Originator: 'codex_cli_rs',
},
codexSessionCacheKey: 'gpt-5.4:user-123',
} as any);
expect(firstRequest.headers.Session_id).toBe(secondRequest.headers.Session_id);
expect(firstRequest.headers.Conversation_id).toBe(secondRequest.headers.Conversation_id);
expect(firstRequest.body.prompt_cache_key).toBe(secondRequest.body.prompt_cache_key);
});
it('does not synthesize prompt_cache_key or conversation_id for native codex responses requests without one', () => {
const request = buildUpstreamEndpointRequest({
endpoint: 'responses',
modelName: 'gpt-5.4',
stream: false,
tokenValue: 'oauth-access-token',
sitePlatform: 'codex',
siteUrl: 'https://chatgpt.com/backend-api/codex',
openaiBody: {},
downstreamFormat: 'responses',
responsesOriginalBody: {
model: 'gpt-5.4',
input: 'hello codex',
},
providerHeaders: {
Originator: 'codex_cli_rs',
},
} as any);
expect(request.headers.Session_id).toMatch(/^[0-9a-f-]{36}$/i);
expect(request.headers.Conversation_id).toBeUndefined();
expect(request.body.prompt_cache_key).toBeUndefined();
});
it('preserves explicit prompt_cache_key for native codex responses requests', () => {
const request = buildUpstreamEndpointRequest({
endpoint: 'responses',
modelName: 'gpt-5.4',
stream: false,
tokenValue: 'oauth-access-token',
sitePlatform: 'codex',
siteUrl: 'https://chatgpt.com/backend-api/codex',
openaiBody: {},
downstreamFormat: 'responses',
responsesOriginalBody: {
model: 'gpt-5.4',
prompt_cache_key: 'codex-cache-123',
input: 'hello codex',
},
providerHeaders: {
Originator: 'codex_cli_rs',
},
codexExplicitSessionId: 'codex-cache-123',
} as any);
expect(request.headers.Session_id).toBe('codex-cache-123');
expect(request.headers.Conversation_id).toBe('codex-cache-123');
expect(request.body.prompt_cache_key).toBe('codex-cache-123');
});
it('builds gemini-cli native requests with project envelope and bearer headers', () => {
const request = buildUpstreamEndpointRequest({
endpoint: 'responses',
modelName: 'gemini-2.5-pro',
stream: true,
tokenValue: 'oauth-access-token',
oauthProvider: 'gemini-cli',
oauthProjectId: 'project-demo',
sitePlatform: 'gemini-cli',
siteUrl: 'https://cloudcode-pa.googleapis.com',
openaiBody: {
model: 'gemini-2.5-pro',
messages: [
{ role: 'system', content: 'be concise' },
{ role: 'user', content: 'hello gemini cli' },
],
temperature: 0.4,
},
downstreamFormat: 'openai',
providerHeaders: {
'User-Agent': 'GeminiCLI/0.31.0/unknown (win32; x64)',
'X-Goog-Api-Client': 'google-genai-sdk/1.41.0 gl-node/v22.19.0',
},
});
expect(request.path).toBe('/v1internal:streamGenerateContent?alt=sse');
expect(request.headers.Authorization).toBe('Bearer oauth-access-token');
expect(request.headers['User-Agent']).toBe('GeminiCLI/0.31.0/gemini-2.5-pro (win32; x64)');
expect(request.headers['X-Goog-Api-Client']).toContain('google-genai-sdk/');
expect(request.body.project).toBe('project-demo');
expect(request.body.model).toBe('gemini-2.5-pro');
expect(request.body.request).toMatchObject({
generationConfig: {
temperature: 0.4,
},
systemInstruction: {
role: 'user',
},
contents: [
{
role: 'user',
parts: [{ text: 'hello gemini cli' }],
},
],
});
});
it('builds antigravity native requests with the same internal Gemini envelope', () => {
const request = buildUpstreamEndpointRequest({
endpoint: 'chat',
modelName: 'gemini-3-pro-preview',
stream: false,
tokenValue: 'oauth-access-token',
oauthProvider: 'antigravity',
oauthProjectId: 'project-demo',
sitePlatform: 'antigravity',
siteUrl: 'https://cloudcode-pa.googleapis.com',
openaiBody: {
model: 'gemini-3-pro-preview',
messages: [
{ role: 'system', content: 'be concise' },
{ role: 'user', content: 'hello antigravity' },
],
},
downstreamFormat: 'openai',
providerHeaders: {
'User-Agent': 'google-api-nodejs-client/9.15.1',
'X-Goog-Api-Client': 'google-cloud-sdk vscode_cloudshelleditor/0.1',
},
});
expect(request.path).toBe('/v1internal:generateContent');
expect(request.headers.Authorization).toBe('Bearer oauth-access-token');
expect(request.headers['User-Agent']).toBe('antigravity/1.19.6 darwin/arm64');
expect(request.headers['X-Goog-Api-Client']).toBeUndefined();
expect(request.headers['Client-Metadata']).toBeUndefined();
expect(request.body).toEqual({
project: 'project-demo',
model: 'gemini-3-pro-preview',
request: {
systemInstruction: {
role: 'user',
parts: [{ text: 'be concise' }],
},
contents: [
{
role: 'user',
parts: [{ text: 'hello antigravity' }],
},
],
},
});
});
it('uses claude-code runtime headers for claude oauth upstream requests', () => {
const request = buildUpstreamEndpointRequest({
endpoint: 'messages',
modelName: 'claude-opus-4-6',
stream: true,
tokenValue: 'oauth-access-token',
oauthProvider: 'claude',
sitePlatform: 'claude',
siteUrl: 'https://api.anthropic.com',
openaiBody: {
model: 'claude-opus-4-6',
messages: [{ role: 'user', content: 'hello claude oauth' }],
},
downstreamFormat: 'openai',
});
expect(request.path).toBe('/v1/messages');
expect(request.headers.Authorization).toBe('Bearer oauth-access-token');
expect(request.headers['x-api-key']).toBeUndefined();
expect(request.headers['anthropic-version']).toBe('2023-06-01');
expect(request.headers['Anthropic-Dangerous-Direct-Browser-Access']).toBe('true');
expect(request.headers['X-App']).toBe('cli');
expect(request.headers['X-Stainless-Retry-Count']).toBe('0');
expect(request.headers['X-Stainless-Runtime-Version']).toBe('v24.3.0');
expect(request.headers['X-Stainless-Package-Version']).toBe('0.74.0');
expect(request.headers['X-Stainless-Runtime']).toBe('node');
expect(request.headers['X-Stainless-Lang']).toBe('js');
expect(request.headers['X-Stainless-Arch']).toBe('x64');
expect(request.headers['X-Stainless-Os']).toBe('Windows');
expect(request.headers['X-Stainless-Timeout']).toBe('600');
expect(request.headers['User-Agent']).toBe('claude-cli/2.1.63 (external, cli)');
expect(request.headers.Connection).toBe('keep-alive');
expect(request.headers.Accept).toBe('text/event-stream');
expect(request.headers['Accept-Encoding']).toBe('identity');
});
it('uses claude-code beta headers and compressed non-stream responses for claude upstream requests', () => {
const request = buildUpstreamEndpointRequest({
endpoint: 'messages',
modelName: 'claude-opus-4-6',
stream: false,
tokenValue: 'oauth-access-token',
oauthProvider: 'claude',
sitePlatform: 'claude',
siteUrl: 'https://api.anthropic.com',
openaiBody: {
model: 'claude-opus-4-6',
messages: [{ role: 'user', content: 'hello claude oauth' }],
},
downstreamFormat: 'openai',
});
expect(request.headers['anthropic-beta']).toContain('claude-code-20250219');
expect(request.headers['anthropic-beta']).toContain('oauth-2025-04-20');
expect(request.headers['anthropic-beta']).toContain('context-management-2025-06-27');
expect(request.headers.Accept).toBe('application/json');
expect(request.headers['Accept-Encoding']).toBe('gzip, deflate, br, zstd');
});
it('converts system roles to developer in native codex responses bodies', () => {
const request = buildUpstreamEndpointRequest({
endpoint: 'responses',
modelName: 'gpt-5.4',
stream: false,
tokenValue: 'oauth-access-token',
sitePlatform: 'codex',
siteUrl: 'https://chatgpt.com/backend-api/codex',
openaiBody: {},
downstreamFormat: 'responses',
responsesOriginalBody: {
model: 'gpt-5.4',
input: [
{
type: 'message',
role: 'system',
content: [{ type: 'input_text', text: 'be careful' }],
},
{
type: 'message',
role: 'user',
content: [{ type: 'input_text', text: 'hello' }],
},
],
},
});
expect(request.body.input).toEqual([
{
type: 'message',
role: 'developer',
content: [{ type: 'input_text', text: 'be careful' }],
},
{
type: 'message',
role: 'user',
content: [{ type: 'input_text', text: 'hello' }],
},
]);
});
it('normalizes downstream responses input string before forwarding upstream', () => {
const request = buildUpstreamEndpointRequest({
endpoint: 'responses',
src/server/routes/proxy/upstreamEndpoint.ts
+538 -13
View File
@@ -1,3 +1,4 @@
import { createHash, randomUUID } from 'node:crypto';
import { fetchModelPricingCatalog } from '../../services/modelPricingService.js';
import type { DownstreamFormat } from '../../transformers/shared/normalized.js';
import {
@@ -8,6 +9,10 @@ import {
convertOpenAiBodyToAnthropicMessagesBody,
sanitizeAnthropicMessagesBody,
} from '../../transformers/anthropic/messages/conversion.js';
import {
buildGeminiGenerateContentRequestFromOpenAi,
wrapGeminiCliRequest,
} from './geminiCliCompat.js';
export {
buildMinimalJsonHeadersForCompatibility,
isEndpointDispatchDeniedError,
@@ -94,6 +99,12 @@ const BLOCKED_PASSTHROUGH_HEADERS = new Set([
'sec-websocket-extensions',
]);
const CODEX_CLIENT_VERSION = '0.101.0';
const CODEX_DEFAULT_USER_AGENT = 'codex_cli_rs/0.101.0 (Mac OS 26.0.1; arm64) Apple_Terminal/464';
const ANTIGRAVITY_RUNTIME_USER_AGENT = 'antigravity/1.19.6 darwin/arm64';
const CLAUDE_DEFAULT_USER_AGENT = 'claude-cli/2.1.63 (external, cli)';
const CLAUDE_DEFAULT_BETA_HEADER = 'claude-code-20250219,oauth-2025-04-20,interleaved-thinking-2025-05-14,context-management-2025-06-27,prompt-caching-scope-2026-01-05';
function shouldSkipPassthroughHeader(key: string): boolean {
return HOP_BY_HOP_HEADERS.has(key) || BLOCKED_PASSTHROUGH_HEADERS.has(key);
}
@@ -164,6 +175,239 @@ function extractResponsesPassthroughHeaders(
return forwarded;
}
function getInputHeader(
headers: Record<string, unknown> | Record<string, string> | undefined,
key: string,
): string | null {
if (!headers) return null;
for (const [candidateKey, candidateValue] of Object.entries(headers)) {
if (candidateKey.toLowerCase() !== key.toLowerCase()) continue;
return headerValueToString(candidateValue);
}
return null;
}
function parseGeminiCliUserAgentRuntime(userAgent: string | null): {
version: string;
platform: string;
arch: string;
} | null {
if (!userAgent) return null;
const match = /^GeminiCLI\/([^/]+)\/[^ ]+ \(([^;]+); ([^)]+)\)$/i.exec(userAgent.trim());
if (!match) return null;
return {
version: match[1] || '0.31.0',
platform: match[2] || 'win32',
arch: match[3] || 'x64',
};
}
function buildGeminiCLIUserAgent(modelName: string, existingUserAgent?: string | null): string {
const parsed = parseGeminiCliUserAgentRuntime(existingUserAgent ?? null);
const version = parsed?.version || '0.31.0';
const platform = parsed?.platform || 'win32';
const arch = parsed?.arch || 'x64';
const effectiveModel = asTrimmedString(modelName) || 'unknown';
return `GeminiCLI/${version}/${effectiveModel} (${platform}; ${arch})`;
}
function uuidFromSeed(seed: string): string {
const hash = createHash('sha1').update(seed).digest();
const bytes = new Uint8Array(hash.subarray(0, 16));
bytes[6] = (bytes[6]! & 0x0f) | 0x50;
bytes[8] = (bytes[8]! & 0x3f) | 0x80;
const hex = Array.from(bytes, (value) => value.toString(16).padStart(2, '0')).join('');
return [
hex.slice(0, 8),
hex.slice(8, 12),
hex.slice(12, 16),
hex.slice(16, 20),
hex.slice(20, 32),
].join('-');
}
function mergeClaudeBetaHeader(
explicitValue: string | null,
extraBetas: string[] = [],
): string {
const source = explicitValue || CLAUDE_DEFAULT_BETA_HEADER;
const seen = new Set<string>();
const merged: string[] = [];
for (const entry of source.split(',')) {
const normalized = entry.trim();
if (!normalized || seen.has(normalized)) continue;
seen.add(normalized);
merged.push(normalized);
}
if (!explicitValue) {
for (const entry of extraBetas) {
const normalized = entry.trim();
if (!normalized || seen.has(normalized)) continue;
seen.add(normalized);
merged.push(normalized);
}
}
return merged.join(',');
}
function extractClaudeBetasFromBody(body: Record<string, unknown>): {
body: Record<string, unknown>;
betas: string[];
} {
const next = { ...body };
const rawBetas = next.betas;
delete next.betas;
if (typeof rawBetas === 'string') {
return {
body: next,
betas: rawBetas.split(',').map((entry) => entry.trim()).filter(Boolean),
};
}
if (Array.isArray(rawBetas)) {
return {
body: next,
betas: rawBetas
.map((entry) => asTrimmedString(entry))
.filter(Boolean),
};
}
return {
body: next,
betas: [],
};
}
function buildCodexRuntimeHeaders(input: {
baseHeaders: Record<string, string>;
providerHeaders?: Record<string, string>;
explicitSessionId?: string | null;
continuityKey?: string | null;
}): Record<string, string> {
const authorization = (
getInputHeader(input.baseHeaders, 'authorization')
|| getInputHeader(input.baseHeaders, 'Authorization')
|| ''
);
const originator = getInputHeader(input.providerHeaders, 'originator') || 'codex_cli_rs';
const accountId = getInputHeader(input.providerHeaders, 'chatgpt-account-id');
const version = getInputHeader(input.baseHeaders, 'version') || CODEX_CLIENT_VERSION;
const userAgent = getInputHeader(input.baseHeaders, 'user-agent') || CODEX_DEFAULT_USER_AGENT;
const explicitSessionId = asTrimmedString(input.explicitSessionId);
const continuityKey = asTrimmedString(input.continuityKey);
const sessionId = (
getInputHeader(input.baseHeaders, 'session_id')
|| getInputHeader(input.baseHeaders, 'session-id')
|| explicitSessionId
|| (continuityKey ? uuidFromSeed(`metapi:codex:${continuityKey}`) : null)
|| randomUUID()
);
const conversationId = (
getInputHeader(input.baseHeaders, 'conversation_id')
|| getInputHeader(input.baseHeaders, 'conversation-id')
|| explicitSessionId
|| (continuityKey ? sessionId : null)
);
return {
Authorization: authorization,
'Content-Type': 'application/json',
...(accountId ? { 'Chatgpt-Account-Id': accountId } : {}),
Originator: originator,
Version: version,
Session_id: sessionId,
...(conversationId ? { Conversation_id: conversationId } : {}),
'User-Agent': userAgent,
Accept: 'text/event-stream',
Connection: 'Keep-Alive',
};
}
function buildGeminiCliRuntimeHeaders(input: {
baseHeaders: Record<string, string>;
providerHeaders?: Record<string, string>;
modelName: string;
stream: boolean;
}): Record<string, string> {
const apiClient = (
getInputHeader(input.providerHeaders, 'x-goog-api-client')
|| getInputHeader(input.baseHeaders, 'x-goog-api-client')
);
const userAgent = buildGeminiCLIUserAgent(
input.modelName,
getInputHeader(input.providerHeaders, 'user-agent') || getInputHeader(input.baseHeaders, 'user-agent'),
);
const headers: Record<string, string> = {
Authorization: input.baseHeaders.Authorization,
'Content-Type': 'application/json',
'User-Agent': userAgent,
};
if (apiClient) {
headers['X-Goog-Api-Client'] = apiClient;
}
if (input.stream) {
headers.Accept = 'text/event-stream';
}
return headers;
}
function buildAntigravityRuntimeHeaders(input: {
baseHeaders: Record<string, string>;
stream: boolean;
}): Record<string, string> {
const headers: Record<string, string> = {
Authorization: input.baseHeaders.Authorization,
'Content-Type': 'application/json',
Accept: input.stream ? 'text/event-stream' : 'application/json',
'User-Agent': ANTIGRAVITY_RUNTIME_USER_AGENT,
};
return headers;
}
function buildClaudeRuntimeHeaders(input: {
baseHeaders: Record<string, string>;
claudeHeaders: Record<string, string>;
anthropicVersion: string;
stream: boolean;
isClaudeOauthUpstream: boolean;
tokenValue: string;
extraBetas?: string[];
}): Record<string, string> {
const anthropicBeta = mergeClaudeBetaHeader(
getInputHeader(input.claudeHeaders, 'anthropic-beta'),
input.extraBetas,
);
const headers: Record<string, string> = {
...input.baseHeaders,
...input.claudeHeaders,
'anthropic-version': input.anthropicVersion,
...(anthropicBeta ? { 'anthropic-beta': anthropicBeta } : {}),
'Anthropic-Dangerous-Direct-Browser-Access': 'true',
'X-App': 'cli',
'X-Stainless-Retry-Count': getInputHeader(input.claudeHeaders, 'x-stainless-retry-count') || '0',
'X-Stainless-Runtime-Version': getInputHeader(input.claudeHeaders, 'x-stainless-runtime-version') || 'v24.3.0',
'X-Stainless-Package-Version': getInputHeader(input.claudeHeaders, 'x-stainless-package-version') || '0.74.0',
'X-Stainless-Runtime': getInputHeader(input.claudeHeaders, 'x-stainless-runtime') || 'node',
'X-Stainless-Lang': getInputHeader(input.claudeHeaders, 'x-stainless-lang') || 'js',
'X-Stainless-Arch': getInputHeader(input.claudeHeaders, 'x-stainless-arch') || 'x64',
'X-Stainless-Os': getInputHeader(input.claudeHeaders, 'x-stainless-os') || 'Windows',
'X-Stainless-Timeout': getInputHeader(input.claudeHeaders, 'x-stainless-timeout') || '600',
'User-Agent': getInputHeader(input.claudeHeaders, 'user-agent') || CLAUDE_DEFAULT_USER_AGENT,
Connection: 'keep-alive',
Accept: input.stream ? 'text/event-stream' : 'application/json',
'Accept-Encoding': input.stream ? 'identity' : 'gzip, deflate, br, zstd',
};
if (input.isClaudeOauthUpstream) {
headers.Authorization = `Bearer ${input.tokenValue}`;
} else {
headers['x-api-key'] = input.tokenValue;
}
return headers;
}
function ensureStreamAcceptHeader(
headers: Record<string, string>,
stream: boolean,
@@ -186,6 +430,89 @@ function toFiniteNumber(value: unknown): number | null {
return typeof value === 'number' && Number.isFinite(value) ? value : null;
}
function ensureCodexResponsesInstructions(
body: Record<string, unknown>,
sitePlatform: string,
): Record<string, unknown> {
if (sitePlatform !== 'codex') return body;
if (typeof body.instructions === 'string') return body;
return {
...body,
instructions: '',
};
}
function ensureCodexResponsesStoreFalse(
body: Record<string, unknown>,
sitePlatform: string,
): Record<string, unknown> {
if (sitePlatform !== 'codex') return body;
if (body.store === false) return body;
return {
...body,
store: false,
};
}
function convertCodexSystemRoleToDeveloper(input: unknown): unknown {
if (!Array.isArray(input)) return input;
return input.map((item) => {
if (!isRecord(item)) return item;
if (asTrimmedString(item.type).toLowerCase() !== 'message') return item;
if (asTrimmedString(item.role).toLowerCase() !== 'system') return item;
return {
...item,
role: 'developer',
};
});
}
function applyCodexResponsesCompatibility(
body: Record<string, unknown>,
sitePlatform: string,
options?: {
preservePreviousResponseId?: boolean;
},
): Record<string, unknown> {
if (sitePlatform !== 'codex') return body;
const next: Record<string, unknown> = {
...body,
stream: true,
store: false,
parallel_tool_calls: true,
include: ['reasoning.encrypted_content'],
input: convertCodexSystemRoleToDeveloper(body.input),
};
if (typeof next.instructions !== 'string') {
next.instructions = '';
}
for (const key of [
'max_output_tokens',
'max_completion_tokens',
'temperature',
'top_p',
'truncation',
'user',
'context_management',
'prompt_cache_retention',
'safety_identifier',
]) {
delete next[key];
}
if (!options?.preservePreviousResponseId) {
delete next.previous_response_id;
}
if (asTrimmedString(next.service_tier).toLowerCase() !== 'priority') {
delete next.service_tier;
}
return next;
}
function normalizeEndpointTypes(value: unknown): UpstreamEndpoint[] {
const raw = asTrimmedString(value).toLowerCase();
@@ -237,11 +564,23 @@ function preferredEndpointOrder(
): UpstreamEndpoint[] {
const platform = normalizePlatformName(sitePlatform);
if (platform === 'codex') {
return ['responses'];
}
if (platform === 'gemini') {
// Gemini upstream is routed through OpenAI-compatible chat endpoint.
return ['chat'];
}
if (platform === 'gemini-cli') {
return ['chat'];
}
if (platform === 'antigravity') {
return ['chat'];
}
if (platform === 'openai') {
if (preferMessagesForClaudeModel && downstreamFormat !== 'responses') {
// Some OpenAI-compatible gateways expose Claude natively via /v1/messages.
@@ -319,6 +658,7 @@ export async function resolveUpstreamEndpointCandidates(
? (() => {
if (sitePlatform === 'claude') return ['messages'] as UpstreamEndpoint[];
if (sitePlatform === 'gemini') return ['responses', 'chat'] as UpstreamEndpoint[];
if (sitePlatform === 'gemini-cli' || sitePlatform === 'antigravity') return ['chat'] as UpstreamEndpoint[];
if (preferMessagesForClaudeModel) return ['messages', 'responses', 'chat'] as UpstreamEndpoint[];
return ['responses', 'messages', 'chat'] as UpstreamEndpoint[];
})()
@@ -338,6 +678,8 @@ export async function resolveUpstreamEndpointCandidates(
&& preferMessagesForClaudeModel
&& sitePlatform !== 'openai'
&& sitePlatform !== 'gemini'
&& sitePlatform !== 'antigravity'
&& sitePlatform !== 'gemini-cli'
);
try {
@@ -422,6 +764,8 @@ export function buildUpstreamEndpointRequest(input: {
modelName: string;
stream: boolean;
tokenValue: string;
oauthProvider?: string;
oauthProjectId?: string;
sitePlatform?: string;
siteUrl?: string;
openaiBody: Record<string, unknown>;
@@ -430,14 +774,28 @@ export function buildUpstreamEndpointRequest(input: {
forceNormalizeClaudeBody?: boolean;
responsesOriginalBody?: Record<string, unknown>;
downstreamHeaders?: Record<string, unknown>;
providerHeaders?: Record<string, string>;
codexSessionCacheKey?: string | null;
codexExplicitSessionId?: string | null;
}): {
path: string;
headers: Record<string, string>;
body: Record<string, unknown>;
runtime?: {
executor: 'default' | 'codex' | 'gemini-cli' | 'antigravity' | 'claude';
modelName?: string;
stream?: boolean;
oauthProjectId?: string | null;
action?: 'generateContent' | 'streamGenerateContent' | 'countTokens';
};
} {
const sitePlatform = normalizePlatformName(input.sitePlatform);
const isClaudeUpstream = sitePlatform === 'claude';
const isGeminiUpstream = sitePlatform === 'gemini';
const isGeminiCliUpstream = sitePlatform === 'gemini-cli';
const isAntigravityUpstream = sitePlatform === 'antigravity';
const isInternalGeminiUpstream = isGeminiCliUpstream || isAntigravityUpstream;
const isClaudeOauthUpstream = isClaudeUpstream && input.oauthProvider === 'claude';
const resolveGeminiEndpointPath = (endpoint: UpstreamEndpoint): string => {
const normalizedSiteUrl = asTrimmedString(input.siteUrl).toLowerCase();
@@ -463,6 +821,16 @@ export function buildUpstreamEndpointRequest(input: {
return '/v1/chat/completions';
}
if (sitePlatform === 'codex') {
return '/responses';
}
if (sitePlatform === 'gemini-cli' || sitePlatform === 'antigravity') {
return input.stream
? '/v1internal:streamGenerateContent?alt=sse'
: '/v1internal:generateContent';
}
if (sitePlatform === 'claude') {
return '/v1/messages';
}
@@ -476,6 +844,7 @@ export function buildUpstreamEndpointRequest(input: {
const commonHeaders: Record<string, string> = {
...passthroughHeaders,
'Content-Type': 'application/json',
...(input.providerHeaders || {}),
};
if (!isClaudeUpstream) {
commonHeaders.Authorization = `Bearer ${input.tokenValue}`;
@@ -483,7 +852,7 @@ export function buildUpstreamEndpointRequest(input: {
const stripGeminiUnsupportedFields = (body: Record<string, unknown>) => {
const next = { ...body };
if (isGeminiUpstream) {
if (isGeminiUpstream || isInternalGeminiUpstream) {
for (const key of [
'frequency_penalty',
'presence_penalty',
@@ -499,6 +868,61 @@ export function buildUpstreamEndpointRequest(input: {
};
const openaiBody = stripGeminiUnsupportedFields(input.openaiBody);
const runtime = {
executor: (
sitePlatform === 'codex'
? 'codex'
: sitePlatform === 'gemini-cli'
? 'gemini-cli'
: sitePlatform === 'antigravity'
? 'antigravity'
: sitePlatform === 'claude'
? 'claude'
: 'default'
) as 'default' | 'codex' | 'gemini-cli' | 'antigravity' | 'claude',
modelName: input.modelName,
stream: input.stream,
oauthProjectId: asTrimmedString(input.oauthProjectId) || null,
};
if (isInternalGeminiUpstream) {
const projectId = asTrimmedString(input.oauthProjectId);
if (isGeminiCliUpstream && !projectId) {
throw new Error('gemini-cli oauth project id missing');
}
const instructions = (
input.downstreamFormat === 'responses'
&& typeof input.responsesOriginalBody?.instructions === 'string'
)
? input.responsesOriginalBody.instructions
: undefined;
const geminiRequest = buildGeminiGenerateContentRequestFromOpenAi({
body: openaiBody,
modelName: input.modelName,
instructions,
});
const headers = isGeminiCliUpstream
? buildGeminiCliRuntimeHeaders({
baseHeaders: commonHeaders,
providerHeaders: input.providerHeaders,
modelName: input.modelName,
stream: input.stream,
})
: buildAntigravityRuntimeHeaders({
baseHeaders: commonHeaders,
stream: input.stream,
});
return {
path: resolveEndpointPath(input.endpoint),
headers,
body: wrapGeminiCliRequest({
modelName: input.modelName,
projectId: projectId || '',
request: geminiRequest,
}) as Record<string, unknown>,
runtime,
};
}
if (input.endpoint === 'messages') {
const claudeHeaders = input.downstreamFormat === 'claude'
@@ -537,21 +961,26 @@ export function buildUpstreamEndpointRequest(input: {
convertOpenAiBodyToAnthropicMessagesBody(openaiBody, input.modelName, input.stream),
);
const headers = ensureStreamAcceptHeader({
...commonHeaders,
...claudeHeaders,
'x-api-key': input.tokenValue,
'anthropic-version': anthropicVersion,
}, input.stream);
const headers = buildClaudeRuntimeHeaders({
baseHeaders: commonHeaders,
claudeHeaders,
anthropicVersion,
stream: input.stream,
isClaudeOauthUpstream,
tokenValue: input.tokenValue,
});
return {
path: resolveEndpointPath('messages'),
headers,
body: sanitizedBody,
runtime,
};
}
if (input.endpoint === 'responses') {
const websocketMode = Object.entries(input.downstreamHeaders || {}).find(([rawKey]) => rawKey.trim().toLowerCase() === 'x-metapi-responses-websocket-mode');
const preserveWebsocketIncrementalMode = asTrimmedString(websocketMode?.[1]).toLowerCase() === 'incremental';
const responsesHeaders = input.downstreamFormat === 'responses'
? extractResponsesPassthroughHeaders(input.downstreamHeaders)
: {};
@@ -564,17 +993,56 @@ export function buildUpstreamEndpointRequest(input: {
}
: convertOpenAiBodyToResponsesBodyViaTransformer(openaiBody, input.modelName, input.stream)
);
const body = sanitizeResponsesBodyForProxyViaTransformer(rawBody, input.modelName, input.stream);
const sanitizedResponsesBody = sanitizeResponsesBodyForProxyViaTransformer(rawBody, input.modelName, input.stream);
if (preserveWebsocketIncrementalMode && rawBody.generate === false) {
sanitizedResponsesBody.generate = false;
}
const body = ensureCodexResponsesStoreFalse(
ensureCodexResponsesInstructions(
applyCodexResponsesCompatibility(
sanitizedResponsesBody,
sitePlatform,
{ preservePreviousResponseId: preserveWebsocketIncrementalMode },
),
sitePlatform,
),
sitePlatform,
);
const headers = ensureStreamAcceptHeader({
...commonHeaders,
...responsesHeaders,
}, input.stream);
const headers = sitePlatform === 'codex'
? buildCodexRuntimeHeaders({
baseHeaders: {
...commonHeaders,
...responsesHeaders,
},
providerHeaders: input.providerHeaders,
explicitSessionId: asTrimmedString(input.codexExplicitSessionId) || null,
continuityKey: asTrimmedString(input.codexSessionCacheKey) || null,
})
: ensureStreamAcceptHeader({
...commonHeaders,
...responsesHeaders,
}, input.stream);
const codexSessionId = sitePlatform === 'codex'
? (getInputHeader(headers, 'session_id') || getInputHeader(headers, 'session-id'))
: null;
const shouldInjectDerivedPromptCacheKey = sitePlatform === 'codex'
&& !!codexSessionId
&& !asTrimmedString((body as Record<string, unknown>).prompt_cache_key)
&& !asTrimmedString(input.codexExplicitSessionId)
&& !!asTrimmedString(input.codexSessionCacheKey);
const runtimeBody = shouldInjectDerivedPromptCacheKey
? {
...body,
prompt_cache_key: codexSessionId,
}
: body;
return {
path: resolveEndpointPath('responses'),
headers,
body,
body: runtimeBody,
runtime,
};
}
@@ -587,6 +1055,63 @@ export function buildUpstreamEndpointRequest(input: {
model: input.modelName,
stream: input.stream,
},
runtime,
};
}
export function buildClaudeCountTokensUpstreamRequest(input: {
modelName: string;
tokenValue: string;
oauthProvider?: string;
sitePlatform?: string;
claudeBody: Record<string, unknown>;
downstreamHeaders?: Record<string, unknown>;
}): {
path: string;
headers: Record<string, string>;
body: Record<string, unknown>;
runtime: {
executor: 'claude';
modelName: string;
stream: false;
action: 'countTokens';
};
} {
const sitePlatform = normalizePlatformName(input.sitePlatform);
const isClaudeOauthUpstream = sitePlatform === 'claude' && input.oauthProvider === 'claude';
const claudeHeaders = extractClaudePassthroughHeaders(input.downstreamHeaders);
const anthropicVersion = claudeHeaders['anthropic-version'] || '2023-06-01';
const { body: bodyWithoutBetas, betas } = extractClaudeBetasFromBody({
...input.claudeBody,
model: input.modelName,
});
const sanitizedBody = sanitizeAnthropicMessagesBody(bodyWithoutBetas);
delete sanitizedBody.max_tokens;
delete sanitizedBody.maxTokens;
delete sanitizedBody.stream;
const headers = buildClaudeRuntimeHeaders({
baseHeaders: {
'Content-Type': 'application/json',
},
claudeHeaders,
anthropicVersion,
stream: false,
isClaudeOauthUpstream,
tokenValue: input.tokenValue,
extraBetas: betas,
});
return {
path: '/v1/messages/count_tokens?beta=true',
headers,
body: sanitizedBody,
runtime: {
executor: 'claude',
modelName: input.modelName,
stream: false,
action: 'countTokens',
},
};
}