Nvidia/BossAI
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into feat/account-level-proxy

Browse Source 
# Conflicts:
#	src/server/proxy-core/surfaces/chatSurface.ts
#	src/server/proxy-core/surfaces/openAiResponsesSurface.ts
#	src/server/routes/proxy/embeddings.ts
#	src/server/routes/proxy/images.ts
#	src/server/services/modelService.ts
This commit is contained in:
Hureru
2026-03-21 10:09:19 +08:00
parent 382c6b22d8 4ba7749391
commit 8df3b1431f
198 changed files with 20039 additions and 1105 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/server/proxy-core/capabilities/conversationFileCapabilities.test.ts
+79
View File
@@ -0,0 +1,79 @@
import { describe, expect, it } from 'vitest';
import {
rankConversationFileEndpoints,
resolveConversationFileEndpointCapability,
summarizeConversationFileInputsInOpenAiBody,
} from './conversationFileCapabilities.js';
describe('conversationFileCapabilities', () => {
it('summarizes image, audio, document, and remote file inputs from OpenAI bodies', () => {
const summary = summarizeConversationFileInputsInOpenAiBody({
messages: [
{
role: 'user',
content: [
{ type: 'image_url', image_url: { url: 'data:image/png;base64,AAAA' } },
{ type: 'input_audio', input_audio: { data: 'UklGRg==', format: 'wav' } },
{
type: 'file',
file: {
filename: 'brief.pdf',
file_data: 'JVBERi0xLjc=',
},
},
{
type: 'input_file',
filename: 'remote.pdf',
file_url: 'https://example.com/remote.pdf',
},
],
},
],
});
expect(summary).toEqual({
hasImage: true,
hasAudio: true,
hasDocument: true,
hasRemoteDocumentUrl: true,
});
});
it('describes native remote-document support for claude messages and inline-only support for gemini chat paths', () => {
expect(resolveConversationFileEndpointCapability({
sitePlatform: 'claude',
endpoint: 'messages',
})).toMatchObject({
image: 'native',
audio: 'unsupported',
document: 'native',
preservesRemoteDocumentUrl: true,
});
expect(resolveConversationFileEndpointCapability({
sitePlatform: 'gemini-cli',
endpoint: 'chat',
})).toMatchObject({
image: 'native',
audio: 'native',
document: 'inline_only',
preservesRemoteDocumentUrl: false,
});
});
it('ranks document-capable endpoints ahead of lossy fallbacks', () => {
const ranked = rankConversationFileEndpoints({
sitePlatform: 'new-api',
requestedOrder: ['chat', 'messages', 'responses'],
summary: {
hasImage: false,
hasAudio: false,
hasDocument: true,
hasRemoteDocumentUrl: false,
},
});
expect(ranked).toEqual(['responses', 'messages', 'chat']);
});
});
src/server/proxy-core/capabilities/conversationFileCapabilities.ts
+267
View File
@@ -0,0 +1,267 @@
import { inferInputFileMimeType, normalizeInputFileBlock } from '../../transformers/shared/inputFile.js';
export type ConversationFileTransport = 'unsupported' | 'inline_only' | 'native';
export type ConversationFileEndpoint = 'chat' | 'messages' | 'responses';
export type ConversationFileInputSummary = {
hasImage: boolean;
hasAudio: boolean;
hasDocument: boolean;
hasRemoteDocumentUrl: boolean;
};
export type ConversationFileEndpointCapability = {
image: ConversationFileTransport;
audio: ConversationFileTransport;
document: ConversationFileTransport;
preservesRemoteDocumentUrl: boolean;
};
function isRecord(value: unknown): value is Record<string, unknown> {
return !!value && typeof value === 'object' && !Array.isArray(value);
}
function asTrimmedString(value: unknown): string {
return typeof value === 'string' ? value.trim() : '';
}
function classifyMimeFamily(mimeType: string | null): 'image' | 'audio' | 'document' {
const normalized = asTrimmedString(mimeType).toLowerCase();
if (normalized.startsWith('image/')) return 'image';
if (normalized.startsWith('audio/')) return 'audio';
return 'document';
}
function appendConversationFileSummary(
summary: ConversationFileInputSummary,
item: Record<string, unknown>,
): void {
const normalizedFile = normalizeInputFileBlock(item);
if (normalizedFile) {
const mimeType = inferInputFileMimeType(normalizedFile);
const family = classifyMimeFamily(mimeType);
if (family === 'image') {
summary.hasImage = true;
return;
}
if (family === 'audio') {
summary.hasAudio = true;
return;
}
summary.hasDocument = true;
if (normalizedFile.fileUrl) {
summary.hasRemoteDocumentUrl = true;
}
return;
}
const type = asTrimmedString(item.type).toLowerCase();
if (type === 'image_url' || type === 'input_image') {
summary.hasImage = true;
return;
}
if (type === 'input_audio') {
summary.hasAudio = true;
return;
}
if (Array.isArray(item.content)) {
for (const nested of item.content) {
if (isRecord(nested)) {
appendConversationFileSummary(summary, nested);
}
}
}
}
function createEmptySummary(): ConversationFileInputSummary {
return {
hasImage: false,
hasAudio: false,
hasDocument: false,
hasRemoteDocumentUrl: false,
};
}
export function summarizeConversationFileInputsInOpenAiBody(
body: Record<string, unknown>,
): ConversationFileInputSummary {
const summary = createEmptySummary();
const messages = Array.isArray(body.messages) ? body.messages : [];
for (const message of messages) {
if (!isRecord(message)) continue;
const content = message.content;
if (Array.isArray(content)) {
for (const item of content) {
if (isRecord(item)) appendConversationFileSummary(summary, item);
}
continue;
}
if (isRecord(content)) {
appendConversationFileSummary(summary, content);
}
}
return summary;
}
export function summarizeConversationFileInputsInResponsesBody(
body: Record<string, unknown>,
): ConversationFileInputSummary {
const summary = createEmptySummary();
const input = body.input;
if (Array.isArray(input)) {
for (const item of input) {
if (isRecord(item)) appendConversationFileSummary(summary, item);
}
return summary;
}
if (isRecord(input)) {
appendConversationFileSummary(summary, input);
}
return summary;
}
export function resolveConversationFileEndpointCapability(input: {
sitePlatform?: string;
endpoint: ConversationFileEndpoint;
}): ConversationFileEndpointCapability {
const platform = asTrimmedString(input.sitePlatform).toLowerCase();
const endpoint = input.endpoint;
if (platform === 'codex') {
if (endpoint === 'responses') {
return {
image: 'native',
audio: 'native',
document: 'native',
preservesRemoteDocumentUrl: true,
};
}
return {
image: 'unsupported',
audio: 'unsupported',
document: 'unsupported',
preservesRemoteDocumentUrl: false,
};
}
if (platform === 'claude') {
if (endpoint === 'messages') {
return {
image: 'native',
audio: 'unsupported',
document: 'native',
preservesRemoteDocumentUrl: true,
};
}
return {
image: 'unsupported',
audio: 'unsupported',
document: 'unsupported',
preservesRemoteDocumentUrl: false,
};
}
if (platform === 'gemini-cli' || platform === 'antigravity') {
if (endpoint === 'chat') {
return {
image: 'native',
audio: 'native',
document: 'inline_only',
preservesRemoteDocumentUrl: false,
};
}
return {
image: 'unsupported',
audio: 'unsupported',
document: 'unsupported',
preservesRemoteDocumentUrl: false,
};
}
if (platform === 'gemini') {
if (endpoint === 'responses') {
return {
image: 'native',
audio: 'native',
document: 'native',
preservesRemoteDocumentUrl: true,
};
}
if (endpoint === 'chat') {
return {
image: 'native',
audio: 'native',
document: 'inline_only',
preservesRemoteDocumentUrl: false,
};
}
return {
image: 'unsupported',
audio: 'unsupported',
document: 'unsupported',
preservesRemoteDocumentUrl: false,
};
}
if (endpoint === 'responses') {
return {
image: 'native',
audio: 'native',
document: 'native',
preservesRemoteDocumentUrl: true,
};
}
if (endpoint === 'messages') {
return {
image: 'native',
audio: 'unsupported',
document: 'inline_only',
preservesRemoteDocumentUrl: false,
};
}
return {
image: 'native',
audio: 'native',
document: 'inline_only',
preservesRemoteDocumentUrl: false,
};
}
export function rankConversationFileEndpoints(input: {
sitePlatform?: string;
requestedOrder: ConversationFileEndpoint[];
summary: ConversationFileInputSummary;
preferMessagesForClaudeModel?: boolean;
}): ConversationFileEndpoint[] {
if (!input.summary.hasDocument) {
return [...input.requestedOrder];
}
const preferredDocumentOrder = input.preferMessagesForClaudeModel === true
? ['messages', 'responses', 'chat']
: ['responses', 'messages', 'chat'];
const supportedDocumentEndpoints = preferredDocumentOrder.filter((endpoint) => {
if (!input.requestedOrder.includes(endpoint as ConversationFileEndpoint)) return false;
const capability = resolveConversationFileEndpointCapability({
sitePlatform: input.sitePlatform,
endpoint: endpoint as ConversationFileEndpoint,
});
if (capability.document === 'unsupported') return false;
if (input.summary.hasRemoteDocumentUrl && !capability.preservesRemoteDocumentUrl) return false;
return true;
}) as ConversationFileEndpoint[];
if (supportedDocumentEndpoints.length <= 0) {
return [...input.requestedOrder];
}
return [
...supportedDocumentEndpoints,
...input.requestedOrder.filter((endpoint) => !supportedDocumentEndpoints.includes(endpoint)),
];
}
src/server/proxy-core/cliProfiles/codexProfile.ts
+30 -1
View File
@@ -1,5 +1,21 @@
import type { CliProfileDefinition, DetectCliProfileInput } from './types.js';
const CODEX_OFFICIAL_CLIENT_USER_AGENT_PREFIXES = [
'codex_cli_rs/',
'codex_vscode/',
'codex_app/',
'codex_chatgpt_desktop/',
'codex_atlas/',
'codex_exec/',
'codex_sdk_ts/',
'codex ',
];
const CODEX_OFFICIAL_CLIENT_ORIGINATOR_PREFIXES = [
'codex_',
'codex ',
];
function headerValueToString(value: unknown): string | null {
if (typeof value === 'string') {
const trimmed = value.trim();
@@ -38,6 +54,18 @@ function hasHeaderPrefix(headers: Record<string, unknown> | undefined, prefix: s
});
}
function matchesHeaderPrefixes(value: string | null, prefixes: string[]): boolean {
const normalizedValue = value?.trim().toLowerCase() || '';
if (!normalizedValue) return false;
return prefixes.some((prefix) => {
const normalizedPrefix = prefix.trim().toLowerCase();
if (!normalizedPrefix) return false;
return normalizedValue.startsWith(normalizedPrefix)
|| normalizedValue.includes(normalizedPrefix);
});
}
function isCodexPath(path: string): boolean {
const normalizedPath = path.trim().toLowerCase();
return normalizedPath.startsWith('/v1/responses')
@@ -62,7 +90,8 @@ export function isCodexRequest(input: DetectCliProfileInput): boolean {
if (!headers) return false;
const originator = getHeaderValue(headers, 'originator');
if (originator?.toLowerCase() === 'codex_cli_rs') return true;
if (matchesHeaderPrefixes(originator, CODEX_OFFICIAL_CLIENT_ORIGINATOR_PREFIXES)) return true;
if (matchesHeaderPrefixes(getHeaderValue(headers, 'user-agent'), CODEX_OFFICIAL_CLIENT_USER_AGENT_PREFIXES)) return true;
if (getHeaderValue(headers, 'openai-beta')) return true;
if (hasHeaderPrefix(headers, 'x-stainless-')) return true;
if (getCodexSessionId(headers)) return true;
src/server/proxy-core/cliProfiles/registry.test.ts
+28
View File
@@ -39,6 +39,34 @@ describe('detectCliProfile', () => {
});
});
it('detects broader Codex official-client headers from user-agent and originator prefixes', () => {
expect(detectCliProfile({
downstreamPath: '/v1/responses',
headers: {
'user-agent': 'Mozilla/5.0 codex_chatgpt_desktop/1.2.3',
},
})).toMatchObject({
id: 'codex',
capabilities: {
supportsResponsesWebsocketIncremental: true,
echoesTurnState: true,
},
});
expect(detectCliProfile({
downstreamPath: '/v1/responses',
headers: {
originator: 'codex_exec',
},
})).toMatchObject({
id: 'codex',
capabilities: {
supportsResponsesWebsocketIncremental: true,
echoesTurnState: true,
},
});
});
it('detects Claude Code requests on the count_tokens surface and exposes token counting support', () => {
expect(detectCliProfile({
downstreamPath: '/v1/messages/count_tokens',
src/server/proxy-core/providers/codexProviderProfile.ts
+24 -1
View File
@@ -1,8 +1,10 @@
import { createHash, randomUUID } from 'node:crypto';
import type { PreparedProviderRequest, PrepareProviderRequestInput, ProviderProfile } from './types.js';
import { config } from '../../config.js';
const CODEX_CLIENT_VERSION = '0.101.0';
const CODEX_DEFAULT_USER_AGENT = 'codex_cli_rs/0.101.0 (Mac OS 26.0.1; arm64) Apple_Terminal/464';
const CODEX_RESPONSES_WEBSOCKET_BETA = 'responses_websockets=2026-02-06';
function asTrimmedString(value: unknown): string {
return typeof value === 'string' ? value.trim() : '';
@@ -53,6 +55,7 @@ function uuidFromSeed(seed: string): string {
function buildCodexRuntimeHeaders(input: {
baseHeaders: Record<string, string>;
providerHeaders?: Record<string, string>;
oauthProvider?: string;
explicitSessionId?: string | null;
continuityKey?: string | null;
}): Record<string, string> {
@@ -64,7 +67,21 @@ function buildCodexRuntimeHeaders(input: {
const originator = getInputHeader(input.providerHeaders, 'originator') || 'codex_cli_rs';
const accountId = getInputHeader(input.providerHeaders, 'chatgpt-account-id');
const version = getInputHeader(input.baseHeaders, 'version') || CODEX_CLIENT_VERSION;
const userAgent = getInputHeader(input.baseHeaders, 'user-agent') || CODEX_DEFAULT_USER_AGENT;
const isCodexOauth = asTrimmedString(input.oauthProvider).toLowerCase() === 'codex';
const configuredUserAgent = isCodexOauth ? asTrimmedString(config.codexHeaderDefaults.userAgent) : '';
const websocketTransport = getInputHeader(input.baseHeaders, 'x-metapi-responses-websocket-transport') === '1';
const configuredBetaFeatures = (
isCodexOauth && websocketTransport
? asTrimmedString(config.codexHeaderDefaults.betaFeatures)
: ''
);
const userAgent = configuredUserAgent || getInputHeader(input.baseHeaders, 'user-agent') || CODEX_DEFAULT_USER_AGENT;
const codexBetaFeatures = getInputHeader(input.baseHeaders, 'x-codex-beta-features') || configuredBetaFeatures;
const codexTurnState = getInputHeader(input.baseHeaders, 'x-codex-turn-state');
const codexTurnMetadata = getInputHeader(input.baseHeaders, 'x-codex-turn-metadata');
const timingMetrics = getInputHeader(input.baseHeaders, 'x-responsesapi-include-timing-metrics');
const openAiBeta = getInputHeader(input.baseHeaders, 'openai-beta')
|| (websocketTransport ? CODEX_RESPONSES_WEBSOCKET_BETA : null);
const explicitSessionId = asTrimmedString(input.explicitSessionId);
const continuityKey = asTrimmedString(input.continuityKey);
const sessionId = (
@@ -87,6 +104,11 @@ function buildCodexRuntimeHeaders(input: {
...(accountId ? { 'Chatgpt-Account-Id': accountId } : {}),
Originator: originator,
Version: version,
...(codexBetaFeatures ? { 'x-codex-beta-features': codexBetaFeatures } : {}),
...(codexTurnState ? { 'x-codex-turn-state': codexTurnState } : {}),
...(codexTurnMetadata ? { 'x-codex-turn-metadata': codexTurnMetadata } : {}),
...(timingMetrics ? { 'x-responsesapi-include-timing-metrics': timingMetrics } : {}),
...(openAiBeta ? { 'OpenAI-Beta': openAiBeta } : {}),
Session_id: sessionId,
...(conversationId ? { Conversation_id: conversationId } : {}),
'User-Agent': userAgent,
@@ -101,6 +123,7 @@ export const codexProviderProfile: ProviderProfile = {
const headers = buildCodexRuntimeHeaders({
baseHeaders: input.baseHeaders,
providerHeaders: input.providerHeaders,
oauthProvider: input.oauthProvider,
explicitSessionId: asTrimmedString(input.codexExplicitSessionId) || null,
continuityKey: asTrimmedString(input.codexSessionCacheKey) || null,
});
src/server/proxy-core/runtime/codexWebsocketHeaders.ts
+37
View File
@@ -0,0 +1,37 @@
const CODEX_RESPONSES_WEBSOCKET_BETA = 'responses_websockets=2026-02-06';
function getHeaderValue(headers: Record<string, string>, key: string): string {
const expected = key.trim().toLowerCase();
for (const [candidateKey, candidateValue] of Object.entries(headers)) {
if (candidateKey.trim().toLowerCase() !== expected) continue;
return candidateValue;
}
return '';
}
export function buildCodexWebsocketHandshakeHeaders(headers: Record<string, string>): Record<string, string> {
const next = { ...headers };
const openAiBeta = getHeaderValue(next, 'openai-beta').trim();
if (!openAiBeta) {
next['OpenAI-Beta'] = CODEX_RESPONSES_WEBSOCKET_BETA;
return next;
}
if (!openAiBeta.includes('responses_websockets=')) {
next['OpenAI-Beta'] = `${openAiBeta},${CODEX_RESPONSES_WEBSOCKET_BETA}`;
}
return next;
}
export function buildCodexWebsocketRequestBody(body: Record<string, unknown>): Record<string, unknown> {
return {
type: 'response.create',
...body,
};
}
export function toCodexWebsocketUrl(requestUrl: string): string {
const parsed = new URL(requestUrl);
if (parsed.protocol === 'https:') parsed.protocol = 'wss:';
if (parsed.protocol === 'http:') parsed.protocol = 'ws:';
return parsed.toString();
}
src/server/proxy-core/runtime/codexWebsocketRuntime.test.ts
+137
View File
@@ -0,0 +1,137 @@
import { AddressInfo } from 'node:net';
import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest';
import { WebSocketServer } from 'ws';
describe('codexWebsocketRuntime', () => {
let upstreamServer: WebSocketServer;
let upstreamWsUrl: string;
let upstreamConnectionCount = 0;
let upstreamRequests: Record<string, unknown>[] = [];
beforeAll(async () => {
upstreamServer = new WebSocketServer({ port: 0 });
upstreamServer.on('connection', (socket) => {
upstreamConnectionCount += 1;
socket.on('message', (payload) => {
const parsed = JSON.parse(String(payload)) as Record<string, unknown>;
upstreamRequests.push(parsed);
const responseId = `resp-${upstreamRequests.length}`;
socket.send(JSON.stringify({
type: 'response.completed',
response: {
id: responseId,
object: 'response',
model: parsed.model || 'gpt-5.4',
status: 'completed',
output: [],
usage: {
input_tokens: 1,
output_tokens: 1,
total_tokens: 2,
},
},
}));
});
});
await new Promise<void>((resolve) => upstreamServer.once('listening', () => resolve()));
const address = upstreamServer.address() as AddressInfo;
upstreamWsUrl = `ws://127.0.0.1:${address.port}/backend-api/codex/responses`;
});
beforeEach(() => {
upstreamConnectionCount = 0;
upstreamRequests = [];
});
afterAll(async () => {
await new Promise<void>((resolve) => upstreamServer.close(() => resolve()));
});
it('reuses the same upstream websocket connection across turns for one execution session', async () => {
const { createCodexWebsocketRuntime } = await import('./codexWebsocketRuntime.js');
const runtime = createCodexWebsocketRuntime();
const first = await runtime.sendRequest({
sessionId: 'exec-session-1',
requestUrl: upstreamWsUrl,
headers: {
Authorization: 'Bearer oauth-access-token',
'OpenAI-Beta': 'responses_websockets=2026-02-06',
},
body: {
model: 'gpt-5.4',
input: [],
},
});
const second = await runtime.sendRequest({
sessionId: 'exec-session-1',
requestUrl: upstreamWsUrl,
headers: {
Authorization: 'Bearer oauth-access-token',
'OpenAI-Beta': 'responses_websockets=2026-02-06',
},
body: {
model: 'gpt-5.4',
previous_response_id: 'resp-1',
input: [],
},
});
expect(first.events[0]).toMatchObject({
type: 'response.completed',
response: { id: 'resp-1' },
});
expect(second.events[0]).toMatchObject({
type: 'response.completed',
response: { id: 'resp-2' },
});
expect(upstreamConnectionCount).toBe(1);
expect(upstreamRequests).toHaveLength(2);
expect(upstreamRequests[0]).toMatchObject({
type: 'response.create',
model: 'gpt-5.4',
});
expect(upstreamRequests[1]).toMatchObject({
type: 'response.create',
previous_response_id: 'resp-1',
});
await runtime.closeSession('exec-session-1');
});
it('closes the upstream websocket when the execution session is closed explicitly', async () => {
const { createCodexWebsocketRuntime } = await import('./codexWebsocketRuntime.js');
const runtime = createCodexWebsocketRuntime();
await runtime.sendRequest({
sessionId: 'exec-session-close',
requestUrl: upstreamWsUrl,
headers: {
Authorization: 'Bearer oauth-access-token',
'OpenAI-Beta': 'responses_websockets=2026-02-06',
},
body: {
model: 'gpt-5.4',
input: [],
},
});
await runtime.closeSession('exec-session-close');
await runtime.sendRequest({
sessionId: 'exec-session-close',
requestUrl: upstreamWsUrl,
headers: {
Authorization: 'Bearer oauth-access-token',
'OpenAI-Beta': 'responses_websockets=2026-02-06',
},
body: {
model: 'gpt-5.4',
input: [],
},
});
expect(upstreamConnectionCount).toBe(2);
await runtime.closeSession('exec-session-close');
});
});
src/server/proxy-core/runtime/codexWebsocketRuntime.ts
+284
View File
@@ -0,0 +1,284 @@
import type { IncomingMessage } from 'node:http';
import WebSocket from 'ws';
import {
buildCodexWebsocketHandshakeHeaders,
buildCodexWebsocketRequestBody,
toCodexWebsocketUrl,
} from './codexWebsocketHeaders.js';
import { createCodexWebsocketSessionStore } from './codexWebsocketSessionStore.js';
import type {
CodexWebsocketRuntimeResult,
CodexWebsocketRuntimeSendInput,
CodexWebsocketSession,
CodexWebsocketSessionStore,
} from './types.js';
function asTrimmedString(value: unknown): string {
return typeof value === 'string' ? value.trim() : '';
}
function isRecord(value: unknown): value is Record<string, unknown> {
return !!value && typeof value === 'object' && !Array.isArray(value);
}
function isTerminalEvent(payload: Record<string, unknown>): boolean {
const type = asTrimmedString(payload.type);
return type === 'response.completed' || type === 'response.failed';
}
export class CodexWebsocketRuntimeError extends Error {
events: Array<Record<string, unknown>>;
status?: number;
payload?: unknown;
constructor(
message: string,
options?: {
events?: Array<Record<string, unknown>>;
status?: number;
payload?: unknown;
},
) {
super(message);
this.name = 'CodexWebsocketRuntimeError';
this.events = options?.events ?? [];
this.status = options?.status;
this.payload = options?.payload;
}
}
function tryParseJson(raw: string): unknown {
const trimmed = raw.trim();
if (!trimmed) return undefined;
try {
return JSON.parse(trimmed);
} catch {
return undefined;
}
}
async function readUnexpectedResponseBody(response: IncomingMessage): Promise<string> {
return await new Promise<string>((resolve) => {
const chunks: Buffer[] = [];
response.on('data', (chunk) => {
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
});
response.once('end', () => {
resolve(Buffer.concat(chunks).toString('utf8'));
});
response.once('error', () => {
resolve('');
});
});
}
async function waitForSocketOpen(socket: WebSocket): Promise<void> {
if (socket.readyState === WebSocket.OPEN) return;
if (socket.readyState !== WebSocket.CONNECTING) {
throw new Error('upstream websocket is not open');
}
await new Promise<void>((resolve, reject) => {
const cleanup = () => {
socket.off('open', onOpen);
socket.off('error', onError);
socket.off('close', onClose);
socket.off('unexpected-response', onUnexpectedResponse);
};
const onOpen = () => {
cleanup();
resolve();
};
const onError = (error: Error) => {
cleanup();
reject(error);
};
const onClose = () => {
cleanup();
reject(new Error('upstream websocket closed before opening'));
};
const onUnexpectedResponse = (_request: unknown, response: IncomingMessage) => {
void readUnexpectedResponseBody(response).then((body) => {
cleanup();
reject(new CodexWebsocketRuntimeError(
body.trim() || response.statusMessage || `upstream websocket upgrade failed with status ${response.statusCode || 502}`,
{
status: response.statusCode || 502,
payload: tryParseJson(body),
},
));
});
};
socket.once('open', onOpen);
socket.once('error', onError);
socket.once('close', onClose);
socket.once('unexpected-response', onUnexpectedResponse);
});
}
async function closeSocket(socket: WebSocket | null): Promise<void> {
if (!socket) return;
if (socket.readyState === WebSocket.CLOSED) return;
await new Promise<void>((resolve) => {
const onClose = () => resolve();
socket.once('close', onClose);
try {
socket.close();
} catch {
socket.off('close', onClose);
resolve();
}
setTimeout(() => {
socket.off('close', onClose);
resolve();
}, 200);
});
}
function clearSessionSocket(session: CodexWebsocketSession, socket: WebSocket): void {
if (session.socket !== socket) return;
session.socket = null;
session.socketUrl = null;
}
async function ensureSessionSocket(
session: CodexWebsocketSession,
input: CodexWebsocketRuntimeSendInput,
): Promise<{ socket: WebSocket; reusedSession: boolean }> {
const requestUrl = toCodexWebsocketUrl(input.requestUrl);
const existing = session.socket;
if (
existing
&& session.socketUrl === requestUrl
&& existing.readyState === WebSocket.OPEN
) {
return {
socket: existing,
reusedSession: true,
};
}
if (existing) {
await closeSocket(existing);
clearSessionSocket(session, existing);
}
const nextSocket = new WebSocket(requestUrl, {
headers: buildCodexWebsocketHandshakeHeaders(input.headers),
});
await waitForSocketOpen(nextSocket);
session.socket = nextSocket;
session.socketUrl = requestUrl;
nextSocket.on('close', () => {
clearSessionSocket(session, nextSocket);
});
nextSocket.on('error', () => {
clearSessionSocket(session, nextSocket);
});
return {
socket: nextSocket,
reusedSession: false,
};
}
async function sendSessionRequest(
session: CodexWebsocketSession,
input: CodexWebsocketRuntimeSendInput,
): Promise<CodexWebsocketRuntimeResult> {
const { socket, reusedSession } = await ensureSessionSocket(session, input);
const events: Array<Record<string, unknown>> = [];
return new Promise<CodexWebsocketRuntimeResult>((resolve, reject) => {
let settled = false;
const cleanup = () => {
socket.off('message', onMessage);
socket.off('error', onError);
socket.off('close', onClose);
};
const rejectWith = (message: string) => {
if (settled) return;
settled = true;
cleanup();
reject(new CodexWebsocketRuntimeError(message, { events: [...events] }));
};
const onMessage = (payload: WebSocket.RawData) => {
try {
const parsed = JSON.parse(String(payload));
if (!isRecord(parsed)) return;
events.push(parsed);
if (!isTerminalEvent(parsed)) return;
if (settled) return;
settled = true;
cleanup();
resolve({
events: [...events],
reusedSession,
});
} catch {
// Ignore malformed frames and wait for a terminal event.
}
};
const onError = (error: Error) => {
clearSessionSocket(session, socket);
rejectWith(error.message || 'upstream websocket error');
};
const onClose = () => {
clearSessionSocket(session, socket);
rejectWith('stream closed before response.completed');
};
socket.on('message', onMessage);
socket.once('error', onError);
socket.once('close', onClose);
socket.send(JSON.stringify(buildCodexWebsocketRequestBody(input.body)), (error) => {
if (!error) return;
clearSessionSocket(session, socket);
rejectWith(error.message || 'failed to send upstream websocket request');
});
});
}
export function createCodexWebsocketRuntime(input?: {
sessionStore?: CodexWebsocketSessionStore;
}) {
const sessionStore = input?.sessionStore || createCodexWebsocketSessionStore();
return {
async sendRequest(payload: CodexWebsocketRuntimeSendInput): Promise<CodexWebsocketRuntimeResult> {
const sessionId = payload.sessionId.trim();
if (!sessionId) {
throw new CodexWebsocketRuntimeError('missing websocket session id');
}
const session = sessionStore.getOrCreate(sessionId);
const run = session.queue
.catch(() => undefined)
.then(() => sendSessionRequest(session, payload));
session.queue = run.then(() => undefined, () => undefined);
return run;
},
async closeSession(sessionId: string): Promise<void> {
const session = sessionStore.take(sessionId);
if (!session) return;
await session.queue.catch(() => undefined);
await closeSocket(session.socket);
session.socket = null;
session.socketUrl = null;
},
async closeAllSessions(): Promise<void> {
const sessions = sessionStore.list();
for (const session of sessions) {
await this.closeSession(session.sessionId);
}
},
};
}
src/server/proxy-core/runtime/codexWebsocketSessionStore.ts
+34
View File
@@ -0,0 +1,34 @@
import type { CodexWebsocketSession, CodexWebsocketSessionStore } from './types.js';
export function createCodexWebsocketSessionStore(): CodexWebsocketSessionStore {
const sessions = new Map<string, CodexWebsocketSession>();
return {
getOrCreate(sessionId) {
const normalized = sessionId.trim();
const existing = sessions.get(normalized);
if (existing) return existing;
const created: CodexWebsocketSession = {
sessionId: normalized,
socket: null,
socketUrl: null,
queue: Promise.resolve(),
};
sessions.set(normalized, created);
return created;
},
take(sessionId) {
const normalized = sessionId.trim();
if (!normalized) return null;
const existing = sessions.get(normalized) || null;
if (existing) {
sessions.delete(normalized);
}
return existing;
},
list() {
return [...sessions.values()];
},
};
}
src/server/proxy-core/runtime/types.ts
+26
View File
@@ -0,0 +1,26 @@
import type WebSocket from 'ws';
export type CodexWebsocketRuntimeSendInput = {
sessionId: string;
requestUrl: string;
headers: Record<string, string>;
body: Record<string, unknown>;
};
export type CodexWebsocketRuntimeResult = {
events: Array<Record<string, unknown>>;
reusedSession: boolean;
};
export type CodexWebsocketSession = {
sessionId: string;
socket: WebSocket | null;
socketUrl: string | null;
queue: Promise<unknown>;
};
export type CodexWebsocketSessionStore = {
getOrCreate(sessionId: string): CodexWebsocketSession;
take(sessionId: string): CodexWebsocketSession | null;
list(): CodexWebsocketSession[];
};
src/server/proxy-core/surfaces/chatSurface.ts
+140 -61
View File
@@ -1,7 +1,6 @@
import { TextDecoder } from 'node:util';
import type { FastifyRequest, FastifyReply } from 'fastify';
import { tokenRouter } from '../../services/tokenRouter.js';
import { db, hasProxyLogDownstreamApiKeyIdColumn, schema } from '../../db/index.js';
import { refreshModelsAndRebuildRoutes } from '../../services/modelService.js';
import { reportProxyAllFailed, reportTokenExpired } from '../../services/alertService.js';
import { isTokenExpiredError } from '../../services/alertRules.js';
@@ -13,6 +12,8 @@ import { type DownstreamFormat } from '../../transformers/shared/normalized.js';
import {
buildClaudeCountTokensUpstreamRequest,
buildUpstreamEndpointRequest,
recordUpstreamEndpointFailure,
recordUpstreamEndpointSuccess,
resolveUpstreamEndpointCandidates,
} from '../../routes/proxy/upstreamEndpoint.js';
import {
@@ -23,6 +24,7 @@ import {
import { composeProxyLogMessage } from '../../routes/proxy/logPathMeta.js';
import { executeEndpointFlow, type BuiltEndpointRequest } from '../../routes/proxy/endpointFlow.js';
import { detectProxyFailure } from '../../routes/proxy/proxyFailureJudge.js';
import { buildUpstreamUrl } from '../../routes/proxy/upstreamUrl.js';
import { formatUtcSqlDateTime } from '../../services/localTimeService.js';
import { resolveProxyLogBilling } from '../../routes/proxy/proxyBilling.js';
import { openAiChatTransformer } from '../../transformers/openai/chat/index.js';
@@ -30,7 +32,6 @@ import { anthropicMessagesTransformer } from '../../transformers/anthropic/messa
import { getProxyAuthContext, getProxyResourceOwner } from '../../middleware/auth.js';
import {
ProxyInputFileResolutionError,
hasNonImageFileInputInOpenAiBody,
resolveOpenAiBodyInputFiles,
} from '../../services/proxyInputFileResolver.js';
import {
@@ -39,14 +40,20 @@ import {
import { getOauthInfoFromExtraConfig } from '../../services/oauth/oauthAccount.js';
import { recordOauthQuotaResetHint } from '../../services/oauth/quota.js';
import { refreshOauthAccessTokenSingleflight } from '../../services/oauth/refreshSingleflight.js';
import { collectResponsesFinalPayloadFromSse } from '../../routes/proxy/responsesSseFinal.js';
import {
collectResponsesFinalPayloadFromSse,
collectResponsesFinalPayloadFromSseText,
createSingleChunkStreamReader,
looksLikeResponsesSseText,
} from '../../routes/proxy/responsesSseFinal.js';
import {
createGeminiCliStreamReader,
unwrapGeminiCliPayload,
} from '../../routes/proxy/geminiCliCompat.js';
import { dispatchRuntimeRequest } from '../../routes/proxy/runtimeExecutor.js';
import { detectCliProfile } from '../cliProfiles/registry.js';
import type { CliProfileId } from '../cliProfiles/types.js';
import { summarizeConversationFileInputsInOpenAiBody } from '../capabilities/conversationFileCapabilities.js';
import { detectDownstreamClientContext, type DownstreamClientContext } from '../../routes/proxy/downstreamClientContext.js';
import { insertProxyLog } from '../../services/proxyLogStore.js';
const MAX_RETRIES = 2;
@@ -58,25 +65,6 @@ function asTrimmedString(value: unknown): string {
return typeof value === 'string' ? value.trim() : '';
}
type DownstreamClientContext = {
clientKind: CliProfileId;
sessionId?: string;
traceHint?: string;
};
function detectDownstreamClientContext(input: {
downstreamPath: string;
headers?: Record<string, unknown>;
body?: unknown;
}): DownstreamClientContext {
const detected = detectCliProfile(input);
return {
clientKind: detected.id,
...(detected.sessionId ? { sessionId: detected.sessionId } : {}),
...(detected.traceHint ? { traceHint: detected.traceHint } : {}),
};
}
export async function handleChatSurfaceRequest(
request: FastifyRequest,
reply: FastifyReply,
@@ -117,7 +105,8 @@ export async function handleChatSurfaceRequest(
throw error;
}
}
const hasNonImageFileInput = hasNonImageFileInputInOpenAiBody(resolvedOpenAiBody);
const conversationFileSummary = summarizeConversationFileInputsInOpenAiBody(resolvedOpenAiBody);
const hasNonImageFileInput = conversationFileSummary.hasDocument;
const codexSessionCacheKey = deriveCodexSessionCacheKey({
downstreamFormat,
body: downstreamFormat === 'claude' ? claudeOriginalBody : request.body,
@@ -125,8 +114,6 @@ export async function handleChatSurfaceRequest(
proxyToken: getProxyAuthContext(request)?.token || null,
});
const downstreamApiKeyId = getProxyAuthContext(request)?.keyId ?? null;
const logDownstreamApiKeyId = downstreamApiKeyId !== null
&& await hasProxyLogDownstreamApiKeyIdColumn();
const excludeChannelIds: number[] = [];
let retryCount = 0;
@@ -167,9 +154,20 @@ export async function handleChatSurfaceRequest(
requestedModel,
{
hasNonImageFileInput,
conversationFileSummary,
},
),
];
const endpointRuntimeContext = {
siteId: selected.site.id,
modelName,
downstreamFormat,
requestedModelHint: requestedModel,
requestCapabilities: {
hasNonImageFileInput,
conversationFileSummary,
},
};
const buildProviderHeaders = () => (
buildOauthProviderHeaders({
extraConfig: typeof selected.account.extraConfig === 'string' ? selected.account.extraConfig : null,
@@ -245,7 +243,7 @@ export async function handleChatSurfaceRequest(
extraConfig: refreshed.extraConfig ?? selected.account.extraConfig,
};
const refreshedRequest = buildEndpointRequest(ctx.request.endpoint);
const refreshedTargetUrl = `${selected.site.url}${refreshedRequest.path}`;
const refreshedTargetUrl = buildUpstreamUrl(selected.site.url, refreshedRequest.path);
const refreshedResponse = await dispatchRequest(refreshedRequest, refreshedTargetUrl);
if (refreshedResponse.ok) {
return {
@@ -272,6 +270,20 @@ export async function handleChatSurfaceRequest(
buildRequest: (endpoint) => buildEndpointRequest(endpoint),
dispatchRequest,
tryRecover,
onAttemptFailure: (ctx) => {
recordUpstreamEndpointFailure({
...endpointRuntimeContext,
endpoint: ctx.request.endpoint,
status: ctx.response.status,
errorText: ctx.rawErrText,
});
},
onAttemptSuccess: (ctx) => {
recordUpstreamEndpointSuccess({
...endpointRuntimeContext,
endpoint: ctx.request.endpoint,
});
},
shouldDowngrade: endpointStrategy.shouldDowngrade,
onDowngrade: (ctx) => {
logProxy(
@@ -290,7 +302,7 @@ export async function handleChatSurfaceRequest(
null,
null,
clientContext,
logDownstreamApiKeyId ? downstreamApiKeyId : null,
downstreamApiKeyId,
);
},
});
@@ -299,7 +311,11 @@ export async function handleChatSurfaceRequest(
const status = endpointResult.status || 502;
const errText = endpointResult.errText || 'unknown error';
const rawErrText = endpointResult.rawErrText || errText;
tokenRouter.recordFailure(selected.channel.id);
tokenRouter.recordFailure(selected.channel.id, {
status,
errorText: rawErrText,
modelName,
});
logProxy(
selected,
requestedModel,
@@ -316,7 +332,7 @@ export async function handleChatSurfaceRequest(
null,
null,
clientContext,
logDownstreamApiKeyId ? downstreamApiKeyId : null,
downstreamApiKeyId,
);
await recordOauthQuotaResetHint({
accountId: selected.account.id,
@@ -394,6 +410,37 @@ export async function handleChatSurfaceRequest(
if (!upstreamContentType.includes('text/event-stream')) {
const fallbackText = await upstream.text();
rawText = fallbackText;
if (looksLikeResponsesSseText(fallbackText)) {
startSseResponse();
const streamResult = await streamSession.run(
createSingleChunkStreamReader(fallbackText),
reply.raw,
);
const latency = Date.now() - startTime;
if (streamResult.status === 'failed') {
tokenRouter.recordFailure(selected.channel.id, modelName);
logProxy(
selected,
requestedModel,
'failed',
200,
latency,
streamResult.errorMessage,
retryCount,
downstreamPath,
parsedUsage.promptTokens,
parsedUsage.completionTokens,
parsedUsage.totalTokens,
0,
null,
successfulUpstreamPath,
clientContext,
downstreamApiKeyId,
);
return;
}
return;
}
let fallbackData: unknown = null;
try {
fallbackData = JSON.parse(fallbackText);
@@ -407,7 +454,11 @@ export async function handleChatSurfaceRequest(
const latency = Date.now() - startTime;
const failure = detectProxyFailure({ rawText, usage: parsedUsage });
if (failure) {
tokenRouter.recordFailure(selected.channel.id);
tokenRouter.recordFailure(selected.channel.id, {
status: failure.status,
errorText: failure.reason,
modelName,
});
logProxy(
selected,
requestedModel,
@@ -424,7 +475,7 @@ export async function handleChatSurfaceRequest(
null,
successfulUpstreamPath,
clientContext,
logDownstreamApiKeyId ? downstreamApiKeyId : null,
downstreamApiKeyId,
);
if (shouldRetryProxyRequest(failure.status, failure.reason) && retryCount < MAX_RETRIES) {
@@ -445,7 +496,11 @@ export async function handleChatSurfaceRequest(
startSseResponse();
const streamResult = streamSession.consumeUpstreamFinalPayload(fallbackData, fallbackText, reply.raw);
if (streamResult.status === 'failed') {
tokenRouter.recordFailure(selected.channel.id);
tokenRouter.recordFailure(selected.channel.id, {
status: 502,
errorText: streamResult.errorMessage,
modelName,
});
logProxy(
selected,
requestedModel,
@@ -462,7 +517,7 @@ export async function handleChatSurfaceRequest(
null,
successfulUpstreamPath,
clientContext,
logDownstreamApiKeyId ? downstreamApiKeyId : null,
downstreamApiKeyId,
);
return;
}
@@ -495,7 +550,11 @@ export async function handleChatSurfaceRequest(
const latency = Date.now() - startTime;
if (streamResult.status === 'failed') {
tokenRouter.recordFailure(selected.channel.id);
tokenRouter.recordFailure(selected.channel.id, {
status: 502,
errorText: streamResult.errorMessage,
modelName,
});
logProxy(
selected,
requestedModel,
@@ -512,7 +571,7 @@ export async function handleChatSurfaceRequest(
null,
successfulUpstreamPath,
clientContext,
logDownstreamApiKeyId ? downstreamApiKeyId : null,
downstreamApiKeyId,
);
return;
}
@@ -548,7 +607,7 @@ export async function handleChatSurfaceRequest(
resolvedUsage,
});
tokenRouter.recordSuccess(selected.channel.id, latency, estimatedCost);
tokenRouter.recordSuccess(selected.channel.id, latency, estimatedCost, modelName);
recordDownstreamCostUsage(request, estimatedCost);
logProxy(
selected,
@@ -566,7 +625,7 @@ export async function handleChatSurfaceRequest(
billingDetails,
successfulUpstreamPath,
clientContext,
logDownstreamApiKeyId ? downstreamApiKeyId : null,
downstreamApiKeyId,
);
return;
}
@@ -580,11 +639,15 @@ export async function handleChatSurfaceRequest(
upstreamData = collected.payload;
} else {
rawText = await upstream.text();
upstreamData = rawText;
try {
upstreamData = JSON.parse(rawText);
} catch {
if (looksLikeResponsesSseText(rawText)) {
upstreamData = collectResponsesFinalPayloadFromSseText(rawText, modelName).payload;
} else {
upstreamData = rawText;
try {
upstreamData = JSON.parse(rawText);
} catch {
upstreamData = rawText;
}
}
}
if (String(selected.site.platform || '').trim().toLowerCase() === 'gemini-cli') {
@@ -595,7 +658,11 @@ export async function handleChatSurfaceRequest(
const parsedUsage = parseProxyUsage(upstreamData);
const failure = detectProxyFailure({ rawText, usage: parsedUsage });
if (failure) {
tokenRouter.recordFailure(selected.channel.id);
tokenRouter.recordFailure(selected.channel.id, {
status: failure.status,
errorText: failure.reason,
modelName,
});
logProxy(
selected,
requestedModel,
@@ -612,7 +679,7 @@ export async function handleChatSurfaceRequest(
null,
successfulUpstreamPath,
clientContext,
logDownstreamApiKeyId ? downstreamApiKeyId : null,
downstreamApiKeyId,
);
if (shouldRetryProxyRequest(failure.status, failure.reason) && retryCount < MAX_RETRIES) {
@@ -656,7 +723,7 @@ export async function handleChatSurfaceRequest(
resolvedUsage,
});
tokenRouter.recordSuccess(selected.channel.id, latency, estimatedCost);
tokenRouter.recordSuccess(selected.channel.id, latency, estimatedCost, modelName);
recordDownstreamCostUsage(request, estimatedCost);
logProxy(
selected,
@@ -674,12 +741,15 @@ export async function handleChatSurfaceRequest(
billingDetails,
successfulUpstreamPath,
clientContext,
logDownstreamApiKeyId ? downstreamApiKeyId : null,
downstreamApiKeyId,
);
return reply.send(downstreamResponse);
} catch (err: any) {
tokenRouter.recordFailure(selected.channel.id);
tokenRouter.recordFailure(selected.channel.id, {
errorText: err?.message,
modelName,
});
logProxy(
selected,
requestedModel,
@@ -696,7 +766,7 @@ export async function handleChatSurfaceRequest(
null,
null,
clientContext,
logDownstreamApiKeyId ? downstreamApiKeyId : null,
downstreamApiKeyId,
);
if (retryCount < MAX_RETRIES) {
@@ -775,8 +845,6 @@ export async function handleClaudeCountTokensSurfaceRequest(
});
const downstreamPolicy = getDownstreamRoutingPolicy(request);
const downstreamApiKeyId = getProxyAuthContext(request)?.keyId ?? null;
const logDownstreamApiKeyId = downstreamApiKeyId !== null
&& await hasProxyLogDownstreamApiKeyIdColumn();
const excludeChannelIds: number[] = [];
let retryCount = 0;
@@ -883,7 +951,11 @@ export async function handleClaudeCountTokensSurfaceRequest(
}
if (!upstream.ok) {
tokenRouter.recordFailure(selected.channel.id);
tokenRouter.recordFailure(selected.channel.id, {
status: upstream.status,
errorText: typeof payload === 'string' ? payload : text,
modelName,
});
logProxy(
selected,
requestedModel,
@@ -900,7 +972,7 @@ export async function handleClaudeCountTokensSurfaceRequest(
null,
upstreamRequest.path,
clientContext,
logDownstreamApiKeyId ? downstreamApiKeyId : null,
downstreamApiKeyId,
);
if (shouldRetryProxyRequest(upstream.status, typeof payload === 'string' ? payload : text) && retryCount < MAX_RETRIES) {
retryCount += 1;
@@ -909,7 +981,7 @@ export async function handleClaudeCountTokensSurfaceRequest(
return reply.code(upstream.status).type(contentType).send(payload);
}
tokenRouter.recordSuccess(selected.channel.id, latency, 0);
tokenRouter.recordSuccess(selected.channel.id, latency, 0, modelName);
recordDownstreamCostUsage(request, 0);
logProxy(
selected,
@@ -927,11 +999,14 @@ export async function handleClaudeCountTokensSurfaceRequest(
null,
upstreamRequest.path,
clientContext,
logDownstreamApiKeyId ? downstreamApiKeyId : null,
downstreamApiKeyId,
);
return reply.code(upstream.status).type(contentType).send(payload);
} catch (error: any) {
tokenRouter.recordFailure(selected.channel.id);
tokenRouter.recordFailure(selected.channel.id, {
errorText: error?.message,
modelName,
});
logProxy(
selected,
requestedModel,
@@ -948,7 +1023,7 @@ export async function handleClaudeCountTokensSurfaceRequest(
null,
null,
clientContext,
logDownstreamApiKeyId ? downstreamApiKeyId : null,
downstreamApiKeyId,
);
if (retryCount < MAX_RETRIES) {
retryCount += 1;
@@ -994,11 +1069,11 @@ async function logProxy(
upstreamPath,
errorMessage,
});
await db.insert(schema.proxyLogs).values({
await insertProxyLog({
routeId: selected.channel.routeId,
channelId: selected.channel.id,
accountId: selected.account.id,
...(downstreamApiKeyId !== null ? { downstreamApiKeyId } : {}),
downstreamApiKeyId,
modelRequested,
modelActual: selected.actualModel,
status,
@@ -1008,11 +1083,15 @@ async function logProxy(
completionTokens,
totalTokens,
estimatedCost,
billingDetails: billingDetails ? JSON.stringify(billingDetails) : null,
billingDetails,
clientFamily: clientContext?.clientKind || null,
clientAppId: clientContext?.clientAppId || null,
clientAppName: clientContext?.clientAppName || null,
clientConfidence: clientContext?.clientConfidence || null,
errorMessage: normalizedErrorMessage,
retryCount,
createdAt,
}).run();
});
} catch (error) {
console.warn('[proxy/chat] failed to write proxy log', error);
}
src/server/proxy-core/surfaces/geminiSurface.ts
+84 -11
View File
@@ -16,6 +16,8 @@ import { executeEndpointFlow, type BuiltEndpointRequest } from '../../routes/pro
import { composeProxyLogMessage } from '../../routes/proxy/logPathMeta.js';
import {
buildUpstreamEndpointRequest,
recordUpstreamEndpointFailure,
recordUpstreamEndpointSuccess,
resolveUpstreamEndpointCandidates,
} from '../../routes/proxy/upstreamEndpoint.js';
import {
@@ -29,6 +31,9 @@ import {
wrapGeminiCliRequest,
} from '../../routes/proxy/geminiCliCompat.js';
import { dispatchRuntimeRequest } from '../../routes/proxy/runtimeExecutor.js';
import { detectDownstreamClientContext, type DownstreamClientContext } from '../../routes/proxy/downstreamClientContext.js';
import { insertProxyLog } from '../../services/proxyLogStore.js';
import { summarizeConversationFileInputsInOpenAiBody } from '../capabilities/conversationFileCapabilities.js';
const MAX_RETRIES = 2;
const GEMINI_MODEL_PROBES = [
@@ -114,10 +119,10 @@ function resolveUpstreamPath(apiVersion: string, modelActionPath: string): strin
return `/${normalizedVersion}/${normalizedAction}`;
}
function hasDownstreamModelRestrictions(policy: { supportedModels?: unknown; allowedRouteIds?: unknown }): boolean {
function hasDownstreamModelRestrictions(policy: { supportedModels?: unknown; allowedRouteIds?: unknown; denyAllWhenEmpty?: unknown }): boolean {
const supportedModels = Array.isArray(policy.supportedModels) ? policy.supportedModels : [];
const allowedRouteIds = Array.isArray(policy.allowedRouteIds) ? policy.allowedRouteIds : [];
return supportedModels.length > 0 || allowedRouteIds.length > 0;
return supportedModels.length > 0 || allowedRouteIds.length > 0 || policy.denyAllWhenEmpty === true;
}
function extractGeminiListedModelName(item: unknown): string {
@@ -203,6 +208,7 @@ async function logProxy(
retryCount: number,
downstreamPath: string,
upstreamPath: string | null,
clientContext: DownstreamClientContext | null = null,
promptTokens = 0,
completionTokens = 0,
totalTokens = 0,
@@ -210,11 +216,16 @@ async function logProxy(
try {
const createdAt = formatUtcSqlDateTime(new Date());
const normalizedErrorMessage = composeProxyLogMessage({
clientKind: clientContext?.clientKind && clientContext.clientKind !== 'generic'
? clientContext.clientKind
: null,
sessionId: clientContext?.sessionId || null,
traceHint: clientContext?.traceHint || null,
downstreamPath,
upstreamPath,
errorMessage,
});
await db.insert(schema.proxyLogs).values({
await insertProxyLog({
routeId: selected.channel.routeId,
channelId: selected.channel.id,
accountId: selected.account.id,
@@ -227,11 +238,14 @@ async function logProxy(
completionTokens,
totalTokens,
estimatedCost: 0,
billingDetails: null,
clientFamily: clientContext?.clientKind || null,
clientAppId: clientContext?.clientAppId || null,
clientAppName: clientContext?.clientAppName || null,
clientConfidence: clientContext?.clientConfidence || null,
errorMessage: normalizedErrorMessage,
retryCount,
createdAt,
}).run();
});
} catch (error) {
console.warn('[proxy/gemini] failed to write proxy log', error);
}
@@ -285,7 +299,10 @@ export async function geminiProxyRoute(app: FastifyInstance) {
lastStatus = upstream.status;
lastText = text;
lastContentType = upstream.headers.get('content-type') || 'application/json';
await tokenRouter.recordFailure?.(selected.channel.id);
await tokenRouter.recordFailure?.(selected.channel.id, {
status: upstream.status,
errorText: text,
});
if (retryCount < MAX_RETRIES) {
retryCount += 1;
continue;
@@ -300,7 +317,9 @@ export async function geminiProxyRoute(app: FastifyInstance) {
return reply.code(upstream.status).type(upstream.headers.get('content-type') || 'application/json').send(text);
}
} catch (error) {
await tokenRouter.recordFailure?.(selected.channel.id);
await tokenRouter.recordFailure?.(selected.channel.id, {
errorText: error instanceof Error ? error.message : 'Gemini upstream request failed',
});
lastStatus = 502;
lastContentType = 'application/json';
lastText = JSON.stringify({
@@ -355,6 +374,11 @@ export async function geminiProxyRoute(app: FastifyInstance) {
const policy = getDownstreamRoutingPolicy(request);
const downstreamPath = resolveDownstreamPath(request);
const clientContext = detectDownstreamClientContext({
downstreamPath,
headers: request.headers as Record<string, unknown>,
body: request.body,
});
const excludeChannelIds: number[] = [];
let retryCount = 0;
let lastStatus = 503;
@@ -394,7 +418,10 @@ export async function geminiProxyRoute(app: FastifyInstance) {
type: 'server_error',
},
});
await tokenRouter.recordFailure?.(selected.channel.id);
await tokenRouter.recordFailure?.(selected.channel.id, {
status: 500,
errorText: 'Gemini CLI OAuth project is missing',
});
if (retryCount < MAX_RETRIES) {
retryCount += 1;
continue;
@@ -478,7 +505,10 @@ export async function geminiProxyRoute(app: FastifyInstance) {
lastStatus = upstream.status;
lastContentType = contentType;
lastText = await upstream.text();
await tokenRouter.recordFailure?.(selected.channel.id);
await tokenRouter.recordFailure?.(selected.channel.id, {
status: upstream.status,
errorText: lastText,
});
await logProxy(
selected,
requestedModel,
@@ -489,6 +519,7 @@ export async function geminiProxyRoute(app: FastifyInstance) {
retryCount,
downstreamPath,
upstreamPath,
clientContext,
);
if (retryCount < MAX_RETRIES) {
retryCount += 1;
@@ -523,6 +554,7 @@ export async function geminiProxyRoute(app: FastifyInstance) {
retryCount,
downstreamPath,
upstreamPath,
clientContext,
);
reply.raw.end();
return;
@@ -572,6 +604,7 @@ export async function geminiProxyRoute(app: FastifyInstance) {
retryCount,
downstreamPath,
upstreamPath,
clientContext,
parsedUsage.promptTokens,
parsedUsage.completionTokens,
parsedUsage.totalTokens,
@@ -607,6 +640,7 @@ export async function geminiProxyRoute(app: FastifyInstance) {
retryCount,
downstreamPath,
upstreamPath,
clientContext,
parsedUsage.promptTokens,
parsedUsage.completionTokens,
parsedUsage.totalTokens,
@@ -629,6 +663,7 @@ export async function geminiProxyRoute(app: FastifyInstance) {
retryCount,
downstreamPath,
upstreamPath,
clientContext,
);
return reply.code(upstream.status).type(contentType || 'application/json').send(text);
}
@@ -651,6 +686,8 @@ export async function geminiProxyRoute(app: FastifyInstance) {
modelName: actualModel,
stream: isStreamAction,
});
const conversationFileSummary = summarizeConversationFileInputsInOpenAiBody(openAiBody);
const hasNonImageFileInput = conversationFileSummary.hasDocument;
const endpointCandidates = await resolveUpstreamEndpointCandidates(
{
site: selected.site,
@@ -659,7 +696,21 @@ export async function geminiProxyRoute(app: FastifyInstance) {
actualModel,
'openai',
requestedModel,
{
hasNonImageFileInput,
conversationFileSummary,
},
);
const endpointRuntimeContext = {
siteId: selected.site.id,
modelName: actualModel,
downstreamFormat: 'openai' as const,
requestedModelHint: requestedModel,
requestCapabilities: {
hasNonImageFileInput,
conversationFileSummary,
},
};
const buildEndpointRequest = (
endpoint: 'chat' | 'messages' | 'responses',
requestOptions: { forceNormalizeClaudeBody?: boolean } = {},
@@ -721,6 +772,20 @@ export async function geminiProxyRoute(app: FastifyInstance) {
buildRequest: (endpoint) => buildEndpointRequest(endpoint),
dispatchRequest,
tryRecover: endpointStrategy.tryRecover,
onAttemptFailure: (ctx) => {
recordUpstreamEndpointFailure({
...endpointRuntimeContext,
endpoint: ctx.request.endpoint,
status: ctx.response.status,
errorText: ctx.rawErrText,
});
},
onAttemptSuccess: (ctx) => {
recordUpstreamEndpointSuccess({
...endpointRuntimeContext,
endpoint: ctx.request.endpoint,
});
},
shouldDowngrade: endpointStrategy.shouldDowngrade,
});
if (!endpointResult.ok) {
@@ -732,7 +797,10 @@ export async function geminiProxyRoute(app: FastifyInstance) {
type: 'upstream_error',
},
});
await tokenRouter.recordFailure?.(selected.channel.id);
await tokenRouter.recordFailure?.(selected.channel.id, {
status: endpointResult.status,
errorText: endpointResult.rawErrText || endpointResult.errText,
});
await logProxy(
selected,
requestedModel,
@@ -743,6 +811,7 @@ export async function geminiProxyRoute(app: FastifyInstance) {
retryCount,
downstreamPath,
null,
clientContext,
);
if (retryCount < MAX_RETRIES) {
retryCount += 1;
@@ -780,6 +849,7 @@ export async function geminiProxyRoute(app: FastifyInstance) {
retryCount,
downstreamPath,
upstreamPath,
clientContext,
parsedUsage.promptTokens,
parsedUsage.completionTokens,
parsedUsage.totalTokens,
@@ -798,7 +868,9 @@ export async function geminiProxyRoute(app: FastifyInstance) {
type: 'upstream_error',
},
});
await tokenRouter.recordFailure?.(selected.channel.id);
await tokenRouter.recordFailure?.(selected.channel.id, {
errorText: error instanceof Error ? error.message : 'Gemini upstream request failed',
});
await logProxy(
selected,
requestedModel,
@@ -809,6 +881,7 @@ export async function geminiProxyRoute(app: FastifyInstance) {
retryCount,
downstreamPath,
upstreamPath || null,
clientContext,
);
if (retryCount < MAX_RETRIES) {
retryCount += 1;
src/server/proxy-core/surfaces/openAiResponsesSurface.ts
+136 -65
View File
@@ -1,6 +1,5 @@
import { TextDecoder } from 'node:util';
import type { FastifyReply, FastifyRequest } from 'fastify';
import { db, hasProxyLogDownstreamApiKeyIdColumn, schema } from '../../db/index.js';
import { tokenRouter } from '../../services/tokenRouter.js';
import { refreshModelsAndRebuildRoutes } from '../../services/modelService.js';
import { reportProxyAllFailed, reportTokenExpired } from '../../services/alertService.js';
@@ -12,12 +11,15 @@ import { resolveChannelProxyUrl, withSiteRecordProxyRequestInit } from '../../se
import { openAiResponsesTransformer } from '../../transformers/openai/responses/index.js';
import {
buildUpstreamEndpointRequest,
recordUpstreamEndpointFailure,
recordUpstreamEndpointSuccess,
resolveUpstreamEndpointCandidates,
} from '../../routes/proxy/upstreamEndpoint.js';
import { ensureModelAllowedForDownstreamKey, getDownstreamRoutingPolicy, recordDownstreamCostUsage } from '../../routes/proxy/downstreamPolicy.js';
import { composeProxyLogMessage } from '../../routes/proxy/logPathMeta.js';
import { executeEndpointFlow, type BuiltEndpointRequest } from '../../routes/proxy/endpointFlow.js';
import { detectProxyFailure } from '../../routes/proxy/proxyFailureJudge.js';
import { buildUpstreamUrl } from '../../routes/proxy/upstreamUrl.js';
import { formatUtcSqlDateTime } from '../../services/localTimeService.js';
import { resolveProxyLogBilling } from '../../routes/proxy/proxyBilling.js';
import { getProxyAuthContext, getProxyResourceOwner } from '../../middleware/auth.js';
@@ -25,7 +27,6 @@ import { dispatchRuntimeRequest } from '../../routes/proxy/runtimeExecutor.js';
import { normalizeInputFileBlock } from '../../transformers/shared/inputFile.js';
import {
ProxyInputFileResolutionError,
hasNonImageFileInputInOpenAiBody,
resolveResponsesBodyInputFiles,
} from '../../services/proxyInputFileResolver.js';
import {
@@ -34,14 +35,23 @@ import {
import { getOauthInfoFromExtraConfig } from '../../services/oauth/oauthAccount.js';
import { recordOauthQuotaResetHint } from '../../services/oauth/quota.js';
import { refreshOauthAccessTokenSingleflight } from '../../services/oauth/refreshSingleflight.js';
import { collectResponsesFinalPayloadFromSse } from '../../routes/proxy/responsesSseFinal.js';
import {
collectResponsesFinalPayloadFromSse,
collectResponsesFinalPayloadFromSseText,
createSingleChunkStreamReader,
looksLikeResponsesSseText,
} from '../../routes/proxy/responsesSseFinal.js';
import {
createGeminiCliStreamReader,
unwrapGeminiCliPayload,
} from '../../routes/proxy/geminiCliCompat.js';
import { isCodexResponsesSurface } from '../cliProfiles/codexProfile.js';
import { detectCliProfile } from '../cliProfiles/registry.js';
import type { CliProfileId } from '../cliProfiles/types.js';
import {
summarizeConversationFileInputsInOpenAiBody,
summarizeConversationFileInputsInResponsesBody,
} from '../capabilities/conversationFileCapabilities.js';
import { detectDownstreamClientContext, type DownstreamClientContext } from '../../routes/proxy/downstreamClientContext.js';
import { insertProxyLog } from '../../services/proxyLogStore.js';
const MAX_RETRIES = 2;
@@ -127,25 +137,6 @@ function carriesResponsesFileUrlInput(value: unknown): boolean {
type UsageSummary = ReturnType<typeof parseProxyUsage>;
type DownstreamClientContext = {
clientKind: CliProfileId;
sessionId?: string;
traceHint?: string;
};
function detectDownstreamClientContext(input: {
downstreamPath: string;
headers?: Record<string, unknown>;
body?: unknown;
}): DownstreamClientContext {
const detected = detectCliProfile(input);
return {
clientKind: detected.id,
...(detected.sessionId ? { sessionId: detected.sessionId } : {}),
...(detected.traceHint ? { traceHint: detected.traceHint } : {}),
};
}
function deriveCodexExplicitSessionId(body: Record<string, unknown>): string | null {
const promptCacheKey = typeof body.prompt_cache_key === 'string'
? body.prompt_cache_key.trim()
@@ -188,8 +179,6 @@ export async function handleOpenAiResponsesSurfaceRequest(
if (!await ensureModelAllowedForDownstreamKey(request, reply, requestedModel)) return;
const downstreamPolicy = getDownstreamRoutingPolicy(request);
const downstreamApiKeyId = getProxyAuthContext(request)?.keyId ?? null;
const logDownstreamApiKeyId = downstreamApiKeyId !== null
&& await hasProxyLogDownstreamApiKeyIdColumn();
const excludeChannelIds: number[] = [];
let retryCount = 0;
@@ -243,17 +232,12 @@ export async function handleOpenAiResponsesSurfaceRequest(
isStream,
{ defaultEncryptedReasoningInclude },
);
const hasNonImageFileInput = hasNonImageFileInputInOpenAiBody(openAiBody);
const conversationFileSummary = summarizeConversationFileInputsInOpenAiBody(openAiBody);
const hasNonImageFileInput = conversationFileSummary.hasDocument;
const prefersNativeResponsesReasoning = wantsNativeResponsesReasoning(normalizedResponsesBody);
const requiresNativeResponsesFileUrl = carriesResponsesFileUrlInput(normalizedResponsesBody.input);
if (requiresNativeResponsesFileUrl && String(selected.site.platform || '').trim().toLowerCase() === 'claude') {
return reply.code(400).send({
error: {
message: 'Responses input_file.file_url requires an upstream /v1/responses endpoint; current site only supports /v1/messages.',
type: 'invalid_request_error',
},
});
}
const responsesConversationFileSummary = summarizeConversationFileInputsInResponsesBody(normalizedResponsesBody);
const requiresNativeResponsesFileUrl = responsesConversationFileSummary.hasRemoteDocumentUrl
|| carriesResponsesFileUrlInput(normalizedResponsesBody.input);
const endpointCandidates = await resolveUpstreamEndpointCandidates(
{
site: selected.site,
@@ -264,15 +248,24 @@ export async function handleOpenAiResponsesSurfaceRequest(
requestedModel,
{
hasNonImageFileInput,
conversationFileSummary,
wantsNativeResponsesReasoning: prefersNativeResponsesReasoning,
},
);
if (requiresNativeResponsesFileUrl) {
endpointCandidates.splice(0, endpointCandidates.length, 'responses');
}
if (endpointCandidates.length === 0) {
endpointCandidates.push('responses', 'chat', 'messages');
}
const endpointRuntimeContext = {
siteId: selected.site.id,
modelName,
downstreamFormat: 'responses' as const,
requestedModelHint: requestedModel,
requestCapabilities: {
hasNonImageFileInput,
conversationFileSummary,
wantsNativeResponsesReasoning: prefersNativeResponsesReasoning,
},
};
const buildProviderHeaders = () => (
buildOauthProviderHeaders({
extraConfig: typeof selected.account.extraConfig === 'string' ? selected.account.extraConfig : null,
@@ -339,7 +332,7 @@ export async function handleOpenAiResponsesSurfaceRequest(
extraConfig: refreshed.extraConfig ?? selected.account.extraConfig,
};
const refreshedRequest = buildEndpointRequest(ctx.request.endpoint);
const refreshedTargetUrl = `${selected.site.url}${refreshedRequest.path}`;
const refreshedTargetUrl = buildUpstreamUrl(selected.site.url, refreshedRequest.path);
const refreshedResponse = await dispatchRequest(refreshedRequest, refreshedTargetUrl);
if (refreshedResponse.ok) {
return {
@@ -367,6 +360,20 @@ export async function handleOpenAiResponsesSurfaceRequest(
buildRequest: (endpoint) => buildEndpointRequest(endpoint),
dispatchRequest,
tryRecover,
onAttemptFailure: (ctx) => {
recordUpstreamEndpointFailure({
...endpointRuntimeContext,
endpoint: ctx.request.endpoint,
status: ctx.response.status,
errorText: ctx.rawErrText,
});
},
onAttemptSuccess: (ctx) => {
recordUpstreamEndpointSuccess({
...endpointRuntimeContext,
endpoint: ctx.request.endpoint,
});
},
shouldDowngrade: endpointStrategy.shouldDowngrade,
onDowngrade: (ctx) => {
logProxy(
@@ -385,7 +392,7 @@ export async function handleOpenAiResponsesSurfaceRequest(
null,
null,
clientContext,
logDownstreamApiKeyId ? downstreamApiKeyId : null,
downstreamApiKeyId,
);
},
});
@@ -394,7 +401,11 @@ export async function handleOpenAiResponsesSurfaceRequest(
const status = endpointResult.status || 502;
const errText = endpointResult.errText || 'unknown error';
const rawErrText = endpointResult.rawErrText || errText;
tokenRouter.recordFailure(selected.channel.id);
tokenRouter.recordFailure(selected.channel.id, {
status,
errorText: rawErrText,
modelName,
});
logProxy(
selected,
requestedModel,
@@ -411,7 +422,7 @@ export async function handleOpenAiResponsesSurfaceRequest(
null,
null,
clientContext,
logDownstreamApiKeyId ? downstreamApiKeyId : null,
downstreamApiKeyId,
);
await recordOauthQuotaResetHint({
accountId: selected.account.id,
@@ -486,14 +497,14 @@ export async function handleOpenAiResponsesSurfaceRequest(
}
try {
tokenRouter.recordSuccess(selected.channel.id, latency, estimatedCost);
tokenRouter.recordSuccess(selected.channel.id, latency, estimatedCost, modelName);
recordDownstreamCostUsage(request, estimatedCost);
logProxy(
selected, requestedModel, 'success', 200, latency, null, retryCount, downstreamPath,
usageForLog.promptTokens, usageForLog.completionTokens, usageForLog.totalTokens, estimatedCost, billingDetails,
successfulUpstreamPath,
clientContext,
logDownstreamApiKeyId ? downstreamApiKeyId : null,
downstreamApiKeyId,
);
} catch (error) {
console.error('[responses] post-stream success logging failed:', error);
@@ -541,6 +552,39 @@ export async function handleOpenAiResponsesSurfaceRequest(
});
if (!upstreamContentType.includes('text/event-stream')) {
const rawText = await upstream.text();
if (looksLikeResponsesSseText(rawText)) {
startSseResponse();
const streamResult = await streamSession.run(
createSingleChunkStreamReader(rawText),
reply.raw,
);
const latency = Date.now() - startTime;
if (streamResult.status === 'failed') {
tokenRouter.recordFailure(selected.channel.id, modelName);
logProxy(
selected,
requestedModel,
'failed',
200,
latency,
streamResult.errorMessage,
retryCount,
downstreamPath,
parsedUsage.promptTokens,
parsedUsage.completionTokens,
parsedUsage.totalTokens,
0,
null,
successfulUpstreamPath,
clientContext,
downstreamApiKeyId,
);
return;
}
await finalizeStreamSuccess(parsedUsage, latency);
return;
}
let upstreamData: unknown = rawText;
try {
upstreamData = JSON.parse(rawText);
@@ -555,7 +599,11 @@ export async function handleOpenAiResponsesSurfaceRequest(
const latency = Date.now() - startTime;
const failure = detectProxyFailure({ rawText, usage: parsedUsage });
if (failure) {
tokenRouter.recordFailure(selected.channel.id);
tokenRouter.recordFailure(selected.channel.id, {
status: failure.status,
errorText: failure.reason,
modelName,
});
logProxy(
selected,
requestedModel,
@@ -572,7 +620,7 @@ export async function handleOpenAiResponsesSurfaceRequest(
null,
successfulUpstreamPath,
clientContext,
logDownstreamApiKeyId ? downstreamApiKeyId : null,
downstreamApiKeyId,
);
if (shouldRetryProxyRequest(failure.status, failure.reason) && retryCount < MAX_RETRIES) {
@@ -590,7 +638,11 @@ export async function handleOpenAiResponsesSurfaceRequest(
startSseResponse();
const streamResult = streamSession.consumeUpstreamFinalPayload(upstreamData, rawText, reply.raw);
if (streamResult.status === 'failed') {
tokenRouter.recordFailure(selected.channel.id);
tokenRouter.recordFailure(selected.channel.id, {
status: 502,
errorText: streamResult.errorMessage,
modelName,
});
logProxy(
selected,
requestedModel,
@@ -607,7 +659,7 @@ export async function handleOpenAiResponsesSurfaceRequest(
null,
successfulUpstreamPath,
clientContext,
logDownstreamApiKeyId ? downstreamApiKeyId : null,
downstreamApiKeyId,
);
return;
}
@@ -646,7 +698,11 @@ export async function handleOpenAiResponsesSurfaceRequest(
const latency = Date.now() - startTime;
if (streamResult.status === 'failed') {
tokenRouter.recordFailure(selected.channel.id);
tokenRouter.recordFailure(selected.channel.id, {
status: 502,
errorText: streamResult.errorMessage,
modelName,
});
logProxy(
selected,
requestedModel,
@@ -663,7 +719,7 @@ export async function handleOpenAiResponsesSurfaceRequest(
null,
successfulUpstreamPath,
clientContext,
logDownstreamApiKeyId ? downstreamApiKeyId : null,
downstreamApiKeyId,
);
return;
}
@@ -692,11 +748,15 @@ export async function handleOpenAiResponsesSurfaceRequest(
upstreamData = collected.payload;
} else {
rawText = await upstream.text();
upstreamData = rawText;
try {
upstreamData = JSON.parse(rawText);
} catch {
if (looksLikeResponsesSseText(rawText)) {
upstreamData = collectResponsesFinalPayloadFromSseText(rawText, modelName).payload;
} else {
upstreamData = rawText;
try {
upstreamData = JSON.parse(rawText);
} catch {
upstreamData = rawText;
}
}
}
if (String(selected.site.platform || '').trim().toLowerCase() === 'gemini-cli') {
@@ -706,7 +766,11 @@ export async function handleOpenAiResponsesSurfaceRequest(
const parsedUsage = parseProxyUsage(upstreamData);
const failure = detectProxyFailure({ rawText, usage: parsedUsage });
if (failure) {
tokenRouter.recordFailure(selected.channel.id);
tokenRouter.recordFailure(selected.channel.id, {
status: failure.status,
errorText: failure.reason,
modelName,
});
logProxy(
selected,
requestedModel,
@@ -723,7 +787,7 @@ export async function handleOpenAiResponsesSurfaceRequest(
null,
successfulUpstreamPath,
clientContext,
logDownstreamApiKeyId ? downstreamApiKeyId : null,
downstreamApiKeyId,
);
if (shouldRetryProxyRequest(failure.status, failure.reason) && retryCount < MAX_RETRIES) {
retryCount += 1;
@@ -770,18 +834,21 @@ export async function handleOpenAiResponsesSurfaceRequest(
resolvedUsage,
});
tokenRouter.recordSuccess(selected.channel.id, latency, estimatedCost);
tokenRouter.recordSuccess(selected.channel.id, latency, estimatedCost, modelName);
recordDownstreamCostUsage(request, estimatedCost);
logProxy(
selected, requestedModel, 'success', 200, latency, null, retryCount, downstreamPath,
resolvedUsage.promptTokens, resolvedUsage.completionTokens, resolvedUsage.totalTokens, estimatedCost, billingDetails,
successfulUpstreamPath,
clientContext,
logDownstreamApiKeyId ? downstreamApiKeyId : null,
downstreamApiKeyId,
);
return reply.send(downstreamData);
} catch (err: any) {
tokenRouter.recordFailure(selected.channel.id);
tokenRouter.recordFailure(selected.channel.id, {
errorText: err?.message,
modelName,
});
logProxy(
selected,
requestedModel,
@@ -798,7 +865,7 @@ export async function handleOpenAiResponsesSurfaceRequest(
null,
null,
clientContext,
logDownstreamApiKeyId ? downstreamApiKeyId : null,
downstreamApiKeyId,
);
if (retryCount < MAX_RETRIES) {
retryCount += 1;
@@ -845,11 +912,11 @@ async function logProxy(
upstreamPath,
errorMessage,
});
await db.insert(schema.proxyLogs).values({
await insertProxyLog({
routeId: selected.channel.routeId,
channelId: selected.channel.id,
accountId: selected.account.id,
...(downstreamApiKeyId !== null ? { downstreamApiKeyId } : {}),
downstreamApiKeyId,
modelRequested,
modelActual: selected.actualModel,
status,
@@ -859,11 +926,15 @@ async function logProxy(
completionTokens,
totalTokens,
estimatedCost,
billingDetails: billingDetails ? JSON.stringify(billingDetails) : null,
billingDetails,
clientFamily: clientContext?.clientKind || null,
clientAppId: clientContext?.clientAppId || null,
clientAppName: clientContext?.clientAppName || null,
clientConfidence: clientContext?.clientConfidence || null,
errorMessage: normalizedErrorMessage,
retryCount,
createdAt,
}).run();
});
} catch (error) {
console.warn('[proxy/responses] failed to write proxy log', error);
}