Nvidia/BossAI
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[codex] improve proxy stability, routing, and site subscription visibility (#194)

Browse Source 
* wip: preserve local changes before upstream merge

* Improve proxy routing stability and add stable-first strategy
This commit is contained in:
Bainily
2026-03-20 16:18:28 +08:00
committed by GitHub
parent e2595bcab3
commit e9c1bd898e
37 changed files with 3424 additions and 236 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/server/proxy-core/surfaces/chatSurface.ts
+67 -12
View File
@@ -12,6 +12,8 @@ import { type DownstreamFormat } from '../../transformers/shared/normalized.js';
import {
buildClaudeCountTokensUpstreamRequest,
buildUpstreamEndpointRequest,
recordUpstreamEndpointFailure,
recordUpstreamEndpointSuccess,
resolveUpstreamEndpointCandidates,
} from '../../routes/proxy/upstreamEndpoint.js';
import {
@@ -155,6 +157,15 @@ export async function handleChatSurfaceRequest(
},
),
];
const endpointRuntimeContext = {
siteId: selected.site.id,
modelName,
downstreamFormat,
requestedModelHint: requestedModel,
requestCapabilities: {
hasNonImageFileInput,
},
};
const buildProviderHeaders = () => (
buildOauthProviderHeaders({
extraConfig: typeof selected.account.extraConfig === 'string' ? selected.account.extraConfig : null,
@@ -257,6 +268,20 @@ export async function handleChatSurfaceRequest(
buildRequest: (endpoint) => buildEndpointRequest(endpoint),
dispatchRequest,
tryRecover,
onAttemptFailure: (ctx) => {
recordUpstreamEndpointFailure({
...endpointRuntimeContext,
endpoint: ctx.request.endpoint,
status: ctx.response.status,
errorText: ctx.rawErrText,
});
},
onAttemptSuccess: (ctx) => {
recordUpstreamEndpointSuccess({
...endpointRuntimeContext,
endpoint: ctx.request.endpoint,
});
},
shouldDowngrade: endpointStrategy.shouldDowngrade,
onDowngrade: (ctx) => {
logProxy(
@@ -284,7 +309,11 @@ export async function handleChatSurfaceRequest(
const status = endpointResult.status || 502;
const errText = endpointResult.errText || 'unknown error';
const rawErrText = endpointResult.rawErrText || errText;
tokenRouter.recordFailure(selected.channel.id);
tokenRouter.recordFailure(selected.channel.id, {
status,
errorText: rawErrText,
modelName,
});
logProxy(
selected,
requestedModel,
@@ -387,7 +416,7 @@ export async function handleChatSurfaceRequest(
);
const latency = Date.now() - startTime;
if (streamResult.status === 'failed') {
tokenRouter.recordFailure(selected.channel.id);
tokenRouter.recordFailure(selected.channel.id, modelName);
logProxy(
selected,
requestedModel,
@@ -423,7 +452,11 @@ export async function handleChatSurfaceRequest(
const latency = Date.now() - startTime;
const failure = detectProxyFailure({ rawText, usage: parsedUsage });
if (failure) {
tokenRouter.recordFailure(selected.channel.id);
tokenRouter.recordFailure(selected.channel.id, {
status: failure.status,
errorText: failure.reason,
modelName,
});
logProxy(
selected,
requestedModel,
@@ -461,7 +494,11 @@ export async function handleChatSurfaceRequest(
startSseResponse();
const streamResult = streamSession.consumeUpstreamFinalPayload(fallbackData, fallbackText, reply.raw);
if (streamResult.status === 'failed') {
tokenRouter.recordFailure(selected.channel.id);
tokenRouter.recordFailure(selected.channel.id, {
status: 502,
errorText: streamResult.errorMessage,
modelName,
});
logProxy(
selected,
requestedModel,
@@ -511,7 +548,11 @@ export async function handleChatSurfaceRequest(
const latency = Date.now() - startTime;
if (streamResult.status === 'failed') {
tokenRouter.recordFailure(selected.channel.id);
tokenRouter.recordFailure(selected.channel.id, {
status: 502,
errorText: streamResult.errorMessage,
modelName,
});
logProxy(
selected,
requestedModel,
@@ -564,7 +605,7 @@ export async function handleChatSurfaceRequest(
resolvedUsage,
});
tokenRouter.recordSuccess(selected.channel.id, latency, estimatedCost);
tokenRouter.recordSuccess(selected.channel.id, latency, estimatedCost, modelName);
recordDownstreamCostUsage(request, estimatedCost);
logProxy(
selected,
@@ -615,7 +656,11 @@ export async function handleChatSurfaceRequest(
const parsedUsage = parseProxyUsage(upstreamData);
const failure = detectProxyFailure({ rawText, usage: parsedUsage });
if (failure) {
tokenRouter.recordFailure(selected.channel.id);
tokenRouter.recordFailure(selected.channel.id, {
status: failure.status,
errorText: failure.reason,
modelName,
});
logProxy(
selected,
requestedModel,
@@ -676,7 +721,7 @@ export async function handleChatSurfaceRequest(
resolvedUsage,
});
tokenRouter.recordSuccess(selected.channel.id, latency, estimatedCost);
tokenRouter.recordSuccess(selected.channel.id, latency, estimatedCost, modelName);
recordDownstreamCostUsage(request, estimatedCost);
logProxy(
selected,
@@ -699,7 +744,10 @@ export async function handleChatSurfaceRequest(
return reply.send(downstreamResponse);
} catch (err: any) {
tokenRouter.recordFailure(selected.channel.id);
tokenRouter.recordFailure(selected.channel.id, {
errorText: err?.message,
modelName,
});
logProxy(
selected,
requestedModel,
@@ -901,7 +949,11 @@ export async function handleClaudeCountTokensSurfaceRequest(
}
if (!upstream.ok) {
tokenRouter.recordFailure(selected.channel.id);
tokenRouter.recordFailure(selected.channel.id, {
status: upstream.status,
errorText: typeof payload === 'string' ? payload : text,
modelName,
});
logProxy(
selected,
requestedModel,
@@ -927,7 +979,7 @@ export async function handleClaudeCountTokensSurfaceRequest(
return reply.code(upstream.status).type(contentType).send(payload);
}
tokenRouter.recordSuccess(selected.channel.id, latency, 0);
tokenRouter.recordSuccess(selected.channel.id, latency, 0, modelName);
recordDownstreamCostUsage(request, 0);
logProxy(
selected,
@@ -949,7 +1001,10 @@ export async function handleClaudeCountTokensSurfaceRequest(
);
return reply.code(upstream.status).type(contentType).send(payload);
} catch (error: any) {
tokenRouter.recordFailure(selected.channel.id);
tokenRouter.recordFailure(selected.channel.id, {
errorText: error?.message,
modelName,
});
logProxy(
selected,
requestedModel,
src/server/proxy-core/surfaces/geminiSurface.ts
+44 -6
View File
@@ -16,6 +16,8 @@ import { executeEndpointFlow, type BuiltEndpointRequest } from '../../routes/pro
import { composeProxyLogMessage } from '../../routes/proxy/logPathMeta.js';
import {
buildUpstreamEndpointRequest,
recordUpstreamEndpointFailure,
recordUpstreamEndpointSuccess,
resolveUpstreamEndpointCandidates,
} from '../../routes/proxy/upstreamEndpoint.js';
import {
@@ -296,7 +298,10 @@ export async function geminiProxyRoute(app: FastifyInstance) {
lastStatus = upstream.status;
lastText = text;
lastContentType = upstream.headers.get('content-type') || 'application/json';
await tokenRouter.recordFailure?.(selected.channel.id);
await tokenRouter.recordFailure?.(selected.channel.id, {
status: upstream.status,
errorText: text,
});
if (retryCount < MAX_RETRIES) {
retryCount += 1;
continue;
@@ -311,7 +316,9 @@ export async function geminiProxyRoute(app: FastifyInstance) {
return reply.code(upstream.status).type(upstream.headers.get('content-type') || 'application/json').send(text);
}
} catch (error) {
await tokenRouter.recordFailure?.(selected.channel.id);
await tokenRouter.recordFailure?.(selected.channel.id, {
errorText: error instanceof Error ? error.message : 'Gemini upstream request failed',
});
lastStatus = 502;
lastContentType = 'application/json';
lastText = JSON.stringify({
@@ -410,7 +417,10 @@ export async function geminiProxyRoute(app: FastifyInstance) {
type: 'server_error',
},
});
await tokenRouter.recordFailure?.(selected.channel.id);
await tokenRouter.recordFailure?.(selected.channel.id, {
status: 500,
errorText: 'Gemini CLI OAuth project is missing',
});
if (retryCount < MAX_RETRIES) {
retryCount += 1;
continue;
@@ -494,7 +504,10 @@ export async function geminiProxyRoute(app: FastifyInstance) {
lastStatus = upstream.status;
lastContentType = contentType;
lastText = await upstream.text();
await tokenRouter.recordFailure?.(selected.channel.id);
await tokenRouter.recordFailure?.(selected.channel.id, {
status: upstream.status,
errorText: lastText,
});
await logProxy(
selected,
requestedModel,
@@ -681,6 +694,12 @@ export async function geminiProxyRoute(app: FastifyInstance) {
'openai',
requestedModel,
);
const endpointRuntimeContext = {
siteId: selected.site.id,
modelName: actualModel,
downstreamFormat: 'openai' as const,
requestedModelHint: requestedModel,
};
const buildEndpointRequest = (
endpoint: 'chat' | 'messages' | 'responses',
requestOptions: { forceNormalizeClaudeBody?: boolean } = {},
@@ -742,6 +761,20 @@ export async function geminiProxyRoute(app: FastifyInstance) {
buildRequest: (endpoint) => buildEndpointRequest(endpoint),
dispatchRequest,
tryRecover: endpointStrategy.tryRecover,
onAttemptFailure: (ctx) => {
recordUpstreamEndpointFailure({
...endpointRuntimeContext,
endpoint: ctx.request.endpoint,
status: ctx.response.status,
errorText: ctx.rawErrText,
});
},
onAttemptSuccess: (ctx) => {
recordUpstreamEndpointSuccess({
...endpointRuntimeContext,
endpoint: ctx.request.endpoint,
});
},
shouldDowngrade: endpointStrategy.shouldDowngrade,
});
if (!endpointResult.ok) {
@@ -753,7 +786,10 @@ export async function geminiProxyRoute(app: FastifyInstance) {
type: 'upstream_error',
},
});
await tokenRouter.recordFailure?.(selected.channel.id);
await tokenRouter.recordFailure?.(selected.channel.id, {
status: endpointResult.status,
errorText: endpointResult.rawErrText || endpointResult.errText,
});
await logProxy(
selected,
requestedModel,
@@ -821,7 +857,9 @@ export async function geminiProxyRoute(app: FastifyInstance) {
type: 'upstream_error',
},
});
await tokenRouter.recordFailure?.(selected.channel.id);
await tokenRouter.recordFailure?.(selected.channel.id, {
errorText: error instanceof Error ? error.message : 'Gemini upstream request failed',
});
await logProxy(
selected,
requestedModel,
src/server/proxy-core/surfaces/openAiResponsesSurface.ts
+58 -9
View File
@@ -11,6 +11,8 @@ import { resolveProxyUrlForSite, withSiteRecordProxyRequestInit } from '../../se
import { openAiResponsesTransformer } from '../../transformers/openai/responses/index.js';
import {
buildUpstreamEndpointRequest,
recordUpstreamEndpointFailure,
recordUpstreamEndpointSuccess,
resolveUpstreamEndpointCandidates,
} from '../../routes/proxy/upstreamEndpoint.js';
import { ensureModelAllowedForDownstreamKey, getDownstreamRoutingPolicy, recordDownstreamCostUsage } from '../../routes/proxy/downstreamPolicy.js';
@@ -263,6 +265,16 @@ export async function handleOpenAiResponsesSurfaceRequest(
if (endpointCandidates.length === 0) {
endpointCandidates.push('responses', 'chat', 'messages');
}
const endpointRuntimeContext = {
siteId: selected.site.id,
modelName,
downstreamFormat: 'responses' as const,
requestedModelHint: requestedModel,
requestCapabilities: {
hasNonImageFileInput,
wantsNativeResponsesReasoning: prefersNativeResponsesReasoning,
},
};
const buildProviderHeaders = () => (
buildOauthProviderHeaders({
extraConfig: typeof selected.account.extraConfig === 'string' ? selected.account.extraConfig : null,
@@ -357,6 +369,20 @@ export async function handleOpenAiResponsesSurfaceRequest(
buildRequest: (endpoint) => buildEndpointRequest(endpoint),
dispatchRequest,
tryRecover,
onAttemptFailure: (ctx) => {
recordUpstreamEndpointFailure({
...endpointRuntimeContext,
endpoint: ctx.request.endpoint,
status: ctx.response.status,
errorText: ctx.rawErrText,
});
},
onAttemptSuccess: (ctx) => {
recordUpstreamEndpointSuccess({
...endpointRuntimeContext,
endpoint: ctx.request.endpoint,
});
},
shouldDowngrade: endpointStrategy.shouldDowngrade,
onDowngrade: (ctx) => {
logProxy(
@@ -384,7 +410,11 @@ export async function handleOpenAiResponsesSurfaceRequest(
const status = endpointResult.status || 502;
const errText = endpointResult.errText || 'unknown error';
const rawErrText = endpointResult.rawErrText || errText;
tokenRouter.recordFailure(selected.channel.id);
tokenRouter.recordFailure(selected.channel.id, {
status,
errorText: rawErrText,
modelName,
});
logProxy(
selected,
requestedModel,
@@ -476,7 +506,7 @@ export async function handleOpenAiResponsesSurfaceRequest(
}
try {
tokenRouter.recordSuccess(selected.channel.id, latency, estimatedCost);
tokenRouter.recordSuccess(selected.channel.id, latency, estimatedCost, modelName);
recordDownstreamCostUsage(request, estimatedCost);
logProxy(
selected, requestedModel, 'success', 200, latency, null, retryCount, downstreamPath,
@@ -539,7 +569,7 @@ export async function handleOpenAiResponsesSurfaceRequest(
);
const latency = Date.now() - startTime;
if (streamResult.status === 'failed') {
tokenRouter.recordFailure(selected.channel.id);
tokenRouter.recordFailure(selected.channel.id, modelName);
logProxy(
selected,
requestedModel,
@@ -578,7 +608,11 @@ export async function handleOpenAiResponsesSurfaceRequest(
const latency = Date.now() - startTime;
const failure = detectProxyFailure({ rawText, usage: parsedUsage });
if (failure) {
tokenRouter.recordFailure(selected.channel.id);
tokenRouter.recordFailure(selected.channel.id, {
status: failure.status,
errorText: failure.reason,
modelName,
});
logProxy(
selected,
requestedModel,
@@ -613,7 +647,11 @@ export async function handleOpenAiResponsesSurfaceRequest(
startSseResponse();
const streamResult = streamSession.consumeUpstreamFinalPayload(upstreamData, rawText, reply.raw);
if (streamResult.status === 'failed') {
tokenRouter.recordFailure(selected.channel.id);
tokenRouter.recordFailure(selected.channel.id, {
status: 502,
errorText: streamResult.errorMessage,
modelName,
});
logProxy(
selected,
requestedModel,
@@ -669,7 +707,11 @@ export async function handleOpenAiResponsesSurfaceRequest(
const latency = Date.now() - startTime;
if (streamResult.status === 'failed') {
tokenRouter.recordFailure(selected.channel.id);
tokenRouter.recordFailure(selected.channel.id, {
status: 502,
errorText: streamResult.errorMessage,
modelName,
});
logProxy(
selected,
requestedModel,
@@ -733,7 +775,11 @@ export async function handleOpenAiResponsesSurfaceRequest(
const parsedUsage = parseProxyUsage(upstreamData);
const failure = detectProxyFailure({ rawText, usage: parsedUsage });
if (failure) {
tokenRouter.recordFailure(selected.channel.id);
tokenRouter.recordFailure(selected.channel.id, {
status: failure.status,
errorText: failure.reason,
modelName,
});
logProxy(
selected,
requestedModel,
@@ -797,7 +843,7 @@ export async function handleOpenAiResponsesSurfaceRequest(
resolvedUsage,
});
tokenRouter.recordSuccess(selected.channel.id, latency, estimatedCost);
tokenRouter.recordSuccess(selected.channel.id, latency, estimatedCost, modelName);
recordDownstreamCostUsage(request, estimatedCost);
logProxy(
selected, requestedModel, 'success', 200, latency, null, retryCount, downstreamPath,
@@ -808,7 +854,10 @@ export async function handleOpenAiResponsesSurfaceRequest(
);
return reply.send(downstreamData);
} catch (err: any) {
tokenRouter.recordFailure(selected.channel.id);
tokenRouter.recordFailure(selected.channel.id, {
errorText: err?.message,
modelName,
});
logProxy(
selected,
requestedModel,