LaPaoPao/BoosAPI
1
0
Fork 0
forked from boos/BoosAPI
Code Pull Requests 1 Activity

feat(server): add proactive channel probes and load-aware stable routing / 服务端新增主动探活与负载感知稳定路由 (#330)

Browse Source 
* wip: preserve local changes before upstream merge

* Improve proxy routing stability and add stable-first strategy

* Sync explicit-group routing strategy to eligible source routes

* Fix proxy retry and runtime health review issues

* fix: make proxy channel retry attempts configurable

* fix: harden proxy retry recovery and bookkeeping

* fix: address pr 257 review comments

* feat(proxy): add sticky session channel coordination

* feat(server): add proactive channel probes and load-aware stable routing

* test(server): stabilize session load routing selection assertion

* fix(proxy): harden codex forwarding stability

* fix(router): rotate stable-first channels by healthy order

* fix(server): address probe review feedback

* fix(server): address probe review follow-ups

* fix(ci): address CodeQL and core test failures

* feat(server): refine stable-first routing recovery and observation

* Finish PR 330 settings and review fixes

* Fix chat site endpoint CI mock

* Address latest PR 330 review comments

---------

Co-authored-by: Cita <juricek.chen@gmail.com>
This commit is contained in:
Bainily
2026-04-02 09:49:04 +08:00
committed by GitHub
parent 1133846998
commit 596d1b93cb
75 changed files with 6365 additions and 507 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/server/proxy-core/surfaces/chatSurface.ts
+15 -2
View File
@@ -9,7 +9,7 @@ import {
buildClaudeCountTokensUpstreamRequest,
buildUpstreamEndpointRequest,
resolveUpstreamEndpointCandidates,
} from '../../routes/proxy/upstreamEndpoint.js';
} from '../../services/upstreamEndpointRuntime.js';
import {
getUpstreamEndpointRuntimeStateSnapshot,
recordUpstreamEndpointFailure,
@@ -20,10 +20,11 @@ import {
getDownstreamRoutingPolicy,
recordDownstreamCostUsage,
} from '../../routes/proxy/downstreamPolicy.js';
import { executeEndpointFlow, type BuiltEndpointRequest } from '../../routes/proxy/endpointFlow.js';
import { executeEndpointFlow, type BuiltEndpointRequest } from '../orchestration/endpointFlow.js';
import { detectProxyFailure } from '../../routes/proxy/proxyFailureJudge.js';
import { openAiChatTransformer } from '../../transformers/openai/chat/index.js';
import { anthropicMessagesTransformer } from '../../transformers/anthropic/messages/index.js';
import { shouldPreferResponsesForAnthropicContinuation } from '../../transformers/anthropic/messages/compatibility.js';
import { getProxyAuthContext, getProxyResourceOwner } from '../../middleware/auth.js';
import {
ProxyInputFileResolutionError,
@@ -47,6 +48,7 @@ import { summarizeConversationFileInputsInOpenAiBody } from '../capabilities/con
import { getRuntimeResponseReader, readRuntimeResponseText } from '../executors/types.js';
import { detectDownstreamClientContext } from '../../routes/proxy/downstreamClientContext.js';
import { getProxyMaxChannelRetries } from '../../services/proxyChannelRetry.js';
import { shouldAbortSameSiteEndpointFallback } from '../../services/proxyRetryPolicy.js';
import {
acquireSurfaceChannelLease,
bindSurfaceStickyChannel,
@@ -170,6 +172,10 @@ export async function handleChatSurfaceRequest(
}
const conversationFileSummary = summarizeConversationFileInputsInOpenAiBody(resolvedOpenAiBody);
const hasNonImageFileInput = conversationFileSummary.hasDocument;
const wantsContinuationAwareResponses = (
downstreamFormat === 'claude'
&& shouldPreferResponsesForAnthropicContinuation(claudeOriginalBody)
);
const codexSessionCacheKey = deriveCodexSessionCacheKey({
downstreamFormat,
body: downstreamFormat === 'claude' ? claudeOriginalBody : request.body,
@@ -283,6 +289,7 @@ export async function handleChatSurfaceRequest(
{
hasNonImageFileInput,
conversationFileSummary,
wantsContinuationAwareResponses,
},
),
];
@@ -297,6 +304,7 @@ export async function handleChatSurfaceRequest(
requestCapabilities: {
hasNonImageFileInput,
conversationFileSummary,
wantsContinuationAwareResponses,
},
};
await safeUpdateSurfaceProxyDebugCandidates(debugTrace, {
@@ -309,6 +317,7 @@ export async function handleChatSurfaceRequest(
stickyPreferredChannelId,
oauthProvider: oauth?.provider || null,
isCodexSite,
wantsContinuationAwareResponses,
},
});
const buildProviderHeaders = () => (
@@ -389,6 +398,10 @@ export async function handleChatSurfaceRequest(
buildRequest: (endpoint) => buildEndpointRequest(endpoint),
dispatchRequest,
tryRecover,
shouldAbortRemainingEndpoints: (ctx) => shouldAbortSameSiteEndpointFallback(
ctx.response.status,
ctx.rawErrText || ctx.errText,
),
onAttemptFailure: async (ctx) => {
const memoryWrite = recordUpstreamEndpointFailure({
...endpointRuntimeContext,
src/server/proxy-core/surfaces/geminiSurface.ts
+8 -3
View File
@@ -14,12 +14,12 @@ import { refreshOauthAccessTokenSingleflight } from '../../services/oauth/refres
import { resolveChannelProxyUrl, withSiteRecordProxyRequestInit } from '../../services/siteProxy.js';
import * as routeRefreshWorkflow from '../../services/routeRefreshWorkflow.js';
import { getDownstreamRoutingPolicy } from '../../routes/proxy/downstreamPolicy.js';
import { executeEndpointFlow, type BuiltEndpointRequest } from '../../routes/proxy/endpointFlow.js';
import { executeEndpointFlow, type BuiltEndpointRequest } from '../orchestration/endpointFlow.js';
import { composeProxyLogMessage } from '../../routes/proxy/logPathMeta.js';
import {
buildUpstreamEndpointRequest,
resolveUpstreamEndpointCandidates,
} from '../../routes/proxy/upstreamEndpoint.js';
} from '../../services/upstreamEndpointRuntime.js';
import {
getUpstreamEndpointRuntimeStateSnapshot,
recordUpstreamEndpointFailure,
@@ -35,12 +35,13 @@ import {
unwrapGeminiCliPayload,
wrapGeminiCliRequest,
} from '../../routes/proxy/geminiCliCompat.js';
import { dispatchRuntimeRequest } from '../../routes/proxy/runtimeExecutor.js';
import { dispatchRuntimeRequest } from '../../services/runtimeDispatch.js';
import { detectDownstreamClientContext, type DownstreamClientContext } from '../../routes/proxy/downstreamClientContext.js';
import { insertProxyLog } from '../../services/proxyLogStore.js';
import { summarizeConversationFileInputsInOpenAiBody } from '../capabilities/conversationFileCapabilities.js';
import { getRuntimeResponseReader, readRuntimeResponseText } from '../executors/types.js';
import { getProxyMaxChannelRetries } from '../../services/proxyChannelRetry.js';
import { shouldAbortSameSiteEndpointFallback } from '../../services/proxyRetryPolicy.js';
import {
buildSurfaceProxyDebugResponseHeaders,
captureSurfaceProxyDebugSuccessResponseBody,
@@ -1224,6 +1225,10 @@ export async function geminiProxyRoute(app: FastifyInstance) {
buildRequest: (endpoint) => buildEndpointRequest(endpoint),
dispatchRequest,
tryRecover: endpointStrategy.tryRecover,
shouldAbortRemainingEndpoints: (ctx) => shouldAbortSameSiteEndpointFallback(
ctx.response.status,
ctx.rawErrText || ctx.errText,
),
onAttemptFailure: async (ctx) => {
const memoryWrite = recordUpstreamEndpointFailure({
...endpointRuntimeContext,
src/server/proxy-core/surfaces/openAiResponsesSurface.ts
+97 -4
View File
@@ -4,17 +4,24 @@ import { config } from '../../config.js';
import { reportProxyAllFailed } from '../../services/alertService.js';
import { hasProxyUsagePayload, mergeProxyUsage, parseProxyUsage } from '../../services/proxyUsageParser.js';
import { openAiResponsesTransformer } from '../../transformers/openai/responses/index.js';
import {
extractResponsesTerminalResponseId,
isResponsesPreviousResponseNotFoundError,
shouldInferResponsesPreviousResponseId,
stripResponsesPreviousResponseId,
withResponsesPreviousResponseId,
} from '../../transformers/openai/responses/continuation.js';
import {
buildUpstreamEndpointRequest,
resolveUpstreamEndpointCandidates,
} from '../../routes/proxy/upstreamEndpoint.js';
} from '../../services/upstreamEndpointRuntime.js';
import {
getUpstreamEndpointRuntimeStateSnapshot,
recordUpstreamEndpointFailure,
recordUpstreamEndpointSuccess,
} from '../../services/upstreamEndpointRuntimeMemory.js';
import { ensureModelAllowedForDownstreamKey, getDownstreamRoutingPolicy, recordDownstreamCostUsage } from '../../routes/proxy/downstreamPolicy.js';
import { executeEndpointFlow, type BuiltEndpointRequest } from '../../routes/proxy/endpointFlow.js';
import { executeEndpointFlow, type BuiltEndpointRequest } from '../orchestration/endpointFlow.js';
import { detectProxyFailure } from '../../routes/proxy/proxyFailureJudge.js';
import { getProxyAuthContext, getProxyResourceOwner } from '../../middleware/auth.js';
import { normalizeInputFileBlock } from '../../transformers/shared/inputFile.js';
@@ -39,12 +46,19 @@ import {
import { isCodexResponsesSurface } from '../cliProfiles/codexProfile.js';
import { getRuntimeResponseReader, readRuntimeResponseText } from '../executors/types.js';
import { runCodexHttpSessionTask } from '../runtime/codexHttpSessionQueue.js';
import {
buildCodexSessionResponseStoreKey,
clearCodexSessionResponseId,
getCodexSessionResponseId,
setCodexSessionResponseId,
} from '../runtime/codexSessionResponseStore.js';
import {
summarizeConversationFileInputsInOpenAiBody,
summarizeConversationFileInputsInResponsesBody,
} from '../capabilities/conversationFileCapabilities.js';
import { detectDownstreamClientContext } from '../../routes/proxy/downstreamClientContext.js';
import { getProxyMaxChannelRetries } from '../../services/proxyChannelRetry.js';
import { shouldAbortSameSiteEndpointFallback } from '../../services/proxyRetryPolicy.js';
import {
acquireSurfaceChannelLease,
bindSurfaceStickyChannel,
@@ -96,6 +110,12 @@ function isResponsesWebsocketTransportRequest(headers: Record<string, unknown>):
&& String(rawValue).trim() === '1');
}
function rememberCodexSessionResponseId(sessionId: string, payload: unknown): void {
const responseId = extractResponsesTerminalResponseId(payload);
if (!responseId) return;
setCodexSessionResponseId(sessionId, responseId);
}
function normalizeIncludeList(value: unknown): string[] {
if (typeof value === 'string') {
const trimmed = value.trim();
@@ -345,6 +365,20 @@ export async function handleOpenAiResponsesSurfaceRequest(
const modelName = selected.actualModel || requestedModel;
const oauth = getOauthInfoFromAccount(selected.account);
const isCodexSite = String(selected.site.platform || '').trim().toLowerCase() === 'codex';
const codexSessionId = isCodexSite
? getCodexSessionHeaderValue(request.headers as Record<string, string>)
: '';
const codexSessionStoreKey = (
isCodexSite
&& codexSessionId
)
? buildCodexSessionResponseStoreKey({
sessionId: codexSessionId,
siteId: selected.site.id,
accountId: selected.account.id,
channelId: selected.channel.id,
})
: '';
const owner = getProxyResourceOwner(request);
let normalizedResponsesBody: Record<string, unknown> = {
...requestEnvelope.parsed.normalizedBody,
@@ -427,6 +461,20 @@ export async function handleOpenAiResponsesSurfaceRequest(
const executeEndpointResultForSiteApiBaseUrl = async (siteApiBaseUrl: string) => {
const buildEndpointRequest = (endpoint: 'chat' | 'messages' | 'responses') => {
const upstreamStream = isStream || (isCodexSite && endpoint === 'responses');
const responsesOriginalBody = (
endpoint === 'responses'
&& isCodexSite
&& codexSessionStoreKey
&& shouldInferResponsesPreviousResponseId(
normalizedResponsesBody,
getCodexSessionResponseId(codexSessionStoreKey),
)
)
? withResponsesPreviousResponseId(
normalizedResponsesBody,
getCodexSessionResponseId(codexSessionStoreKey)!,
)
: normalizedResponsesBody;
const endpointRequest = buildUpstreamEndpointRequest({
endpoint,
modelName,
@@ -438,7 +486,7 @@ export async function handleOpenAiResponsesSurfaceRequest(
siteUrl: siteApiBaseUrl,
openaiBody: openAiBody,
downstreamFormat: 'responses',
responsesOriginalBody: normalizedResponsesBody,
responsesOriginalBody,
downstreamHeaders: request.headers as Record<string, unknown>,
providerHeaders: buildProviderHeaders(),
});
@@ -469,7 +517,7 @@ export async function handleOpenAiResponsesSurfaceRequest(
}
const sessionId = getCodexSessionHeaderValue(endpointRequest.headers);
return runCodexHttpSessionTask(
sessionId,
codexSessionStoreKey || sessionId,
() => baseDispatchRequest(endpointRequest, targetUrl),
);
};
@@ -496,6 +544,35 @@ export async function handleOpenAiResponsesSurfaceRequest(
return recovered;
}
}
if (
ctx.request.endpoint === 'responses'
&& isResponsesPreviousResponseNotFoundError({
rawErrText: ctx.rawErrText,
})
) {
if (codexSessionStoreKey) {
clearCodexSessionResponseId(codexSessionStoreKey);
}
const previousResponseRecovery = stripResponsesPreviousResponseId(ctx.request.body);
if (previousResponseRecovery.removed) {
const recoveredRequest = {
...ctx.request,
body: previousResponseRecovery.body,
};
const recoveredResponse = await dispatchRequest(recoveredRequest, ctx.targetUrl);
if (recoveredResponse.ok) {
return {
upstream: recoveredResponse,
upstreamPath: recoveredRequest.path,
request: recoveredRequest,
targetUrl: ctx.targetUrl,
};
}
ctx.request = recoveredRequest;
ctx.response = recoveredResponse;
ctx.rawErrText = await readRuntimeResponseText(recoveredResponse).catch(() => 'unknown error');
}
}
return endpointStrategy.tryRecover(ctx);
};
@@ -507,6 +584,10 @@ export async function handleOpenAiResponsesSurfaceRequest(
buildRequest: (endpoint) => buildEndpointRequest(endpoint),
dispatchRequest,
tryRecover,
shouldAbortRemainingEndpoints: (ctx) => shouldAbortSameSiteEndpointFallback(
ctx.response.status,
ctx.rawErrText || ctx.errText,
),
onAttemptFailure: async (ctx) => {
const memoryWrite = recordUpstreamEndpointFailure({
...endpointRuntimeContext,
@@ -699,6 +780,9 @@ export async function handleOpenAiResponsesSurfaceRequest(
if (payload && typeof payload === 'object') {
upstreamUsagePresent = upstreamUsagePresent || hasProxyUsagePayload(payload);
parsedUsage = mergeProxyUsage(parsedUsage, parseProxyUsage(payload));
if (codexSessionStoreKey) {
rememberCodexSessionResponseId(codexSessionStoreKey, payload);
}
}
},
writeLines,
@@ -762,6 +846,9 @@ export async function handleOpenAiResponsesSurfaceRequest(
if (String(selected.site.platform || '').trim().toLowerCase() === 'gemini-cli') {
upstreamData = unwrapGeminiCliPayload(upstreamData);
}
if (codexSessionStoreKey) {
rememberCodexSessionResponseId(codexSessionStoreKey, upstreamData);
}
parsedUsage = parseProxyUsage(upstreamData);
upstreamUsagePresent = upstreamUsagePresent || hasProxyUsagePayload(upstreamData);
@@ -867,6 +954,9 @@ export async function handleOpenAiResponsesSurfaceRequest(
`event: ${terminalEventType}\ndata: ${JSON.stringify({ type: terminalEventType, response: collectedPayload })}\n\n`,
'data: [DONE]\n\n',
]);
if (codexSessionStoreKey) {
rememberCodexSessionResponseId(codexSessionStoreKey, collectedPayload);
}
reply.raw.end();
const latency = Date.now() - startTime;
await finalizeStreamSuccess(
@@ -1025,6 +1115,9 @@ export async function handleOpenAiResponsesSurfaceRequest(
if (String(selected.site.platform || '').trim().toLowerCase() === 'gemini-cli') {
upstreamData = unwrapGeminiCliPayload(upstreamData);
}
if (codexSessionStoreKey) {
rememberCodexSessionResponseId(codexSessionStoreKey, upstreamData);
}
const latency = Date.now() - startTime;
const parsedUsage = parseProxyUsage(upstreamData);
const upstreamUsagePresent = hasProxyUsagePayload(upstreamData);
src/server/proxy-core/surfaces/sharedSurface.test.ts
+1
View File
@@ -85,6 +85,7 @@ vi.mock('../../services/alertRules.js', () => ({
vi.mock('../../services/proxyRetryPolicy.js', () => ({
shouldRetryProxyRequest: (...args: unknown[]) => shouldRetryProxyRequestMock(...args),
shouldAbortSameSiteEndpointFallback: () => false,
}));
vi.mock('../../services/oauth/quota.js', () => ({
src/server/proxy-core/surfaces/sharedSurface.ts
+3 -3
View File
@@ -11,9 +11,9 @@ import { composeProxyLogMessage } from '../../routes/proxy/logPathMeta.js';
import { resolveProxyLogBilling } from '../../routes/proxy/proxyBilling.js';
import type { DownstreamClientContext } from '../../routes/proxy/downstreamClientContext.js';
import { insertProxyLog } from '../../services/proxyLogStore.js';
import { dispatchRuntimeRequest } from '../../routes/proxy/runtimeExecutor.js';
import type { BuiltEndpointRequest } from '../../routes/proxy/endpointFlow.js';
import { buildUpstreamUrl } from '../../routes/proxy/upstreamUrl.js';
import { dispatchRuntimeRequest } from '../../services/runtimeDispatch.js';
import type { BuiltEndpointRequest } from '../orchestration/endpointFlow.js';
import { buildUpstreamUrl } from '../orchestration/upstreamRequest.js';
import { recordOauthQuotaResetHint } from '../../services/oauth/quota.js';
import { refreshOauthAccessTokenSingleflight } from '../../services/oauth/refreshSingleflight.js';
import { proxyChannelCoordinator } from '../../services/proxyChannelCoordinator.js';