feat(server): add proactive channel probes and load-aware stable routing / 服务端新增主动探活与负载感知稳定路由 (#330)

* wip: preserve local changes before upstream merge

* Improve proxy routing stability and add stable-first strategy

* Sync explicit-group routing strategy to eligible source routes

* Fix proxy retry and runtime health review issues

* fix: make proxy channel retry attempts configurable

* fix: harden proxy retry recovery and bookkeeping

* fix: address pr 257 review comments

* feat(proxy): add sticky session channel coordination

* feat(server): add proactive channel probes and load-aware stable routing

* test(server): stabilize session load routing selection assertion

* fix(proxy): harden codex forwarding stability

* fix(router): rotate stable-first channels by healthy order

* fix(server): address probe review feedback

* fix(server): address probe review follow-ups

* fix(ci): address CodeQL and core test failures

* feat(server): refine stable-first routing recovery and observation

* Finish PR 330 settings and review fixes

* Fix chat site endpoint CI mock

* Address latest PR 330 review comments

---------

Co-authored-by: Cita <juricek.chen@gmail.com>

src/server/config.ts

@@ -140,6 +140,10 @@ export function buildConfig(env: NodeJS.ProcessEnv) {
     proxyDebugTargetModel: (env.PROXY_DEBUG_TARGET_MODEL || '').trim(),
     proxyDebugRetentionHours: Math.max(1, Math.trunc(parseNumber(env.PROXY_DEBUG_RETENTION_HOURS, 24))),
     proxyDebugMaxBodyBytes: Math.max(1024, Math.trunc(parseNumber(env.PROXY_DEBUG_MAX_BODY_BYTES, 262_144))),
+    modelAvailabilityProbeEnabled: parseBoolean(env.MODEL_AVAILABILITY_PROBE_ENABLED, false),
+    modelAvailabilityProbeIntervalMs: Math.max(60_000, Math.trunc(parseNumber(env.MODEL_AVAILABILITY_PROBE_INTERVAL_MS, 30 * 60 * 1000))),
+    modelAvailabilityProbeTimeoutMs: Math.max(3_000, Math.trunc(parseNumber(env.MODEL_AVAILABILITY_PROBE_TIMEOUT_MS, 15_000))),
+    modelAvailabilityProbeConcurrency: Math.max(1, Math.min(16, Math.trunc(parseNumber(env.MODEL_AVAILABILITY_PROBE_CONCURRENCY, 1)))),
     proxyLogRetentionDays: Math.max(0, Math.trunc(parseNumber(env.PROXY_LOG_RETENTION_DAYS, 30))),
     proxyLogRetentionPruneIntervalMinutes: Math.max(1, Math.trunc(parseNumber(env.PROXY_LOG_RETENTION_PRUNE_INTERVAL_MINUTES, 30))),
     proxyFileRetentionDays: Math.max(0, Math.trunc(parseNumber(env.PROXY_FILE_RETENTION_DAYS, 30))),

src/server/contracts/settingsRoutePayloads.ts

@@ -4,6 +4,7 @@ const backupExportTypeSchema = z.enum(['all', 'accounts', 'preferences']);
 const migrationDialectSchema = z.enum(['sqlite', 'mysql', 'postgres']);
 
 const runtimeSettingsPayloadSchema = z.object({
+  modelAvailabilityProbeEnabled: z.boolean().optional(),
   webhookEnabled: z.boolean().optional(),
   barkEnabled: z.boolean().optional(),
   serverChanEnabled: z.boolean().optional(),
@@ -86,6 +87,9 @@ function formatSettingsPayloadError(error: z.ZodError): string {
   if (firstPath === 'webhookEnabled') {
     return 'Webhook 开关格式无效：需要 boolean';
   }
+  if (firstPath === 'modelAvailabilityProbeEnabled') {
+    return '批量测活开关格式无效：需要 boolean';
+  }
   if (firstPath === 'barkEnabled') {
     return 'Bark 开关格式无效：需要 boolean';
   }

src/server/index.ts

@@ -38,6 +38,14 @@ import { ensureOauthIdentityBackfill } from './services/oauth/oauthIdentityBackf
 import { ensureOauthProviderSitesExist } from './services/oauth/oauthSiteRegistry.js';
 import { startOAuthLoopbackCallbackServers, stopOAuthLoopbackCallbackServers } from './services/oauth/localCallbackServer.js';
 import { startSiteAnnouncementPolling, stopSiteAnnouncementPolling } from './services/siteAnnouncementPollingService.js';
+import {
+  startModelAvailabilityProbeScheduler,
+  stopModelAvailabilityProbeScheduler,
+} from './services/modelAvailabilityProbeService.js';
+import {
+  startChannelRecoveryProbeScheduler,
+  stopChannelRecoveryProbeScheduler,
+} from './services/channelRecoveryProbeService.js';
 import { startUpdateCenterPolling, stopUpdateCenterPolling } from './services/updateCenterPollingService.js';
 import { reloadBackupWebdavScheduler } from './services/backupService.js';
 import { ensureRuntimeDatabaseReady } from './runtimeDatabaseBootstrap.js';
@@ -144,6 +152,11 @@ function applyRuntimeSettings(settingsMap: Map<string, string>) {
   const systemProxyUrl = parseSettingFromMap<string>(settingsMap, 'system_proxy_url');
   if (typeof systemProxyUrl === 'string') config.systemProxyUrl = systemProxyUrl;
 
+  const modelAvailabilityProbeEnabled = parseSettingFromMap<boolean>(settingsMap, 'model_availability_probe_enabled');
+  if (typeof modelAvailabilityProbeEnabled === 'boolean') {
+    config.modelAvailabilityProbeEnabled = modelAvailabilityProbeEnabled;
+  }
+
   const codexUpstreamWebsocketEnabled = parseSettingFromMap<boolean>(settingsMap, 'codex_upstream_websocket_enabled');
   if (typeof codexUpstreamWebsocketEnabled === 'boolean') {
     config.codexUpstreamWebsocketEnabled = codexUpstreamWebsocketEnabled;
@@ -494,6 +507,8 @@ if (existsSync(webDir)) {
 await startScheduler();
 await reloadBackupWebdavScheduler();
 startSiteAnnouncementPolling();
+startModelAvailabilityProbeScheduler();
+startChannelRecoveryProbeScheduler();
 startUpdateCenterPolling();
 try {
   await startOAuthLoopbackCallbackServers();
@@ -507,6 +522,8 @@ app.addHook('onClose', async () => {
   stopUpdateCenterPolling();
   stopProxyFileRetentionService();
   stopProxyLogRetentionService();
+  stopModelAvailabilityProbeScheduler();
+  stopChannelRecoveryProbeScheduler();
   await stopOAuthLoopbackCallbackServers();
 });
 

src/server/proxy-core/orchestration/endpointFlow.ts

@@ -0,0 +1,233 @@
+import { fetch } from 'undici';
+import { readRuntimeResponseText } from '../executors/types.js';
+import { withSiteProxyRequestInit } from '../../services/siteProxy.js';
+import {
+  buildUpstreamUrl,
+  summarizeUpstreamError,
+  type UpstreamEndpoint,
+} from './upstreamRequest.js';
+
+export type BuiltEndpointRequest = {
+  endpoint: UpstreamEndpoint;
+  path: string;
+  headers: Record<string, string>;
+  body: Record<string, unknown>;
+  runtime?: {
+    executor: 'default' | 'codex' | 'gemini-cli' | 'antigravity' | 'claude';
+    modelName?: string;
+    stream?: boolean;
+    oauthProjectId?: string | null;
+    action?: 'generateContent' | 'streamGenerateContent' | 'countTokens';
+  };
+};
+
+export type EndpointAttemptContext = {
+  endpointIndex: number;
+  endpointCount: number;
+  request: BuiltEndpointRequest;
+  targetUrl: string;
+  response: Awaited<ReturnType<typeof fetch>>;
+  rawErrText: string;
+  recoverApplied?: boolean;
+};
+
+export type EndpointAttemptSuccessContext = {
+  endpointIndex: number;
+  endpointCount: number;
+  request: BuiltEndpointRequest;
+  targetUrl: string;
+  response: Awaited<ReturnType<typeof fetch>>;
+  recoverApplied?: boolean;
+};
+
+export type EndpointRecoverResult = {
+  upstream: Awaited<ReturnType<typeof fetch>>;
+  upstreamPath: string;
+  request?: BuiltEndpointRequest;
+  targetUrl?: string;
+} | null;
+
+export type EndpointFlowResult =
+  | {
+    ok: true;
+    upstream: Awaited<ReturnType<typeof fetch>>;
+    upstreamPath: string;
+  }
+  | {
+    ok: false;
+    status: number;
+    errText: string;
+    rawErrText?: string;
+  };
+
+export type ExecuteEndpointFlowInput = {
+  siteUrl: string;
+  proxyUrl?: string | null;
+  disableCrossProtocolFallback?: boolean;
+  endpointCandidates: UpstreamEndpoint[];
+  buildRequest: (endpoint: UpstreamEndpoint, endpointIndex: number) => BuiltEndpointRequest;
+  dispatchRequest?: (
+    request: BuiltEndpointRequest,
+    targetUrl: string,
+  ) => Promise<Awaited<ReturnType<typeof fetch>>>;
+  tryRecover?: (ctx: EndpointAttemptContext) => Promise<EndpointRecoverResult>;
+  shouldDowngrade?: (ctx: EndpointAttemptContext) => boolean;
+  shouldAbortRemainingEndpoints?: (ctx: EndpointAttemptContext & { errText: string }) => boolean;
+  onDowngrade?: (ctx: EndpointAttemptContext & { errText: string }) => void | Promise<void>;
+  onAttemptFailure?: (ctx: EndpointAttemptContext & { errText: string }) => void | Promise<void>;
+  onAttemptSuccess?: (ctx: EndpointAttemptSuccessContext) => void | Promise<void>;
+};
+
+export function withUpstreamPath(path: string, message: string): string {
+  return `[upstream:${path}] ${message}`;
+}
+
+async function runEndpointFlowHook<T>(
+  hook: ((ctx: T) => void | Promise<void>) | undefined,
+  ctx: T,
+  hookName: string,
+): Promise<void> {
+  if (!hook) return;
+  try {
+    await hook(ctx);
+  } catch (error) {
+    console.error(`endpointFlow ${hookName} hook failed`, error);
+  }
+}
+
+export async function executeEndpointFlow(input: ExecuteEndpointFlowInput): Promise<EndpointFlowResult> {
+  const endpointCount = input.endpointCandidates.length;
+  if (endpointCount <= 0) {
+    return {
+      ok: false,
+      status: 502,
+      errText: 'Upstream request failed',
+    };
+  }
+
+  let finalStatus = 0;
+  let finalErrText = 'unknown error';
+  let finalRawErrText: string | undefined;
+
+  for (let endpointIndex = 0; endpointIndex < endpointCount; endpointIndex += 1) {
+    const endpoint = input.endpointCandidates[endpointIndex] as UpstreamEndpoint;
+    const request = input.buildRequest(endpoint, endpointIndex);
+    const defaultTarget = buildUpstreamUrl(input.siteUrl, request.path);
+    const targetUrl = input.proxyUrl
+      ? buildUpstreamUrl(input.proxyUrl, request.path)
+      : defaultTarget;
+
+    let response = input.dispatchRequest
+      ? await input.dispatchRequest(request, targetUrl)
+      : await fetch(targetUrl, await withSiteProxyRequestInit(targetUrl, {
+        method: 'POST',
+        headers: request.headers,
+        body: JSON.stringify(request.body),
+      }));
+
+    if (response.ok) {
+      await runEndpointFlowHook(input.onAttemptSuccess, {
+        endpointIndex,
+        endpointCount,
+        request,
+        targetUrl,
+        response,
+        recoverApplied: false,
+      }, 'onAttemptSuccess');
+      return {
+        ok: true,
+        upstream: response,
+        upstreamPath: request.path,
+      };
+    }
+
+    let rawErrText = await readRuntimeResponseText(response).catch(() => 'unknown error');
+    const baseContext: EndpointAttemptContext = {
+      endpointIndex,
+      endpointCount,
+      request,
+      targetUrl,
+      response,
+      rawErrText,
+      recoverApplied: false,
+    };
+
+    if (input.tryRecover) {
+      const recovered = await input.tryRecover(baseContext);
+      baseContext.recoverApplied = recovered !== null
+        || baseContext.request !== request
+        || baseContext.response !== response
+        || baseContext.rawErrText !== rawErrText;
+      if (recovered?.upstream?.ok) {
+        const recoveredRequest = recovered.request ?? baseContext.request;
+        const recoveredTargetUrl = recovered.targetUrl ?? (
+          input.proxyUrl
+            ? buildUpstreamUrl(input.proxyUrl, recovered.upstreamPath)
+            : buildUpstreamUrl(input.siteUrl, recovered.upstreamPath)
+        );
+        await runEndpointFlowHook(input.onAttemptSuccess, {
+          endpointIndex,
+          endpointCount,
+          request: recoveredRequest,
+          targetUrl: recoveredTargetUrl,
+          response: recovered.upstream,
+          recoverApplied: true,
+        }, 'onAttemptSuccess');
+        return {
+          ok: true,
+          upstream: recovered.upstream,
+          upstreamPath: recovered.upstreamPath,
+        };
+      }
+    }
+
+    rawErrText = baseContext.rawErrText;
+    response = baseContext.response;
+    const errText = withUpstreamPath(
+      baseContext.request.path,
+      summarizeUpstreamError(response.status, rawErrText),
+    );
+    await runEndpointFlowHook(input.onAttemptFailure, {
+      ...baseContext,
+      errText,
+    }, 'onAttemptFailure');
+
+    const isLastEndpoint = endpointIndex >= endpointCount - 1;
+    if (input.disableCrossProtocolFallback && !isLastEndpoint) {
+      finalStatus = response.status;
+      finalErrText = errText;
+      finalRawErrText = rawErrText;
+      break;
+    }
+    const shouldAbortRemainingEndpoints = !isLastEndpoint && !!input.shouldAbortRemainingEndpoints?.({
+      ...baseContext,
+      errText,
+    });
+    if (shouldAbortRemainingEndpoints) {
+      finalStatus = response.status;
+      finalErrText = errText;
+      finalRawErrText = rawErrText;
+      break;
+    }
+    const shouldDowngrade = !isLastEndpoint && !!input.shouldDowngrade?.(baseContext);
+    if (shouldDowngrade) {
+      await runEndpointFlowHook(input.onDowngrade, {
+        ...baseContext,
+        errText,
+      }, 'onDowngrade');
+      continue;
+    }
+
+    finalStatus = response.status;
+    finalErrText = errText;
+    finalRawErrText = rawErrText;
+    break;
+  }
+
+  return {
+    ok: false,
+    status: finalStatus || 502,
+    errText: finalErrText || 'unknown error',
+    rawErrText: finalRawErrText,
+  };
+}

src/server/proxy-core/orchestration/upstreamRequest.ts

@@ -0,0 +1,138 @@
+export type UpstreamEndpoint = 'chat' | 'messages' | 'responses';
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return !!value && typeof value === 'object';
+}
+
+function collapseWhitespace(text: string): string {
+  return text.replace(/\s+/g, ' ').trim();
+}
+
+function extractJsonErrorMessage(rawText: string): string {
+  try {
+    const parsed = JSON.parse(rawText) as unknown;
+    const root = isRecord(parsed) ? parsed : null;
+    const error = (root && isRecord(root.error)) ? root.error : root;
+    if (!error) return '';
+
+    const message = typeof error.message === 'string' ? collapseWhitespace(error.message) : '';
+    if (message) return message;
+
+    const code = typeof error.code === 'string' ? collapseWhitespace(error.code) : '';
+    const type = typeof error.type === 'string' ? collapseWhitespace(error.type) : '';
+    return [type, code].filter((part) => part.length > 0).join('/');
+  } catch {
+    return '';
+  }
+}
+
+function extractHtmlTitle(rawText: string): string {
+  const match = rawText.match(/<title[^>]*>([^<>]*)<\/title>/i);
+  if (!match?.[1]) return '';
+  return collapseWhitespace(match[1]);
+}
+
+function extractCloudflareHtmlSummary(rawText: string, status: number): string {
+  if (!/cloudflare/i.test(rawText)) return '';
+  const title = extractHtmlTitle(rawText);
+  const codeMatch = (
+    title.match(/\b(\d{3,4})\s*:\s*([^\|<]+)/i)
+    || rawText.match(/Error code\s*(\d{3,4})/i)
+  );
+  const code = codeMatch?.[1] || (status > 0 ? String(status) : '');
+  const reason = collapseWhitespace(
+    (typeof codeMatch?.[2] === 'string' ? codeMatch[2] : '')
+    || (status >= 500 ? 'origin host error' : 'request blocked')
+  );
+  if (code) return `Cloudflare ${code}: ${reason}`;
+  return `Cloudflare: ${reason}`;
+}
+
+function extractHtmlSummary(rawText: string, status: number): string {
+  if (!/(<!doctype|<html)/i.test(rawText)) return '';
+
+  const cloudflareSummary = extractCloudflareHtmlSummary(rawText, status);
+  if (cloudflareSummary) return cloudflareSummary;
+
+  const title = extractHtmlTitle(rawText);
+  if (title) return title;
+
+  const heading = rawText.match(/<h1[^>]*>([^<>]*)<\/h1>/i)?.[1] || '';
+  return collapseWhitespace(heading);
+}
+
+function formatUrlOrigin(url: URL): string {
+  const username = url.username ? encodeURIComponent(url.username) : '';
+  const password = url.password ? encodeURIComponent(url.password) : '';
+  const auth = username
+    ? `${username}${password ? `:${password}` : ''}@`
+    : '';
+
+  return `${url.protocol}//${auth}${url.host}`;
+}
+
+function joinPath(basePath: string, requestPath: string): string {
+  const base = basePath.replace(/\/+$/, '');
+  const path = requestPath.startsWith('/') ? requestPath : `/${requestPath}`;
+
+  if (!base || base === '/') return path || '/';
+  if (!path || path === '/') return base;
+  return `${base}${path}`;
+}
+
+export function summarizeUpstreamError(status: number, rawErrorText: string): string {
+  const statusPrefix = status > 0
+    ? `Upstream returned HTTP ${status}`
+    : 'Upstream request failed';
+
+  const raw = typeof rawErrorText === 'string' ? rawErrorText.trim() : '';
+  if (!raw) return statusPrefix;
+
+  const jsonMessage = extractJsonErrorMessage(raw);
+  if (jsonMessage) return `${statusPrefix}: ${jsonMessage}`;
+
+  const htmlMessage = extractHtmlSummary(raw, status);
+  if (htmlMessage) return `${statusPrefix}: ${htmlMessage}`;
+
+  const compact = collapseWhitespace(raw);
+  if (!compact) return statusPrefix;
+  if (compact.length <= 400) return `${statusPrefix}: ${compact}`;
+  return `${statusPrefix}: ${compact.slice(0, 400)}...(truncated)`;
+}
+
+export function buildUpstreamUrl(siteUrl: string, requestPath: string): string {
+  const baseRaw = typeof siteUrl === 'string' ? siteUrl.trim() : '';
+  const pathRaw = typeof requestPath === 'string' ? requestPath.trim() : '';
+  const fallbackBase = baseRaw.replace(/\/+$/, '');
+  let path = pathRaw.startsWith('/') ? pathRaw : `/${pathRaw}`;
+
+  if (!fallbackBase) return path || '/';
+  if (!path || path === '/') return fallbackBase;
+
+  try {
+    const parsed = new URL(baseRaw);
+    const basePath = parsed.pathname.replace(/\/+$/, '');
+    const baseHasVersionSuffix = /\/(?:api\/)?v1$/i.test(basePath);
+    if (baseHasVersionSuffix) {
+      if (path === '/v1') {
+        path = '/';
+      } else if (path.startsWith('/v1/')) {
+        path = path.slice('/v1'.length) || '/';
+      }
+    }
+
+    const joinedPath = joinPath(basePath, path);
+    return `${formatUrlOrigin(parsed)}${joinedPath}${parsed.search}${parsed.hash}`;
+  } catch {
+    const baseHasVersionSuffix = /\/(?:api\/)?v1$/i.test(fallbackBase);
+    if (baseHasVersionSuffix) {
+      if (path === '/v1') {
+        path = '/';
+      } else if (path.startsWith('/v1/')) {
+        path = path.slice('/v1'.length) || '/';
+      }
+    }
+
+    return `${fallbackBase}${path}`;
+  }
+}

src/server/proxy-core/runtime/codexSessionResponseStore.test.ts

@@ -0,0 +1,48 @@
+import { describe, expect, it } from 'vitest';
+import {
+  buildCodexSessionResponseStoreKey,
+  getCodexSessionResponseId,
+  resetCodexSessionResponseStore,
+  setCodexSessionResponseId,
+} from './codexSessionResponseStore.js';
+
+describe('codexSessionResponseStore', () => {
+  it('evicts the oldest session id when the store exceeds the cap', () => {
+    resetCodexSessionResponseStore();
+
+    for (let index = 0; index <= 10_000; index += 1) {
+      setCodexSessionResponseId(`session-${index}`, `resp-${index}`);
+    }
+
+    expect(getCodexSessionResponseId('session-0')).toBeNull();
+    expect(getCodexSessionResponseId('session-1')).toBe('resp-1');
+    expect(getCodexSessionResponseId('session-10000')).toBe('resp-10000');
+
+    resetCodexSessionResponseStore();
+  });
+
+  it('namespaces identical downstream session ids by channel scope', () => {
+    resetCodexSessionResponseStore();
+
+    const keyA = buildCodexSessionResponseStoreKey({
+      sessionId: 'session-1',
+      siteId: 10,
+      accountId: 20,
+      channelId: 30,
+    });
+    const keyB = buildCodexSessionResponseStoreKey({
+      sessionId: 'session-1',
+      siteId: 10,
+      accountId: 21,
+      channelId: 31,
+    });
+
+    setCodexSessionResponseId(keyA, 'resp-a');
+    setCodexSessionResponseId(keyB, 'resp-b');
+
+    expect(getCodexSessionResponseId(keyA)).toBe('resp-a');
+    expect(getCodexSessionResponseId(keyB)).toBe('resp-b');
+
+    resetCodexSessionResponseStore();
+  });
+});

src/server/proxy-core/runtime/codexSessionResponseStore.ts

@@ -0,0 +1,55 @@
+const MAX_CODEX_SESSION_RESPONSE_IDS = 10_000;
+
+const codexSessionResponseIds = new Map<string, string>();
+
+function normalizeSessionId(sessionId: string): string {
+  return sessionId.trim();
+}
+
+export function buildCodexSessionResponseStoreKey(input: {
+  sessionId: string;
+  siteId?: number | null;
+  accountId?: number | null;
+  channelId?: number | null;
+}): string {
+  const normalizedSessionId = normalizeSessionId(input.sessionId);
+  if (!normalizedSessionId) return '';
+  const parts = [
+    Number.isFinite(input.siteId as number) && Number(input.siteId) > 0 ? `site:${Math.trunc(Number(input.siteId))}` : '',
+    Number.isFinite(input.accountId as number) && Number(input.accountId) > 0 ? `account:${Math.trunc(Number(input.accountId))}` : '',
+    Number.isFinite(input.channelId as number) && Number(input.channelId) > 0 ? `channel:${Math.trunc(Number(input.channelId))}` : '',
+    `session:${normalizedSessionId}`,
+  ].filter(Boolean);
+  return parts.join('|');
+}
+
+export function getCodexSessionResponseId(sessionId: string): string | null {
+  const normalized = normalizeSessionId(sessionId);
+  if (!normalized) return null;
+  return codexSessionResponseIds.get(normalized) ?? null;
+}
+
+export function setCodexSessionResponseId(sessionId: string, responseId: string): void {
+  const normalizedSessionId = normalizeSessionId(sessionId);
+  const normalizedResponseId = responseId.trim();
+  if (!normalizedSessionId || !normalizedResponseId) return;
+  if (codexSessionResponseIds.has(normalizedSessionId)) {
+    codexSessionResponseIds.delete(normalizedSessionId);
+  }
+  codexSessionResponseIds.set(normalizedSessionId, normalizedResponseId);
+  while (codexSessionResponseIds.size > MAX_CODEX_SESSION_RESPONSE_IDS) {
+    const oldestKey = codexSessionResponseIds.keys().next().value;
+    if (!oldestKey) break;
+    codexSessionResponseIds.delete(oldestKey);
+  }
+}
+
+export function clearCodexSessionResponseId(sessionId: string): void {
+  const normalized = normalizeSessionId(sessionId);
+  if (!normalized) return;
+  codexSessionResponseIds.delete(normalized);
+}
+
+export function resetCodexSessionResponseStore(): void {
+  codexSessionResponseIds.clear();
+}

src/server/proxy-core/runtime/codexWebsocketRuntime.test.ts

@@ -1,6 +1,7 @@
 import { AddressInfo } from 'node:net';
 import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest';
 import { WebSocketServer } from 'ws';
+import { resetCodexSessionResponseStore } from './codexSessionResponseStore.js';
 
 describe('codexWebsocketRuntime', () => {
   let upstreamServer: WebSocketServer;
@@ -25,6 +26,7 @@ describe('codexWebsocketRuntime', () => {
   });
 
   beforeEach(() => {
+    resetCodexSessionResponseStore();
     upstreamConnectionCount = 0;
     upstreamRequests = [];
     upstreamMessageHandler = (socket, parsed, requestIndex) => {

src/server/proxy-core/runtime/codexWebsocketRuntime.ts

@@ -1,10 +1,22 @@
 import type { IncomingMessage } from 'node:http';
 import WebSocket from 'ws';
+import {
+  extractResponsesTerminalResponseId,
+  isResponsesPreviousResponseNotFoundError,
+  shouldInferResponsesPreviousResponseId,
+  stripResponsesPreviousResponseId,
+  withResponsesPreviousResponseId,
+} from '../../transformers/openai/responses/continuation.js';
 import {
   buildCodexWebsocketHandshakeHeaders,
   buildCodexWebsocketRequestBody,
   toCodexWebsocketUrl,
 } from './codexWebsocketHeaders.js';
+import {
+  clearCodexSessionResponseId,
+  getCodexSessionResponseId,
+  setCodexSessionResponseId,
+} from './codexSessionResponseStore.js';
 import { createCodexWebsocketSessionStore } from './codexWebsocketSessionStore.js';
 import type {
   CodexWebsocketRuntimeResult,
@@ -190,6 +202,23 @@ function clearSessionSocket(session: CodexWebsocketSession, socket: WebSocket):
   session.socketUrl = null;
 }
 
+function buildContinuationAwareRuntimeBody(
+  sessionId: string,
+  body: Record<string, unknown>,
+): Record<string, unknown> {
+  const rememberedResponseId = getCodexSessionResponseId(sessionId);
+  if (!shouldInferResponsesPreviousResponseId(body, rememberedResponseId)) {
+    return body;
+  }
+  return withResponsesPreviousResponseId(body, rememberedResponseId);
+}
+
+function rememberSessionResponseId(sessionId: string, payload: unknown): void {
+  const responseId = extractResponsesTerminalResponseId(payload);
+  if (!responseId) return;
+  setCodexSessionResponseId(sessionId, responseId);
+}
+
 async function ensureSessionSocket(
   session: CodexWebsocketSession,
   input: CodexWebsocketRuntimeSendInput,
@@ -232,9 +261,11 @@ async function ensureSessionSocket(
   };
 }
 
-async function sendSessionRequest(
+async function sendSessionRequestAttempt(
   session: CodexWebsocketSession,
-  input: CodexWebsocketRuntimeSendInput,
+  input: CodexWebsocketRuntimeSendInput & {
+    body: Record<string, unknown>;
+  },
 ): Promise<CodexWebsocketRuntimeResult> {
   const { socket, reusedSession } = await ensureSessionSocket(session, input);
   const events: Array<Record<string, unknown>> = [];
@@ -262,7 +293,13 @@ async function sendSessionRequest(
         events.push(parsed);
         if (!isTerminalEvent(parsed)) return;
         if (settled) return;
-        if (isRuntimeErrorEvent(parsed)) {
+        if (
+          isRuntimeErrorEvent(parsed)
+          || isResponsesPreviousResponseNotFoundError({
+            payload: parsed,
+            rawErrText: extractTerminalErrorMessage(parsed),
+          })
+        ) {
           settled = true;
           cleanup();
           clearSessionSocket(session, socket);
@@ -274,6 +311,7 @@ async function sendSessionRequest(
           }));
           return;
         }
+        rememberSessionResponseId(session.sessionId, parsed);
         settled = true;
         cleanup();
         resolve({
@@ -307,6 +345,43 @@ async function sendSessionRequest(
   });
 }
 
+async function sendSessionRequest(
+  session: CodexWebsocketSession,
+  input: CodexWebsocketRuntimeSendInput,
+): Promise<CodexWebsocketRuntimeResult> {
+  let currentBody = buildContinuationAwareRuntimeBody(session.sessionId, input.body);
+  let previousResponseRecoveryTried = false;
+
+  for (;;) {
+    try {
+      return await sendSessionRequestAttempt(session, {
+        ...input,
+        body: currentBody,
+      });
+    } catch (error) {
+      if (
+        previousResponseRecoveryTried
+        || !(error instanceof CodexWebsocketRuntimeError)
+        || !isResponsesPreviousResponseNotFoundError({
+          payload: error.payload ?? error.events[error.events.length - 1],
+          rawErrText: error.message,
+        })
+      ) {
+        throw error;
+      }
+
+      const previousResponseRecovery = stripResponsesPreviousResponseId(currentBody);
+      if (!previousResponseRecovery.removed) {
+        throw error;
+      }
+
+      previousResponseRecoveryTried = true;
+      clearCodexSessionResponseId(session.sessionId);
+      currentBody = previousResponseRecovery.body;
+    }
+  }
+}
+
 export function createCodexWebsocketRuntime(input?: {
   sessionStore?: CodexWebsocketSessionStore;
 }) {
@@ -334,6 +409,7 @@ export function createCodexWebsocketRuntime(input?: {
       await closeSocket(session.socket);
       session.socket = null;
       session.socketUrl = null;
+      clearCodexSessionResponseId(sessionId);
     },
 
     async closeAllSessions(): Promise<void> {

src/server/proxy-core/surfaces/chatSurface.ts

@@ -9,7 +9,7 @@ import {
   buildClaudeCountTokensUpstreamRequest,
   buildUpstreamEndpointRequest,
   resolveUpstreamEndpointCandidates,
-} from '../../routes/proxy/upstreamEndpoint.js';
+} from '../../services/upstreamEndpointRuntime.js';
 import {
   getUpstreamEndpointRuntimeStateSnapshot,
   recordUpstreamEndpointFailure,
@@ -20,10 +20,11 @@ import {
   getDownstreamRoutingPolicy,
   recordDownstreamCostUsage,
 } from '../../routes/proxy/downstreamPolicy.js';
-import { executeEndpointFlow, type BuiltEndpointRequest } from '../../routes/proxy/endpointFlow.js';
+import { executeEndpointFlow, type BuiltEndpointRequest } from '../orchestration/endpointFlow.js';
 import { detectProxyFailure } from '../../routes/proxy/proxyFailureJudge.js';
 import { openAiChatTransformer } from '../../transformers/openai/chat/index.js';
 import { anthropicMessagesTransformer } from '../../transformers/anthropic/messages/index.js';
+import { shouldPreferResponsesForAnthropicContinuation } from '../../transformers/anthropic/messages/compatibility.js';
 import { getProxyAuthContext, getProxyResourceOwner } from '../../middleware/auth.js';
 import {
   ProxyInputFileResolutionError,
@@ -47,6 +48,7 @@ import { summarizeConversationFileInputsInOpenAiBody } from '../capabilities/con
 import { getRuntimeResponseReader, readRuntimeResponseText } from '../executors/types.js';
 import { detectDownstreamClientContext } from '../../routes/proxy/downstreamClientContext.js';
 import { getProxyMaxChannelRetries } from '../../services/proxyChannelRetry.js';
+import { shouldAbortSameSiteEndpointFallback } from '../../services/proxyRetryPolicy.js';
 import {
   acquireSurfaceChannelLease,
   bindSurfaceStickyChannel,
@@ -170,6 +172,10 @@ export async function handleChatSurfaceRequest(
   }
   const conversationFileSummary = summarizeConversationFileInputsInOpenAiBody(resolvedOpenAiBody);
   const hasNonImageFileInput = conversationFileSummary.hasDocument;
+  const wantsContinuationAwareResponses = (
+    downstreamFormat === 'claude'
+    && shouldPreferResponsesForAnthropicContinuation(claudeOriginalBody)
+  );
   const codexSessionCacheKey = deriveCodexSessionCacheKey({
     downstreamFormat,
     body: downstreamFormat === 'claude' ? claudeOriginalBody : request.body,
@@ -283,6 +289,7 @@ export async function handleChatSurfaceRequest(
         {
           hasNonImageFileInput,
           conversationFileSummary,
+          wantsContinuationAwareResponses,
         },
       ),
     ];
@@ -297,6 +304,7 @@ export async function handleChatSurfaceRequest(
       requestCapabilities: {
         hasNonImageFileInput,
         conversationFileSummary,
+        wantsContinuationAwareResponses,
       },
     };
     await safeUpdateSurfaceProxyDebugCandidates(debugTrace, {
@@ -309,6 +317,7 @@ export async function handleChatSurfaceRequest(
         stickyPreferredChannelId,
         oauthProvider: oauth?.provider || null,
         isCodexSite,
+        wantsContinuationAwareResponses,
       },
     });
     const buildProviderHeaders = () => (
@@ -389,6 +398,10 @@ export async function handleChatSurfaceRequest(
         buildRequest: (endpoint) => buildEndpointRequest(endpoint),
         dispatchRequest,
         tryRecover,
+        shouldAbortRemainingEndpoints: (ctx) => shouldAbortSameSiteEndpointFallback(
+          ctx.response.status,
+          ctx.rawErrText || ctx.errText,
+        ),
         onAttemptFailure: async (ctx) => {
           const memoryWrite = recordUpstreamEndpointFailure({
             ...endpointRuntimeContext,

src/server/proxy-core/surfaces/geminiSurface.ts

@@ -14,12 +14,12 @@ import { refreshOauthAccessTokenSingleflight } from '../../services/oauth/refres
 import { resolveChannelProxyUrl, withSiteRecordProxyRequestInit } from '../../services/siteProxy.js';
 import * as routeRefreshWorkflow from '../../services/routeRefreshWorkflow.js';
 import { getDownstreamRoutingPolicy } from '../../routes/proxy/downstreamPolicy.js';
-import { executeEndpointFlow, type BuiltEndpointRequest } from '../../routes/proxy/endpointFlow.js';
+import { executeEndpointFlow, type BuiltEndpointRequest } from '../orchestration/endpointFlow.js';
 import { composeProxyLogMessage } from '../../routes/proxy/logPathMeta.js';
 import {
   buildUpstreamEndpointRequest,
   resolveUpstreamEndpointCandidates,
-} from '../../routes/proxy/upstreamEndpoint.js';
+} from '../../services/upstreamEndpointRuntime.js';
 import {
   getUpstreamEndpointRuntimeStateSnapshot,
   recordUpstreamEndpointFailure,
@@ -35,12 +35,13 @@ import {
   unwrapGeminiCliPayload,
   wrapGeminiCliRequest,
 } from '../../routes/proxy/geminiCliCompat.js';
-import { dispatchRuntimeRequest } from '../../routes/proxy/runtimeExecutor.js';
+import { dispatchRuntimeRequest } from '../../services/runtimeDispatch.js';
 import { detectDownstreamClientContext, type DownstreamClientContext } from '../../routes/proxy/downstreamClientContext.js';
 import { insertProxyLog } from '../../services/proxyLogStore.js';
 import { summarizeConversationFileInputsInOpenAiBody } from '../capabilities/conversationFileCapabilities.js';
 import { getRuntimeResponseReader, readRuntimeResponseText } from '../executors/types.js';
 import { getProxyMaxChannelRetries } from '../../services/proxyChannelRetry.js';
+import { shouldAbortSameSiteEndpointFallback } from '../../services/proxyRetryPolicy.js';
 import {
   buildSurfaceProxyDebugResponseHeaders,
   captureSurfaceProxyDebugSuccessResponseBody,
@@ -1224,6 +1225,10 @@ export async function geminiProxyRoute(app: FastifyInstance) {
           buildRequest: (endpoint) => buildEndpointRequest(endpoint),
           dispatchRequest,
           tryRecover: endpointStrategy.tryRecover,
+          shouldAbortRemainingEndpoints: (ctx) => shouldAbortSameSiteEndpointFallback(
+            ctx.response.status,
+            ctx.rawErrText || ctx.errText,
+          ),
           onAttemptFailure: async (ctx) => {
             const memoryWrite = recordUpstreamEndpointFailure({
               ...endpointRuntimeContext,

src/server/proxy-core/surfaces/openAiResponsesSurface.ts

@@ -4,17 +4,24 @@ import { config } from '../../config.js';
 import { reportProxyAllFailed } from '../../services/alertService.js';
 import { hasProxyUsagePayload, mergeProxyUsage, parseProxyUsage } from '../../services/proxyUsageParser.js';
 import { openAiResponsesTransformer } from '../../transformers/openai/responses/index.js';
+import {
+  extractResponsesTerminalResponseId,
+  isResponsesPreviousResponseNotFoundError,
+  shouldInferResponsesPreviousResponseId,
+  stripResponsesPreviousResponseId,
+  withResponsesPreviousResponseId,
+} from '../../transformers/openai/responses/continuation.js';
 import {
   buildUpstreamEndpointRequest,
   resolveUpstreamEndpointCandidates,
-} from '../../routes/proxy/upstreamEndpoint.js';
+} from '../../services/upstreamEndpointRuntime.js';
 import {
   getUpstreamEndpointRuntimeStateSnapshot,
   recordUpstreamEndpointFailure,
   recordUpstreamEndpointSuccess,
 } from '../../services/upstreamEndpointRuntimeMemory.js';
 import { ensureModelAllowedForDownstreamKey, getDownstreamRoutingPolicy, recordDownstreamCostUsage } from '../../routes/proxy/downstreamPolicy.js';
-import { executeEndpointFlow, type BuiltEndpointRequest } from '../../routes/proxy/endpointFlow.js';
+import { executeEndpointFlow, type BuiltEndpointRequest } from '../orchestration/endpointFlow.js';
 import { detectProxyFailure } from '../../routes/proxy/proxyFailureJudge.js';
 import { getProxyAuthContext, getProxyResourceOwner } from '../../middleware/auth.js';
 import { normalizeInputFileBlock } from '../../transformers/shared/inputFile.js';
@@ -39,12 +46,19 @@ import {
 import { isCodexResponsesSurface } from '../cliProfiles/codexProfile.js';
 import { getRuntimeResponseReader, readRuntimeResponseText } from '../executors/types.js';
 import { runCodexHttpSessionTask } from '../runtime/codexHttpSessionQueue.js';
+import {
+  buildCodexSessionResponseStoreKey,
+  clearCodexSessionResponseId,
+  getCodexSessionResponseId,
+  setCodexSessionResponseId,
+} from '../runtime/codexSessionResponseStore.js';
 import {
   summarizeConversationFileInputsInOpenAiBody,
   summarizeConversationFileInputsInResponsesBody,
 } from '../capabilities/conversationFileCapabilities.js';
 import { detectDownstreamClientContext } from '../../routes/proxy/downstreamClientContext.js';
 import { getProxyMaxChannelRetries } from '../../services/proxyChannelRetry.js';
+import { shouldAbortSameSiteEndpointFallback } from '../../services/proxyRetryPolicy.js';
 import {
   acquireSurfaceChannelLease,
   bindSurfaceStickyChannel,
@@ -96,6 +110,12 @@ function isResponsesWebsocketTransportRequest(headers: Record<string, unknown>):
       && String(rawValue).trim() === '1');
 }
 
+function rememberCodexSessionResponseId(sessionId: string, payload: unknown): void {
+  const responseId = extractResponsesTerminalResponseId(payload);
+  if (!responseId) return;
+  setCodexSessionResponseId(sessionId, responseId);
+}
+
 function normalizeIncludeList(value: unknown): string[] {
   if (typeof value === 'string') {
     const trimmed = value.trim();
@@ -345,6 +365,20 @@ export async function handleOpenAiResponsesSurfaceRequest(
       const modelName = selected.actualModel || requestedModel;
       const oauth = getOauthInfoFromAccount(selected.account);
       const isCodexSite = String(selected.site.platform || '').trim().toLowerCase() === 'codex';
+      const codexSessionId = isCodexSite
+        ? getCodexSessionHeaderValue(request.headers as Record<string, string>)
+        : '';
+      const codexSessionStoreKey = (
+        isCodexSite
+        && codexSessionId
+      )
+        ? buildCodexSessionResponseStoreKey({
+          sessionId: codexSessionId,
+          siteId: selected.site.id,
+          accountId: selected.account.id,
+          channelId: selected.channel.id,
+        })
+        : '';
       const owner = getProxyResourceOwner(request);
       let normalizedResponsesBody: Record<string, unknown> = {
         ...requestEnvelope.parsed.normalizedBody,
@@ -427,6 +461,20 @@ export async function handleOpenAiResponsesSurfaceRequest(
       const executeEndpointResultForSiteApiBaseUrl = async (siteApiBaseUrl: string) => {
         const buildEndpointRequest = (endpoint: 'chat' | 'messages' | 'responses') => {
           const upstreamStream = isStream || (isCodexSite && endpoint === 'responses');
+          const responsesOriginalBody = (
+            endpoint === 'responses'
+            && isCodexSite
+            && codexSessionStoreKey
+            && shouldInferResponsesPreviousResponseId(
+              normalizedResponsesBody,
+              getCodexSessionResponseId(codexSessionStoreKey),
+            )
+          )
+            ? withResponsesPreviousResponseId(
+              normalizedResponsesBody,
+              getCodexSessionResponseId(codexSessionStoreKey)!,
+            )
+            : normalizedResponsesBody;
           const endpointRequest = buildUpstreamEndpointRequest({
             endpoint,
             modelName,
@@ -438,7 +486,7 @@ export async function handleOpenAiResponsesSurfaceRequest(
             siteUrl: siteApiBaseUrl,
             openaiBody: openAiBody,
             downstreamFormat: 'responses',
-            responsesOriginalBody: normalizedResponsesBody,
+            responsesOriginalBody,
             downstreamHeaders: request.headers as Record<string, unknown>,
             providerHeaders: buildProviderHeaders(),
           });
@@ -469,7 +517,7 @@ export async function handleOpenAiResponsesSurfaceRequest(
           }
           const sessionId = getCodexSessionHeaderValue(endpointRequest.headers);
           return runCodexHttpSessionTask(
-            sessionId,
+            codexSessionStoreKey || sessionId,
             () => baseDispatchRequest(endpointRequest, targetUrl),
           );
         };
@@ -496,6 +544,35 @@ export async function handleOpenAiResponsesSurfaceRequest(
               return recovered;
             }
           }
+          if (
+            ctx.request.endpoint === 'responses'
+            && isResponsesPreviousResponseNotFoundError({
+              rawErrText: ctx.rawErrText,
+            })
+          ) {
+            if (codexSessionStoreKey) {
+              clearCodexSessionResponseId(codexSessionStoreKey);
+            }
+            const previousResponseRecovery = stripResponsesPreviousResponseId(ctx.request.body);
+            if (previousResponseRecovery.removed) {
+              const recoveredRequest = {
+                ...ctx.request,
+                body: previousResponseRecovery.body,
+              };
+              const recoveredResponse = await dispatchRequest(recoveredRequest, ctx.targetUrl);
+              if (recoveredResponse.ok) {
+                return {
+                  upstream: recoveredResponse,
+                  upstreamPath: recoveredRequest.path,
+                  request: recoveredRequest,
+                  targetUrl: ctx.targetUrl,
+                };
+              }
+              ctx.request = recoveredRequest;
+              ctx.response = recoveredResponse;
+              ctx.rawErrText = await readRuntimeResponseText(recoveredResponse).catch(() => 'unknown error');
+            }
+          }
           return endpointStrategy.tryRecover(ctx);
         };
 
@@ -507,6 +584,10 @@ export async function handleOpenAiResponsesSurfaceRequest(
           buildRequest: (endpoint) => buildEndpointRequest(endpoint),
           dispatchRequest,
           tryRecover,
+          shouldAbortRemainingEndpoints: (ctx) => shouldAbortSameSiteEndpointFallback(
+            ctx.response.status,
+            ctx.rawErrText || ctx.errText,
+          ),
           onAttemptFailure: async (ctx) => {
             const memoryWrite = recordUpstreamEndpointFailure({
               ...endpointRuntimeContext,
@@ -699,6 +780,9 @@ export async function handleOpenAiResponsesSurfaceRequest(
               if (payload && typeof payload === 'object') {
                 upstreamUsagePresent = upstreamUsagePresent || hasProxyUsagePayload(payload);
                 parsedUsage = mergeProxyUsage(parsedUsage, parseProxyUsage(payload));
+                if (codexSessionStoreKey) {
+                  rememberCodexSessionResponseId(codexSessionStoreKey, payload);
+                }
               }
             },
             writeLines,
@@ -762,6 +846,9 @@ export async function handleOpenAiResponsesSurfaceRequest(
             if (String(selected.site.platform || '').trim().toLowerCase() === 'gemini-cli') {
               upstreamData = unwrapGeminiCliPayload(upstreamData);
             }
+            if (codexSessionStoreKey) {
+              rememberCodexSessionResponseId(codexSessionStoreKey, upstreamData);
+            }
 
             parsedUsage = parseProxyUsage(upstreamData);
             upstreamUsagePresent = upstreamUsagePresent || hasProxyUsagePayload(upstreamData);
@@ -867,6 +954,9 @@ export async function handleOpenAiResponsesSurfaceRequest(
                   `event: ${terminalEventType}\ndata: ${JSON.stringify({ type: terminalEventType, response: collectedPayload })}\n\n`,
                   'data: [DONE]\n\n',
                 ]);
+                if (codexSessionStoreKey) {
+                  rememberCodexSessionResponseId(codexSessionStoreKey, collectedPayload);
+                }
                 reply.raw.end();
                 const latency = Date.now() - startTime;
                 await finalizeStreamSuccess(
@@ -1025,6 +1115,9 @@ export async function handleOpenAiResponsesSurfaceRequest(
         if (String(selected.site.platform || '').trim().toLowerCase() === 'gemini-cli') {
           upstreamData = unwrapGeminiCliPayload(upstreamData);
         }
+        if (codexSessionStoreKey) {
+          rememberCodexSessionResponseId(codexSessionStoreKey, upstreamData);
+        }
         const latency = Date.now() - startTime;
         const parsedUsage = parseProxyUsage(upstreamData);
         const upstreamUsagePresent = hasProxyUsagePayload(upstreamData);

src/server/proxy-core/surfaces/sharedSurface.test.ts

@@ -85,6 +85,7 @@ vi.mock('../../services/alertRules.js', () => ({
 
 vi.mock('../../services/proxyRetryPolicy.js', () => ({
   shouldRetryProxyRequest: (...args: unknown[]) => shouldRetryProxyRequestMock(...args),
+  shouldAbortSameSiteEndpointFallback: () => false,
 }));
 
 vi.mock('../../services/oauth/quota.js', () => ({

src/server/proxy-core/surfaces/sharedSurface.ts

@@ -11,9 +11,9 @@ import { composeProxyLogMessage } from '../../routes/proxy/logPathMeta.js';
 import { resolveProxyLogBilling } from '../../routes/proxy/proxyBilling.js';
 import type { DownstreamClientContext } from '../../routes/proxy/downstreamClientContext.js';
 import { insertProxyLog } from '../../services/proxyLogStore.js';
-import { dispatchRuntimeRequest } from '../../routes/proxy/runtimeExecutor.js';
-import type { BuiltEndpointRequest } from '../../routes/proxy/endpointFlow.js';
-import { buildUpstreamUrl } from '../../routes/proxy/upstreamUrl.js';
+import { dispatchRuntimeRequest } from '../../services/runtimeDispatch.js';
+import type { BuiltEndpointRequest } from '../orchestration/endpointFlow.js';
+import { buildUpstreamUrl } from '../orchestration/upstreamRequest.js';
 import { recordOauthQuotaResetHint } from '../../services/oauth/quota.js';
 import { refreshOauthAccessTokenSingleflight } from '../../services/oauth/refreshSingleflight.js';
 import { proxyChannelCoordinator } from '../../services/proxyChannelCoordinator.js';

src/server/routes/api/settings.model-availability-probe.test.ts

@@ -0,0 +1,106 @@
+import Fastify, { type FastifyInstance } from 'fastify';
+import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
+import { mkdtempSync, rmSync } from 'node:fs';
+import { join } from 'node:path';
+import { tmpdir } from 'node:os';
+import { eq } from 'drizzle-orm';
+
+const { startModelAvailabilityProbeSchedulerMock, stopModelAvailabilityProbeSchedulerMock } = vi.hoisted(() => ({
+  startModelAvailabilityProbeSchedulerMock: vi.fn(),
+  stopModelAvailabilityProbeSchedulerMock: vi.fn(),
+}));
+
+vi.mock('../../services/modelAvailabilityProbeService.js', () => ({
+  startModelAvailabilityProbeScheduler: startModelAvailabilityProbeSchedulerMock,
+  stopModelAvailabilityProbeScheduler: stopModelAvailabilityProbeSchedulerMock,
+}));
+
+type ConfigModule = typeof import('../../config.js');
+type DbModule = typeof import('../../db/index.js');
+
+describe('settings model availability probe runtime setting', () => {
+  let app: FastifyInstance;
+  let config: ConfigModule['config'];
+  let db: DbModule['db'];
+  let schema: DbModule['schema'];
+  let dataDir = '';
+
+  beforeAll(async () => {
+    dataDir = mkdtempSync(join(tmpdir(), 'metapi-settings-model-probe-'));
+    process.env.DATA_DIR = dataDir;
+
+    await import('../../db/migrate.js');
+    const dbModule = await import('../../db/index.js');
+    const configModule = await import('../../config.js');
+    const settingsRoutesModule = await import('./settings.js');
+
+    db = dbModule.db;
+    schema = dbModule.schema;
+    config = configModule.config;
+
+    app = Fastify();
+    await app.register(settingsRoutesModule.settingsRoutes);
+  });
+
+  beforeEach(async () => {
+    await db.delete(schema.settings).run();
+    config.modelAvailabilityProbeEnabled = false;
+    startModelAvailabilityProbeSchedulerMock.mockReset();
+    stopModelAvailabilityProbeSchedulerMock.mockReset();
+  });
+
+  afterAll(async () => {
+    await app.close();
+    rmSync(dataDir, { recursive: true, force: true });
+    delete process.env.DATA_DIR;
+  });
+
+  it('persists enabling the model availability probe and starts the scheduler', async () => {
+    const updateResponse = await app.inject({
+      method: 'PUT',
+      url: '/api/settings/runtime',
+      payload: {
+        modelAvailabilityProbeEnabled: true,
+      },
+    });
+
+    expect(updateResponse.statusCode).toBe(200);
+    const updated = updateResponse.json() as { modelAvailabilityProbeEnabled?: boolean };
+    expect(updated.modelAvailabilityProbeEnabled).toBe(true);
+    expect(config.modelAvailabilityProbeEnabled).toBe(true);
+    expect(startModelAvailabilityProbeSchedulerMock).toHaveBeenCalledTimes(1);
+    expect(stopModelAvailabilityProbeSchedulerMock).not.toHaveBeenCalled();
+
+    const saved = await db.select().from(schema.settings).where(eq(schema.settings.key, 'model_availability_probe_enabled')).get();
+    expect(saved?.value).toBe(JSON.stringify(true));
+
+    const getResponse = await app.inject({
+      method: 'GET',
+      url: '/api/settings/runtime',
+    });
+    expect(getResponse.statusCode).toBe(200);
+    expect((getResponse.json() as { modelAvailabilityProbeEnabled?: boolean }).modelAvailabilityProbeEnabled).toBe(true);
+  });
+
+  it('persists disabling the model availability probe and stops the scheduler', async () => {
+    config.modelAvailabilityProbeEnabled = true;
+
+    const updateResponse = await app.inject({
+      method: 'PUT',
+      url: '/api/settings/runtime',
+      payload: {
+        modelAvailabilityProbeEnabled: false,
+      },
+    });
+
+    expect(updateResponse.statusCode).toBe(200);
+    const updated = updateResponse.json() as { modelAvailabilityProbeEnabled?: boolean };
+    expect(updated.modelAvailabilityProbeEnabled).toBe(false);
+    expect(config.modelAvailabilityProbeEnabled).toBe(false);
+    expect(stopModelAvailabilityProbeSchedulerMock).toHaveBeenCalledTimes(1);
+    expect(startModelAvailabilityProbeSchedulerMock).not.toHaveBeenCalled();
+
+    const saved = await db.select().from(schema.settings).where(eq(schema.settings.key, 'model_availability_probe_enabled')).get();
+    expect(saved?.value).toBe(JSON.stringify(false));
+  });
+});

src/server/routes/api/settings.ts

@@ -40,12 +40,17 @@ import { invalidateSiteProxyCache, normalizeSiteProxyUrl, withExplicitProxyReque
 import { performFactoryReset } from '../../services/factoryResetService.js';
 import { normalizeLogCleanupRetentionDays } from '../../services/logCleanupService.js';
 import { stopProxyLogRetentionService } from '../../services/proxyLogRetentionService.js';
+import {
+  startModelAvailabilityProbeScheduler,
+  stopModelAvailabilityProbeScheduler,
+} from '../../services/modelAvailabilityProbeService.js';
 
 type RoutingWeights = typeof config.routingWeights;
 
 interface RuntimeSettingsBody {
   proxyToken?: string;
   systemProxyUrl?: string;
+  modelAvailabilityProbeEnabled?: boolean;
   codexUpstreamWebsocketEnabled?: boolean;
   disableCrossProtocolFallback?: boolean;
   proxySessionChannelConcurrencyLimit?: number;
@@ -404,6 +409,16 @@ function applyImportedSettingToRuntime(key: string, value: unknown) {
       config.systemProxyUrl = normalizeSiteProxyUrl(value) || '';
       return;
     }
+    case 'model_availability_probe_enabled': {
+      if (typeof value !== 'boolean') return;
+      config.modelAvailabilityProbeEnabled = value;
+      if (value) {
+        startModelAvailabilityProbeScheduler();
+      } else {
+        stopModelAvailabilityProbeScheduler();
+      }
+      return;
+    }
     case 'codex_upstream_websocket_enabled': {
       if (typeof value !== 'boolean') return;
       config.codexUpstreamWebsocketEnabled = value;
@@ -687,6 +702,7 @@ function getRuntimeSettingsResponse(currentAdminIp = '') {
     logCleanupUsageLogsEnabled: config.logCleanupUsageLogsEnabled,
     logCleanupProgramLogsEnabled: config.logCleanupProgramLogsEnabled,
     logCleanupRetentionDays: config.logCleanupRetentionDays,
+    modelAvailabilityProbeEnabled: config.modelAvailabilityProbeEnabled,
     codexUpstreamWebsocketEnabled: config.codexUpstreamWebsocketEnabled,
     disableCrossProtocolFallback: config.disableCrossProtocolFallback,
     proxySessionChannelConcurrencyLimit: config.proxySessionChannelConcurrencyLimit,
@@ -1101,6 +1117,29 @@ export async function settingsRoutes(app: FastifyInstance) {
       invalidateSiteProxyCache();
     }
 
+    if (body.modelAvailabilityProbeEnabled !== undefined) {
+      let nextValue = false;
+      try {
+        nextValue = parseBooleanFlag(body.modelAvailabilityProbeEnabled, '批量测活开关');
+      } catch (err: any) {
+        return reply.code(400).send({
+          success: false,
+          message: err?.message || '批量测活开关格式无效',
+        });
+      }
+
+      if (nextValue !== config.modelAvailabilityProbeEnabled) {
+        changedLabels.push(nextValue ? '开启批量测活' : '关闭批量测活');
+      }
+      await upsertSetting('model_availability_probe_enabled', nextValue);
+      config.modelAvailabilityProbeEnabled = nextValue;
+      if (nextValue) {
+        startModelAvailabilityProbeScheduler();
+      } else {
+        stopModelAvailabilityProbeScheduler();
+      }
+    }
+
     if (body.codexUpstreamWebsocketEnabled !== undefined) {
       let nextValue = false;
       try {

src/server/routes/api/stats.model-probe.test.ts

@@ -0,0 +1,157 @@
+import Fastify, { type FastifyInstance } from 'fastify';
+import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
+import { mkdtempSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+
+const buildModelAvailabilityProbeTaskDedupeKeyMock = vi.fn();
+const queueModelAvailabilityProbeTaskMock = vi.fn();
+const getBackgroundTaskMock = vi.fn();
+const getRunningTaskByDedupeKeyMock = vi.fn();
+const waitForBackgroundTaskCompletionMock = vi.fn();
+
+vi.mock('../../services/modelAvailabilityProbeService.js', async () => {
+  const actual = await vi.importActual<typeof import('../../services/modelAvailabilityProbeService.js')>('../../services/modelAvailabilityProbeService.js');
+  return {
+    ...actual,
+    buildModelAvailabilityProbeTaskDedupeKey: (...args: unknown[]) => buildModelAvailabilityProbeTaskDedupeKeyMock(...args),
+    queueModelAvailabilityProbeTask: (...args: unknown[]) => queueModelAvailabilityProbeTaskMock(...args),
+  };
+});
+
+vi.mock('../../services/backgroundTaskService.js', async () => {
+  const actual = await vi.importActual<typeof import('../../services/backgroundTaskService.js')>('../../services/backgroundTaskService.js');
+  return {
+    ...actual,
+    getBackgroundTask: (...args: unknown[]) => getBackgroundTaskMock(...args),
+    getRunningTaskByDedupeKey: (...args: unknown[]) => getRunningTaskByDedupeKeyMock(...args),
+    waitForBackgroundTaskCompletion: (...args: unknown[]) => waitForBackgroundTaskCompletionMock(...args),
+  };
+});
+
+describe('/api/models/probe', () => {
+  let app: FastifyInstance;
+  let dataDir = '';
+  let originalDataDir: string | undefined;
+
+  beforeAll(async () => {
+    dataDir = mkdtempSync(join(tmpdir(), 'metapi-stats-model-probe-'));
+    originalDataDir = process.env.DATA_DIR;
+    process.env.DATA_DIR = dataDir;
+
+    await import('../../db/migrate.js');
+    const routesModule = await import('./stats.js');
+    app = Fastify();
+    await app.register(routesModule.statsRoutes);
+  });
+
+  beforeEach(() => {
+    buildModelAvailabilityProbeTaskDedupeKeyMock.mockReset();
+    queueModelAvailabilityProbeTaskMock.mockReset();
+    getBackgroundTaskMock.mockReset();
+    getRunningTaskByDedupeKeyMock.mockReset();
+    waitForBackgroundTaskCompletionMock.mockReset();
+  });
+
+  afterAll(async () => {
+    await app.close();
+    if (originalDataDir === undefined) {
+      delete process.env.DATA_DIR;
+    } else {
+      process.env.DATA_DIR = originalDataDir;
+    }
+  });
+
+  it('rejects non-object request bodies instead of defaulting to a full probe', async () => {
+    const response = await app.inject({
+      method: 'POST',
+      url: '/api/models/probe',
+      headers: {
+        'content-type': 'application/json',
+      },
+      payload: '"oops"',
+    });
+
+    expect(response.statusCode).toBe(400);
+    expect(queueModelAvailabilityProbeTaskMock).not.toHaveBeenCalled();
+  });
+
+  it('rejects loosely formatted account ids instead of truncating them', async () => {
+    const response = await app.inject({
+      method: 'POST',
+      url: '/api/models/probe',
+      payload: {
+        accountId: '1e2',
+      },
+    });
+
+    expect(response.statusCode).toBe(400);
+    expect(queueModelAvailabilityProbeTaskMock).not.toHaveBeenCalled();
+  });
+
+  it('rejects account ids that exceed the safe integer range', async () => {
+    const response = await app.inject({
+      method: 'POST',
+      url: '/api/models/probe',
+      payload: {
+        accountId: '9007199254740993',
+      },
+    });
+
+    expect(response.statusCode).toBe(400);
+    expect(queueModelAvailabilityProbeTaskMock).not.toHaveBeenCalled();
+  });
+
+  it('reuses the running deduped task for wait=true requests', async () => {
+    buildModelAvailabilityProbeTaskDedupeKeyMock.mockReturnValue('model-availability-probe-7');
+    getRunningTaskByDedupeKeyMock.mockReturnValue({
+      id: 'task-7',
+      status: 'running',
+    });
+    const completedTask = {
+      id: 'task-7',
+      status: 'succeeded',
+      result: {
+        results: [],
+        summary: {
+          totalAccounts: 1,
+          success: 1,
+          failed: 0,
+          skipped: 0,
+          scanned: 3,
+          supported: 2,
+          unsupported: 1,
+          inconclusive: 0,
+          skippedModels: 0,
+          updatedRows: 1,
+          rebuiltRoutes: true,
+        },
+      },
+    };
+    waitForBackgroundTaskCompletionMock.mockResolvedValue(completedTask);
+
+    const response = await app.inject({
+      method: 'POST',
+      url: '/api/models/probe',
+      payload: {
+        accountId: 7,
+        wait: true,
+      },
+    });
+
+    expect(response.statusCode).toBe(200);
+    expect(queueModelAvailabilityProbeTaskMock).not.toHaveBeenCalled();
+    expect(waitForBackgroundTaskCompletionMock).toHaveBeenCalledWith('task-7');
+    expect(response.json()).toMatchObject({
+      success: true,
+      reused: true,
+      jobId: 'task-7',
+      summary: {
+        totalAccounts: 1,
+        scanned: 3,
+        supported: 2,
+        unsupported: 1,
+      },
+    });
+  });
+});

src/server/routes/api/stats.ts

@@ -6,8 +6,18 @@ import { refreshModelsForAccount } from '../../services/modelService.js';
 import * as routeRefreshWorkflow from '../../services/routeRefreshWorkflow.js';
 import { buildModelAnalysis } from '../../services/modelAnalysisService.js';
 import { fallbackTokenCost, fetchModelPricingCatalog } from '../../services/modelPricingService.js';
+import {
+  buildModelAvailabilityProbeTaskDedupeKey,
+  queueModelAvailabilityProbeTask,
+  type ModelAvailabilityProbeExecutionResult,
+} from '../../services/modelAvailabilityProbeService.js';
 import { getUpstreamModelDescriptionsCached } from '../../services/upstreamModelDescriptionService.js';
-import { getRunningTaskByDedupeKey, startBackgroundTask } from '../../services/backgroundTaskService.js';
+import {
+  getBackgroundTask,
+  getRunningTaskByDedupeKey,
+  startBackgroundTask,
+  waitForBackgroundTaskCompletion,
+} from '../../services/backgroundTaskService.js';
 import { parseCheckinRewardAmount } from '../../services/checkinRewardParser.js';
 import { estimateRewardWithTodayIncomeFallback } from '../../services/todayIncomeRewardService.js';
 import {
@@ -39,6 +49,10 @@ function parseBooleanFlag(raw?: string): boolean {
   return normalized === '1' || normalized === 'true' || normalized === 'yes';
 }
 
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return !!value && typeof value === 'object' && !Array.isArray(value);
+}
+
 const MODELS_MARKETPLACE_BASE_TTL_MS = 15_000;
 const MODELS_MARKETPLACE_PRICING_TTL_MS = 90_000;
 const limitModelTokenCandidatesRead = createRateLimitGuard({
@@ -1462,6 +1476,92 @@ export async function statsRoutes(app: FastifyInstance) {
     return { success: true, refresh, rebuild };
   });
 
+  app.post<{ Body?: { accountId?: number; wait?: boolean } }>('/api/models/probe', async (request, reply) => {
+    const requestBody = request.body;
+    if (requestBody !== undefined && !isRecord(requestBody)) {
+      return reply.code(400).send({ success: false, message: '请求体必须是对象' });
+    }
+
+    const rawAccountId = requestBody?.accountId as unknown;
+    const normalizedAccountId = rawAccountId === undefined || rawAccountId === null
+      ? ''
+      : String(rawAccountId).trim();
+    const hasAccountId = normalizedAccountId !== '';
+    const parsedAccountId = hasAccountId && /^[1-9]\d*$/.test(normalizedAccountId)
+      ? Number(normalizedAccountId)
+      : undefined;
+    const accountId = parsedAccountId !== undefined && Number.isSafeInteger(parsedAccountId)
+      ? parsedAccountId
+      : undefined;
+    const wait = requestBody?.wait === true;
+
+    if (hasAccountId && accountId === undefined) {
+      return reply.code(400).send({ success: false, message: '账号 ID 无效' });
+    }
+
+    if (wait) {
+      const taskTitle = accountId ? `探测模型可用性 #${accountId}` : '探测全部模型可用性';
+      const dedupeKey = buildModelAvailabilityProbeTaskDedupeKey(accountId);
+      const runningTask = getRunningTaskByDedupeKey(dedupeKey);
+      const { task, reused } = runningTask
+        ? { task: runningTask, reused: true }
+        : queueModelAvailabilityProbeTask({
+          accountId,
+          title: taskTitle,
+        });
+      const completedTask = await waitForBackgroundTaskCompletion(task.id);
+      if (!completedTask) {
+        return reply.code(500).send({ success: false, message: '模型可用性探测任务不存在或已过期' });
+      }
+      if (completedTask.status === 'failed') {
+        return reply.code(500).send({
+          success: false,
+          reused,
+          jobId: completedTask.id,
+          status: completedTask.status,
+          message: completedTask.error || '模型可用性探测失败',
+        });
+      }
+      const result = completedTask.result as ModelAvailabilityProbeExecutionResult | null;
+      if (!result) {
+        return reply.code(500).send({
+          success: false,
+          reused,
+          jobId: completedTask.id,
+          status: completedTask.status,
+          message: '模型可用性探测结果为空',
+        });
+      }
+      if (accountId && result.summary.totalAccounts === 0) {
+        return reply.code(404).send({ success: false, message: '账号不存在' });
+      }
+      return {
+        success: true,
+        reused,
+        jobId: completedTask.id,
+        status: completedTask.status,
+        ...result,
+      };
+    }
+
+    const taskTitle = accountId ? `探测模型可用性 #${accountId}` : '探测全部模型可用性';
+    const { task, reused } = queueModelAvailabilityProbeTask({
+      accountId,
+      title: taskTitle,
+    });
+
+    return reply.code(202).send({
+      success: true,
+      queued: true,
+      reused,
+      jobId: task.id,
+      status: task.status,
+      message: reused
+        ? '模型可用性探测任务进行中，请稍后查看任务列表'
+        : '已开始模型可用性探测，请稍后查看任务列表',
+    });
+  });
+
   // Site distribution – per-site aggregate data
   app.get('/api/stats/site-distribution', async () => {
     const accountRows = await db.select({

src/server/routes/proxy/architecture-boundaries.test.ts

@@ -104,6 +104,13 @@ describe('proxy route architecture boundaries', () => {
     expect(source).not.toContain('export function getUpstreamEndpointRuntimeStateSnapshot(');
   });
 
+  it('keeps endpoint flow orchestration owned by proxy-core instead of the route layer', () => {
+    const source = readSource('./endpointFlow.ts');
+    expect(source).toContain("from '../../proxy-core/orchestration/endpointFlow.js'");
+    expect(source).not.toContain('async function runEndpointFlowHook<');
+    expect(source).not.toContain('export async function executeEndpointFlow(');
+  });
+
   it('keeps gemini runtime closure in transformer-owned helpers', () => {
     const source = readSource('./gemini.ts');
     const surfaceSource = readSource('../../proxy-core/surfaces/geminiSurface.ts');

src/server/routes/proxy/chat.codex-oauth.test.ts

@@ -55,6 +55,7 @@ vi.mock('../../services/modelPricingService.js', () => ({
 
 vi.mock('../../services/proxyRetryPolicy.js', () => ({
   shouldRetryProxyRequest: () => false,
+  shouldAbortSameSiteEndpointFallback: () => false,
   RETRYABLE_TIMEOUT_PATTERNS: [/(request timed out|connection timed out|read timeout|\btimed out\b)/i],
 }));
 

src/server/routes/proxy/chat.count-tokens.test.ts

@@ -59,6 +59,7 @@ vi.mock('../../services/modelPricingService.js', () => ({
 
 vi.mock('../../services/proxyRetryPolicy.js', () => ({
   shouldRetryProxyRequest: () => false,
+  shouldAbortSameSiteEndpointFallback: () => false,
   RETRYABLE_TIMEOUT_PATTERNS: [/(request timed out|connection timed out|read timeout|\btimed out\b)/i],
 }));
 

src/server/routes/proxy/chat.siteApiEndpoint.test.ts

@@ -59,6 +59,7 @@ vi.mock('../../services/modelPricingService.js', () => ({
 
 vi.mock('../../services/proxyRetryPolicy.js', () => ({
   shouldRetryProxyRequest: () => false,
+  shouldAbortSameSiteEndpointFallback: () => false,
   RETRYABLE_TIMEOUT_PATTERNS: [/(request timed out|connection timed out|read timeout|\btimed out\b)/i],
 }));
 

src/server/routes/proxy/chat.stream.test.ts

@@ -60,6 +60,7 @@ vi.mock('../../services/modelPricingService.js', () => ({
 
 vi.mock('../../services/proxyRetryPolicy.js', () => ({
   shouldRetryProxyRequest: () => false,
+  shouldAbortSameSiteEndpointFallback: () => false,
   RETRYABLE_TIMEOUT_PATTERNS: [/(request timed out|connection timed out|read timeout|\btimed out\b)/i],
 }));
 
@@ -817,6 +818,122 @@ describe('chat proxy stream behavior', () => {
     expect(response.body).toContain('event: message_stop');
   });
 
+  it('normalizes null Claude message content before proxying on /v1/messages', async () => {
+    selectChannelMock.mockReturnValue({
+      channel: { id: 11, routeId: 22 },
+      site: {
+        name: 'claude-site',
+        url: 'https://upstream.example.com',
+        platform: 'claude',
+      },
+      account: { id: 33, username: 'demo-user' },
+      tokenName: 'default',
+      tokenValue: 'sk-demo',
+      actualModel: 'claude-opus-4-6',
+    });
+    fetchMock.mockResolvedValue(new Response(JSON.stringify({
+      id: 'msg_1',
+      type: 'message',
+      role: 'assistant',
+      model: 'claude-opus-4-6',
+      content: [{ type: 'text', text: 'ok' }],
+      stop_reason: 'end_turn',
+      usage: { input_tokens: 1, output_tokens: 1 },
+    }), {
+      status: 200,
+      headers: { 'content-type': 'application/json' },
+    }));
+
+    const response = await app.inject({
+      method: 'POST',
+      url: '/v1/messages',
+      payload: {
+        model: 'claude-opus-4-6',
+        max_tokens: 256,
+        messages: [
+          { role: 'user', content: 'hello' },
+          { role: 'assistant', content: null },
+        ],
+      },
+    });
+
+    expect(response.statusCode).toBe(200);
+    const forwardedBody = JSON.parse(fetchMock.mock.calls[0]?.[1]?.body as string);
+    expect(forwardedBody.messages).toEqual([
+      { role: 'user', content: 'hello' },
+    ]);
+  });
+
+  it('prefers responses for Claude tool_result follow-ups that include continuation hints', async () => {
+    selectChannelMock.mockReturnValue({
+      channel: { id: 11, routeId: 22 },
+      site: {
+        name: 'openai-site',
+        url: 'https://upstream.example.com',
+        platform: 'openai',
+      },
+      account: { id: 33, username: 'demo-user' },
+      tokenName: 'default',
+      tokenValue: 'sk-demo',
+      actualModel: 'gpt-5.4',
+    });
+    fetchMock.mockResolvedValue(new Response(JSON.stringify({
+      id: 'resp_1',
+      object: 'response',
+      model: 'gpt-5.4',
+      created_at: 1_706_000_000,
+      status: 'completed',
+      output: [{
+        id: 'msg_1',
+        type: 'message',
+        role: 'assistant',
+        status: 'completed',
+        content: [{ type: 'output_text', text: 'done' }],
+      }],
+      usage: {
+        input_tokens: 5,
+        output_tokens: 1,
+        total_tokens: 6,
+      },
+    }), {
+      status: 200,
+      headers: { 'content-type': 'application/json' },
+    }));
+
+    const response = await app.inject({
+      method: 'POST',
+      url: '/v1/messages',
+      payload: {
+        model: 'claude-opus-4-6',
+        max_tokens: 256,
+        previous_response_id: 'resp_prev_1',
+        messages: [
+          {
+            role: 'user',
+            content: [
+              {
+                type: 'tool_result',
+                tool_use_id: 'toolu_missing',
+                content: [{ type: 'text', text: '{"matches":1}' }],
+              },
+              { type: 'text', text: 'continue' },
+            ],
+          },
+        ],
+      },
+    });
+
+    expect(response.statusCode).toBe(200);
+    expect(String(fetchMock.mock.calls[0]?.[0])).toContain('/v1/responses');
+    const forwardedBody = JSON.parse(fetchMock.mock.calls[0]?.[1]?.body as string);
+    expect(forwardedBody.previous_response_id).toBe('resp_prev_1');
+    expect(forwardedBody.input[0]).toEqual({
+      type: 'function_call_output',
+      call_id: 'toolu_missing',
+      output: '{"matches":1}',
+    });
+  });
+
   it('converts OpenAI tool_calls SSE into Claude tool_use stream events on /v1/messages', async () => {
     const encoder = new TextEncoder();
     const upstreamBody = new ReadableStream<Uint8Array>({

src/server/routes/proxy/downstreamClientContext.routes.test.ts

@@ -58,6 +58,7 @@ vi.mock('../../services/modelPricingService.js', () => ({
 
 vi.mock('../../services/proxyRetryPolicy.js', () => ({
   shouldRetryProxyRequest: () => false,
+  shouldAbortSameSiteEndpointFallback: () => false,
   RETRYABLE_TIMEOUT_PATTERNS: [/(request timed out|connection timed out|read timeout|\btimed out\b)/i],
 }));
 

src/server/routes/proxy/endpointFlow.test.ts

@@ -224,6 +224,37 @@ describe('executeEndpointFlow', () => {
     expect(onAttemptSuccess.mock.calls[0]?.[0]?.request?.path).toBe('/v1/chat/completions');
   });
 
+  it('stops same-site endpoint fallback when the failure is classified as a site outage', async () => {
+    fetchMock
+      .mockResolvedValueOnce(toUndiciResponse(new Response(JSON.stringify({
+        error: { message: 'Service temporarily unavailable', type: 'upstream_error' },
+      }), {
+        status: 503,
+        headers: { 'content-type': 'application/json' },
+      })))
+      .mockResolvedValueOnce(toUndiciResponse(new Response(JSON.stringify({ ok: true }), {
+        status: 200,
+        headers: { 'content-type': 'application/json' },
+      })));
+
+    const result = await executeEndpointFlow({
+      siteUrl: 'https://example.com',
+      endpointCandidates: ['responses', 'chat'],
+      buildRequest: (endpoint) => endpoint === 'responses'
+        ? requestFor('/v1/responses')
+        : { ...requestFor('/v1/chat/completions'), endpoint },
+      shouldAbortRemainingEndpoints: () => true,
+      shouldDowngrade: () => true,
+    });
+
+    expect(result.ok).toBe(false);
+    if (!result.ok) {
+      expect(result.status).toBe(503);
+      expect(result.errText).toContain('Service temporarily unavailable');
+    }
+    expect(fetchMock).toHaveBeenCalledTimes(1);
+  });
+
   it('accepts recovered response from tryRecover hook', async () => {
     fetchMock.mockResolvedValueOnce(toUndiciResponse(new Response(JSON.stringify({
       error: { message: 'upstream_error', type: 'upstream_error' },

src/server/routes/proxy/endpointFlow.ts

@@ -1,220 +1,13 @@
-import { fetch } from 'undici';
-import { readRuntimeResponseText } from '../../proxy-core/executors/types.js';
-import { withSiteProxyRequestInit } from '../../services/siteProxy.js';
-import { summarizeUpstreamError } from './upstreamError.js';
-import type { UpstreamEndpoint } from './upstreamEndpoint.js';
-import { buildUpstreamUrl } from './upstreamUrl.js';
+export {
+  executeEndpointFlow,
+  withUpstreamPath,
+} from '../../proxy-core/orchestration/endpointFlow.js';
 
-export type BuiltEndpointRequest = {
-  endpoint: UpstreamEndpoint;
-  path: string;
-  headers: Record<string, string>;
-  body: Record<string, unknown>;
-  runtime?: {
-    executor: 'default' | 'codex' | 'gemini-cli' | 'antigravity' | 'claude';
-    modelName?: string;
-    stream?: boolean;
-    oauthProjectId?: string | null;
-    action?: 'generateContent' | 'streamGenerateContent' | 'countTokens';
-  };
-};
-
-export type EndpointAttemptContext = {
-  endpointIndex: number;
-  endpointCount: number;
-  request: BuiltEndpointRequest;
-  targetUrl: string;
-  response: Awaited<ReturnType<typeof fetch>>;
-  rawErrText: string;
-  recoverApplied?: boolean;
-};
-
-export type EndpointAttemptSuccessContext = {
-  endpointIndex: number;
-  endpointCount: number;
-  request: BuiltEndpointRequest;
-  targetUrl: string;
-  response: Awaited<ReturnType<typeof fetch>>;
-  recoverApplied?: boolean;
-};
-
-export type EndpointRecoverResult = {
-  upstream: Awaited<ReturnType<typeof fetch>>;
-  upstreamPath: string;
-  request?: BuiltEndpointRequest;
-  targetUrl?: string;
-} | null;
-
-export type EndpointFlowResult =
-  | {
-    ok: true;
-    upstream: Awaited<ReturnType<typeof fetch>>;
-    upstreamPath: string;
-  }
-  | {
-    ok: false;
-    status: number;
-    errText: string;
-    rawErrText?: string;
-  };
-
-export function withUpstreamPath(path: string, message: string): string {
-  return `[upstream:${path}] ${message}`;
-}
-
-type ExecuteEndpointFlowInput = {
-  siteUrl: string;
-  proxyUrl?: string | null;
-  disableCrossProtocolFallback?: boolean;
-  endpointCandidates: UpstreamEndpoint[];
-  buildRequest: (endpoint: UpstreamEndpoint, endpointIndex: number) => BuiltEndpointRequest;
-  dispatchRequest?: (
-    request: BuiltEndpointRequest,
-    targetUrl: string,
-  ) => Promise<Awaited<ReturnType<typeof fetch>>>;
-  tryRecover?: (ctx: EndpointAttemptContext) => Promise<EndpointRecoverResult>;
-  shouldDowngrade?: (ctx: EndpointAttemptContext) => boolean;
-  onDowngrade?: (ctx: EndpointAttemptContext & { errText: string }) => void | Promise<void>;
-  onAttemptFailure?: (ctx: EndpointAttemptContext & { errText: string }) => void | Promise<void>;
-  onAttemptSuccess?: (ctx: EndpointAttemptSuccessContext) => void | Promise<void>;
-};
-
-async function runEndpointFlowHook<T>(
-  hook: ((ctx: T) => void | Promise<void>) | undefined,
-  ctx: T,
-  hookName: string,
-): Promise<void> {
-  if (!hook) return;
-  try {
-    await hook(ctx);
-  } catch (error) {
-    console.error(`endpointFlow ${hookName} hook failed`, error);
-  }
-}
-export async function executeEndpointFlow(input: ExecuteEndpointFlowInput): Promise<EndpointFlowResult> {
-  const endpointCount = input.endpointCandidates.length;
-  if (endpointCount <= 0) {
-    return {
-      ok: false,
-      status: 502,
-      errText: 'Upstream request failed',
-    };
-  }
-
-  let finalStatus = 0;
-  let finalErrText = 'unknown error';
-  let finalRawErrText: string | undefined;
-
-  for (let endpointIndex = 0; endpointIndex < endpointCount; endpointIndex += 1) {
-    const endpoint = input.endpointCandidates[endpointIndex] as UpstreamEndpoint;
-    const request = input.buildRequest(endpoint, endpointIndex);
-    const defaultTarget = buildUpstreamUrl(input.siteUrl, request.path);
-    const targetUrl = input.proxyUrl
-      ? buildUpstreamUrl(input.proxyUrl, request.path)
-      : defaultTarget;
-
-    let response = input.dispatchRequest
-      ? await input.dispatchRequest(request, targetUrl)
-      : await fetch(targetUrl, await withSiteProxyRequestInit(targetUrl, {
-        method: 'POST',
-        headers: request.headers,
-        body: JSON.stringify(request.body),
-      }));
-
-    if (response.ok) {
-      await runEndpointFlowHook(input.onAttemptSuccess, {
-        endpointIndex,
-        endpointCount,
-        request,
-        targetUrl,
-        response,
-        recoverApplied: false,
-      }, 'onAttemptSuccess');
-      return {
-        ok: true,
-        upstream: response,
-        upstreamPath: request.path,
-      };
-    }
-
-    let rawErrText = await readRuntimeResponseText(response).catch(() => 'unknown error');
-    const baseContext: EndpointAttemptContext = {
-      endpointIndex,
-      endpointCount,
-      request,
-      targetUrl,
-      response,
-      rawErrText,
-      recoverApplied: false,
-    };
-
-    if (input.tryRecover) {
-      const recovered = await input.tryRecover(baseContext);
-      baseContext.recoverApplied = recovered !== null
-        || baseContext.request !== request
-        || baseContext.response !== response
-        || baseContext.rawErrText !== rawErrText;
-      if (recovered?.upstream?.ok) {
-        const recoveredRequest = recovered.request ?? baseContext.request;
-        const recoveredTargetUrl = recovered.targetUrl ?? (
-          input.proxyUrl
-            ? buildUpstreamUrl(input.proxyUrl, recovered.upstreamPath)
-            : buildUpstreamUrl(input.siteUrl, recovered.upstreamPath)
-        );
-        await runEndpointFlowHook(input.onAttemptSuccess, {
-          endpointIndex,
-          endpointCount,
-          request: recoveredRequest,
-          targetUrl: recoveredTargetUrl,
-          response: recovered.upstream,
-          recoverApplied: true,
-        }, 'onAttemptSuccess');
-        return {
-          ok: true,
-          upstream: recovered.upstream,
-          upstreamPath: recovered.upstreamPath,
-        };
-      }
-    }
-
-    // Normalize again in case recoverer performed additional probes and updated the response text.
-    rawErrText = baseContext.rawErrText;
-    response = baseContext.response;
-    const errText = withUpstreamPath(
-      baseContext.request.path,
-      summarizeUpstreamError(response.status, rawErrText),
-    );
-    await runEndpointFlowHook(input.onAttemptFailure, {
-      ...baseContext,
-      errText,
-    }, 'onAttemptFailure');
-
-    const isLastEndpoint = endpointIndex >= endpointCount - 1;
-    if (input.disableCrossProtocolFallback && !isLastEndpoint) {
-      finalStatus = response.status;
-      finalErrText = errText;
-      finalRawErrText = rawErrText;
-      break;
-    }
-    const shouldDowngrade = !isLastEndpoint && !!input.shouldDowngrade?.(baseContext);
-    if (shouldDowngrade) {
-      await runEndpointFlowHook(input.onDowngrade, {
-        ...baseContext,
-        errText,
-      }, 'onDowngrade');
-      continue;
-    }
-
-    finalStatus = response.status;
-    finalErrText = errText;
-    finalRawErrText = rawErrText;
-    break;
-  }
-
-  return {
-    ok: false,
-    status: finalStatus || 502,
-    errText: finalErrText || 'unknown error',
-    rawErrText: finalRawErrText,
-  };
-}
+export type {
+  BuiltEndpointRequest,
+  EndpointAttemptContext,
+  EndpointAttemptSuccessContext,
+  EndpointRecoverResult,
+  EndpointFlowResult,
+  ExecuteEndpointFlowInput,
+} from '../../proxy-core/orchestration/endpointFlow.js';

src/server/routes/proxy/images.edits.test.ts

@@ -52,6 +52,7 @@ vi.mock('../../services/modelPricingService.js', () => ({
 
 vi.mock('../../services/proxyRetryPolicy.js', () => ({
   shouldRetryProxyRequest: () => false,
+  shouldAbortSameSiteEndpointFallback: () => false,
   RETRYABLE_TIMEOUT_PATTERNS: [/(request timed out|connection timed out|read timeout|\btimed out\b)/i],
 }));
 

src/server/routes/proxy/responses.codex-oauth.test.ts

@@ -3,6 +3,7 @@ import Fastify, { type FastifyInstance } from 'fastify';
 import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
 import { config } from '../../config.js';
 import { resetCodexHttpSessionQueue } from '../../proxy-core/runtime/codexHttpSessionQueue.js';
+import { resetCodexSessionResponseStore } from '../../proxy-core/runtime/codexSessionResponseStore.js';
 
 const fetchMock = vi.fn();
 const selectChannelMock = vi.fn();
@@ -69,6 +70,7 @@ vi.mock('../../services/modelPricingService.js', () => ({
 
 vi.mock('../../services/proxyRetryPolicy.js', () => ({
   shouldRetryProxyRequest: () => false,
+  shouldAbortSameSiteEndpointFallback: () => false,
   RETRYABLE_TIMEOUT_PATTERNS: [/(request timed out|connection timed out|read timeout|\btimed out\b)/i],
 }));
 
@@ -174,6 +176,7 @@ describe('responses proxy codex oauth refresh', () => {
 
   beforeEach(() => {
     resetCodexHttpSessionQueue();
+    resetCodexSessionResponseStore();
     config.proxyEmptyContentFailEnabled = false;
     config.proxyStickySessionEnabled = originalProxyStickySessionEnabled;
     config.proxySessionChannelConcurrencyLimit = originalProxySessionChannelConcurrencyLimit;
@@ -215,6 +218,7 @@ describe('responses proxy codex oauth refresh', () => {
     });
     selectPreferredChannelMock.mockReturnValue(null);
     selectNextChannelMock.mockReturnValue(null);
+    selectPreferredChannelMock.mockReturnValue(null);
     refreshOauthAccessTokenSingleflightMock.mockResolvedValue({
       accessToken: 'fresh-access-token',
       accountId: 33,
@@ -488,6 +492,137 @@ describe('responses proxy codex oauth refresh', () => {
     expect(forwardedBody.prompt_cache_key).toBe('codex-cache-123');
   });
 
+  it('infers previous_response_id for codex tool-output follow-up turns on the same downstream session', async () => {
+    fetchMock
+      .mockResolvedValueOnce(new Response(JSON.stringify({
+        id: 'resp_codex_prev_1',
+        object: 'response',
+        model: 'gpt-5.2-codex',
+        status: 'completed',
+        output_text: 'tool call issued',
+        usage: { input_tokens: 4, output_tokens: 2, total_tokens: 6 },
+      }), {
+        status: 200,
+        headers: { 'content-type': 'application/json' },
+      }))
+      .mockResolvedValueOnce(new Response(JSON.stringify({
+        id: 'resp_codex_prev_2',
+        object: 'response',
+        model: 'gpt-5.2-codex',
+        status: 'completed',
+        output_text: 'tool result accepted',
+        usage: { input_tokens: 5, output_tokens: 2, total_tokens: 7 },
+      }), {
+        status: 200,
+        headers: { 'content-type': 'application/json' },
+      }));
+
+    const headers = {
+      session_id: 'session-http-prev-1',
+      'user-agent': 'CodexClient/1.0',
+    };
+
+    const firstResponse = await app.inject({
+      method: 'POST',
+      url: '/v1/responses',
+      headers,
+      payload: {
+        model: 'gpt-5.2-codex',
+        input: 'start codex tool flow',
+      },
+    });
+    expect(firstResponse.statusCode).toBe(200);
+
+    const secondResponse = await app.inject({
+      method: 'POST',
+      url: '/v1/responses',
+      headers,
+      payload: {
+        model: 'gpt-5.2-codex',
+        input: [
+          {
+            id: 'tool_out_1',
+            type: 'function_call_output',
+            call_id: 'call_1',
+            output: '{"ok":true}',
+          },
+        ],
+      },
+    });
+
+    expect(secondResponse.statusCode).toBe(200);
+    expect(fetchMock).toHaveBeenCalledTimes(2);
+
+    const [, secondOptions] = fetchMock.mock.calls[1] as [string, any];
+    const secondBody = JSON.parse(secondOptions.body);
+    expect(secondBody.previous_response_id).toBe('resp_codex_prev_1');
+    expect(secondBody.input).toEqual([
+      {
+        id: 'tool_out_1',
+        type: 'function_call_output',
+        call_id: 'call_1',
+        output: '{"ok":true}',
+      },
+    ]);
+  });
+
+  it('drops stale previous_response_id and retries codex responses requests once', async () => {
+    fetchMock
+      .mockResolvedValueOnce(new Response(JSON.stringify({
+        error: {
+          message: 'previous_response_not_found',
+          code: 'previous_response_not_found',
+          type: 'invalid_request_error',
+        },
+      }), {
+        status: 400,
+        headers: { 'content-type': 'application/json' },
+      }))
+      .mockResolvedValueOnce(new Response(JSON.stringify({
+        id: 'resp_codex_prev_recovered',
+        object: 'response',
+        model: 'gpt-5.2-codex',
+        status: 'completed',
+        output_text: 'recovered after dropping stale previous_response_id',
+        usage: { input_tokens: 5, output_tokens: 2, total_tokens: 7 },
+      }), {
+        status: 200,
+        headers: { 'content-type': 'application/json' },
+      }));
+
+    const response = await app.inject({
+      method: 'POST',
+      url: '/v1/responses',
+      headers: {
+        session_id: 'session-http-prev-recovery',
+      },
+      payload: {
+        model: 'gpt-5.2-codex',
+        previous_response_id: 'resp_stale',
+        input: [
+          {
+            id: 'tool_out_retry_1',
+            type: 'function_call_output',
+            call_id: 'call_retry_1',
+            output: '{"retry":true}',
+          },
+        ],
+      },
+    });
+
+    expect(response.statusCode).toBe(200);
+    expect(fetchMock).toHaveBeenCalledTimes(2);
+
+    const [, firstOptions] = fetchMock.mock.calls[0] as [string, any];
+    const [, secondOptions] = fetchMock.mock.calls[1] as [string, any];
+    const firstBody = JSON.parse(firstOptions.body);
+    const secondBody = JSON.parse(secondOptions.body);
+
+    expect(firstBody.previous_response_id).toBe('resp_stale');
+    expect(secondBody.previous_response_id).toBeUndefined();
+    expect(secondBody.input).toEqual(firstBody.input);
+  });
+
   it('strips generic downstream headers before forwarding codex responses upstream', async () => {
     fetchMock.mockResolvedValue(new Response(JSON.stringify({
       id: 'resp_codex_header_filter',

src/server/routes/proxy/responses.compact-upstream.test.ts

@@ -57,6 +57,7 @@ vi.mock('../../services/modelPricingService.js', () => ({
 
 vi.mock('../../services/proxyRetryPolicy.js', () => ({
   shouldRetryProxyRequest: () => false,
+  shouldAbortSameSiteEndpointFallback: () => false,
   RETRYABLE_TIMEOUT_PATTERNS: [/(request timed out|connection timed out|read timeout|\btimed out\b)/i],
 }));
 

src/server/routes/proxy/responses.websocket.test.ts

@@ -2,12 +2,14 @@ import Fastify, { type FastifyInstance } from 'fastify';
 import { createServer, type Server } from 'node:http';
 import { AddressInfo } from 'node:net';
 import WebSocket, { WebSocketServer } from 'ws';
-import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
+import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
 import { config } from '../../config.js';
+import { resetCodexSessionResponseStore } from '../../proxy-core/runtime/codexSessionResponseStore.js';
 
 const fetchMock = vi.fn();
 const selectChannelMock = vi.fn();
 const selectNextChannelMock = vi.fn();
+const selectPreferredChannelMock = vi.fn();
 const previewSelectedChannelMock = vi.fn();
 const recordSuccessMock = vi.fn();
 const recordFailureMock = vi.fn();
@@ -21,6 +23,7 @@ const resolveProxyUsageWithSelfLogFallbackMock = vi.fn(async ({ usage }: any) =>
   estimatedCostFromQuota: 0,
   recoveredFromSelfLog: false,
 }));
+const trackedClientSockets = new Set<WebSocket>();
 const dbInsertMock = vi.fn((_arg?: any) => ({
   values: () => ({
     run: () => undefined,
@@ -35,6 +38,7 @@ vi.mock('../../services/tokenRouter.js', () => ({
   tokenRouter: {
     selectChannel: (...args: unknown[]) => selectChannelMock(...args),
     selectNextChannel: (...args: unknown[]) => selectNextChannelMock(...args),
+    selectPreferredChannel: (...args: unknown[]) => selectPreferredChannelMock(...args),
     previewSelectedChannel: (...args: unknown[]) => previewSelectedChannelMock(...args),
     recordSuccess: (...args: unknown[]) => recordSuccessMock(...args),
     recordFailure: (...args: unknown[]) => recordFailureMock(...args),
@@ -78,6 +82,7 @@ vi.mock('../../services/modelPricingService.js', () => ({
 
 vi.mock('../../services/proxyRetryPolicy.js', () => ({
   shouldRetryProxyRequest: () => false,
+  shouldAbortSameSiteEndpointFallback: () => false,
   RETRYABLE_TIMEOUT_PATTERNS: [/(request timed out|connection timed out|read timeout|\btimed out\b)/i],
 }));
 
@@ -256,16 +261,26 @@ function createDeferred<T>() {
 }
 
 function createClientSocket(baseUrl: string, headers: Record<string, string> = {}) {
-  return new WebSocket(`${baseUrl}/v1/responses`, {
+  const socket = new WebSocket(`${baseUrl}/v1/responses`, {
     headers: {
       Authorization: 'Bearer sk-global-proxy-token',
       ...headers,
     },
   });
+  trackedClientSockets.add(socket);
+  socket.once('close', () => {
+    trackedClientSockets.delete(socket);
+  });
+  return socket;
 }
 
 function createClientSocketForPath(path: string, headers: Record<string, string> = {}) {
-  return new WebSocket(path, { headers });
+  const socket = new WebSocket(path, { headers });
+  trackedClientSockets.add(socket);
+  socket.once('close', () => {
+    trackedClientSockets.delete(socket);
+  });
+  return socket;
 }
 
 describe('responses websocket transport', () => {
@@ -274,6 +289,7 @@ describe('responses websocket transport', () => {
   let app: FastifyInstance;
   let baseUrl: string;
   let upstreamServer: WebSocketServer;
+  let upstreamSockets: Set<WebSocket>;
   let upstreamSiteUrl: string;
   let upstreamConnectionCount: number;
   let upstreamUpgradeHeaders: Record<string, string>;
@@ -294,7 +310,12 @@ describe('responses websocket transport', () => {
     baseUrl = `ws://127.0.0.1:${address.port}`;
 
     upstreamServer = new WebSocketServer({ port: 0 });
+    upstreamSockets = new Set();
     upstreamServer.on('connection', (socket, request) => {
+      upstreamSockets.add(socket);
+      socket.once('close', () => {
+        upstreamSockets.delete(socket);
+      });
       upstreamConnectionCount += 1;
       upstreamUpgradeHeaders = Object.fromEntries(
         Object.entries(request.headers)
@@ -329,9 +350,11 @@ describe('responses websocket transport', () => {
   });
 
   beforeEach(() => {
+    resetCodexSessionResponseStore();
     fetchMock.mockReset();
     selectChannelMock.mockReset();
     selectNextChannelMock.mockReset();
+    selectPreferredChannelMock.mockReset();
     previewSelectedChannelMock.mockReset();
     recordSuccessMock.mockReset();
     recordFailureMock.mockReset();
@@ -346,6 +369,7 @@ describe('responses websocket transport', () => {
     const selectedChannel = createSelectedChannel();
     selectChannelMock.mockReturnValue(selectedChannel);
     selectNextChannelMock.mockReturnValue(null);
+    selectPreferredChannelMock.mockReturnValue(null);
     previewSelectedChannelMock.mockResolvedValue(selectedChannel);
     upstreamConnectionCount = 0;
     upstreamUpgradeHeaders = {};
@@ -386,8 +410,29 @@ describe('responses websocket transport', () => {
     };
   });
 
+  afterEach(() => {
+    for (const socket of trackedClientSockets) {
+      try {
+        socket.terminate();
+      } catch {}
+    }
+    trackedClientSockets.clear();
+  });
+
   afterAll(async () => {
     (config as any).codexUpstreamWebsocketEnabled = originalCodexUpstreamWebsocketEnabled;
+    for (const socket of trackedClientSockets) {
+      try {
+        socket.terminate();
+      } catch {}
+    }
+    trackedClientSockets.clear();
+    for (const socket of upstreamSockets || []) {
+      try {
+        socket.terminate();
+      } catch {}
+    }
+    upstreamSockets?.clear();
     if (rejectedUpgradeServer) {
       await new Promise<void>((resolve) => rejectedUpgradeServer.close(() => resolve()));
     }
@@ -551,6 +596,143 @@ describe('responses websocket transport', () => {
     expect(upstreamUpgradeHeaders['openai-beta']).toContain('responses_websockets=2099-01-01');
   });
 
+  it('infers previous_response_id for websocket tool-output follow-up turns when the client omits it', async () => {
+    const selectedChannel = createSelectedChannel({
+      siteUrl: upstreamSiteUrl,
+    });
+    selectChannelMock.mockReturnValue(selectedChannel);
+    previewSelectedChannelMock.mockResolvedValue(selectedChannel);
+
+    const socket = createClientSocket(baseUrl, {
+      session_id: 'ws-session-prev-infer',
+    });
+    await waitForSocketOpen(socket);
+
+    const firstResponsePromise = waitForSocketMessageMatching(
+      socket,
+      (message) => message?.type === 'response.completed',
+    );
+    socket.send(JSON.stringify({
+      type: 'response.create',
+      model: 'gpt-5.4',
+      input: [],
+    }));
+    const firstResponse = await firstResponsePromise;
+
+    const secondResponsePromise = waitForSocketMessageMatching(
+      socket,
+      (message) => message?.type === 'response.completed' && message?.response?.id === 'resp_upstream_2',
+    );
+    socket.send(JSON.stringify({
+      type: 'response.create',
+      model: 'gpt-5.4',
+      input: [
+        {
+          id: 'tool_out_ws_1',
+          type: 'function_call_output',
+          call_id: 'call_ws_1',
+          output: '{"ok":true}',
+        },
+      ],
+    }));
+    await secondResponsePromise;
+    socket.close();
+
+    expect(firstResponse?.response?.id).toBe('resp_upstream_1');
+    expect(upstreamRequests).toHaveLength(2);
+    expect(upstreamRequests[1]).toMatchObject({
+      type: 'response.create',
+      previous_response_id: 'resp_upstream_1',
+      input: [
+        {
+          id: 'tool_out_ws_1',
+          type: 'function_call_output',
+          call_id: 'call_ws_1',
+          output: '{"ok":true}',
+        },
+      ],
+    });
+  });
+
+  it('retries websocket turns once without previous_response_id when the upstream reports previous_response_not_found', async () => {
+    const selectedChannel = createSelectedChannel({
+      siteUrl: upstreamSiteUrl,
+    });
+    selectChannelMock.mockReturnValue(selectedChannel);
+    previewSelectedChannelMock.mockResolvedValue(selectedChannel);
+    upstreamMessageHandler = (socket, parsed, requestIndex) => {
+      if (requestIndex === 1) {
+        socket.send(JSON.stringify({
+          type: 'error',
+          error: {
+            message: 'previous_response_not_found',
+            code: 'previous_response_not_found',
+            type: 'invalid_request_error',
+          },
+        }));
+        return;
+      }
+      socket.send(JSON.stringify({
+        type: 'response.completed',
+        response: {
+          id: 'resp_upstream_recovered',
+          object: 'response',
+          model: parsed.model || 'gpt-5.4',
+          status: 'completed',
+          output: [],
+          usage: {
+            input_tokens: 1,
+            output_tokens: 1,
+            total_tokens: 2,
+          },
+        },
+      }));
+    };
+
+    const socket = createClientSocket(baseUrl, {
+      session_id: 'ws-session-prev-recovery',
+    });
+    await waitForSocketOpen(socket);
+
+    const responsePromise = waitForSocketMessageMatching(
+      socket,
+      (message) => message?.type === 'response.completed' && message?.response?.id === 'resp_upstream_recovered',
+    );
+    socket.send(JSON.stringify({
+      type: 'response.create',
+      model: 'gpt-5.4',
+      previous_response_id: 'resp_stale_ws',
+      input: [
+        {
+          id: 'tool_out_ws_retry_1',
+          type: 'function_call_output',
+          call_id: 'call_ws_retry_1',
+          output: '{"retry":true}',
+        },
+      ],
+    }));
+    await responsePromise;
+    socket.close();
+
+    expect(upstreamRequests).toHaveLength(2);
+    expect(upstreamRequests[0]).toMatchObject({
+      type: 'response.create',
+      previous_response_id: 'resp_stale_ws',
+    });
+    expect(upstreamRequests[1]).toMatchObject({
+      type: 'response.create',
+      input: [
+        {
+          id: 'tool_out_ws_retry_1',
+          type: 'function_call_output',
+          call_id: 'call_ws_retry_1',
+          output: '{"retry":true}',
+        },
+      ],
+    });
+    expect(upstreamRequests[1]?.previous_response_id).toBeUndefined();
+  });
+
   it('falls back to the HTTP responses executor when the upstream codex websocket upgrade returns 426', async () => {
     const selectedChannel = createSelectedChannel({
       siteUrl: rejectedUpgradeSiteUrl,

src/server/routes/proxy/responsesWebsocket.ts

@@ -4,6 +4,7 @@ import type { IncomingMessage } from 'node:http';
 import type { Duplex } from 'node:stream';
 import { WebSocketServer, type RawData, type WebSocket } from 'ws';
 import { createCodexWebsocketRuntime, CodexWebsocketRuntimeError } from '../../proxy-core/runtime/codexWebsocketRuntime.js';
+import { buildCodexSessionResponseStoreKey } from '../../proxy-core/runtime/codexSessionResponseStore.js';
 import {
   authorizeDownstreamToken,
   consumeManagedKeyRequest,
@@ -541,13 +542,23 @@ async function handleResponsesWebsocketConnection(
   const websocketSessionId = headerValueToTrimmedString(request.headers['session_id'])
     || headerValueToTrimmedString(request.headers['session-id'])
     || randomUUID();
+  const runtimeSessionKeys = new Set<string>();
   let lastRequest: Record<string, unknown> | null = null;
   let lastResponseOutput: unknown[] = [];
   let selectedChannel: SelectedChannel | null = null;
   let messageQueue = Promise.resolve();
 
   socket.once('close', () => {
-    void codexWebsocketRuntime.closeSession(websocketSessionId);
+    const sessionKeys = runtimeSessionKeys.size > 0
+      ? Array.from(runtimeSessionKeys)
+      : [websocketSessionId];
+    void Promise.all(sessionKeys.map(async (sessionKey) => {
+      try {
+        await codexWebsocketRuntime.closeSession(sessionKey);
+      } catch {
+        // Ignore close-time cleanup failures after downstream disconnects.
+      }
+    }));
   });
 
   socket.on('message', (raw) => {
@@ -634,8 +645,15 @@ async function handleResponsesWebsocketConnection(
             const requestUrl = `${codexWebsocketChannel.site.url.replace(/\/+$/, '')}${prepared.path}`;
 
             try {
-              const runtimeResult = await codexWebsocketRuntime.sendRequest({
+              const websocketRuntimeSessionKey = buildCodexSessionResponseStoreKey({
                 sessionId: websocketSessionId,
+                siteId: codexWebsocketChannel.site.id,
+                accountId: codexWebsocketChannel.account.id,
+                channelId: codexWebsocketChannel.channel.id,
+              }) || websocketSessionId;
+              runtimeSessionKeys.add(websocketRuntimeSessionKey);
+              const runtimeResult = await codexWebsocketRuntime.sendRequest({
+                sessionId: websocketRuntimeSessionKey,
                 requestUrl,
                 headers: prepared.headers,
                 body: prepared.body,

src/server/routes/proxy/search.test.ts

@@ -48,6 +48,7 @@ vi.mock('../../services/modelPricingService.js', () => ({
 
 vi.mock('../../services/proxyRetryPolicy.js', () => ({
   shouldRetryProxyRequest: () => false,
+  shouldAbortSameSiteEndpointFallback: () => false,
   RETRYABLE_TIMEOUT_PATTERNS: [/(request timed out|connection timed out|read timeout|\btimed out\b)/i],
 }));
 

src/server/routes/proxy/upstreamEndpoint.test.ts

@@ -8,6 +8,7 @@ vi.mock('../../services/modelPricingService.js', () => ({
 }));
 
 import {
+  buildClaudeCountTokensUpstreamRequest,
   buildMinimalJsonHeadersForCompatibility,
   buildUpstreamEndpointRequest,
   isUnsupportedMediaTypeError,
@@ -188,6 +189,23 @@ describe('resolveUpstreamEndpointCandidates', () => {
     expect(codexOrder).toEqual(['responses']);
   });
 
+  it('prefers responses for claude continuation follow-ups that carry orphan tool results', async () => {
+    const order = await resolveUpstreamEndpointCandidates(
+      {
+        ...baseContext,
+        site: { ...baseContext.site, platform: 'openai' },
+      },
+      'gpt-5.4',
+      'claude',
+      'claude-opus-4-6',
+      {
+        wantsContinuationAwareResponses: true,
+      },
+    );
+
+    expect(order).toEqual(['responses', 'messages', 'chat']);
+  });
+
   it('prefers document-capable endpoints when downstream content contains non-image files', async () => {
     const order = await resolveUpstreamEndpointCandidates(
       {
@@ -2053,6 +2071,60 @@ describe('buildUpstreamEndpointRequest', () => {
     });
   });
 
+  it('drops responses-style continuation fields before proxying native Claude messages upstream', () => {
+    const request = buildUpstreamEndpointRequest({
+      endpoint: 'messages',
+      modelName: 'claude-opus-4-6',
+      stream: false,
+      tokenValue: 'sk-test',
+      sitePlatform: 'claude',
+      siteUrl: 'https://example.com',
+      downstreamFormat: 'claude',
+      openaiBody: {
+        model: 'ignored',
+        messages: [{ role: 'user', content: 'ignored' }],
+      },
+      claudeOriginalBody: {
+        model: 'claude-opus-4-6',
+        max_tokens: 256,
+        previous_response_id: 'resp_prev_1',
+        prompt_cache_key: 'cache-key-1',
+        messages: [{ role: 'user', content: 'hello' }],
+      },
+    });
+
+    expect(request.body).toEqual({
+      model: 'claude-opus-4-6',
+      max_tokens: 256,
+      messages: [{ role: 'user', content: 'hello' }],
+      stream: false,
+    });
+  });
+
+  it('drops responses-style continuation fields before proxying Claude count_tokens upstream', () => {
+    const request = buildClaudeCountTokensUpstreamRequest({
+      modelName: 'claude-opus-4-6',
+      tokenValue: 'sk-test',
+      sitePlatform: 'claude',
+      claudeBody: {
+        model: 'claude-opus-4-6',
+        max_tokens: 256,
+        previous_response_id: 'resp_prev_1',
+        prompt_cache_key: 'cache-key-1',
+        messages: [{ role: 'user', content: 'hello' }],
+      },
+    });
+
+    expect(request.body).toMatchObject({
+      model: 'claude-opus-4-6',
+      messages: [{ role: 'user' }],
+    });
+    expect(request.body).not.toHaveProperty('previous_response_id');
+    expect(request.body).not.toHaveProperty('prompt_cache_key');
+    expect(request.body).not.toHaveProperty('max_tokens');
+    expect(request.body).not.toHaveProperty('maxTokens');
+  });
+
   it('preserves multimodal OpenAI user content when converting to Responses input', () => {
     const request = buildUpstreamEndpointRequest({
       endpoint: 'responses',

src/server/routes/proxy/upstreamEndpoint.ts

@@ -3,6 +3,7 @@ import {
   rankConversationFileEndpoints,
   type ConversationFileInputSummary,
 } from '../../proxy-core/capabilities/conversationFileCapabilities.js';
+import type { UpstreamEndpoint } from '../../proxy-core/orchestration/upstreamRequest.js';
 import { resolveProviderProfile } from '../../proxy-core/providers/registry.js';
 import { config } from '../../config.js';
 import { fetchModelPricingCatalog } from '../../services/modelPricingService.js';
@@ -46,7 +47,7 @@ export {
   shouldPreferResponsesAfterLegacyChatError,
 };
 
-export type UpstreamEndpoint = 'chat' | 'messages' | 'responses';
+export type { UpstreamEndpoint } from '../../proxy-core/orchestration/upstreamRequest.js';
 export type EndpointPreference = DownstreamFormat | 'responses';
 
 type ChannelContext = {
@@ -225,6 +226,15 @@ function extractClaudeBetasFromBody(body: Record<string, unknown>): {
   };
 }
 
+function stripClaudeMessagesContinuationFields(
+  body: Record<string, unknown>,
+): Record<string, unknown> {
+  const next = { ...body };
+  delete next.previous_response_id;
+  delete next.prompt_cache_key;
+  return next;
+}
+
 function buildAntigravityRuntimeHeaders(input: {
   baseHeaders: Record<string, string>;
   stream: boolean;
@@ -529,6 +539,7 @@ export async function resolveUpstreamEndpointCandidates(
     hasNonImageFileInput?: boolean;
     conversationFileSummary?: ConversationFileInputSummary;
     wantsNativeResponsesReasoning?: boolean;
+    wantsContinuationAwareResponses?: boolean;
   },
 ): Promise<UpstreamEndpoint[]> {
   const sitePlatform = normalizePlatformName(context.site.platform);
@@ -540,6 +551,7 @@ export async function resolveUpstreamEndpointCandidates(
   const preferMessagesForClaudeModel = capabilityProfile.preferMessagesForClaudeModel;
   const hasNonImageFileInput = capabilityProfile.hasNonImageFileInput;
   const wantsNativeResponsesReasoning = capabilityProfile.wantsNativeResponsesReasoning;
+  const wantsContinuationAwareResponses = capabilityProfile.wantsContinuationAwareResponses;
   const applyRuntimePreference = (candidates: UpstreamEndpoint[]) => (
     applyUpstreamEndpointRuntimePreference(candidates, {
       siteId: context.site.id,
@@ -587,9 +599,11 @@ export async function resolveUpstreamEndpointCandidates(
     })()
     : preferred;
   const prioritizedPreferredEndpoints: UpstreamEndpoint[] = (
-    wantsNativeResponsesReasoning
-    && preferMessagesForClaudeModel
-    && preferredWithCapabilities.includes('responses')
+    preferredWithCapabilities.includes('responses')
+    && (
+      wantsContinuationAwareResponses
+      || (wantsNativeResponsesReasoning && preferMessagesForClaudeModel)
+    )
   )
     ? [
       'responses',
@@ -865,7 +879,7 @@ export function buildUpstreamEndpointRequest(input: {
       && input.forceNormalizeClaudeBody !== true
     )
       ? {
-        ...input.claudeOriginalBody,
+        ...stripClaudeMessagesContinuationFields(input.claudeOriginalBody),
         model: input.modelName,
         stream: input.stream,
       }
@@ -876,7 +890,7 @@ export function buildUpstreamEndpointRequest(input: {
       && input.forceNormalizeClaudeBody === true
     )
       ? sanitizeAnthropicMessagesBody({
-        ...input.claudeOriginalBody,
+        ...stripClaudeMessagesContinuationFields(input.claudeOriginalBody),
         model: input.modelName,
         stream: input.stream,
       })
@@ -1034,7 +1048,7 @@ export function buildClaudeCountTokensUpstreamRequest(input: {
   const sitePlatform = normalizePlatformName(input.sitePlatform);
   const claudeHeaders = extractClaudePassthroughHeaders(input.downstreamHeaders);
   const { body: bodyWithoutBetas, betas } = extractClaudeBetasFromBody({
-    ...input.claudeBody,
+    ...stripClaudeMessagesContinuationFields(input.claudeBody),
     model: input.modelName,
   });
   const sanitizedBody = sanitizeAnthropicMessagesBody(bodyWithoutBetas);

src/server/routes/proxy/upstreamError.ts

@@ -1,81 +1 @@
-function isRecord(value: unknown): value is Record<string, unknown> {
-  return !!value && typeof value === 'object';
-}
-
-function collapseWhitespace(text: string): string {
-  return text.replace(/\s+/g, ' ').trim();
-}
-
-function extractJsonErrorMessage(rawText: string): string {
-  try {
-    const parsed = JSON.parse(rawText) as unknown;
-    const root = isRecord(parsed) ? parsed : null;
-    const error = (root && isRecord(root.error)) ? root.error : root;
-    if (!error) return '';
-
-    const message = typeof error.message === 'string' ? collapseWhitespace(error.message) : '';
-    if (message) return message;
-
-    const code = typeof error.code === 'string' ? collapseWhitespace(error.code) : '';
-    const type = typeof error.type === 'string' ? collapseWhitespace(error.type) : '';
-    return [type, code].filter((part) => part.length > 0).join('/');
-  } catch {
-    return '';
-  }
-}
-
-function extractHtmlTitle(rawText: string): string {
-  const match = rawText.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
-  if (!match?.[1]) return '';
-  return collapseWhitespace(match[1].replace(/<[^>]+>/g, ''));
-}
-
-function extractCloudflareHtmlSummary(rawText: string, status: number): string {
-  if (!/cloudflare/i.test(rawText)) return '';
-  const title = extractHtmlTitle(rawText);
-  const codeMatch = (
-    title.match(/\b(\d{3,4})\s*:\s*([^\|<]+)/i)
-    || rawText.match(/Error code\s*(\d{3,4})/i)
-  );
-  const code = codeMatch?.[1] || (status > 0 ? String(status) : '');
-  const reason = collapseWhitespace(
-    (typeof codeMatch?.[2] === 'string' ? codeMatch[2] : '')
-    || (status >= 500 ? 'origin host error' : 'request blocked')
-  );
-  if (code) return `Cloudflare ${code}: ${reason}`;
-  return `Cloudflare: ${reason}`;
-}
-
-function extractHtmlSummary(rawText: string, status: number): string {
-  if (!/(<!doctype|<html)/i.test(rawText)) return '';
-
-  const cloudflareSummary = extractCloudflareHtmlSummary(rawText, status);
-  if (cloudflareSummary) return cloudflareSummary;
-
-  const title = extractHtmlTitle(rawText);
-  if (title) return title;
-
-  const heading = rawText.match(/<h1[^>]*>([\s\S]*?)<\/h1>/i)?.[1] || '';
-  return collapseWhitespace(heading.replace(/<[^>]+>/g, ''));
-}
-
-export function summarizeUpstreamError(status: number, rawErrorText: string): string {
-  const statusPrefix = status > 0
-    ? `Upstream returned HTTP ${status}`
-    : 'Upstream request failed';
-
-  const raw = typeof rawErrorText === 'string' ? rawErrorText.trim() : '';
-  if (!raw) return statusPrefix;
-
-  const jsonMessage = extractJsonErrorMessage(raw);
-  if (jsonMessage) return `${statusPrefix}: ${jsonMessage}`;
-
-  const htmlMessage = extractHtmlSummary(raw, status);
-  if (htmlMessage) return `${statusPrefix}: ${htmlMessage}`;
-
-  const compact = collapseWhitespace(raw);
-  if (!compact) return statusPrefix;
-  if (compact.length <= 400) return `${statusPrefix}: ${compact}`;
-  return `${statusPrefix}: ${compact.slice(0, 400)}...(truncated)`;
-}
-
+export { summarizeUpstreamError } from '../../proxy-core/orchestration/upstreamRequest.js';

src/server/routes/proxy/upstreamUrl.ts

@@ -1,58 +1 @@
-function formatUrlOrigin(url: URL): string {
-  // URL.origin drops credentials; preserve them if present.
-  const username = url.username ? encodeURIComponent(url.username) : '';
-  const password = url.password ? encodeURIComponent(url.password) : '';
-  const auth = username
-    ? `${username}${password ? `:${password}` : ''}@`
-    : '';
-
-  return `${url.protocol}//${auth}${url.host}`;
-}
-
-function joinPath(basePath: string, requestPath: string): string {
-  const base = basePath.replace(/\/+$/, '');
-  const path = requestPath.startsWith('/') ? requestPath : `/${requestPath}`;
-
-  if (!base || base === '/') return path || '/';
-  if (!path || path === '/') return base;
-  return `${base}${path}`;
-}
-
-export function buildUpstreamUrl(siteUrl: string, requestPath: string): string {
-  const baseRaw = typeof siteUrl === 'string' ? siteUrl.trim() : '';
-  const pathRaw = typeof requestPath === 'string' ? requestPath.trim() : '';
-  const fallbackBase = baseRaw.replace(/\/+$/, '');
-  let path = pathRaw.startsWith('/') ? pathRaw : `/${pathRaw}`;
-
-  if (!fallbackBase) return path || '/';
-  if (!path || path === '/') return fallbackBase;
-
-  try {
-    const parsed = new URL(baseRaw);
-    const basePath = parsed.pathname.replace(/\/+$/, '');
-    const baseHasVersionSuffix = /\/(?:api\/)?v1$/i.test(basePath);
-    if (baseHasVersionSuffix) {
-      if (path === '/v1') {
-        path = '/';
-      } else if (path.startsWith('/v1/')) {
-        path = path.slice('/v1'.length) || '/';
-      }
-    }
-
-    const joinedPath = joinPath(basePath, path);
-    return `${formatUrlOrigin(parsed)}${joinedPath}${parsed.search}${parsed.hash}`;
-  } catch {
-    // Ignore URL parsing errors and fall back to naive join.
-    const baseHasVersionSuffix = /\/(?:api\/)?v1$/i.test(fallbackBase);
-    if (baseHasVersionSuffix) {
-      if (path === '/v1') {
-        path = '/';
-      } else if (path.startsWith('/v1/')) {
-        path = path.slice('/v1'.length) || '/';
-      }
-    }
-
-    return `${fallbackBase}${path}`;
-  }
-}
-
+export { buildUpstreamUrl } from '../../proxy-core/orchestration/upstreamRequest.js';

src/server/routes/proxy/videos.test.ts

@@ -53,6 +53,7 @@ vi.mock('../../services/modelPricingService.js', () => ({
 }));
 
 vi.mock('../../services/proxyRetryPolicy.js', () => ({
+  shouldAbortSameSiteEndpointFallback: () => false,
   shouldRetryProxyRequest: (...args: unknown[]) => shouldRetryProxyRequestMock(...args),
   RETRYABLE_TIMEOUT_PATTERNS: [/(request timed out|connection timed out|read timeout|\btimed out\b)/i],
 }));

src/server/services/backgroundTaskService.ts

@@ -301,6 +301,21 @@ export function getRunningTaskByDedupeKey(key: string): BackgroundTask | null {
   return task;
 }
 
+export async function waitForBackgroundTaskCompletion(taskId: string, pollIntervalMs = 25): Promise<BackgroundTask | null> {
+  const safePollIntervalMs = Math.max(5, Math.trunc(pollIntervalMs || 0));
+  while (true) {
+    const task = getBackgroundTask(taskId);
+    if (!task) return null;
+    if (task.status !== 'pending' && task.status !== 'running') {
+      return task;
+    }
+    await new Promise<void>((resolve) => {
+      const timer = setTimeout(resolve, safePollIntervalMs);
+      timer.unref?.();
+    });
+  }
+}
+
 export function summarizeCheckinResults(results: Array<{ result?: any }>): { total: number; success: number; skipped: number; failed: number } {
   const summary = { total: results.length, success: 0, skipped: 0, failed: 0 };
   for (const item of results) {

src/server/services/channelRecoveryProbeService.test.ts

@@ -0,0 +1,348 @@
+import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
+import { mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { eq } from 'drizzle-orm';
+
+const probeRuntimeModelMock = vi.fn();
+
+vi.mock('./runtimeModelProbe.js', () => ({
+  probeRuntimeModel: (...args: unknown[]) => probeRuntimeModelMock(...args),
+}));
+
+type DbModule = typeof import('../db/index.js');
+type RecoveryModule = typeof import('./channelRecoveryProbeService.js');
+type CoordinatorModule = typeof import('./proxyChannelCoordinator.js');
+type ConfigModule = typeof import('../config.js');
+type TokenRouterModule = typeof import('./tokenRouter.js');
+
+describe('channelRecoveryProbeService', () => {
+  let db: DbModule['db'];
+  let schema: DbModule['schema'];
+  let runChannelRecoveryProbeSweep: RecoveryModule['runChannelRecoveryProbeSweep'];
+  let resetChannelRecoveryProbeState: RecoveryModule['resetChannelRecoveryProbeState'];
+  let proxyChannelCoordinator: CoordinatorModule['proxyChannelCoordinator'];
+  let resetProxyChannelCoordinatorState: CoordinatorModule['resetProxyChannelCoordinatorState'];
+  let invalidateTokenRouterCache: TokenRouterModule['invalidateTokenRouterCache'];
+  let resetSiteRuntimeHealthState: TokenRouterModule['resetSiteRuntimeHealthState'];
+  let config: ConfigModule['config'];
+  let dataDir = '';
+  let originalDataDir: string | undefined;
+  let originalConcurrencyLimit = 0;
+
+  beforeAll(async () => {
+    dataDir = mkdtempSync(join(tmpdir(), 'metapi-channel-recovery-probe-'));
+    originalDataDir = process.env.DATA_DIR;
+    process.env.DATA_DIR = dataDir;
+
+    await import('../db/migrate.js');
+    const dbModule = await import('../db/index.js');
+    const recoveryModule = await import('./channelRecoveryProbeService.js');
+    const coordinatorModule = await import('./proxyChannelCoordinator.js');
+    const configModule = await import('../config.js');
+    const tokenRouterModule = await import('./tokenRouter.js');
+
+    db = dbModule.db;
+    schema = dbModule.schema;
+    runChannelRecoveryProbeSweep = recoveryModule.runChannelRecoveryProbeSweep;
+    resetChannelRecoveryProbeState = recoveryModule.resetChannelRecoveryProbeState;
+    proxyChannelCoordinator = coordinatorModule.proxyChannelCoordinator;
+    resetProxyChannelCoordinatorState = coordinatorModule.resetProxyChannelCoordinatorState;
+    invalidateTokenRouterCache = tokenRouterModule.invalidateTokenRouterCache;
+    resetSiteRuntimeHealthState = tokenRouterModule.resetSiteRuntimeHealthState;
+    config = configModule.config;
+    originalConcurrencyLimit = config.proxySessionChannelConcurrencyLimit;
+  });
+
+  beforeEach(async () => {
+    probeRuntimeModelMock.mockReset();
+    probeRuntimeModelMock.mockResolvedValue({
+      status: 'supported',
+      latencyMs: 320,
+      reason: 'probe succeeded',
+    });
+    config.proxySessionChannelConcurrencyLimit = 1;
+    resetChannelRecoveryProbeState();
+    resetProxyChannelCoordinatorState();
+    invalidateTokenRouterCache();
+    resetSiteRuntimeHealthState();
+
+    await db.delete(schema.routeChannels).run();
+    await db.delete(schema.tokenRoutes).run();
+    await db.delete(schema.settings).run();
+    await db.delete(schema.accountTokens).run();
+    await db.delete(schema.accounts).run();
+    await db.delete(schema.sites).run();
+  });
+
+  afterAll(() => {
+    config.proxySessionChannelConcurrencyLimit = originalConcurrencyLimit;
+    resetChannelRecoveryProbeState();
+    resetProxyChannelCoordinatorState();
+    invalidateTokenRouterCache();
+    resetSiteRuntimeHealthState();
+    rmSync(dataDir, { recursive: true, force: true });
+    if (originalDataDir === undefined) {
+      delete process.env.DATA_DIR;
+    } else {
+      process.env.DATA_DIR = originalDataDir;
+    }
+  });
+
+  it('clears cooldown markers when a background probe succeeds', async () => {
+    const site = await db.insert(schema.sites).values({
+      name: 'recovery-site',
+      url: 'https://recovery-site.example.com',
+      platform: 'new-api',
+      status: 'active',
+    }).returning().get();
+
+    const account = await db.insert(schema.accounts).values({
+      siteId: site.id,
+      username: 'recovery-user',
+      accessToken: 'access-recovery',
+      apiToken: 'sk-recovery',
+      status: 'active',
+    }).returning().get();
+
+    const token = await db.insert(schema.accountTokens).values({
+      accountId: account.id,
+      name: 'default',
+      token: 'sk-recovery-token',
+      enabled: true,
+      isDefault: true,
+    }).returning().get();
+
+    const route = await db.insert(schema.tokenRoutes).values({
+      modelPattern: 'gpt-5.4',
+      enabled: true,
+    }).returning().get();
+
+    const channel = await db.insert(schema.routeChannels).values({
+      routeId: route.id,
+      accountId: account.id,
+      tokenId: token.id,
+      enabled: true,
+      cooldownUntil: new Date(Date.now() + 10 * 60 * 1000).toISOString(),
+      lastFailAt: new Date().toISOString(),
+      consecutiveFailCount: 2,
+      cooldownLevel: 1,
+    }).returning().get();
+
+    await runChannelRecoveryProbeSweep();
+
+    expect(probeRuntimeModelMock).toHaveBeenCalledTimes(1);
+    expect(probeRuntimeModelMock.mock.calls[0]?.[0]).toMatchObject({
+      modelName: 'gpt-5.4',
+      tokenValue: 'sk-recovery-token',
+    });
+
+    const refreshed = await db.select().from(schema.routeChannels)
+      .where(eq(schema.routeChannels.id, channel.id))
+      .get();
+    expect(refreshed?.cooldownUntil).toBeNull();
+    expect(refreshed?.lastFailAt).toBeNull();
+    expect(refreshed?.consecutiveFailCount).toBe(0);
+    expect(refreshed?.cooldownLevel).toBe(0);
+  });
+
+  it('also probes active leased channels in the background', async () => {
+    const site = await db.insert(schema.sites).values({
+      name: 'active-site',
+      url: 'https://active-site.example.com',
+      platform: 'new-api',
+      status: 'active',
+    }).returning().get();
+
+    const account = await db.insert(schema.accounts).values({
+      siteId: site.id,
+      username: 'active-user',
+      accessToken: 'access-active',
+      apiToken: 'sk-active',
+      status: 'active',
+      extraConfig: JSON.stringify({
+        credentialMode: 'session',
+      }),
+    }).returning().get();
+
+    const token = await db.insert(schema.accountTokens).values({
+      accountId: account.id,
+      name: 'default',
+      token: 'sk-active-token',
+      enabled: true,
+      isDefault: true,
+    }).returning().get();
+
+    const route = await db.insert(schema.tokenRoutes).values({
+      modelPattern: 'gpt-5.2',
+      enabled: true,
+    }).returning().get();
+
+    const channel = await db.insert(schema.routeChannels).values({
+      routeId: route.id,
+      accountId: account.id,
+      tokenId: token.id,
+      enabled: true,
+    }).returning().get();
+
+    const lease = await proxyChannelCoordinator.acquireChannelLease({
+      channelId: channel.id,
+      accountExtraConfig: account.extraConfig,
+    });
+    expect(lease.status).toBe('acquired');
+    if (lease.status !== 'acquired') return;
+
+    await runChannelRecoveryProbeSweep();
+
+    expect(probeRuntimeModelMock).toHaveBeenCalledTimes(1);
+    expect(probeRuntimeModelMock.mock.calls[0]?.[0]).toMatchObject({
+      modelName: 'gpt-5.2',
+      tokenValue: 'sk-active-token',
+    });
+
+    lease.lease.release();
+  });
+
+  it('skips provider-directed quota cooldown channels during recovery sweeps', async () => {
+    const site = await db.insert(schema.sites).values({
+      name: 'quota-site',
+      url: 'https://quota-site.example.com',
+      platform: 'codex',
+      status: 'active',
+    }).returning().get();
+
+    const account = await db.insert(schema.accounts).values({
+      siteId: site.id,
+      username: 'quota-user',
+      accessToken: 'access-quota',
+      apiToken: 'sk-quota',
+      status: 'active',
+    }).returning().get();
+
+    const quotaToken = await db.insert(schema.accountTokens).values({
+      accountId: account.id,
+      name: 'quota-token',
+      token: 'sk-quota-token',
+      enabled: true,
+      isDefault: false,
+    }).returning().get();
+
+    const retryToken = await db.insert(schema.accountTokens).values({
+      accountId: account.id,
+      name: 'retry-token',
+      token: 'sk-retry-token',
+      enabled: true,
+      isDefault: false,
+    }).returning().get();
+
+    const route = await db.insert(schema.tokenRoutes).values({
+      modelPattern: 'gpt-5.4',
+      enabled: true,
+    }).returning().get();
+
+    await db.insert(schema.routeChannels).values([
+      {
+        routeId: route.id,
+        accountId: account.id,
+        tokenId: quotaToken.id,
+        enabled: true,
+        cooldownUntil: new Date(Date.now() + 10 * 60 * 1000).toISOString(),
+        lastFailAt: new Date().toISOString(),
+        failCount: 0,
+        consecutiveFailCount: 0,
+        cooldownLevel: 0,
+      },
+      {
+        routeId: route.id,
+        accountId: account.id,
+        tokenId: retryToken.id,
+        enabled: true,
+        cooldownUntil: new Date(Date.now() + 10 * 60 * 1000).toISOString(),
+        lastFailAt: new Date().toISOString(),
+        failCount: 2,
+        consecutiveFailCount: 0,
+        cooldownLevel: 0,
+      },
+    ]).run();
+
+    await runChannelRecoveryProbeSweep();
+
+    expect(probeRuntimeModelMock).toHaveBeenCalledTimes(1);
+    expect(probeRuntimeModelMock.mock.calls[0]?.[0]).toMatchObject({
+      tokenValue: 'sk-retry-token',
+      modelName: 'gpt-5.4',
+    });
+  });
+
+  it('prioritizes never-probed active channels before reprobing recently started ones', async () => {
+    const site = await db.insert(schema.sites).values({
+      name: 'priority-site',
+      url: 'https://priority-site.example.com',
+      platform: 'new-api',
+      status: 'active',
+    }).returning().get();
+
+    const leases: Array<{ release: () => void }> = [];
+    try {
+      for (let index = 1; index <= 5; index += 1) {
+        const account = await db.insert(schema.accounts).values({
+          siteId: site.id,
+          username: `priority-user-${index}`,
+          accessToken: `access-priority-${index}`,
+          apiToken: `sk-priority-${index}`,
+          status: 'active',
+          extraConfig: JSON.stringify({
+            credentialMode: 'session',
+          }),
+        }).returning().get();
+
+        const token = await db.insert(schema.accountTokens).values({
+          accountId: account.id,
+          name: `token-${index}`,
+          token: `sk-priority-token-${index}`,
+          enabled: true,
+          isDefault: true,
+        }).returning().get();
+
+        const route = await db.insert(schema.tokenRoutes).values({
+          modelPattern: `gpt-5.4-${index}`,
+          enabled: true,
+        }).returning().get();
+
+        const channel = await db.insert(schema.routeChannels).values({
+          routeId: route.id,
+          accountId: account.id,
+          tokenId: token.id,
+          enabled: true,
+        }).returning().get();
+
+        const lease = await proxyChannelCoordinator.acquireChannelLease({
+          channelId: channel.id,
+          accountExtraConfig: account.extraConfig,
+        });
+        expect(lease.status).toBe('acquired');
+        if (lease.status === 'acquired') {
+          leases.push(lease.lease);
+        }
+      }
+
+      const startedAt = Date.UTC(2026, 3, 1, 0, 0, 0);
+      await runChannelRecoveryProbeSweep(startedAt);
+
+      expect(probeRuntimeModelMock).toHaveBeenCalledTimes(4);
+      expect(probeRuntimeModelMock.mock.calls.map((call) => call[0]?.tokenValue)).not.toContain('sk-priority-token-5');
+
+      probeRuntimeModelMock.mockClear();
+
+      await runChannelRecoveryProbeSweep(startedAt + 5 * 60 * 1000);
+
+      expect(probeRuntimeModelMock).toHaveBeenCalledTimes(4);
+      const secondSweepTokens = probeRuntimeModelMock.mock.calls.map((call) => call[0]?.tokenValue);
+      expect(secondSweepTokens).toContain('sk-priority-token-5');
+    } finally {
+      for (const lease of leases) {
+        lease.release();
+      }
+    }
+  });
+});

src/server/services/channelRecoveryProbeService.ts

@@ -0,0 +1,299 @@
+import { and, eq, gt, inArray, isNotNull } from 'drizzle-orm';
+import { db, schema } from '../db/index.js';
+import { isUsableAccountToken } from './accountTokenService.js';
+import { getOauthInfoFromAccount } from './oauth/oauthAccount.js';
+import { proxyChannelCoordinator } from './proxyChannelCoordinator.js';
+import { probeRuntimeModel } from './runtimeModelProbe.js';
+import { tokenRouter } from './tokenRouter.js';
+import { isExactTokenRouteModelPattern } from '../../shared/tokenRoutePatterns.js';
+
+type RecoveryProbeSource = 'cooldown' | 'active';
+
+type RecoveryProbeCandidate = {
+  source: RecoveryProbeSource;
+  channelId: number;
+  modelName: string;
+  tokenValue: string;
+  account: typeof schema.accounts.$inferSelect;
+  site: typeof schema.sites.$inferSelect;
+};
+
+const CHANNEL_RECOVERY_SWEEP_INTERVAL_MS = 30_000;
+const CHANNEL_RECOVERY_PROBE_TIMEOUT_MS = 12_000;
+// Keep recovery probes conservative so they do not look like bulk health checks to upstream providers.
+const CHANNEL_RECOVERY_PROBE_CONCURRENCY = 1;
+const CHANNEL_RECOVERY_MAX_BATCH = 4;
+const CHANNEL_RECOVERY_COOLDOWN_RECHECK_MS = 30_000;
+const CHANNEL_RECOVERY_ACTIVE_RECHECK_MS = 5 * 60_000;
+
+let recoveryProbeSchedulerTimer: ReturnType<typeof setInterval> | null = null;
+let recoveryProbeSweepInFlight: Promise<void> | null = null;
+const recoveryProbeInFlightKeys = new Set<string>();
+const recoveryProbeLastStartedAtByKey = new Map<string, number>();
+
+function shouldUnrefTimer(timer: ReturnType<typeof setTimeout> | ReturnType<typeof setInterval>) {
+  if (typeof (timer as { unref?: () => void }).unref === 'function') {
+    (timer as { unref: () => void }).unref();
+  }
+}
+
+function buildRecoveryProbeKey(channelId: number, modelName: string): string {
+  return `${channelId}:${String(modelName || '').trim().toLowerCase()}`;
+}
+
+function resolveRecoveryProbeWindowMs(source: RecoveryProbeSource): number {
+  return source === 'cooldown'
+    ? CHANNEL_RECOVERY_COOLDOWN_RECHECK_MS
+    : CHANNEL_RECOVERY_ACTIVE_RECHECK_MS;
+}
+
+function resolveProbeModelName(row: {
+  route_channels: typeof schema.routeChannels.$inferSelect;
+  token_routes: typeof schema.tokenRoutes.$inferSelect;
+}): string {
+  const sourceModel = String(row.route_channels.sourceModel || '').trim();
+  if (sourceModel) return sourceModel;
+  const routeModelPattern = String(row.token_routes.modelPattern || '').trim();
+  return isExactTokenRouteModelPattern(routeModelPattern) ? routeModelPattern : '';
+}
+
+function resolveProbeTokenValue(row: {
+  route_channels: typeof schema.routeChannels.$inferSelect;
+  accounts: typeof schema.accounts.$inferSelect;
+  account_tokens: typeof schema.accountTokens.$inferSelect | null;
+}): string | null {
+  if (typeof row.route_channels.tokenId === 'number' && row.route_channels.tokenId > 0) {
+    if (!row.account_tokens || !isUsableAccountToken(row.account_tokens)) return null;
+    const tokenValue = String(row.account_tokens.token || '').trim();
+    return tokenValue || null;
+  }
+
+  if (getOauthInfoFromAccount(row.accounts)) {
+    const accessToken = String(row.accounts.accessToken || '').trim();
+    return accessToken || null;
+  }
+
+  const fallbackApiToken = String(row.accounts.apiToken || '').trim();
+  return fallbackApiToken || null;
+}
+
+function isProviderDirectedCooldown(row: {
+  route_channels: typeof schema.routeChannels.$inferSelect;
+}): boolean {
+  return !!row.route_channels.cooldownUntil
+    && (row.route_channels.failCount ?? 0) <= 0
+    && (row.route_channels.consecutiveFailCount ?? 0) <= 0
+    && (row.route_channels.cooldownLevel ?? 0) <= 0;
+}
+
+async function mapWithConcurrency<T>(
+  items: T[],
+  concurrency: number,
+  worker: (item: T, index: number) => Promise<void>,
+): Promise<void> {
+  const safeConcurrency = Math.max(1, Math.min(items.length || 1, Math.trunc(concurrency || 1)));
+  let nextIndex = 0;
+  const runWorker = async () => {
+    while (true) {
+      const currentIndex = nextIndex;
+      nextIndex += 1;
+      if (currentIndex >= items.length) return;
+      await worker(items[currentIndex] as T, currentIndex);
+    }
+  };
+
+  await Promise.all(Array.from({ length: safeConcurrency }, () => runWorker()));
+}
+
+async function loadCoolingProbeCandidates(nowIso: string): Promise<RecoveryProbeCandidate[]> {
+  const rows = await db.select()
+    .from(schema.routeChannels)
+    .innerJoin(schema.accounts, eq(schema.routeChannels.accountId, schema.accounts.id))
+    .innerJoin(schema.sites, eq(schema.accounts.siteId, schema.sites.id))
+    .innerJoin(schema.tokenRoutes, eq(schema.routeChannels.routeId, schema.tokenRoutes.id))
+    .leftJoin(schema.accountTokens, eq(schema.routeChannels.tokenId, schema.accountTokens.id))
+    .where(and(
+      eq(schema.routeChannels.enabled, true),
+      eq(schema.accounts.status, 'active'),
+      eq(schema.sites.status, 'active'),
+      isNotNull(schema.routeChannels.cooldownUntil),
+      gt(schema.routeChannels.cooldownUntil, nowIso),
+    ))
+    .all();
+
+  return rows.flatMap((row) => {
+    if (isProviderDirectedCooldown(row)) return [];
+    const modelName = resolveProbeModelName(row);
+    const tokenValue = resolveProbeTokenValue(row);
+    if (!modelName || !tokenValue) return [];
+    return [{
+      source: 'cooldown' as const,
+      channelId: row.route_channels.id,
+      modelName,
+      tokenValue,
+      account: row.accounts,
+      site: row.sites,
+    }];
+  });
+}
+
+async function loadActiveProbeCandidates(activeChannelIds: number[]): Promise<RecoveryProbeCandidate[]> {
+  if (activeChannelIds.length <= 0) return [];
+
+  const rows = await db.select()
+    .from(schema.routeChannels)
+    .innerJoin(schema.accounts, eq(schema.routeChannels.accountId, schema.accounts.id))
+    .innerJoin(schema.sites, eq(schema.accounts.siteId, schema.sites.id))
+    .innerJoin(schema.tokenRoutes, eq(schema.routeChannels.routeId, schema.tokenRoutes.id))
+    .leftJoin(schema.accountTokens, eq(schema.routeChannels.tokenId, schema.accountTokens.id))
+    .where(and(
+      eq(schema.routeChannels.enabled, true),
+      eq(schema.accounts.status, 'active'),
+      eq(schema.sites.status, 'active'),
+      inArray(schema.routeChannels.id, activeChannelIds),
+    ))
+    .all();
+
+  return rows.flatMap((row) => {
+    const modelName = resolveProbeModelName(row);
+    const tokenValue = resolveProbeTokenValue(row);
+    if (!modelName || !tokenValue) return [];
+    return [{
+      source: 'active' as const,
+      channelId: row.route_channels.id,
+      modelName,
+      tokenValue,
+      account: row.accounts,
+      site: row.sites,
+    }];
+  });
+}
+
+function mergeRecoveryProbeCandidates(candidates: RecoveryProbeCandidate[]): RecoveryProbeCandidate[] {
+  const merged = new Map<number, RecoveryProbeCandidate>();
+  for (const candidate of candidates) {
+    const existing = merged.get(candidate.channelId);
+    if (!existing || (existing.source === 'active' && candidate.source === 'cooldown')) {
+      merged.set(candidate.channelId, candidate);
+    }
+  }
+  return Array.from(merged.values());
+}
+
+function shouldProbeCandidate(candidate: RecoveryProbeCandidate, nowMs: number): boolean {
+  const key = buildRecoveryProbeKey(candidate.channelId, candidate.modelName);
+  if (recoveryProbeInFlightKeys.has(key)) return false;
+  const lastStartedAt = recoveryProbeLastStartedAtByKey.get(key) ?? 0;
+  return (nowMs - lastStartedAt) >= resolveRecoveryProbeWindowMs(candidate.source);
+}
+
+function compareRecoveryProbeCandidatePriority(left: RecoveryProbeCandidate, right: RecoveryProbeCandidate): number {
+  const leftKey = buildRecoveryProbeKey(left.channelId, left.modelName);
+  const rightKey = buildRecoveryProbeKey(right.channelId, right.modelName);
+  const leftLastStartedAt = recoveryProbeLastStartedAtByKey.get(leftKey);
+  const rightLastStartedAt = recoveryProbeLastStartedAtByKey.get(rightKey);
+
+  if (leftLastStartedAt == null && rightLastStartedAt == null) {
+    return left.channelId - right.channelId;
+  }
+  if (leftLastStartedAt == null) return -1;
+  if (rightLastStartedAt == null) return 1;
+  if (leftLastStartedAt !== rightLastStartedAt) {
+    return leftLastStartedAt - rightLastStartedAt;
+  }
+  return left.channelId - right.channelId;
+}
+
+async function runRecoveryProbeCandidate(candidate: RecoveryProbeCandidate, nowMs: number): Promise<void> {
+  const key = buildRecoveryProbeKey(candidate.channelId, candidate.modelName);
+  recoveryProbeInFlightKeys.add(key);
+  recoveryProbeLastStartedAtByKey.set(key, nowMs);
+  try {
+    const result = await probeRuntimeModel({
+      site: candidate.site,
+      account: candidate.account,
+      modelName: candidate.modelName,
+      tokenValue: candidate.tokenValue,
+      timeoutMs: CHANNEL_RECOVERY_PROBE_TIMEOUT_MS,
+    });
+    if (result.status === 'supported') {
+      await tokenRouter.recordProbeSuccess(
+        candidate.channelId,
+        result.latencyMs ?? 0,
+        candidate.modelName,
+      );
+    }
+  } catch (error) {
+    console.warn(`[channel-recovery-probe] channel ${candidate.channelId} probe failed`, error);
+  } finally {
+    recoveryProbeInFlightKeys.delete(key);
+  }
+}
+
+export async function runChannelRecoveryProbeSweep(nowMs = Date.now()): Promise<void> {
+  if (recoveryProbeSweepInFlight) {
+    await recoveryProbeSweepInFlight;
+    return;
+  }
+
+  recoveryProbeSweepInFlight = (async () => {
+    const nowIso = new Date(nowMs).toISOString();
+    const activeChannelIds = proxyChannelCoordinator.getActiveChannelIds();
+    const [coolingCandidates, activeCandidates] = await Promise.all([
+      loadCoolingProbeCandidates(nowIso),
+      loadActiveProbeCandidates(activeChannelIds),
+    ]);
+
+    const merged = mergeRecoveryProbeCandidates([
+      ...coolingCandidates,
+      ...activeCandidates,
+    ]);
+    const dueCandidates = merged
+      .filter((candidate) => shouldProbeCandidate(candidate, nowMs))
+      .sort(compareRecoveryProbeCandidatePriority)
+      .slice(0, CHANNEL_RECOVERY_MAX_BATCH);
+    if (dueCandidates.length <= 0) return;
+
+    await mapWithConcurrency(
+      dueCandidates,
+      CHANNEL_RECOVERY_PROBE_CONCURRENCY,
+      async (candidate) => runRecoveryProbeCandidate(candidate, nowMs),
+    );
+  })().finally(() => {
+    recoveryProbeSweepInFlight = null;
+  });
+
+  await recoveryProbeSweepInFlight;
+}
+
+export function startChannelRecoveryProbeScheduler(intervalMs = CHANNEL_RECOVERY_SWEEP_INTERVAL_MS) {
+  stopChannelRecoveryProbeScheduler();
+  const safeIntervalMs = Math.max(10_000, Math.trunc(intervalMs || 0));
+  recoveryProbeSchedulerTimer = setInterval(() => {
+    void runChannelRecoveryProbeSweep().catch((error) => {
+      console.warn('[channel-recovery-probe] background sweep failed', error);
+    });
+  }, safeIntervalMs);
+  shouldUnrefTimer(recoveryProbeSchedulerTimer);
+  void runChannelRecoveryProbeSweep().catch((error) => {
+    console.warn('[channel-recovery-probe] initial sweep failed', error);
+  });
+  return {
+    enabled: true,
+    intervalMs: safeIntervalMs,
+  };
+}
+
+export function stopChannelRecoveryProbeScheduler() {
+  if (recoveryProbeSchedulerTimer) {
+    clearInterval(recoveryProbeSchedulerTimer);
+    recoveryProbeSchedulerTimer = null;
+  }
+}
+
+export function resetChannelRecoveryProbeState() {
+  stopChannelRecoveryProbeScheduler();
+  recoveryProbeSweepInFlight = null;
+  recoveryProbeInFlightKeys.clear();
+  recoveryProbeLastStartedAtByKey.clear();
+}

src/server/services/modelAvailabilityProbeService.test.ts

@@ -0,0 +1,507 @@
+import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
+import { mkdtempSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { eq } from 'drizzle-orm';
+
+const resolveUpstreamEndpointCandidatesMock = vi.fn();
+const buildUpstreamEndpointRequestMock = vi.fn();
+const dispatchRuntimeRequestMock = vi.fn();
+const resolveChannelProxyUrlMock = vi.fn();
+const withSiteRecordProxyRequestInitMock = vi.fn();
+const rebuildRoutesOnlyMock = vi.fn();
+
+vi.mock('./upstreamEndpointRuntime.js', () => ({
+  resolveUpstreamEndpointCandidates: (...args: unknown[]) => resolveUpstreamEndpointCandidatesMock(...args),
+  buildUpstreamEndpointRequest: (...args: unknown[]) => buildUpstreamEndpointRequestMock(...args),
+}));
+
+vi.mock('./runtimeDispatch.js', () => ({
+  dispatchRuntimeRequest: (...args: unknown[]) => dispatchRuntimeRequestMock(...args),
+}));
+
+vi.mock('./siteProxy.js', () => ({
+  resolveChannelProxyUrl: (...args: unknown[]) => resolveChannelProxyUrlMock(...args),
+  withSiteRecordProxyRequestInit: (...args: unknown[]) => withSiteRecordProxyRequestInitMock(...args),
+}));
+
+vi.mock('./routeRefreshWorkflow.js', () => ({
+  rebuildRoutesOnly: (...args: unknown[]) => rebuildRoutesOnlyMock(...args),
+}));
+
+type DbModule = typeof import('../db/index.js');
+type ProbeModule = typeof import('./modelAvailabilityProbeService.js');
+
+describe('modelAvailabilityProbeService', () => {
+  let db: DbModule['db'];
+  let schema: DbModule['schema'];
+  let executeModelAvailabilityProbe: ProbeModule['executeModelAvailabilityProbe'];
+  let resetProbeExecutionState: ProbeModule['__resetModelAvailabilityProbeExecutionStateForTests'];
+  let dataDir = '';
+  let originalDataDir: string | undefined;
+
+  beforeAll(async () => {
+    dataDir = mkdtempSync(join(tmpdir(), 'metapi-model-probe-'));
+    originalDataDir = process.env.DATA_DIR;
+    process.env.DATA_DIR = dataDir;
+
+    await import('../db/migrate.js');
+    const dbModule = await import('../db/index.js');
+    const probeModule = await import('./modelAvailabilityProbeService.js');
+
+    db = dbModule.db;
+    schema = dbModule.schema;
+    executeModelAvailabilityProbe = probeModule.executeModelAvailabilityProbe;
+    resetProbeExecutionState = probeModule.__resetModelAvailabilityProbeExecutionStateForTests;
+  });
+
+  beforeEach(async () => {
+    resolveUpstreamEndpointCandidatesMock.mockReset();
+    buildUpstreamEndpointRequestMock.mockReset();
+    dispatchRuntimeRequestMock.mockReset();
+    resolveChannelProxyUrlMock.mockReset();
+    withSiteRecordProxyRequestInitMock.mockReset();
+    rebuildRoutesOnlyMock.mockReset();
+    rebuildRoutesOnlyMock.mockResolvedValue(undefined);
+    resetProbeExecutionState();
+
+    resolveUpstreamEndpointCandidatesMock.mockResolvedValue(['chat']);
+    buildUpstreamEndpointRequestMock.mockImplementation((input: { modelName?: string; endpoint?: string }) => ({
+      path: '/v1/chat/completions',
+      headers: {
+        'content-type': 'application/json',
+      },
+      body: {
+        model: input.modelName || 'unknown-model',
+        endpoint: input.endpoint || 'chat',
+      },
+      runtime: {
+        executor: 'default',
+        modelName: input.modelName || 'unknown-model',
+        stream: false,
+      },
+    }));
+    dispatchRuntimeRequestMock.mockImplementation(async (input: { request?: { body?: { model?: string } } }) => {
+      const modelName = String(input?.request?.body?.model || '');
+      if (modelName === 'gpt-ghost') {
+        return new Response(JSON.stringify({
+          error: {
+            message: 'The model `gpt-ghost` does not exist',
+          },
+        }), {
+          status: 404,
+          headers: {
+            'content-type': 'application/json',
+          },
+        });
+      }
+      return new Response(JSON.stringify({
+        id: 'resp_probe',
+        model: modelName,
+        choices: [
+          {
+            message: {
+              role: 'assistant',
+              content: 'OK',
+            },
+          },
+        ],
+      }), {
+        status: 200,
+        headers: {
+          'content-type': 'application/json',
+        },
+      });
+    });
+    resolveChannelProxyUrlMock.mockReturnValue(null);
+    withSiteRecordProxyRequestInitMock.mockImplementation((_site: unknown, init: unknown) => init);
+
+    await db.delete(schema.routeChannels).run();
+    await db.delete(schema.tokenRoutes).run();
+    await db.delete(schema.tokenModelAvailability).run();
+    await db.delete(schema.modelAvailability).run();
+    await db.delete(schema.accountTokens).run();
+    await db.delete(schema.accounts).run();
+    await db.delete(schema.sites).run();
+  });
+
+  afterAll(() => {
+    if (originalDataDir === undefined) {
+      delete process.env.DATA_DIR;
+    } else {
+      process.env.DATA_DIR = originalDataDir;
+    }
+  });
+
+  it('marks supported rows true and unsupported rows false after probing', async () => {
+    const site = await db.insert(schema.sites).values({
+      name: 'probe-site',
+      url: 'https://probe.example.com',
+      platform: 'new-api',
+      status: 'active',
+    }).returning().get();
+
+    const account = await db.insert(schema.accounts).values({
+      siteId: site.id,
+      username: 'probe-user',
+      accessToken: '',
+      apiToken: 'sk-account-probe',
+      status: 'active',
+      extraConfig: JSON.stringify({
+        credentialMode: 'apikey',
+      }),
+    }).returning().get();
+
+    const token = await db.insert(schema.accountTokens).values({
+      accountId: account.id,
+      name: 'default',
+      token: 'sk-token-probe',
+      enabled: true,
+      isDefault: true,
+      valueStatus: 'ready',
+    }).returning().get();
+
+    await db.insert(schema.modelAvailability).values([
+      {
+        accountId: account.id,
+        modelName: 'gpt-real',
+        available: true,
+        checkedAt: '2026-03-20T00:00:00.000Z',
+      },
+    ]).run();
+
+    await db.insert(schema.tokenModelAvailability).values([
+      {
+        tokenId: token.id,
+        modelName: 'gpt-ghost',
+        available: true,
+        checkedAt: '2026-03-20T00:00:00.000Z',
+      },
+    ]).run();
+
+    const result = await executeModelAvailabilityProbe({
+      accountId: account.id,
+      rebuildRoutes: false,
+    });
+
+    expect(result.summary).toMatchObject({
+      totalAccounts: 1,
+      scanned: 2,
+      supported: 1,
+      unsupported: 1,
+      inconclusive: 0,
+      skippedModels: 0,
+      updatedRows: 2,
+      rebuiltRoutes: false,
+    });
+
+    const accountRows = await db.select().from(schema.modelAvailability)
+      .where(eq(schema.modelAvailability.accountId, account.id))
+      .all();
+    expect(accountRows).toHaveLength(1);
+    expect(accountRows[0]?.available).toBe(true);
+
+    const tokenRows = await db.select().from(schema.tokenModelAvailability)
+      .where(eq(schema.tokenModelAvailability.tokenId, token.id))
+      .all();
+    expect(tokenRows).toHaveLength(1);
+    expect(tokenRows[0]?.available).toBe(false);
+  });
+
+  it('skips probing obvious non-conversation models', async () => {
+    const site = await db.insert(schema.sites).values({
+      name: 'probe-site',
+      url: 'https://probe.example.com',
+      platform: 'new-api',
+      status: 'active',
+    }).returning().get();
+
+    const account = await db.insert(schema.accounts).values({
+      siteId: site.id,
+      username: 'probe-user',
+      accessToken: '',
+      apiToken: 'sk-account-probe',
+      status: 'active',
+      extraConfig: JSON.stringify({
+        credentialMode: 'apikey',
+      }),
+    }).returning().get();
+
+    await db.insert(schema.modelAvailability).values([
+      {
+        accountId: account.id,
+        modelName: 'text-embedding-3-large',
+        available: true,
+        checkedAt: '2026-03-20T00:00:00.000Z',
+      },
+    ]).run();
+
+    const result = await executeModelAvailabilityProbe({
+      accountId: account.id,
+      rebuildRoutes: false,
+    });
+
+    expect(result.summary).toMatchObject({
+      totalAccounts: 1,
+      scanned: 1,
+      supported: 0,
+      unsupported: 0,
+      inconclusive: 0,
+      skippedModels: 1,
+      updatedRows: 0,
+    });
+    expect(dispatchRuntimeRequestMock).not.toHaveBeenCalled();
+  });
+
+  it('does not mark a model unavailable when probe result is inconclusive', async () => {
+    dispatchRuntimeRequestMock.mockImplementation(async (input: { request?: { body?: { model?: string } } }) => {
+      const modelName = String(input?.request?.body?.model || '');
+      if (modelName === 'gpt-flaky') {
+        return new Response(JSON.stringify({
+          error: {
+            message: 'upstream temporary failure',
+          },
+        }), {
+          status: 502,
+          headers: {
+            'content-type': 'application/json',
+          },
+        });
+      }
+      return new Response(JSON.stringify({
+        id: 'resp_probe',
+        model: modelName,
+        choices: [
+          {
+            message: {
+              role: 'assistant',
+              content: 'OK',
+            },
+          },
+        ],
+      }), {
+        status: 200,
+        headers: {
+          'content-type': 'application/json',
+        },
+      });
+    });
+
+    const site = await db.insert(schema.sites).values({
+      name: 'probe-site',
+      url: 'https://probe.example.com',
+      platform: 'new-api',
+      status: 'active',
+    }).returning().get();
+
+    const account = await db.insert(schema.accounts).values({
+      siteId: site.id,
+      username: 'probe-user',
+      accessToken: '',
+      apiToken: 'sk-account-probe',
+      status: 'active',
+      extraConfig: JSON.stringify({
+        credentialMode: 'apikey',
+      }),
+    }).returning().get();
+
+    const checkedAt = '2026-03-20T00:00:00.000Z';
+    await db.insert(schema.modelAvailability).values([
+      {
+        accountId: account.id,
+        modelName: 'gpt-flaky',
+        available: true,
+        checkedAt,
+      },
+    ]).run();
+
+    const result = await executeModelAvailabilityProbe({
+      accountId: account.id,
+      rebuildRoutes: false,
+    });
+
+    expect(result.summary).toMatchObject({
+      totalAccounts: 1,
+      scanned: 1,
+      supported: 0,
+      unsupported: 0,
+      inconclusive: 1,
+      skippedModels: 0,
+      updatedRows: 0,
+    });
+
+    const accountRows = await db.select().from(schema.modelAvailability)
+      .where(eq(schema.modelAvailability.accountId, account.id))
+      .all();
+    expect(accountRows).toHaveLength(1);
+    expect(accountRows[0]?.available).toBe(true);
+    expect(accountRows[0]?.checkedAt).toBe(checkedAt);
+  });
+
+  it('does not rebuild routes when probe only refreshes checkedAt and latency', async () => {
+    const site = await db.insert(schema.sites).values({
+      name: 'stable-site',
+      url: 'https://stable.example.com',
+      platform: 'new-api',
+      status: 'active',
+    }).returning().get();
+
+    const account = await db.insert(schema.accounts).values({
+      siteId: site.id,
+      username: 'stable-user',
+      accessToken: '',
+      apiToken: 'sk-stable',
+      status: 'active',
+    }).returning().get();
+
+    await db.insert(schema.modelAvailability).values({
+      accountId: account.id,
+      modelName: 'gpt-stable',
+      available: true,
+      checkedAt: '2026-03-20T00:00:00.000Z',
+      latencyMs: 123,
+    }).run();
+
+    const result = await executeModelAvailabilityProbe({
+      accountId: account.id,
+      rebuildRoutes: true,
+    });
+
+    expect(result.summary.updatedRows).toBe(1);
+    expect(result.summary.rebuiltRoutes).toBe(false);
+    expect(rebuildRoutesOnlyMock).not.toHaveBeenCalled();
+  });
+
+  it('rebuilds routes when probe flips availability state', async () => {
+    const site = await db.insert(schema.sites).values({
+      name: 'changed-site',
+      url: 'https://changed.example.com',
+      platform: 'new-api',
+      status: 'active',
+    }).returning().get();
+
+    const account = await db.insert(schema.accounts).values({
+      siteId: site.id,
+      username: 'changed-user',
+      accessToken: '',
+      apiToken: 'sk-changed',
+      status: 'active',
+    }).returning().get();
+
+    await db.insert(schema.modelAvailability).values({
+      accountId: account.id,
+      modelName: 'gpt-changed',
+      available: false,
+      checkedAt: '2026-03-20T00:00:00.000Z',
+    }).run();
+
+    const result = await executeModelAvailabilityProbe({
+      accountId: account.id,
+      rebuildRoutes: true,
+    });
+
+    expect(result.summary.updatedRows).toBe(1);
+    expect(result.summary.rebuiltRoutes).toBe(true);
+    expect(rebuildRoutesOnlyMock).toHaveBeenCalledTimes(1);
+  });
+
+  it('does not count inactive accounts in probe results', async () => {
+    const site = await db.insert(schema.sites).values({
+      name: 'inactive-site',
+      url: 'https://inactive.example.com',
+      platform: 'new-api',
+      status: 'active',
+    }).returning().get();
+
+    const account = await db.insert(schema.accounts).values({
+      siteId: site.id,
+      username: 'inactive-user',
+      accessToken: '',
+      apiToken: 'sk-inactive',
+      status: 'disabled',
+    }).returning().get();
+
+    await db.insert(schema.modelAvailability).values({
+      accountId: account.id,
+      modelName: 'gpt-inactive',
+      available: true,
+      checkedAt: '2026-03-20T00:00:00.000Z',
+    }).run();
+
+    const result = await executeModelAvailabilityProbe({
+      accountId: account.id,
+      rebuildRoutes: false,
+    });
+
+    expect(result.summary.totalAccounts).toBe(0);
+    expect(result.results).toEqual([]);
+    expect(dispatchRuntimeRequestMock).not.toHaveBeenCalled();
+  });
+
+  it('skips an account when another probe already holds the execution lease', async () => {
+    const site = await db.insert(schema.sites).values({
+      name: 'leased-site',
+      url: 'https://leased.example.com',
+      platform: 'new-api',
+      status: 'active',
+    }).returning().get();
+
+    const account = await db.insert(schema.accounts).values({
+      siteId: site.id,
+      username: 'leased-user',
+      accessToken: '',
+      apiToken: 'sk-leased',
+      status: 'active',
+    }).returning().get();
+
+    await db.insert(schema.modelAvailability).values({
+      accountId: account.id,
+      modelName: 'gpt-leased',
+      available: true,
+      checkedAt: '2026-03-20T00:00:00.000Z',
+    }).run();
+
+    let releaseFirstProbe: (() => void) | null = null;
+    const firstProbeStarted = new Promise<void>((resolve) => {
+      dispatchRuntimeRequestMock.mockImplementationOnce(async () => {
+        resolve();
+        await new Promise<void>((release) => {
+          releaseFirstProbe = release;
+        });
+        return new Response(JSON.stringify({
+          id: 'resp_probe',
+          choices: [{ message: { role: 'assistant', content: 'OK' } }],
+        }), {
+          status: 200,
+          headers: { 'content-type': 'application/json' },
+        });
+      });
+    });
+
+    const firstProbe = executeModelAvailabilityProbe({
+      accountId: account.id,
+      rebuildRoutes: false,
+    });
+    await firstProbeStarted;
+
+    const overlappingProbe = await executeModelAvailabilityProbe({
+      accountId: account.id,
+      rebuildRoutes: false,
+    });
+
+    releaseFirstProbe?.();
+    await firstProbe;
+
+    expect(overlappingProbe.summary).toMatchObject({
+      totalAccounts: 1,
+      skipped: 1,
+      scanned: 0,
+    });
+    expect(overlappingProbe.results[0]).toMatchObject({
+      accountId: account.id,
+      siteId: site.id,
+      status: 'skipped',
+      message: 'model availability probe already running for account',
+    });
+  });
+});

src/server/services/modelAvailabilityProbeService.ts

@@ -0,0 +1,447 @@
+import { and, asc, eq } from 'drizzle-orm';
+import { config } from '../config.js';
+import { db, schema } from '../db/index.js';
+import { startBackgroundTask } from './backgroundTaskService.js';
+import { isUsableAccountToken, ACCOUNT_TOKEN_VALUE_STATUS_READY } from './accountTokenService.js';
+import { probeRuntimeModel } from './runtimeModelProbe.js';
+import * as routeRefreshWorkflow from './routeRefreshWorkflow.js';
+
+type ProbeStatus = 'supported' | 'unsupported' | 'inconclusive' | 'skipped';
+
+type ProbeAccountTarget = {
+  kind: 'account';
+  rowId: number;
+  modelName: string;
+  lastKnownAvailable: boolean;
+  account: typeof schema.accounts.$inferSelect;
+  site: typeof schema.sites.$inferSelect;
+};
+
+type ProbeTokenTarget = {
+  kind: 'token';
+  rowId: number;
+  tokenId: number;
+  modelName: string;
+  tokenValue: string;
+  lastKnownAvailable: boolean;
+  account: typeof schema.accounts.$inferSelect;
+  site: typeof schema.sites.$inferSelect;
+};
+
+type ProbeTarget = ProbeAccountTarget | ProbeTokenTarget;
+type ProbeAccountContext = {
+  account: typeof schema.accounts.$inferSelect;
+  site: typeof schema.sites.$inferSelect;
+};
+
+export type ModelAvailabilityProbeAccountResult = {
+  accountId: number;
+  siteId: number;
+  status: 'success' | 'failed' | 'skipped';
+  scanned: number;
+  supported: number;
+  unsupported: number;
+  inconclusive: number;
+  skipped: number;
+  updatedRows: number;
+  message: string;
+};
+
+export type ModelAvailabilityProbeExecutionResult = {
+  results: ModelAvailabilityProbeAccountResult[];
+  summary: {
+    totalAccounts: number;
+    success: number;
+    failed: number;
+    skipped: number;
+    scanned: number;
+    supported: number;
+    unsupported: number;
+    inconclusive: number;
+    skippedModels: number;
+    updatedRows: number;
+    rebuiltRoutes: boolean;
+  };
+};
+
+let probeSchedulerTimer: ReturnType<typeof setInterval> | null = null;
+const probeAccountLeases = new Set<number>();
+
+async function mapWithConcurrency<T, R>(
+  items: T[],
+  concurrency: number,
+  worker: (item: T, index: number) => Promise<R>,
+): Promise<R[]> {
+  const safeConcurrency = Math.max(1, Math.min(items.length || 1, Math.trunc(concurrency || 1)));
+  const results = new Array<R>(items.length);
+  let nextIndex = 0;
+
+  const runWorker = async () => {
+    while (true) {
+      const currentIndex = nextIndex;
+      nextIndex += 1;
+      if (currentIndex >= items.length) return;
+      results[currentIndex] = await worker(items[currentIndex] as T, currentIndex);
+    }
+  };
+
+  await Promise.all(Array.from({ length: safeConcurrency }, () => runWorker()));
+  return results;
+}
+
+async function probeSingleTarget(target: ProbeTarget): Promise<{
+  status: ProbeStatus;
+  latencyMs: number | null;
+  reason: string;
+}> {
+  return await probeRuntimeModel({
+    site: target.site,
+    account: target.account,
+    modelName: target.modelName,
+    timeoutMs: config.modelAvailabilityProbeTimeoutMs,
+    tokenValue: target.kind === 'token' ? target.tokenValue : undefined,
+  });
+}
+
+async function updateProbeRow(target: ProbeTarget, status: ProbeStatus, latencyMs: number | null): Promise<{
+  touched: boolean;
+  availabilityChanged: boolean;
+}> {
+  if (status === 'inconclusive' || status === 'skipped') {
+    return {
+      touched: false,
+      availabilityChanged: false,
+    };
+  }
+  const nextAvailable = status === 'supported';
+  const patch = {
+    available: nextAvailable,
+    latencyMs,
+    checkedAt: new Date().toISOString(),
+  };
+
+  if (target.kind === 'account') {
+    await db.update(schema.modelAvailability)
+      .set(patch)
+      .where(eq(schema.modelAvailability.id, target.rowId))
+      .run();
+    return {
+      touched: true,
+      availabilityChanged: target.lastKnownAvailable !== nextAvailable,
+    };
+  }
+
+  await db.update(schema.tokenModelAvailability)
+    .set(patch)
+    .where(eq(schema.tokenModelAvailability.id, target.rowId))
+    .run();
+  return {
+    touched: true,
+    availabilityChanged: target.lastKnownAvailable !== nextAvailable,
+  };
+}
+
+async function loadActiveProbeAccountContext(accountId: number): Promise<ProbeAccountContext | null> {
+  const accountRow = await db.select()
+    .from(schema.accounts)
+    .innerJoin(schema.sites, eq(schema.accounts.siteId, schema.sites.id))
+    .where(eq(schema.accounts.id, accountId))
+    .get();
+  if (!accountRow) return null;
+  if ((accountRow.accounts.status || 'active') !== 'active') return null;
+  if ((accountRow.sites.status || 'active') !== 'active') return null;
+  return {
+    account: accountRow.accounts,
+    site: accountRow.sites,
+  };
+}
+
+async function loadProbeTargetsForAccount(context: ProbeAccountContext): Promise<ProbeTarget[]> {
+  const targets: ProbeTarget[] = [];
+  const accountModels = await db.select()
+    .from(schema.modelAvailability)
+    .where(eq(schema.modelAvailability.accountId, context.account.id))
+    .orderBy(asc(schema.modelAvailability.checkedAt))
+    .all();
+  for (const row of accountModels) {
+    if (row.isManual) continue;
+    targets.push({
+      kind: 'account',
+      rowId: row.id,
+      modelName: row.modelName,
+      lastKnownAvailable: !!row.available,
+      account: context.account,
+      site: context.site,
+    });
+  }
+
+  const tokenRows = await db.select()
+    .from(schema.tokenModelAvailability)
+    .innerJoin(schema.accountTokens, eq(schema.tokenModelAvailability.tokenId, schema.accountTokens.id))
+    .where(and(
+      eq(schema.accountTokens.accountId, context.account.id),
+      eq(schema.accountTokens.enabled, true),
+      eq(schema.accountTokens.valueStatus, ACCOUNT_TOKEN_VALUE_STATUS_READY),
+    ))
+    .orderBy(asc(schema.tokenModelAvailability.checkedAt))
+    .all();
+  for (const row of tokenRows) {
+    if (!isUsableAccountToken(row.account_tokens)) continue;
+    const tokenValue = String(row.account_tokens.token || '').trim();
+    if (!tokenValue) continue;
+    targets.push({
+      kind: 'token',
+      rowId: row.token_model_availability.id,
+      tokenId: row.account_tokens.id,
+      modelName: row.token_model_availability.modelName,
+      tokenValue,
+      lastKnownAvailable: !!row.token_model_availability.available,
+      account: context.account,
+      site: context.site,
+    });
+  }
+
+  return targets;
+}
+
+function tryAcquireProbeAccountLease(accountId: number): boolean {
+  if (!Number.isFinite(accountId) || accountId <= 0) return false;
+  if (probeAccountLeases.has(accountId)) return false;
+  probeAccountLeases.add(accountId);
+  return true;
+}
+
+function releaseProbeAccountLease(accountId: number): void {
+  probeAccountLeases.delete(accountId);
+}
+
+function buildSkippedProbeAccountResult(input: {
+  accountId: number;
+  siteId: number;
+  message: string;
+}): ModelAvailabilityProbeAccountResult {
+  return {
+    accountId: input.accountId,
+    siteId: input.siteId,
+    status: 'skipped',
+    scanned: 0,
+    supported: 0,
+    unsupported: 0,
+    inconclusive: 0,
+    skipped: 0,
+    updatedRows: 0,
+    message: input.message,
+  };
+}
+
+function summarizeProbeResults(results: ModelAvailabilityProbeAccountResult[], rebuiltRoutes: boolean): ModelAvailabilityProbeExecutionResult {
+  return {
+    results,
+    summary: {
+      totalAccounts: results.length,
+      success: results.filter((item) => item.status === 'success').length,
+      failed: results.filter((item) => item.status === 'failed').length,
+      skipped: results.filter((item) => item.status === 'skipped').length,
+      scanned: results.reduce((sum, item) => sum + item.scanned, 0),
+      supported: results.reduce((sum, item) => sum + item.supported, 0),
+      unsupported: results.reduce((sum, item) => sum + item.unsupported, 0),
+      inconclusive: results.reduce((sum, item) => sum + item.inconclusive, 0),
+      skippedModels: results.reduce((sum, item) => sum + item.skipped, 0),
+      updatedRows: results.reduce((sum, item) => sum + item.updatedRows, 0),
+      rebuiltRoutes,
+    },
+  };
+}
+
+export async function executeModelAvailabilityProbe(input: {
+  accountId?: number;
+  rebuildRoutes?: boolean;
+} = {}): Promise<ModelAvailabilityProbeExecutionResult> {
+  const accountIds = input.accountId
+    ? [input.accountId]
+    : (await db.select({ id: schema.accounts.id })
+      .from(schema.accounts)
+      .where(eq(schema.accounts.status, 'active'))
+      .all()).map((row) => row.id);
+
+  const results: ModelAvailabilityProbeAccountResult[] = [];
+  let shouldRebuildRoutes = false;
+
+  for (const accountId of accountIds) {
+    const context = await loadActiveProbeAccountContext(accountId);
+    if (!context) {
+      continue;
+    }
+    if (!tryAcquireProbeAccountLease(accountId)) {
+      results.push(buildSkippedProbeAccountResult({
+        accountId,
+        siteId: context.site.id,
+        message: 'model availability probe already running for account',
+      }));
+      continue;
+    }
+
+    try {
+      const targets = await loadProbeTargetsForAccount(context);
+      if (targets.length <= 0) {
+        results.push(buildSkippedProbeAccountResult({
+          accountId,
+          siteId: context.site.id,
+          message: 'no discovered models to probe',
+        }));
+        continue;
+      }
+
+      let supported = 0;
+      let unsupported = 0;
+      let inconclusive = 0;
+      let skipped = 0;
+      let updatedRows = 0;
+      let failed = false;
+
+      const probeOutcomes = await mapWithConcurrency(
+        targets,
+        config.modelAvailabilityProbeConcurrency,
+        async (target) => {
+          try {
+            const probe = await probeSingleTarget(target);
+            const update = await updateProbeRow(target, probe.status, probe.latencyMs);
+            return {
+              target,
+              probe,
+              touched: update.touched,
+              availabilityChanged: update.availabilityChanged,
+              failed: false,
+            };
+          } catch (error) {
+            console.warn(`[model-probe] account ${accountId} model ${target.modelName} probe failed`, error);
+            return {
+              target,
+              probe: {
+                status: 'inconclusive' as const,
+                latencyMs: null,
+                reason: error instanceof Error ? error.message : 'probe failed',
+              },
+              touched: false,
+              availabilityChanged: false,
+              failed: true,
+            };
+          }
+        },
+      );
+
+      for (const outcome of probeOutcomes) {
+        if (outcome.probe.status === 'supported') supported += 1;
+        if (outcome.probe.status === 'unsupported') unsupported += 1;
+        if (outcome.probe.status === 'inconclusive') inconclusive += 1;
+        if (outcome.probe.status === 'skipped') skipped += 1;
+        if (outcome.touched) {
+          updatedRows += 1;
+        }
+        if (outcome.availabilityChanged) {
+          shouldRebuildRoutes = true;
+        }
+        if (outcome.failed) {
+          failed = true;
+        }
+      }
+
+      results.push({
+        accountId,
+        siteId: context.site.id,
+        status: failed ? 'failed' : 'success',
+        scanned: targets.length,
+        supported,
+        unsupported,
+        inconclusive,
+        skipped,
+        updatedRows,
+        message: failed
+          ? 'model availability probe finished with partial failures'
+          : 'model availability probe finished',
+      });
+    } finally {
+      releaseProbeAccountLease(accountId);
+    }
+  }
+
+  let rebuiltRoutes = false;
+  if (input.rebuildRoutes !== false && shouldRebuildRoutes) {
+    await routeRefreshWorkflow.rebuildRoutesOnly();
+    rebuiltRoutes = true;
+  }
+
+  return summarizeProbeResults(results, rebuiltRoutes);
+}
+
+export function buildModelAvailabilityProbeTaskDedupeKey(accountId?: number | null): string {
+  const normalizedAccountId = Number.isFinite(accountId as number) && Number(accountId) > 0
+    ? Math.trunc(Number(accountId))
+    : null;
+  return normalizedAccountId
+    ? `model-availability-probe-${normalizedAccountId}`
+    : 'model-availability-probe-all';
+}
+
+export function queueModelAvailabilityProbeTask(input: {
+  accountId?: number;
+  title?: string;
+}) {
+  const accountId = Number.isFinite(input.accountId as number) ? Math.trunc(input.accountId as number) : null;
+  const title = input.title || (accountId
+    ? `探测模型可用性 #${accountId}`
+    : '探测模型可用性');
+  const dedupeKey = buildModelAvailabilityProbeTaskDedupeKey(accountId);
+
+  return startBackgroundTask(
+    {
+      type: 'model-probe',
+      title,
+      dedupeKey,
+      notifyOnFailure: true,
+      successMessage: (currentTask) => {
+        const summary = (currentTask.result as ModelAvailabilityProbeExecutionResult | undefined)?.summary;
+        if (!summary) return `${title}已完成`;
+        return `${title}完成：探测 ${summary.scanned}，可用 ${summary.supported}，不可用 ${summary.unsupported}，不确定 ${summary.inconclusive}`;
+      },
+      failureMessage: (currentTask) => `${title}失败：${currentTask.error || 'unknown error'}`,
+    },
+    async () => executeModelAvailabilityProbe({
+      accountId: accountId ?? undefined,
+      rebuildRoutes: true,
+    }),
+  );
+}
+
+export function startModelAvailabilityProbeScheduler(intervalMs = config.modelAvailabilityProbeIntervalMs) {
+  stopModelAvailabilityProbeScheduler();
+  if (!config.modelAvailabilityProbeEnabled) {
+    return {
+      enabled: false,
+      intervalMs: 0,
+    };
+  }
+
+  const safeIntervalMs = Math.max(60_000, Math.trunc(intervalMs || 0));
+  probeSchedulerTimer = setInterval(() => {
+    void queueModelAvailabilityProbeTask({
+      title: '后台模型可用性探测',
+    });
+  }, safeIntervalMs);
+  probeSchedulerTimer.unref?.();
+  return {
+    enabled: true,
+    intervalMs: safeIntervalMs,
+  };
+}
+
+export function stopModelAvailabilityProbeScheduler() {
+  if (probeSchedulerTimer) {
+    clearInterval(probeSchedulerTimer);
+    probeSchedulerTimer = null;
+  }
+}
+
+export function __resetModelAvailabilityProbeExecutionStateForTests(): void {
+  probeAccountLeases.clear();
+}

src/server/services/proxyChannelCoordinator.test.ts

@@ -162,4 +162,71 @@ describe('proxyChannelCoordinator', () => {
       second.lease.release();
     }
   });
+
+  it('exposes the set of currently active leased channels', async () => {
+    const lease = await proxyChannelCoordinator.acquireChannelLease({
+      channelId: 23,
+      accountExtraConfig: JSON.stringify({ credentialMode: 'session' }),
+    });
+    expect(lease.status).toBe('acquired');
+    if (lease.status !== 'acquired') return;
+
+    expect(proxyChannelCoordinator.getActiveChannelIds()).toEqual([23]);
+
+    lease.lease.release();
+    expect(proxyChannelCoordinator.getActiveChannelIds()).toEqual([]);
+  });
+
+  it('reports active and waiting load for a guarded session channel', async () => {
+    const first = await proxyChannelCoordinator.acquireChannelLease({
+      channelId: 31,
+      accountExtraConfig: JSON.stringify({ credentialMode: 'session' }),
+    });
+    expect(first.status).toBe('acquired');
+    if (first.status !== 'acquired') return;
+
+    const secondPromise = proxyChannelCoordinator.acquireChannelLease({
+      channelId: 31,
+      accountExtraConfig: JSON.stringify({ credentialMode: 'session' }),
+    });
+    await vi.advanceTimersByTimeAsync(0);
+
+    expect(proxyChannelCoordinator.getChannelLoadSnapshot({
+      channelId: 31,
+      accountExtraConfig: JSON.stringify({ credentialMode: 'session' }),
+    })).toEqual({
+      channelId: 31,
+      sessionScoped: true,
+      concurrencyLimit: 1,
+      activeLeaseCount: 1,
+      waitingCount: 1,
+      loadRatio: 2,
+      saturated: true,
+    });
+
+    first.lease.release();
+    await vi.advanceTimersByTimeAsync(0);
+
+    const second = await secondPromise;
+    expect(second.status).toBe('acquired');
+    if (second.status === 'acquired') {
+      second.lease.release();
+    }
+  });
+
+  it('treats structured oauth providers as session-scoped in load snapshots', () => {
+    expect(proxyChannelCoordinator.getChannelLoadSnapshot({
+      channelId: 41,
+      accountExtraConfig: JSON.stringify({ credentialMode: 'session' }),
+      accountOauthProvider: 'codex',
+    })).toEqual({
+      channelId: 41,
+      sessionScoped: true,
+      concurrencyLimit: 1,
+      activeLeaseCount: 0,
+      waitingCount: 0,
+      loadRatio: 0,
+      saturated: false,
+    });
+  });
 });

src/server/services/proxyChannelCoordinator.ts

@@ -24,6 +24,16 @@ type ChannelRuntimeState = {
   queue: ChannelWaiter[];
 };
 
+export type ProxyChannelLoadSnapshot = {
+  channelId: number;
+  sessionScoped: boolean;
+  concurrencyLimit: number;
+  activeLeaseCount: number;
+  waitingCount: number;
+  loadRatio: number;
+  saturated: boolean;
+};
+
 export type ProxyChannelLease = {
   channelId: number;
   isActive(): boolean;
@@ -104,9 +114,15 @@ function getOrCreateChannelRuntimeState(channelId: number): ChannelRuntimeState
   return state;
 }
 
+function pruneCancelledWaiters(state: ChannelRuntimeState): void {
+  if (state.queue.length <= 0) return;
+  state.queue = state.queue.filter((waiter) => !waiter.cancelled);
+}
+
 function maybeDeleteChannelRuntimeState(channelId: number): void {
   const state = channelRuntimeStates.get(channelId);
   if (!state) return;
+  pruneCancelledWaiters(state);
   if (state.activeLeaseIds.size <= 0 && state.queue.every((waiter) => waiter.cancelled)) {
     channelRuntimeStates.delete(channelId);
   }
@@ -177,6 +193,62 @@ class ProxyChannelCoordinator {
     stickySessionBindings.delete(normalizedKey);
   }
 
+  getActiveChannelIds(): number[] {
+    const ids: number[] = [];
+    for (const [channelId, state] of channelRuntimeStates.entries()) {
+      pruneCancelledWaiters(state);
+      if (state.activeLeaseIds.size > 0) {
+        ids.push(channelId);
+      }
+    }
+    return ids;
+  }
+
+  getChannelLoadSnapshot(input: {
+    channelId: number;
+    accountExtraConfig?: string | null;
+    accountOauthProvider?: string | null;
+  }): ProxyChannelLoadSnapshot {
+    const channelId = Math.trunc(input.channelId || 0);
+    const sessionScoped = isSessionScopedChannel({
+      extraConfig: input.accountExtraConfig,
+      oauthProvider: input.accountOauthProvider,
+    });
+    const concurrencyLimit = getChannelConcurrencyLimit({
+      extraConfig: input.accountExtraConfig,
+      oauthProvider: input.accountOauthProvider,
+    });
+    const state = channelId > 0 ? channelRuntimeStates.get(channelId) : null;
+    if (state) {
+      pruneCancelledWaiters(state);
+    }
+    const activeLeaseCount = state?.activeLeaseIds.size ?? 0;
+    const waitingCount = state?.queue.length ?? 0;
+    const denominator = concurrencyLimit > 0 ? concurrencyLimit : 1;
+    return {
+      channelId,
+      sessionScoped,
+      concurrencyLimit,
+      activeLeaseCount,
+      waitingCount,
+      loadRatio: (activeLeaseCount + waitingCount) / denominator,
+      saturated: concurrencyLimit > 0 && activeLeaseCount >= concurrencyLimit,
+    };
+  }
+
+  getChannelLoadSnapshots(input: Array<{
+    channelId: number;
+    accountExtraConfig?: string | null;
+    accountOauthProvider?: string | null;
+  }>): Map<number, ProxyChannelLoadSnapshot> {
+    const snapshots = new Map<number, ProxyChannelLoadSnapshot>();
+    for (const item of input) {
+      const snapshot = this.getChannelLoadSnapshot(item);
+      snapshots.set(snapshot.channelId, snapshot);
+    }
+    return snapshots;
+  }
+
   async acquireChannelLease(input: {
     channelId: number;
     accountExtraConfig?: string | null;
@@ -202,6 +274,7 @@ class ProxyChannelCoordinator {
     }
 
     const state = getOrCreateChannelRuntimeState(channelId);
+    pruneCancelledWaiters(state);
     if (state.activeLeaseIds.size < concurrencyLimit) {
       return {
         status: 'acquired',
@@ -226,6 +299,7 @@ class ProxyChannelCoordinator {
       waiter.timer = setTimeout(() => {
         waiter.cancelled = true;
         waiter.timer = null;
+        pruneCancelledWaiters(state);
         maybeDeleteChannelRuntimeState(channelId);
         resolve({
           status: 'timeout',
@@ -285,6 +359,7 @@ class ProxyChannelCoordinator {
   private drainQueue(channelId: number): void {
     const state = channelRuntimeStates.get(channelId);
     if (!state) return;
+    pruneCancelledWaiters(state);
     const concurrencyLimit = Math.max(0, Math.trunc(config.proxySessionChannelConcurrencyLimit || 0));
     while (state.activeLeaseIds.size < concurrencyLimit && state.queue.length > 0) {
       const waiter = state.queue.shift();

src/server/services/proxyDebugTraceRuntime.ts

@@ -1,7 +1,7 @@
 import { readRuntimeResponseText } from '../proxy-core/executors/types.js';
 import type {
   EndpointAttemptSuccessContext,
-} from '../routes/proxy/endpointFlow.js';
+} from '../proxy-core/orchestration/endpointFlow.js';
 import {
   finalizeProxyDebugTrace,
   insertProxyDebugAttempt,

src/server/services/proxyRetryPolicy.test.ts

@@ -1,5 +1,5 @@
 import { describe, expect, it } from 'vitest';
-import { shouldRetryProxyRequest } from './proxyRetryPolicy.js';
+import { shouldAbortSameSiteEndpointFallback, shouldRetryProxyRequest } from './proxyRetryPolicy.js';
 
 describe('proxyRetryPolicy', () => {
   it('retries on rate limit and server errors', () => {
@@ -52,4 +52,16 @@ describe('proxyRetryPolicy', () => {
       shouldRetryProxyRequest(400, '{"error":{"message":"invalid timeout parameter"}}'),
     ).toBe(false);
   });
+
+  it('aborts same-site endpoint fallback on rate-limit and quota responses', () => {
+    expect(
+      shouldAbortSameSiteEndpointFallback(429, '{"error":{"message":"rate limit exceeded"}}'),
+    ).toBe(true);
+    expect(
+      shouldAbortSameSiteEndpointFallback(429, '{"error":{"message":"quota exceeded"}}'),
+    ).toBe(true);
+    expect(
+      shouldAbortSameSiteEndpointFallback(429, '{"error":{"message":"too many requests"}}'),
+    ).toBe(true);
+  });
 });

src/server/services/proxyRetryPolicy.ts

@@ -9,6 +9,7 @@ const MODEL_UNSUPPORTED_PATTERNS: RegExp[] = [
   /model.*does\s+not\s+exist/i,
   /no\s+such\s+model/i,
   /unknown\s+model/i,
+  /unknown\s+provider\s+for\s+model/i,
   /invalid\s+model/i,
   /model[_\s-]?not[_\s-]?found/i,
   /you\s+do\s+not\s+have\s+access\s+to\s+the\s+model/i,
@@ -54,6 +55,23 @@ const NON_RETRYABLE_REQUEST_PATTERNS: RegExp[] = [
   /unsupported\s+media\s+type/i,
 ];
 
+const SAME_SITE_ENDPOINT_ABORT_PATTERNS: RegExp[] = [
+  /\b429\b/i,
+  /too\s+many\s+requests/i,
+  /rate\s+limit/i,
+  /quota(?:\s+exceeded)?/i,
+  /bad\s+gateway/i,
+  /gateway\s+time-?out/i,
+  /service\s+unavailable/i,
+  /temporar(?:y|ily)\s+unavailable/i,
+  /cpu\s+overloaded/i,
+  /connection\s+reset/i,
+  /connection\s+refused/i,
+  /econnreset/i,
+  /econnrefused/i,
+  ...RETRYABLE_TIMEOUT_PATTERNS,
+];
+
 function isModelUnsupportedErrorMessage(rawMessage?: string | null): boolean {
   const text = (rawMessage || '').trim();
   if (!text) return false;
@@ -76,3 +94,10 @@ export function shouldRetryProxyRequest(status: number, upstreamErrorText?: stri
   if (status === 400 || status === 404 || status === 422) return false;
   return false;
 }
+
+export function shouldAbortSameSiteEndpointFallback(status: number, upstreamErrorText?: string | null): boolean {
+  if (status < 500 && status !== 408 && status !== 429) {
+    return false;
+  }
+  return matchesAnyPattern(SAME_SITE_ENDPOINT_ABORT_PATTERNS, upstreamErrorText);
+}

src/server/services/runtimeDispatch.ts

@@ -0,0 +1 @@
+export { dispatchRuntimeRequest } from '../routes/proxy/runtimeExecutor.js';

src/server/services/runtimeModelProbe.test.ts

@@ -0,0 +1,116 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+
+const resolveUpstreamEndpointCandidatesMock = vi.fn();
+const buildUpstreamEndpointRequestMock = vi.fn();
+const dispatchRuntimeRequestMock = vi.fn();
+const resolveChannelProxyUrlMock = vi.fn();
+const withSiteRecordProxyRequestInitMock = vi.fn();
+
+vi.mock('../routes/proxy/upstreamEndpoint.js', () => ({
+  resolveUpstreamEndpointCandidates: (...args: unknown[]) => resolveUpstreamEndpointCandidatesMock(...args),
+  buildUpstreamEndpointRequest: (...args: unknown[]) => buildUpstreamEndpointRequestMock(...args),
+}));
+
+vi.mock('../routes/proxy/runtimeExecutor.js', () => ({
+  dispatchRuntimeRequest: (...args: unknown[]) => dispatchRuntimeRequestMock(...args),
+}));
+
+vi.mock('./siteProxy.js', () => ({
+  resolveChannelProxyUrl: (...args: unknown[]) => resolveChannelProxyUrlMock(...args),
+  withSiteRecordProxyRequestInit: (...args: unknown[]) => withSiteRecordProxyRequestInitMock(...args),
+}));
+
+describe('probeRuntimeModel', () => {
+  const site = {
+    id: 1,
+    name: 'probe-site',
+    url: 'https://probe.example.com',
+    platform: 'new-api',
+    status: 'active',
+  } as any;
+
+  const account = {
+    id: 1,
+    siteId: 1,
+    username: 'probe-user',
+    accessToken: '',
+    apiToken: 'sk-probe',
+    status: 'active',
+    extraConfig: null,
+  } as any;
+
+  beforeEach(() => {
+    vi.resetModules();
+    resolveUpstreamEndpointCandidatesMock.mockReset();
+    buildUpstreamEndpointRequestMock.mockReset();
+    dispatchRuntimeRequestMock.mockReset();
+    resolveChannelProxyUrlMock.mockReset();
+    withSiteRecordProxyRequestInitMock.mockReset();
+
+    buildUpstreamEndpointRequestMock.mockReturnValue({
+      path: '/v1/chat/completions',
+      headers: { 'content-type': 'application/json' },
+      body: { model: 'gpt-5.4' },
+      runtime: {
+        executor: 'default',
+        modelName: 'gpt-5.4',
+        stream: false,
+      },
+    });
+    resolveChannelProxyUrlMock.mockReturnValue(null);
+    withSiteRecordProxyRequestInitMock.mockImplementation(async (_site: unknown, init: RequestInit) => init);
+  });
+
+  it('returns an inconclusive result instead of throwing when endpoint resolution fails', async () => {
+    resolveUpstreamEndpointCandidatesMock.mockRejectedValue(new Error('resolution failed'));
+
+    const { probeRuntimeModel } = await import('./runtimeModelProbe.js');
+    const result = await probeRuntimeModel({
+      site,
+      account,
+      modelName: 'gpt-5.4',
+      timeoutMs: 10,
+    });
+
+    expect(result.status).toBe('inconclusive');
+    expect(result.reason).toContain('resolution failed');
+    expect(result.latencyMs).not.toBeNull();
+  });
+
+  it('uses the remaining timeout budget for the runtime request phase', async () => {
+    resolveUpstreamEndpointCandidatesMock.mockImplementation(async () => {
+      await new Promise((resolve) => setTimeout(resolve, 15));
+      return ['chat'];
+    });
+    dispatchRuntimeRequestMock.mockImplementation(async (input: {
+      buildInit: (requestUrl: string, request: Record<string, unknown>) => Promise<RequestInit>;
+      request: Record<string, unknown>;
+      targetUrl?: string;
+    }) => {
+      const init = await input.buildInit(input.targetUrl || 'https://probe.example.com/v1/chat/completions', input.request);
+      const signal = init.signal as AbortSignal | undefined;
+      await new Promise<void>((resolve, reject) => {
+        const timer = setTimeout(resolve, 40);
+        signal?.addEventListener('abort', () => {
+          clearTimeout(timer);
+          reject(signal.reason ?? new Error('aborted'));
+        }, { once: true });
+      });
+      return new Response(JSON.stringify({ ok: true }), { status: 200 });
+    });
+
+    const { probeRuntimeModel } = await import('./runtimeModelProbe.js');
+    const startedAt = Date.now();
+    const result = await probeRuntimeModel({
+      site,
+      account,
+      modelName: 'gpt-5.4',
+      timeoutMs: 30,
+    });
+    const elapsedMs = Date.now() - startedAt;
+
+    expect(result.status).toBe('inconclusive');
+    expect(result.latencyMs).not.toBeNull();
+    expect(elapsedMs).toBeLessThan(200);
+  });
+});

src/server/services/runtimeModelProbe.ts

@@ -0,0 +1,250 @@
+import { getOauthInfoFromAccount } from './oauth/oauthAccount.js';
+import { buildOauthProviderHeaders } from './oauth/service.js';
+import { resolveChannelProxyUrl, withSiteRecordProxyRequestInit } from './siteProxy.js';
+import { dispatchRuntimeRequest } from './runtimeDispatch.js';
+import {
+  buildUpstreamEndpointRequest,
+  resolveUpstreamEndpointCandidates,
+  type UpstreamEndpoint,
+} from './upstreamEndpointRuntime.js';
+import { executeEndpointFlow, type BuiltEndpointRequest } from '../proxy-core/orchestration/endpointFlow.js';
+import type { schema } from '../db/index.js';
+
+export type RuntimeModelProbeStatus = 'supported' | 'unsupported' | 'inconclusive' | 'skipped';
+
+export type RuntimeModelProbeResult = {
+  status: RuntimeModelProbeStatus;
+  latencyMs: number | null;
+  reason: string;
+};
+
+const NON_CONVERSATION_MODEL_PATTERNS = [
+  /(^|[-_/])embedding(s)?($|[-_/])/i,
+  /(^|[-_/])rerank($|[-_/])/i,
+  /(^|[-_/])moderation($|[-_/])/i,
+  /(^|[-_/])whisper($|[-_/])/i,
+  /(^|[-_/])tts($|[-_/])/i,
+  /(^|[-_/])transcribe|transcription/i,
+  /(^|[-_/])dall-e($|[-_/])/i,
+  /(^|[-_/])imagen($|[-_/])/i,
+  /(^|[-_/])veo($|[-_/])/i,
+  /(^|[-_/])cogvideo($|[-_/])/i,
+];
+
+const DEFINITE_UNSUPPORTED_PATTERNS = [
+  /no such model/i,
+  /unknown model/i,
+  /unsupported model/i,
+  /invalid model/i,
+  /model[^]{0,80}(does not exist|not found|not available|unavailable|unsupported|invalid|disabled)/i,
+  /(does not exist|not found|not available|unavailable|unsupported|invalid|disabled)[^]{0,40}model/i,
+  /模型[^]{0,40}(不存在|不可用|不支持|无效|禁用|未开通|未开放)/,
+  /(不存在|不可用|不支持|无效|禁用)[^]{0,20}模型/,
+  /model[^]{0,80}(access denied|permission|forbidden|not allowed)/i,
+  /模型[^]{0,40}(无权限|未授权|禁止访问)/,
+];
+
+function isLikelyConversationModel(modelName: string): boolean {
+  const normalized = String(modelName || '').trim();
+  if (!normalized) return false;
+  if (normalized.startsWith('__')) return false;
+  return !NON_CONVERSATION_MODEL_PATTERNS.some((pattern) => pattern.test(normalized));
+}
+
+function classifyUnsupportedFailure(status: number, rawErrorText: string): boolean {
+  if (![400, 403, 404, 422].includes(status)) return false;
+  const normalized = String(rawErrorText || '').trim();
+  if (!normalized) return false;
+  return DEFINITE_UNSUPPORTED_PATTERNS.some((pattern) => pattern.test(normalized));
+}
+
+function buildProbeBody(modelName: string): Record<string, unknown> {
+  return {
+    model: modelName,
+    messages: [
+      {
+        role: 'user',
+        content: 'Reply with OK.',
+      },
+    ],
+    max_tokens: 8,
+    stream: false,
+  };
+}
+
+async function withTimeout<T>(fn: () => Promise<T>, timeoutMs: number, timeoutMessage: string): Promise<T> {
+  let timer: ReturnType<typeof setTimeout> | null = null;
+  try {
+    return await Promise.race([
+      fn(),
+      new Promise<T>((_, reject) => {
+        timer = setTimeout(() => reject(new Error(timeoutMessage)), timeoutMs);
+        timer.unref?.();
+      }),
+    ]);
+  } finally {
+    if (timer) clearTimeout(timer);
+  }
+}
+
+function resolveRemainingTimeoutMs(deadlineAtMs: number, timeoutLabel: string): number {
+  const remainingMs = deadlineAtMs - Date.now();
+  if (remainingMs <= 0) {
+    throw new Error(timeoutLabel);
+  }
+  return remainingMs;
+}
+
+export async function probeRuntimeModel(input: {
+  site: typeof schema.sites.$inferSelect;
+  account: typeof schema.accounts.$inferSelect;
+  modelName: string;
+  timeoutMs: number;
+  tokenValue?: string | null;
+}): Promise<RuntimeModelProbeResult> {
+  if (!isLikelyConversationModel(input.modelName)) {
+    return {
+      status: 'skipped',
+      latencyMs: null,
+      reason: 'skipped non-conversation model probe',
+    };
+  }
+
+  const oauth = getOauthInfoFromAccount(input.account);
+  const tokenValue = String(
+    input.tokenValue
+    || (oauth ? input.account.accessToken : input.account.apiToken)
+    || '',
+  ).trim();
+  if (!tokenValue) {
+    return {
+      status: 'inconclusive',
+      latencyMs: null,
+      reason: 'missing credential for probe',
+    };
+  }
+
+  const startedAt = Date.now();
+  const deadlineAtMs = startedAt + Math.max(1, input.timeoutMs);
+  try {
+    const endpointCandidates = await withTimeout(
+      () => resolveUpstreamEndpointCandidates(
+        {
+          site: input.site,
+          account: input.account,
+        },
+        input.modelName,
+        'openai',
+        input.modelName,
+      ),
+      resolveRemainingTimeoutMs(
+        deadlineAtMs,
+        `runtime model probe candidate resolution timeout (${Math.round(input.timeoutMs / 1000)}s)`,
+      ),
+      `runtime model probe candidate resolution timeout (${Math.round(input.timeoutMs / 1000)}s)`,
+    );
+    if (endpointCandidates.length <= 0) {
+      return {
+        status: 'inconclusive',
+        latencyMs: Date.now() - startedAt,
+        reason: 'no compatible probe endpoint candidates',
+      };
+    }
+
+    const providerHeaders = buildOauthProviderHeaders({
+      account: input.account,
+      downstreamHeaders: {},
+    });
+    const openaiBody = buildProbeBody(input.modelName);
+    const channelProxyUrl = resolveChannelProxyUrl(input.site, input.account.extraConfig);
+    const abortController = new AbortController();
+    const remainingExecutionTimeoutMs = resolveRemainingTimeoutMs(
+      deadlineAtMs,
+      `runtime model probe timeout (${Math.round(input.timeoutMs / 1000)}s)`,
+    );
+    const abortTimer = setTimeout(() => {
+      abortController.abort(new Error(`runtime model probe timeout (${Math.round(input.timeoutMs / 1000)}s)`));
+    }, remainingExecutionTimeoutMs);
+    abortTimer.unref?.();
+
+    const buildRequest = (endpoint: UpstreamEndpoint): BuiltEndpointRequest => {
+      const request = buildUpstreamEndpointRequest({
+        endpoint,
+        modelName: input.modelName,
+        stream: false,
+        tokenValue,
+        oauthProvider: oauth?.provider,
+        oauthProjectId: oauth?.projectId,
+        sitePlatform: input.site.platform,
+        siteUrl: input.site.url,
+        openaiBody,
+        downstreamFormat: 'openai',
+        downstreamHeaders: {},
+        providerHeaders,
+      });
+      return {
+        endpoint,
+        path: request.path,
+        headers: request.headers,
+        body: request.body as Record<string, unknown>,
+        runtime: request.runtime,
+      };
+    };
+    const dispatchRequest = async (
+      request: BuiltEndpointRequest,
+      targetUrl: string,
+    ) => (
+      dispatchRuntimeRequest({
+        siteUrl: input.site.url,
+        targetUrl,
+        request,
+        buildInit: (_requestUrl, requestForFetch) => withSiteRecordProxyRequestInit(
+          input.site,
+          {
+            method: 'POST',
+            headers: requestForFetch.headers,
+            body: JSON.stringify(requestForFetch.body),
+            signal: abortController.signal,
+          },
+          channelProxyUrl,
+        ),
+      })
+    );
+
+    let result: Awaited<ReturnType<typeof executeEndpointFlow>>;
+    try {
+      result = await executeEndpointFlow({
+        siteUrl: input.site.url,
+        proxyUrl: channelProxyUrl,
+        endpointCandidates,
+        buildRequest,
+        dispatchRequest,
+      });
+    } finally {
+      clearTimeout(abortTimer);
+    }
+    const latencyMs = Date.now() - startedAt;
+
+    if (result.ok) {
+      await result.upstream.text().catch(() => undefined);
+      return {
+        status: 'supported',
+        latencyMs,
+        reason: 'probe succeeded',
+      };
+    }
+
+    const rawErrorText = String(result.rawErrText || result.errText || '').trim();
+    return {
+      status: classifyUnsupportedFailure(result.status || 0, rawErrorText) ? 'unsupported' : 'inconclusive',
+      latencyMs,
+      reason: rawErrorText || `probe failed with status ${result.status || 0}`,
+    };
+  } catch (error) {
+    return {
+      status: 'inconclusive',
+      latencyMs: Date.now() - startedAt,
+      reason: error instanceof Error ? error.message : 'probe failed',
+    };
+  }
+}

src/server/services/tokenRouter.cache.test.ts

@@ -2,7 +2,7 @@ import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest';
 import { mkdtempSync } from 'node:fs';
 import { tmpdir } from 'node:os';
 import { join } from 'node:path';
-import { eq } from 'drizzle-orm';
+import { eq, inArray } from 'drizzle-orm';
 
 type DbModule = typeof import('../db/index.js');
 type TokenRouterModule = typeof import('./tokenRouter.js');
@@ -615,6 +615,90 @@ describe('TokenRouter runtime cache', () => {
     expect(await router.selectChannel('gpt-4o-mini')).toBeNull();
   });
 
+  it('clears short-window cooldown on sibling channels after a successful recovery probe', async () => {
+    const site = await db.insert(schema.sites).values({
+      name: 'shared-credential-recovery-site',
+      url: 'https://shared-credential-recovery.example.com',
+      platform: 'codex',
+      status: 'active',
+    }).returning().get();
+
+    const account = await db.insert(schema.accounts).values({
+      siteId: site.id,
+      username: 'shared-credential-recovery-user',
+      accessToken: 'shared-credential-recovery-access-token',
+      status: 'active',
+      oauthProvider: 'codex',
+      extraConfig: JSON.stringify({
+        credentialMode: 'session',
+        oauth: {
+          provider: 'codex',
+        },
+      }),
+    }).returning().get();
+
+    const primaryRoute = await db.insert(schema.tokenRoutes).values({
+      modelPattern: 'gpt-5.4',
+      routingStrategy: 'weighted',
+      enabled: true,
+    }).returning().get();
+
+    const siblingRoute = await db.insert(schema.tokenRoutes).values({
+      modelPattern: 'gpt-4o-mini',
+      routingStrategy: 'weighted',
+      enabled: true,
+    }).returning().get();
+
+    const primaryChannel = await db.insert(schema.routeChannels).values({
+      routeId: primaryRoute.id,
+      accountId: account.id,
+      priority: 0,
+      weight: 10,
+      enabled: true,
+    }).returning().get();
+
+    const siblingChannel = await db.insert(schema.routeChannels).values({
+      routeId: siblingRoute.id,
+      accountId: account.id,
+      priority: 0,
+      weight: 10,
+      enabled: true,
+    }).returning().get();
+
+    const router = new TokenRouter();
+    await router.recordFailure(primaryChannel.id, {
+      status: 429,
+      errorText: JSON.stringify({
+        error: {
+          type: 'usage_limit_reached',
+          message: 'The usage limit has been reached',
+        },
+      }),
+      modelName: 'gpt-5.4',
+    });
+
+    await router.recordProbeSuccess(primaryChannel.id, 180, 'gpt-5.4');
+
+    const refreshedChannels = await db.select().from(schema.routeChannels)
+      .where(inArray(schema.routeChannels.id, [primaryChannel.id, siblingChannel.id]))
+      .all();
+
+    expect(refreshedChannels).toEqual(expect.arrayContaining([
+      expect.objectContaining({
+        id: primaryChannel.id,
+        cooldownUntil: null,
+        consecutiveFailCount: 0,
+        cooldownLevel: 0,
+      }),
+      expect.objectContaining({
+        id: siblingChannel.id,
+        cooldownUntil: null,
+        consecutiveFailCount: 0,
+        cooldownLevel: 0,
+      }),
+    ]));
+  });
+
   it('round robins across all available channels regardless of priority', async () => {
     const site = await db.insert(schema.sites).values({
       name: 'round-robin-site',

src/server/services/tokenRouter.selection.test.ts

@@ -7,6 +7,7 @@ import { eq } from 'drizzle-orm';
 type DbModule = typeof import('../db/index.js');
 type TokenRouterModule = typeof import('./tokenRouter.js');
 type ConfigModule = typeof import('../config.js');
+type ProxyChannelCoordinatorModule = typeof import('./proxyChannelCoordinator.js');
 
 const mockedCatalogRoutingCost = vi.fn<(
   input: { siteId: number; accountId: number; modelName: string }
@@ -24,14 +25,20 @@ describe('TokenRouter selection scoring', () => {
   let db: DbModule['db'];
   let schema: DbModule['schema'];
   let TokenRouter: TokenRouterModule['TokenRouter'];
+  let tokenRouterTestUtils: TokenRouterModule['__tokenRouterTestUtils'];
   let invalidateTokenRouterCache: TokenRouterModule['invalidateTokenRouterCache'];
   let resetSiteRuntimeHealthState: TokenRouterModule['resetSiteRuntimeHealthState'];
   let flushSiteRuntimeHealthPersistence: TokenRouterModule['flushSiteRuntimeHealthPersistence'];
+  let filterRecentlyFailedCandidates: TokenRouterModule['filterRecentlyFailedCandidates'];
   let config: ConfigModule['config'];
+  let proxyChannelCoordinator: ProxyChannelCoordinatorModule['proxyChannelCoordinator'];
+  let resetProxyChannelCoordinatorState: ProxyChannelCoordinatorModule['resetProxyChannelCoordinatorState'];
   let dataDir = '';
   let idSeed = 0;
   let originalRoutingWeights: typeof config.routingWeights;
   let originalRoutingFallbackUnitCost: number;
+  let originalProxySessionChannelConcurrencyLimit: number;
+  let originalProxySessionChannelQueueWaitMs: number;
 
   const nextId = () => {
     idSeed += 1;
@@ -46,21 +53,30 @@ describe('TokenRouter selection scoring', () => {
     const dbModule = await import('../db/index.js');
     const tokenRouterModule = await import('./tokenRouter.js');
     const configModule = await import('../config.js');
+    const coordinatorModule = await import('./proxyChannelCoordinator.js');
     db = dbModule.db;
     schema = dbModule.schema;
     TokenRouter = tokenRouterModule.TokenRouter;
+    tokenRouterTestUtils = tokenRouterModule.__tokenRouterTestUtils;
     invalidateTokenRouterCache = tokenRouterModule.invalidateTokenRouterCache;
     resetSiteRuntimeHealthState = tokenRouterModule.resetSiteRuntimeHealthState;
     flushSiteRuntimeHealthPersistence = tokenRouterModule.flushSiteRuntimeHealthPersistence;
+    filterRecentlyFailedCandidates = tokenRouterModule.filterRecentlyFailedCandidates;
     config = configModule.config;
+    proxyChannelCoordinator = coordinatorModule.proxyChannelCoordinator;
+    resetProxyChannelCoordinatorState = coordinatorModule.resetProxyChannelCoordinatorState;
     originalRoutingWeights = { ...config.routingWeights };
     originalRoutingFallbackUnitCost = config.routingFallbackUnitCost;
+    originalProxySessionChannelConcurrencyLimit = config.proxySessionChannelConcurrencyLimit;
+    originalProxySessionChannelQueueWaitMs = config.proxySessionChannelQueueWaitMs;
   });
 
   beforeEach(async () => {
     idSeed = 0;
     mockedCatalogRoutingCost.mockReset();
     mockedCatalogRoutingCost.mockReturnValue(null);
+    config.proxySessionChannelConcurrencyLimit = originalProxySessionChannelConcurrencyLimit;
+    config.proxySessionChannelQueueWaitMs = originalProxySessionChannelQueueWaitMs;
     await db.delete(schema.routeChannels).run();
     await db.delete(schema.tokenRoutes).run();
     await db.delete(schema.settings).run();
@@ -69,13 +85,17 @@ describe('TokenRouter selection scoring', () => {
     await db.delete(schema.sites).run();
     invalidateTokenRouterCache();
     resetSiteRuntimeHealthState();
+    resetProxyChannelCoordinatorState();
   });
 
   afterAll(() => {
     config.routingWeights = { ...originalRoutingWeights };
     config.routingFallbackUnitCost = originalRoutingFallbackUnitCost;
+    config.proxySessionChannelConcurrencyLimit = originalProxySessionChannelConcurrencyLimit;
+    config.proxySessionChannelQueueWaitMs = originalProxySessionChannelQueueWaitMs;
     invalidateTokenRouterCache();
     resetSiteRuntimeHealthState();
+    resetProxyChannelCoordinatorState();
     delete process.env.DATA_DIR;
   });
 
@@ -96,7 +116,7 @@ describe('TokenRouter selection scoring', () => {
     }).returning().get();
   }
 
-  async function createAccount(siteId: number, usernamePrefix: string) {
+  async function createAccount(siteId: number, usernamePrefix: string, options: { extraConfig?: string | null } = {}) {
     const id = nextId();
     return await db.insert(schema.accounts).values({
       siteId,
@@ -104,6 +124,7 @@ describe('TokenRouter selection scoring', () => {
       accessToken: `access-${id}`,
       apiToken: `sk-${id}`,
       status: 'active',
+      extraConfig: options.extraConfig ?? null,
     }).returning().get();
   }
 
@@ -126,7 +147,11 @@ describe('TokenRouter selection scoring', () => {
       usageWeight: 0,
     };
 
-    const route = await createRoute('gpt-5.2');
+    const route = await db.insert(schema.tokenRoutes).values({
+      modelPattern: 'gpt-5.2',
+      routingStrategy: 'stable_first',
+      enabled: true,
+    }).returning().get();
     const site = await createSite('sticky-site');
     const account = await createAccount(site.id, 'sticky-user');
     const tokenA = await createToken(account.id, 'sticky-a');
@@ -165,6 +190,27 @@ describe('TokenRouter selection scoring', () => {
     await expect(router.selectPreferredChannel('gpt-5.2', preferredChannel.id)).resolves.toBeNull();
   });
 
+  it('avoids recently failed candidates by default when healthy alternatives exist', () => {
+    const nowMs = Date.now();
+    const filtered = filterRecentlyFailedCandidates([
+      {
+        channel: {
+          failCount: 3,
+          lastFailAt: new Date(nowMs).toISOString(),
+        },
+      },
+      {
+        channel: {
+          failCount: 0,
+          lastFailAt: null,
+        },
+      },
+    ], nowMs);
+
+    expect(filtered).toHaveLength(1);
+    expect(filtered[0]?.channel.failCount).toBe(0);
+  });
+
   it('normalizes probability across channels on the same site', async () => {
     config.routingWeights = {
       baseWeightFactor: 1,
@@ -790,6 +836,66 @@ describe('TokenRouter selection scoring', () => {
     expect(weakCandidate?.reason || '').toContain('成功率=40.0%');
   });
 
+  it('stable_first ranks recent and fallback success rate ahead of balance-heavy weak sites', async () => {
+    config.routingWeights = {
+      baseWeightFactor: 0,
+      valueScoreFactor: 1,
+      costWeight: 0.1,
+      balanceWeight: 10,
+      usageWeight: 0,
+    };
+
+    const route = await db.insert(schema.tokenRoutes).values({
+      modelPattern: 'gpt-5.4',
+      routingStrategy: 'stable_first',
+      enabled: true,
+    }).returning().get();
+
+    const siteStable = await createSite('stable-rate-front');
+    const accountStable = await createAccount(siteStable.id, 'stable-rate-user-front');
+    const tokenStable = await createToken(accountStable.id, 'stable-rate-token-front');
+    const stableChannel = await db.insert(schema.routeChannels).values({
+      routeId: route.id,
+      accountId: accountStable.id,
+      tokenId: tokenStable.id,
+      priority: 0,
+      weight: 10,
+      enabled: true,
+      successCount: 80,
+      failCount: 4,
+      totalLatencyMs: 80 * 380,
+    }).returning().get();
+
+    const siteWeak = await createSite('stable-rate-back');
+    const accountWeak = await createAccount(siteWeak.id, 'stable-rate-user-back');
+    await db.update(schema.accounts).set({
+      balance: 999999,
+    }).where(eq(schema.accounts.id, accountWeak.id)).run();
+    const tokenWeak = await createToken(accountWeak.id, 'stable-rate-token-back');
+    const weakChannel = await db.insert(schema.routeChannels).values({
+      routeId: route.id,
+      accountId: accountWeak.id,
+      tokenId: tokenWeak.id,
+      priority: 0,
+      weight: 10,
+      enabled: true,
+      successCount: 5,
+      failCount: 21,
+      totalLatencyMs: 5 * 1600,
+    }).returning().get();
+
+    const decision = await new TokenRouter().explainSelection('gpt-5.4');
+    const stableCandidate = decision.candidates.find((candidate) => candidate.channelId === stableChannel.id);
+    const weakCandidate = decision.candidates.find((candidate) => candidate.channelId === weakChannel.id);
+
+    expect(stableCandidate).toBeTruthy();
+    expect(weakCandidate).toBeTruthy();
+    expect(decision.selectedChannelId).toBe(stableChannel.id);
+    expect((stableCandidate?.probability || 0)).toBeGreaterThan(weakCandidate?.probability || 0);
+    expect(stableCandidate?.reason || '').toContain('近期成功率=');
+    expect(weakCandidate?.reason || '').toContain('综合近期成功率=');
+  });
+
   it('reloads persisted runtime health after in-memory reset', async () => {
     config.routingWeights = {
       baseWeightFactor: 1,
@@ -856,6 +962,79 @@ describe('TokenRouter selection scoring', () => {
     expect(candidateA?.reason || '').toContain('运行时健康=');
   });
 
+  it('keeps a recovered stable_first site behind healthier peers until recent success rebuilds', async () => {
+    config.routingWeights = {
+      baseWeightFactor: 1,
+      valueScoreFactor: 0,
+      costWeight: 0,
+      balanceWeight: 0,
+      usageWeight: 0,
+    };
+
+    const route = await db.insert(schema.tokenRoutes).values({
+      modelPattern: 'gpt-5.3',
+      routingStrategy: 'stable_first',
+      enabled: true,
+    }).returning().get();
+
+    const siteRecovered = await createSite('stable-recovery-a');
+    const accountRecovered = await createAccount(siteRecovered.id, 'stable-recovery-user-a');
+    const tokenRecovered = await createToken(accountRecovered.id, 'stable-recovery-token-a');
+    const recoveredChannel = await db.insert(schema.routeChannels).values({
+      routeId: route.id,
+      accountId: accountRecovered.id,
+      tokenId: tokenRecovered.id,
+      priority: 0,
+      weight: 10,
+      enabled: true,
+    }).returning().get();
+
+    const siteHealthy = await createSite('stable-recovery-b');
+    const accountHealthy = await createAccount(siteHealthy.id, 'stable-recovery-user-b');
+    const tokenHealthy = await createToken(accountHealthy.id, 'stable-recovery-token-b');
+    const healthyChannel = await db.insert(schema.routeChannels).values({
+      routeId: route.id,
+      accountId: accountHealthy.id,
+      tokenId: tokenHealthy.id,
+      priority: 0,
+      weight: 10,
+      enabled: true,
+    }).returning().get();
+
+    const router = new TokenRouter();
+    for (let index = 0; index < 3; index += 1) {
+      await router.recordFailure(recoveredChannel.id, {
+        status: 502,
+        errorText: 'Gateway timeout',
+        modelName: 'gpt-5.3',
+      });
+    }
+    await db.update(schema.routeChannels).set({
+      cooldownUntil: null,
+      lastFailAt: null,
+      failCount: 0,
+      consecutiveFailCount: 0,
+      cooldownLevel: 0,
+    }).where(eq(schema.routeChannels.id, recoveredChannel.id)).run();
+
+    await router.recordSuccess(recoveredChannel.id, 900, 0, 'gpt-5.3');
+    for (let index = 0; index < 4; index += 1) {
+      await router.recordSuccess(healthyChannel.id, 320, 0, 'gpt-5.3');
+    }
+    invalidateTokenRouterCache();
+
+    const preview = await router.previewSelectedChannel('gpt-5.3');
+    const decision = await router.explainSelection('gpt-5.3');
+    const recoveredCandidate = decision.candidates.find((candidate) => candidate.channelId === recoveredChannel.id);
+    const healthyCandidate = decision.candidates.find((candidate) => candidate.channelId === healthyChannel.id);
+
+    expect(preview?.channel.id).toBe(healthyChannel.id);
+    expect(decision.selectedChannelId).toBe(healthyChannel.id);
+    expect((recoveredCandidate?.probability || 0)).toBeLessThan(healthyCandidate?.probability || 0);
+    expect(recoveredCandidate?.reason || '').toContain('近期成功率=');
+    expect(healthyCandidate?.reason || '').toContain('近期成功率=');
+  });
+
   it('penalizes the failed model more than unrelated models on the same site', async () => {
     config.routingWeights = {
       baseWeightFactor: 1,
@@ -934,6 +1113,90 @@ describe('TokenRouter selection scoring', () => {
     expect(gptCandidateA?.reason || '').toContain('模型=');
   });
 
+  it('treats unknown provider for model as model-scoped degradation instead of opening a site breaker', async () => {
+    config.routingWeights = {
+      baseWeightFactor: 1,
+      valueScoreFactor: 0,
+      costWeight: 0,
+      balanceWeight: 0,
+      usageWeight: 0,
+    };
+
+    const gptRoute = await createRoute('gpt-5.4');
+    const claudeRoute = await createRoute('claude-sonnet-4-6');
+
+    const siteA = await createSite('unknown-provider-a');
+    const accountA = await createAccount(siteA.id, 'unknown-provider-user-a');
+    const tokenA = await createToken(accountA.id, 'unknown-provider-token-a');
+    const gptChannelA = await db.insert(schema.routeChannels).values({
+      routeId: gptRoute.id,
+      accountId: accountA.id,
+      tokenId: tokenA.id,
+      priority: 0,
+      weight: 10,
+      enabled: true,
+    }).returning().get();
+    await db.insert(schema.routeChannels).values({
+      routeId: claudeRoute.id,
+      accountId: accountA.id,
+      tokenId: tokenA.id,
+      priority: 0,
+      weight: 10,
+      enabled: true,
+    }).run();
+
+    const siteB = await createSite('unknown-provider-b');
+    const accountB = await createAccount(siteB.id, 'unknown-provider-user-b');
+    const tokenB = await createToken(accountB.id, 'unknown-provider-token-b');
+    await db.insert(schema.routeChannels).values([
+      {
+        routeId: gptRoute.id,
+        accountId: accountB.id,
+        tokenId: tokenB.id,
+        priority: 0,
+        weight: 10,
+        enabled: true,
+      },
+      {
+        routeId: claudeRoute.id,
+        accountId: accountB.id,
+        tokenId: tokenB.id,
+        priority: 0,
+        weight: 10,
+        enabled: true,
+      },
+    ]).run();
+
+    const router = new TokenRouter();
+    for (let index = 0; index < 3; index += 1) {
+      await router.recordFailure(gptChannelA.id, {
+        status: 502,
+        errorText: 'unknown provider for model gpt-5.4',
+        modelName: 'gpt-5.4',
+      });
+    }
+    await db.update(schema.routeChannels).set({
+      cooldownUntil: null,
+      lastFailAt: null,
+      failCount: 0,
+    }).where(eq(schema.routeChannels.id, gptChannelA.id)).run();
+    invalidateTokenRouterCache();
+
+    const gptDecision = await router.explainSelection('gpt-5.4');
+    const claudeDecision = await router.explainSelection('claude-sonnet-4-6');
+    const gptCandidateA = gptDecision.candidates.find((candidate) => candidate.siteName.startsWith('unknown-provider-a'));
+    const claudeCandidateA = claudeDecision.candidates.find((candidate) => candidate.siteName.startsWith('unknown-provider-a'));
+
+    expect(gptCandidateA).toBeTruthy();
+    expect(claudeCandidateA).toBeTruthy();
+    expect(gptDecision.summary.join(' ')).not.toContain('站点熔断避让');
+    expect(claudeDecision.summary.join(' ')).not.toContain('站点熔断避让');
+    expect(gptCandidateA?.reason || '').not.toContain('站点熔断');
+    expect(claudeCandidateA?.reason || '').not.toContain('站点熔断');
+    expect((gptCandidateA?.probability || 0)).toBeLessThan((claudeCandidateA?.probability || 0));
+    expect(gptCandidateA?.reason || '').toContain('模型=');
+  });
+
   it('stable_first deterministically chooses the healthiest candidate', async () => {
     config.routingWeights = {
       baseWeightFactor: 1,
@@ -993,4 +1256,282 @@ describe('TokenRouter selection scoring', () => {
     expect(decision.summary.join(' ')).toContain('稳定优先');
     expect(decision.selectedChannelId).toBe(channelB.id);
   });
+
+  it('stable_first rotates across sites that remain inside the stable pool', async () => {
+    config.routingWeights = {
+      baseWeightFactor: 1,
+      valueScoreFactor: 0,
+      costWeight: 0,
+      balanceWeight: 0,
+      usageWeight: 0,
+    };
+
+    const route = await db.insert(schema.tokenRoutes).values({
+      modelPattern: 'gpt-5.4',
+      routingStrategy: 'stable_first',
+      enabled: true,
+    }).returning().get();
+
+    const siteA = await createSite('stable-pool-a');
+    const accountA = await createAccount(siteA.id, 'stable-pool-user-a');
+    const tokenA = await createToken(accountA.id, 'stable-pool-token-a');
+    const channelA = await db.insert(schema.routeChannels).values({
+      routeId: route.id,
+      accountId: accountA.id,
+      tokenId: tokenA.id,
+      priority: 0,
+      weight: 10,
+      enabled: true,
+    }).returning().get();
+
+    const siteB = await createSite('stable-pool-b');
+    const accountB = await createAccount(siteB.id, 'stable-pool-user-b');
+    const tokenB = await createToken(accountB.id, 'stable-pool-token-b');
+    const channelB = await db.insert(schema.routeChannels).values({
+      routeId: route.id,
+      accountId: accountB.id,
+      tokenId: tokenB.id,
+      priority: 0,
+      weight: 9,
+      enabled: true,
+    }).returning().get();
+
+    const router = new TokenRouter();
+    const first = await router.selectChannel('gpt-5.4');
+    const second = await router.selectChannel('gpt-5.4');
+    const third = await router.selectChannel('gpt-5.4');
+    const decision = await router.explainSelection('gpt-5.4');
+
+    expect(first?.channel.id).toBe(channelA.id);
+    expect(second?.channel.id).toBe(channelB.id);
+    expect(third?.channel.id).toBe(channelA.id);
+    expect(decision.summary.join(' ')).toContain('主池站点 2');
+  });
+
+  it('stable_first rotates across healthy sites in configured priority order instead of stopping at the first layer', async () => {
+    config.routingWeights = {
+      baseWeightFactor: 1,
+      valueScoreFactor: 0,
+      costWeight: 0,
+      balanceWeight: 0,
+      usageWeight: 0,
+    };
+
+    const route = await db.insert(schema.tokenRoutes).values({
+      modelPattern: 'gpt-4.1',
+      routingStrategy: 'stable_first',
+      enabled: true,
+    }).returning().get();
+
+    const siteA = await createSite('ordered-stable-a');
+    const accountA = await createAccount(siteA.id, 'ordered-stable-user-a');
+    const tokenA = await createToken(accountA.id, 'ordered-stable-token-a');
+    const channelA = await db.insert(schema.routeChannels).values({
+      routeId: route.id,
+      accountId: accountA.id,
+      tokenId: tokenA.id,
+      priority: 0,
+      weight: 10,
+      enabled: true,
+    }).returning().get();
+
+    const siteB = await createSite('ordered-stable-b');
+    const accountB = await createAccount(siteB.id, 'ordered-stable-user-b');
+    const tokenB = await createToken(accountB.id, 'ordered-stable-token-b');
+    const channelB = await db.insert(schema.routeChannels).values({
+      routeId: route.id,
+      accountId: accountB.id,
+      tokenId: tokenB.id,
+      priority: 4,
+      weight: 10,
+      enabled: true,
+    }).returning().get();
+
+    const siteC = await createSite('ordered-stable-c');
+    const accountC = await createAccount(siteC.id, 'ordered-stable-user-c');
+    const tokenC = await createToken(accountC.id, 'ordered-stable-token-c');
+    const channelC = await db.insert(schema.routeChannels).values({
+      routeId: route.id,
+      accountId: accountC.id,
+      tokenId: tokenC.id,
+      priority: 8,
+      weight: 10,
+      enabled: true,
+    }).returning().get();
+
+    const router = new TokenRouter();
+    const first = await router.selectChannel('gpt-4.1');
+    const second = await router.selectChannel('gpt-4.1');
+    const third = await router.selectChannel('gpt-4.1');
+    const fourth = await router.selectChannel('gpt-4.1');
+    const decision = await router.explainSelection('gpt-4.1');
+
+    expect(first?.channel.id).toBe(channelA.id);
+    expect(second?.channel.id).toBe(channelB.id);
+    expect(third?.channel.id).toBe(channelC.id);
+    expect(fourth?.channel.id).toBe(channelA.id);
+    expect(decision.summary.join(' ')).toContain('按配置顺序轮询站点');
+  });
+
+  it('stable_first gives observation-pool sites occasional real traffic without background probing', async () => {
+    config.routingWeights = {
+      baseWeightFactor: 1,
+      valueScoreFactor: 0,
+      costWeight: 0,
+      balanceWeight: 0,
+      usageWeight: 0,
+    };
+
+    const route = await db.insert(schema.tokenRoutes).values({
+      modelPattern: 'gpt-5.4-probe-free',
+      routingStrategy: 'stable_first',
+      enabled: true,
+    }).returning().get();
+
+    const sitePrimary = await createSite('observation-primary');
+    const accountPrimary = await createAccount(sitePrimary.id, 'observation-user-primary');
+    const tokenPrimary = await createToken(accountPrimary.id, 'observation-token-primary');
+    const primaryChannel = await db.insert(schema.routeChannels).values({
+      routeId: route.id,
+      accountId: accountPrimary.id,
+      tokenId: tokenPrimary.id,
+      priority: 0,
+      weight: 10,
+      enabled: true,
+      successCount: 48,
+      failCount: 0,
+      totalLatencyMs: 48 * 320,
+    }).returning().get();
+
+    const siteObservation = await createSite('observation-candidate');
+    const accountObservation = await createAccount(siteObservation.id, 'observation-user-candidate');
+    const tokenObservation = await createToken(accountObservation.id, 'observation-token-candidate');
+    const observationChannel = await db.insert(schema.routeChannels).values({
+      routeId: route.id,
+      accountId: accountObservation.id,
+      tokenId: tokenObservation.id,
+      priority: 1,
+      weight: 10,
+      enabled: true,
+    }).returning().get();
+
+    const router = new TokenRouter();
+    const selectedChannelIds: number[] = [];
+    for (let index = 0; index < 23; index += 1) {
+      const selected = await router.selectChannel('gpt-5.4-probe-free');
+      selectedChannelIds.push(selected?.channel.id ?? 0);
+    }
+    let decision = await router.explainSelection('gpt-5.4-probe-free');
+    let observationCandidate = decision.candidates.find((candidate) => candidate.channelId === observationChannel.id);
+    let primaryCandidate = decision.candidates.find((candidate) => candidate.channelId === primaryChannel.id);
+
+    expect(observationCandidate?.probability).toBe(100);
+    expect(primaryCandidate?.probability).toBe(0);
+    expect(decision.summary.join(' ')).toContain('本次命中观察池灰度流量');
+
+    const observationSelected = await router.selectChannel('gpt-5.4-probe-free');
+    selectedChannelIds.push(observationSelected?.channel.id ?? 0);
+
+    decision = await router.explainSelection('gpt-5.4-probe-free');
+    observationCandidate = decision.candidates.find((candidate) => candidate.channelId === observationChannel.id);
+    primaryCandidate = decision.candidates.find((candidate) => candidate.channelId === primaryChannel.id);
+
+    expect(selectedChannelIds.filter((channelId) => channelId === observationChannel.id)).toHaveLength(1);
+    expect(selectedChannelIds.filter((channelId) => channelId === primaryChannel.id).length).toBeGreaterThan(20);
+    expect(decision.summary.join(' ')).toContain('观察池站点 1');
+    expect(decision.summary.join(' ')).toContain('还需 23 次主池请求');
+    expect(observationCandidate?.reason || '').toContain('观察池');
+    expect(observationCandidate?.probability).toBe(0);
+    expect(primaryCandidate?.reason || '').toContain('主池');
+    expect((primaryCandidate?.probability || 0)).toBeGreaterThan(0);
+  });
+
+  it('caps the stable_first rotation cache size', () => {
+    invalidateTokenRouterCache();
+
+    for (let index = 0; index < 1200; index += 1) {
+      tokenRouterTestUtils.rememberStableFirstSiteSelectionForKey(`route:${index}`, (index % 7) + 1);
+    }
+
+    expect(tokenRouterTestUtils.getStableFirstRotationCacheSize()).toBeLessThanOrEqual(1024);
+  });
+
+  it('penalizes saturated session-scoped channels using runtime load snapshots', async () => {
+    config.routingWeights = {
+      baseWeightFactor: 1,
+      valueScoreFactor: 0,
+      costWeight: 0,
+      balanceWeight: 0,
+      usageWeight: 0,
+    };
+    config.proxySessionChannelConcurrencyLimit = 1;
+    config.proxySessionChannelQueueWaitMs = 5_000;
+
+    const route = await db.insert(schema.tokenRoutes).values({
+      modelPattern: 'gpt-5.2',
+      routingStrategy: 'stable_first',
+      enabled: true,
+    }).returning().get();
+    const sessionExtraConfig = JSON.stringify({ credentialMode: 'session' });
+
+    const siteBusy = await createSite('runtime-load-busy');
+    const accountBusy = await createAccount(siteBusy.id, 'runtime-load-user-busy', {
+      extraConfig: sessionExtraConfig,
+    });
+    const tokenBusy = await createToken(accountBusy.id, 'runtime-load-token-busy');
+    const channelBusy = await db.insert(schema.routeChannels).values({
+      routeId: route.id,
+      accountId: accountBusy.id,
+      tokenId: tokenBusy.id,
+      priority: 0,
+      weight: 10,
+      enabled: true,
+    }).returning().get();
+
+    const siteFree = await createSite('runtime-load-free');
+    const accountFree = await createAccount(siteFree.id, 'runtime-load-user-free', {
+      extraConfig: sessionExtraConfig,
+    });
+    const tokenFree = await createToken(accountFree.id, 'runtime-load-token-free');
+    const channelFree = await db.insert(schema.routeChannels).values({
+      routeId: route.id,
+      accountId: accountFree.id,
+      tokenId: tokenFree.id,
+      priority: 0,
+      weight: 10,
+      enabled: true,
+    }).returning().get();
+
+    const activeLease = await proxyChannelCoordinator.acquireChannelLease({
+      channelId: channelBusy.id,
+      accountExtraConfig: accountBusy.extraConfig,
+    });
+    expect(activeLease.status).toBe('acquired');
+    if (activeLease.status !== 'acquired') return;
+
+    const queuedLeasePromise = proxyChannelCoordinator.acquireChannelLease({
+      channelId: channelBusy.id,
+      accountExtraConfig: accountBusy.extraConfig,
+    });
+    await Promise.resolve();
+
+    const router = new TokenRouter();
+    const preview = await router.previewSelectedChannel('gpt-5.2');
+    const decision = await router.explainSelection('gpt-5.2');
+    const busyCandidate = decision.candidates.find((candidate) => candidate.channelId === channelBusy.id);
+    const freeCandidate = decision.candidates.find((candidate) => candidate.channelId === channelFree.id);
+
+    expect(preview?.channel.id).toBe(channelFree.id);
+    expect(busyCandidate?.reason || '').toContain('会话负载=');
+    expect(busyCandidate?.reason || '').toContain('活跃=1/1');
+    expect(busyCandidate?.reason || '').toContain('等待=1');
+    expect((busyCandidate?.probability || 0)).toBeLessThan((freeCandidate?.probability || 0));
+
+    activeLease.lease.release();
+    const queuedLease = await queuedLeasePromise;
+    expect(queuedLease.status).toBe('acquired');
+    if (queuedLease.status === 'acquired') {
+      queuedLease.lease.release();
+    }
+  });
 });

src/server/services/tokenRouter.test.ts

@@ -81,7 +81,7 @@ describe('filterRecentlyFailedCandidates', () => {
     expect(result.map((c) => c.id).sort()).toEqual(['a', 'b']);
   });
 
-  it('keeps candidates unchanged when avoidSec is omitted', () => {
+  it('uses the default recent-failure backoff when avoidSec is omitted', () => {
     const nowMs = Date.now();
     const candidates: Candidate[] = [
       {
@@ -101,7 +101,7 @@ describe('filterRecentlyFailedCandidates', () => {
     ];
 
     const result = filterRecentlyFailedCandidates(candidates, nowMs);
-    expect(result).toEqual(candidates);
+    expect(result.map((candidate) => candidate.id)).toEqual(['healthy']);
   });
 
   it('does not penalize stale failures outside the avoidance window', () => {

src/server/services/upstreamEndpointRuntime.ts

@@ -0,0 +1,6 @@
+export {
+  buildClaudeCountTokensUpstreamRequest,
+  buildUpstreamEndpointRequest,
+  resolveUpstreamEndpointCandidates,
+  type UpstreamEndpoint,
+} from '../routes/proxy/upstreamEndpoint.js';

src/server/services/upstreamEndpointRuntimeMemory.ts

@@ -33,6 +33,7 @@ export type EndpointCapabilityProfile = {
   hasNonImageFileInput: boolean;
   hasRemoteDocumentUrl: boolean;
   wantsNativeResponsesReasoning: boolean;
+  wantsContinuationAwareResponses: boolean;
 };
 
 type EndpointRuntimeState = {
@@ -88,6 +89,7 @@ export function buildEndpointCapabilityProfile(input?: {
     hasNonImageFileInput?: boolean;
     conversationFileSummary?: ConversationFileInputSummary;
     wantsNativeResponsesReasoning?: boolean;
+    wantsContinuationAwareResponses?: boolean;
   };
 }): EndpointCapabilityProfile {
   const conversationFileSummary = input?.requestCapabilities?.conversationFileSummary;
@@ -107,6 +109,7 @@ export function buildEndpointCapabilityProfile(input?: {
       conversationFileSummary?.hasRemoteDocumentUrl === true
     ),
     wantsNativeResponsesReasoning: input?.requestCapabilities?.wantsNativeResponsesReasoning === true,
+    wantsContinuationAwareResponses: input?.requestCapabilities?.wantsContinuationAwareResponses === true,
   };
 }
 
@@ -131,6 +134,7 @@ function buildEndpointRuntimeStateKey(input: {
     capabilityProfile.hasNonImageFileInput ? 'files' : 'nofiles',
     capabilityProfile.hasRemoteDocumentUrl ? 'remoteurl' : 'noremoteurl',
     capabilityProfile.wantsNativeResponsesReasoning ? 'reasoning' : 'noreasoning',
+    capabilityProfile.wantsContinuationAwareResponses ? 'continuation' : 'nocontinuation',
   ].join(':');
 }
 
@@ -247,6 +251,7 @@ export function getUpstreamEndpointRuntimeStateSnapshot(input: {
     hasNonImageFileInput?: boolean;
     conversationFileSummary?: ConversationFileInputSummary;
     wantsNativeResponsesReasoning?: boolean;
+    wantsContinuationAwareResponses?: boolean;
   };
 }) {
   const capabilityProfile = buildEndpointCapabilityProfile({
@@ -349,6 +354,7 @@ export function recordUpstreamEndpointSuccess(input: {
     hasNonImageFileInput?: boolean;
     conversationFileSummary?: ConversationFileInputSummary;
     wantsNativeResponsesReasoning?: boolean;
+    wantsContinuationAwareResponses?: boolean;
   };
 }): UpstreamEndpointRuntimeMemoryWrite | null {
   const capabilityProfile = buildEndpointCapabilityProfile({
@@ -390,6 +396,7 @@ export function recordUpstreamEndpointFailure(input: {
     hasNonImageFileInput?: boolean;
     conversationFileSummary?: ConversationFileInputSummary;
     wantsNativeResponsesReasoning?: boolean;
+    wantsContinuationAwareResponses?: boolean;
   };
 }): UpstreamEndpointRuntimeMemoryWrite | null {
   const capabilityProfile = buildEndpointCapabilityProfile({

src/server/transformers/anthropic/messages/compatibility.ts

@@ -1,5 +1,57 @@
 import { type DownstreamFormat } from '../../shared/normalized.js';
 
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return !!value && typeof value === 'object' && !Array.isArray(value);
+}
+
+function asTrimmedString(value: unknown): string {
+  return typeof value === 'string' ? value.trim() : '';
+}
+
+function hasAnthropicContinuationHint(body: Record<string, unknown>): boolean {
+  if (asTrimmedString(body.previous_response_id)) return true;
+  if (asTrimmedString(body.prompt_cache_key)) return true;
+  if (isRecord(body.metadata) && asTrimmedString(body.metadata.user_id)) return true;
+  return false;
+}
+
+function hasOrphanAnthropicToolResult(body: Record<string, unknown>): boolean {
+  const messages = Array.isArray(body.messages) ? body.messages : [];
+  const seenToolUseIds = new Set<string>();
+
+  for (const message of messages) {
+    if (!isRecord(message) || !Array.isArray(message.content)) continue;
+
+    const role = asTrimmedString(message.role).toLowerCase();
+    for (const block of message.content) {
+      if (!isRecord(block)) continue;
+      const type = asTrimmedString(block.type).toLowerCase();
+      if (!type) continue;
+
+      if (role === 'assistant' && type === 'tool_use') {
+        const toolUseId = asTrimmedString(block.id);
+        if (toolUseId) seenToolUseIds.add(toolUseId);
+        continue;
+      }
+
+      if (type !== 'tool_result') continue;
+      const toolUseId = asTrimmedString(block.tool_use_id ?? block.toolUseId);
+      if (toolUseId && !seenToolUseIds.has(toolUseId)) {
+        return true;
+      }
+    }
+  }
+
+  return false;
+}
+
+export function shouldPreferResponsesForAnthropicContinuation(
+  body: Record<string, unknown> | null | undefined,
+): boolean {
+  if (!body) return false;
+  return hasAnthropicContinuationHint(body) && hasOrphanAnthropicToolResult(body);
+}
+
 export function shouldRetryNormalizedMessagesBody(input: {
   downstreamFormat: DownstreamFormat;
   endpointPath: string;

src/server/transformers/anthropic/messages/conversion.test.ts

@@ -62,6 +62,30 @@ describe('sanitizeAnthropicMessagesBody', () => {
     ]);
   });
 
+  it('normalizes primitive content arrays into text blocks and drops unsupported entries', () => {
+    const result = sanitizeAnthropicMessagesBody({
+      model: 'claude-opus-4-6',
+      messages: [
+        {
+          role: 'user',
+          content: ['hello', true, 42, null, { type: 'text', text: 'done' }],
+        },
+      ],
+    });
+
+    expect(result.messages).toEqual([
+      {
+        role: 'user',
+        content: [
+          { type: 'text', text: 'hello' },
+          { type: 'text', text: 'true' },
+          { type: 'text', text: '42' },
+          { type: 'text', text: 'done', cache_control: { type: 'ephemeral' } },
+        ],
+      },
+    ]);
+  });
+
   it('drops output_config.effort unless thinking stays adaptive', () => {
     const enabled = sanitizeAnthropicMessagesBody({
       model: 'claude-opus-4-6',

src/server/transformers/anthropic/messages/conversion.ts

@@ -425,6 +425,43 @@ function sanitizeAnthropicContentBlock(item: Record<string, unknown>): Record<st
   return { ...item };
 }
 
+function sanitizeAnthropicMessage(message: Record<string, unknown>): Record<string, unknown> | null {
+  const next: Record<string, unknown> = { ...message };
+  const rawContent = next.content;
+
+  if (Array.isArray(rawContent)) {
+    const content = rawContent
+      .map((item) => {
+        if (isRecord(item)) return sanitizeAnthropicContentBlock(item);
+        if (typeof item === 'string' || typeof item === 'number' || typeof item === 'boolean') {
+          return sanitizeAnthropicContentBlock(toAnthropicTextBlock(String(item)) ?? {});
+        }
+        return null;
+      })
+      .filter((item): item is Record<string, unknown> => item !== null);
+    if (content.length <= 0) return null;
+    next.content = content;
+    return next;
+  }
+
+  if (rawContent === null || rawContent === undefined) {
+    return null;
+  }
+
+  if (isRecord(rawContent)) {
+    const block = sanitizeAnthropicContentBlock(rawContent);
+    if (!block) return null;
+    next.content = [block];
+    return next;
+  }
+
+  if (typeof rawContent === 'number' || typeof rawContent === 'boolean') {
+    next.content = String(rawContent);
+  }
+
+  return typeof next.content === 'string' ? next : null;
+}
+
 function isCacheableAnthropicMessageBlock(item: Record<string, unknown>): boolean {
   const type = asTrimmedString(item.type).toLowerCase();
   if (type === 'thinking' || type === 'redacted_thinking') return false;
@@ -710,15 +747,9 @@ export function sanitizeAnthropicMessagesBody(
   const rawMessages = Array.isArray(sanitized.messages) ? sanitized.messages : [];
   if (rawMessages.length > 0) {
     sanitized.messages = rawMessages.map((message) => {
-      if (!isRecord(message)) return message;
-      const next = { ...message };
-      if (Array.isArray(next.content)) {
-        next.content = next.content
-          .map((item) => (isRecord(item) ? sanitizeAnthropicContentBlock(item) : item))
-          .filter((item) => item !== null);
-      }
-      return next;
-    });
+      if (!isRecord(message)) return null;
+      return sanitizeAnthropicMessage(message);
+    }).filter((message): message is Record<string, unknown> => message !== null);
   }
 
   if (autoOptimizeCacheControls) {

src/server/transformers/anthropic/messages/inbound.test.ts

@@ -43,6 +43,66 @@ describe('anthropicMessagesInbound', () => {
     });
   });
 
+  it('drops null-content turns instead of rejecting the whole Claude request', () => {
+    const result = anthropicMessagesInbound.parse({
+      model: 'claude-opus-4-6',
+      max_tokens: 256,
+      messages: [
+        { role: 'user', content: 'hello' },
+        { role: 'assistant', content: null },
+      ],
+    });
+
+    expect(result.error).toBeUndefined();
+    expect(result.value?.parsed.upstreamBody.messages).toEqual([
+      { role: 'user', content: 'hello' },
+    ]);
+    expect(result.value?.parsed.claudeOriginalBody).toMatchObject({
+      messages: [
+        { role: 'user', content: 'hello' },
+      ],
+    });
+  });
+
+  it('preserves continuation hints for tool_result-only follow-up turns', () => {
+    const result = anthropicMessagesInbound.parse({
+      model: 'claude-opus-4-6',
+      max_tokens: 256,
+      previous_response_id: 'resp_prev_1',
+      prompt_cache_key: 'cache-key-1',
+      messages: [
+        {
+          role: 'user',
+          content: [
+            {
+              type: 'tool_result',
+              tool_use_id: 'toolu_missing',
+              content: [{ type: 'text', text: '{"matches":1}' }],
+            },
+            { type: 'text', text: 'continue' },
+          ],
+        },
+      ],
+    });
+
+    expect(result.error).toBeUndefined();
+    expect(result.value?.parsed.upstreamBody).toMatchObject({
+      previous_response_id: 'resp_prev_1',
+      prompt_cache_key: 'cache-key-1',
+      messages: [
+        {
+          role: 'tool',
+          tool_call_id: 'toolu_missing',
+          content: '{"matches":1}',
+        },
+        {
+          role: 'user',
+          content: 'continue',
+        },
+      ],
+    });
+  });
+
   it('normalizes adaptive effort and tool choice at the inbound boundary', () => {
     const result = anthropicMessagesInbound.parse({
       model: 'claude-opus-4-6',

src/server/transformers/openai/responses/continuation.test.ts

@@ -0,0 +1,125 @@
+import { describe, expect, it } from 'vitest';
+
+import {
+  extractResponsesTerminalResponseId,
+  isResponsesPreviousResponseNotFoundError,
+  shouldInferResponsesPreviousResponseId,
+  stripResponsesPreviousResponseId,
+  withResponsesPreviousResponseId,
+} from './continuation.js';
+
+describe('responses continuation helpers', () => {
+  it('infers previous_response_id only for tool-output follow-up turns that omit it', () => {
+    expect(shouldInferResponsesPreviousResponseId({
+      input: [{
+        type: 'function_call_output',
+        call_id: 'call_1',
+        output: '{"ok":true}',
+      }],
+    }, 'resp_prev_1')).toBe(true);
+
+    expect(shouldInferResponsesPreviousResponseId({
+      previous_response_id: 'resp_existing',
+      input: [{
+        type: 'function_call_output',
+        call_id: 'call_1',
+        output: '{"ok":true}',
+      }],
+    }, 'resp_prev_1')).toBe(false);
+
+    expect(shouldInferResponsesPreviousResponseId({
+      input: [{
+        type: 'message',
+        role: 'user',
+        content: [{ type: 'input_text', text: 'hello' }],
+      }],
+    }, 'resp_prev_1')).toBe(false);
+  });
+
+  it('adds and removes previous_response_id without mutating other fields', () => {
+    const body = {
+      model: 'gpt-5.4',
+      input: [{
+        type: 'function_call_output',
+        call_id: 'call_1',
+        output: '{"ok":true}',
+      }],
+    };
+
+    const withPrevious = withResponsesPreviousResponseId(body, 'resp_prev_1');
+    expect(withPrevious).toMatchObject({
+      model: 'gpt-5.4',
+      previous_response_id: 'resp_prev_1',
+    });
+
+    const stripped = stripResponsesPreviousResponseId(withPrevious);
+    expect(stripped.removed).toBe(true);
+    expect(stripped.body.previous_response_id).toBeUndefined();
+    expect(stripped.body.input).toEqual(body.input);
+  });
+
+  it('detects previous_response_not_found from either raw text or structured payloads', () => {
+    expect(isResponsesPreviousResponseNotFoundError({
+      rawErrText: JSON.stringify({
+        error: {
+          code: 'previous_response_not_found',
+          message: 'previous_response_not_found',
+        },
+      }),
+    })).toBe(true);
+
+    expect(isResponsesPreviousResponseNotFoundError({
+      payload: {
+        type: 'response.failed',
+        response: {
+          error: {
+            message: 'Previous response id not found',
+            type: 'invalid_request_error',
+          },
+        },
+      },
+    })).toBe(true);
+
+    expect(isResponsesPreviousResponseNotFoundError({
+      rawErrText: JSON.stringify({
+        error: {
+          code: 'rate_limit_exceeded',
+          message: 'too many requests',
+        },
+      }),
+    })).toBe(false);
+  });
+
+  it('extracts terminal response ids from responses payloads and websocket terminal events', () => {
+    expect(extractResponsesTerminalResponseId({
+      id: 'resp_1',
+      object: 'response',
+      status: 'completed',
+    })).toBe('resp_1');
+
+    expect(extractResponsesTerminalResponseId({
+      type: 'response.failed',
+      response: {
+        id: 'resp_2',
+        status: 'failed',
+        error: {
+          message: 'tool failed',
+        },
+      },
+    })).toBe('resp_2');
+
+    expect(extractResponsesTerminalResponseId({
+      type: 'response.created',
+      response: {
+        id: 'resp_3',
+        status: 'in_progress',
+      },
+    })).toBeNull();
+
+    expect(extractResponsesTerminalResponseId({
+      id: 'resp_in_progress',
+      object: 'response',
+      status: 'in_progress',
+    })).toBeNull();
+  });
+});

src/server/transformers/openai/responses/continuation.ts

@@ -0,0 +1,143 @@
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return !!value && typeof value === 'object' && !Array.isArray(value);
+}
+
+function asTrimmedString(value: unknown): string {
+  return typeof value === 'string' ? value.trim() : '';
+}
+
+const RESPONSES_TOOL_OUTPUT_TYPES = new Set([
+  'function_call_output',
+  'custom_tool_call_output',
+]);
+
+const RESPONSES_TERMINAL_STATUSES = new Set([
+  'completed',
+  'failed',
+  'incomplete',
+]);
+
+function collectResponsesErrorFragments(value: unknown): string[] {
+  if (!value) return [];
+  if (typeof value === 'string') return [value];
+  if (!isRecord(value)) return [];
+
+  const fragments = [
+    value.type,
+    value.code,
+    value.message,
+    value.reason,
+  ]
+    .map((entry) => asTrimmedString(entry))
+    .filter(Boolean);
+
+  if (isRecord(value.error)) {
+    fragments.push(...collectResponsesErrorFragments(value.error));
+  }
+
+  if (isRecord(value.response)) {
+    fragments.push(...collectResponsesErrorFragments(value.response));
+  }
+
+  if (isRecord(value.incomplete_details)) {
+    fragments.push(...collectResponsesErrorFragments(value.incomplete_details));
+  }
+
+  return fragments;
+}
+
+function hasResponsesToolOutput(input: unknown): boolean {
+  if (!Array.isArray(input)) return false;
+  return input.some((item) => {
+    if (!isRecord(item)) return false;
+    const type = asTrimmedString(item.type).toLowerCase();
+    if (!RESPONSES_TOOL_OUTPUT_TYPES.has(type)) return false;
+    return asTrimmedString(item.call_id ?? item.id).length > 0;
+  });
+}
+
+export function shouldInferResponsesPreviousResponseId(
+  body: Record<string, unknown> | null | undefined,
+  candidatePreviousResponseId: unknown,
+): candidatePreviousResponseId is string {
+  if (!body) return false;
+  if (asTrimmedString(body.previous_response_id)) return false;
+  const candidate = asTrimmedString(candidatePreviousResponseId);
+  if (!candidate) return false;
+  return hasResponsesToolOutput(body.input);
+}
+
+export function withResponsesPreviousResponseId(
+  body: Record<string, unknown>,
+  previousResponseId: string,
+): Record<string, unknown> {
+  return {
+    ...body,
+    previous_response_id: previousResponseId.trim(),
+  };
+}
+
+export function stripResponsesPreviousResponseId(
+  body: Record<string, unknown>,
+): { body: Record<string, unknown>; removed: boolean } {
+  if (!Object.prototype.hasOwnProperty.call(body, 'previous_response_id')) {
+    return { body, removed: false };
+  }
+  const next = { ...body };
+  delete next.previous_response_id;
+  return { body: next, removed: true };
+}
+
+export function isResponsesPreviousResponseNotFoundError(input: {
+  rawErrText?: string | null;
+  payload?: unknown;
+}): boolean {
+  const fragments = [
+    ...collectResponsesErrorFragments(input.payload),
+  ];
+  const rawErrText = asTrimmedString(input.rawErrText);
+  if (rawErrText) fragments.push(rawErrText);
+  const combined = fragments.join(' ').toLowerCase();
+  if (!combined) return false;
+  return (
+    combined.includes('previous_response_not_found')
+    || /previous[\s_-]*response(?:[\s_-]*(?:id|identifier))?[\s_-]*not[\s_-]*found/i.test(combined)
+  );
+}
+
+export function extractResponsesTerminalResponseId(payload: unknown): string | null {
+  if (!isRecord(payload)) return null;
+
+  const type = asTrimmedString(payload.type).toLowerCase();
+  if (
+    type === 'response.completed'
+    || type === 'response.failed'
+    || type === 'response.incomplete'
+  ) {
+    if (isRecord(payload.response)) {
+      const responseId = asTrimmedString(payload.response.id);
+      return responseId || null;
+    }
+    return null;
+  }
+
+  const object = asTrimmedString(payload.object).toLowerCase();
+  const status = asTrimmedString(payload.status).toLowerCase();
+  if (
+    (object === 'response' && RESPONSES_TERMINAL_STATUSES.has(status))
+    || RESPONSES_TERMINAL_STATUSES.has(status)
+  ) {
+    const responseId = asTrimmedString(payload.id);
+    return responseId || null;
+  }
+
+  if (isRecord(payload.response)) {
+    const nestedStatus = asTrimmedString(payload.response.status).toLowerCase();
+    if (RESPONSES_TERMINAL_STATUSES.has(nestedStatus)) {
+      const responseId = asTrimmedString(payload.response.id);
+      return responseId || null;
+    }
+  }
+
+  return null;
+}

src/server/transformers/shared/chatFormatsCore.ts

@@ -1234,6 +1234,12 @@ export function convertClaudeRequestToOpenAiBody(body: Record<string, unknown>):
   if (body.tools !== undefined) payload.tools = convertClaudeToolsToOpenAiChat(body.tools);
   if (body.tool_choice !== undefined) payload.tool_choice = convertClaudeToolChoiceToOpenAiChat(body.tool_choice);
 
+  const promptCacheKey = asTrimmedString(body.prompt_cache_key);
+  if (promptCacheKey) payload.prompt_cache_key = promptCacheKey;
+
+  const previousResponseId = asTrimmedString(body.previous_response_id);
+  if (previousResponseId) payload.previous_response_id = previousResponseId;
+
   return { model, stream, messages, payload };
 }