[codex] add proxy debug tracing (#299)

* add proxy debug tracing

* fix proxy debug response header typing

* extend proxy debug tracing to side paths

* Refine proxy debug trace review fixes

* Stabilize proxy debug attempt indexing

* Tighten proxy debug trace accuracy

src/server/routes/api/settings.events.test.ts

@@ -55,6 +55,15 @@ describe('settings and auth events', () => {
     config.codexUpstreamWebsocketEnabled = false;
     config.proxySessionChannelConcurrencyLimit = 2;
     config.proxySessionChannelQueueWaitMs = 1500;
+    (config as any).proxyDebugTraceEnabled = false;
+    (config as any).proxyDebugCaptureHeaders = true;
+    (config as any).proxyDebugCaptureBodies = false;
+    (config as any).proxyDebugCaptureStreamChunks = false;
+    (config as any).proxyDebugTargetSessionId = '';
+    (config as any).proxyDebugTargetClientKind = '';
+    (config as any).proxyDebugTargetModel = '';
+    (config as any).proxyDebugRetentionHours = 24;
+    (config as any).proxyDebugMaxBodyBytes = 262144;
     config.routingFallbackUnitCost = 1;
     (config as any).telegramEnabled = false;
     (config as any).telegramApiBaseUrl = 'https://api.telegram.org';
@@ -146,6 +155,67 @@ describe('settings and auth events', () => {
     expect(savedQueueWait?.value).toBe(JSON.stringify(4200));
   });
 
+  it('persists proxy debug runtime settings from runtime settings', async () => {
+    const updateResponse = await app.inject({
+      method: 'PUT',
+      url: '/api/settings/runtime',
+      payload: {
+        proxyDebugTraceEnabled: true,
+        proxyDebugCaptureHeaders: true,
+        proxyDebugCaptureBodies: true,
+        proxyDebugCaptureStreamChunks: true,
+        proxyDebugTargetSessionId: 'sess-debug-1',
+        proxyDebugTargetClientKind: 'codex',
+        proxyDebugTargetModel: 'gpt-4o',
+        proxyDebugRetentionHours: 12,
+        proxyDebugMaxBodyBytes: 131072,
+      },
+    });
+
+    expect(updateResponse.statusCode).toBe(200);
+    const updated = updateResponse.json() as {
+      proxyDebugTraceEnabled?: boolean;
+      proxyDebugCaptureHeaders?: boolean;
+      proxyDebugCaptureBodies?: boolean;
+      proxyDebugCaptureStreamChunks?: boolean;
+      proxyDebugTargetSessionId?: string;
+      proxyDebugTargetClientKind?: string;
+      proxyDebugTargetModel?: string;
+      proxyDebugRetentionHours?: number;
+      proxyDebugMaxBodyBytes?: number;
+    };
+    expect(updated).toMatchObject({
+      proxyDebugTraceEnabled: true,
+      proxyDebugCaptureHeaders: true,
+      proxyDebugCaptureBodies: true,
+      proxyDebugCaptureStreamChunks: true,
+      proxyDebugTargetSessionId: 'sess-debug-1',
+      proxyDebugTargetClientKind: 'codex',
+      proxyDebugTargetModel: 'gpt-4o',
+      proxyDebugRetentionHours: 12,
+      proxyDebugMaxBodyBytes: 131072,
+    });
+
+    const savedEnabled = await db.select().from(schema.settings).where(eq(schema.settings.key, 'proxy_debug_trace_enabled')).get();
+    const savedHeaders = await db.select().from(schema.settings).where(eq(schema.settings.key, 'proxy_debug_capture_headers')).get();
+    const savedBodies = await db.select().from(schema.settings).where(eq(schema.settings.key, 'proxy_debug_capture_bodies')).get();
+    const savedStreamChunks = await db.select().from(schema.settings).where(eq(schema.settings.key, 'proxy_debug_capture_stream_chunks')).get();
+    const savedTargetSessionId = await db.select().from(schema.settings).where(eq(schema.settings.key, 'proxy_debug_target_session_id')).get();
+    const savedTargetClientKind = await db.select().from(schema.settings).where(eq(schema.settings.key, 'proxy_debug_target_client_kind')).get();
+    const savedTargetModel = await db.select().from(schema.settings).where(eq(schema.settings.key, 'proxy_debug_target_model')).get();
+    const savedRetentionHours = await db.select().from(schema.settings).where(eq(schema.settings.key, 'proxy_debug_retention_hours')).get();
+    const savedMaxBodyBytes = await db.select().from(schema.settings).where(eq(schema.settings.key, 'proxy_debug_max_body_bytes')).get();
+    expect(savedEnabled?.value).toBe(JSON.stringify(true));
+    expect(savedHeaders?.value).toBe(JSON.stringify(true));
+    expect(savedBodies?.value).toBe(JSON.stringify(true));
+    expect(savedStreamChunks?.value).toBe(JSON.stringify(true));
+    expect(savedTargetSessionId?.value).toBe(JSON.stringify('sess-debug-1'));
+    expect(savedTargetClientKind?.value).toBe(JSON.stringify('codex'));
+    expect(savedTargetModel?.value).toBe(JSON.stringify('gpt-4o'));
+    expect(savedRetentionHours?.value).toBe(JSON.stringify(12));
+    expect(savedMaxBodyBytes?.value).toBe(JSON.stringify(131072));
+  });
+
   it('returns current recognized admin IP in runtime settings response', async () => {
     const response = await app.inject({
       method: 'GET',

src/server/routes/api/settings.ts

@@ -41,6 +41,15 @@ interface RuntimeSettingsBody {
   codexUpstreamWebsocketEnabled?: boolean;
   proxySessionChannelConcurrencyLimit?: number;
   proxySessionChannelQueueWaitMs?: number;
+  proxyDebugTraceEnabled?: boolean;
+  proxyDebugCaptureHeaders?: boolean;
+  proxyDebugCaptureBodies?: boolean;
+  proxyDebugCaptureStreamChunks?: boolean;
+  proxyDebugTargetSessionId?: string;
+  proxyDebugTargetClientKind?: string;
+  proxyDebugTargetModel?: string;
+  proxyDebugRetentionHours?: number;
+  proxyDebugMaxBodyBytes?: number;
   checkinCron?: string;
   checkinScheduleMode?: 'cron' | 'interval';
   checkinIntervalHours?: number;
@@ -409,6 +418,62 @@ function applyImportedSettingToRuntime(key: string, value: unknown) {
       config.proxySessionChannelQueueWaitMs = Math.trunc(queueWaitMs);
       return;
     }
+    case 'proxy_debug_trace_enabled': {
+      try {
+        config.proxyDebugTraceEnabled = parseBooleanFlag(value, '代理调试追踪开关');
+      } catch {
+        return;
+      }
+      return;
+    }
+    case 'proxy_debug_capture_headers': {
+      try {
+        config.proxyDebugCaptureHeaders = parseBooleanFlag(value, '代理调试请求头采集');
+      } catch {
+        return;
+      }
+      return;
+    }
+    case 'proxy_debug_capture_bodies': {
+      try {
+        config.proxyDebugCaptureBodies = parseBooleanFlag(value, '代理调试请求体采集');
+      } catch {
+        return;
+      }
+      return;
+    }
+    case 'proxy_debug_capture_stream_chunks': {
+      try {
+        config.proxyDebugCaptureStreamChunks = parseBooleanFlag(value, '代理调试流式分片采集');
+      } catch {
+        return;
+      }
+      return;
+    }
+    case 'proxy_debug_target_session_id': {
+      config.proxyDebugTargetSessionId = typeof value === 'string' ? value.trim() : '';
+      return;
+    }
+    case 'proxy_debug_target_client_kind': {
+      config.proxyDebugTargetClientKind = typeof value === 'string' ? value.trim() : '';
+      return;
+    }
+    case 'proxy_debug_target_model': {
+      config.proxyDebugTargetModel = typeof value === 'string' ? value.trim() : '';
+      return;
+    }
+    case 'proxy_debug_retention_hours': {
+      const retentionHours = Number(value);
+      if (!Number.isFinite(retentionHours) || retentionHours < 1) return;
+      config.proxyDebugRetentionHours = Math.trunc(retentionHours);
+      return;
+    }
+    case 'proxy_debug_max_body_bytes': {
+      const maxBodyBytes = Number(value);
+      if (!Number.isFinite(maxBodyBytes) || maxBodyBytes < 1024) return;
+      config.proxyDebugMaxBodyBytes = Math.trunc(maxBodyBytes);
+      return;
+    }
     case 'proxy_empty_content_fail_enabled': {
       try {
         config.proxyEmptyContentFailEnabled = parseBooleanFlag(value, '空内容判定失败开关');
@@ -580,6 +645,15 @@ function getRuntimeSettingsResponse(currentAdminIp = '') {
     codexUpstreamWebsocketEnabled: config.codexUpstreamWebsocketEnabled,
     proxySessionChannelConcurrencyLimit: config.proxySessionChannelConcurrencyLimit,
     proxySessionChannelQueueWaitMs: config.proxySessionChannelQueueWaitMs,
+    proxyDebugTraceEnabled: config.proxyDebugTraceEnabled,
+    proxyDebugCaptureHeaders: config.proxyDebugCaptureHeaders,
+    proxyDebugCaptureBodies: config.proxyDebugCaptureBodies,
+    proxyDebugCaptureStreamChunks: config.proxyDebugCaptureStreamChunks,
+    proxyDebugTargetSessionId: config.proxyDebugTargetSessionId,
+    proxyDebugTargetClientKind: config.proxyDebugTargetClientKind,
+    proxyDebugTargetModel: config.proxyDebugTargetModel,
+    proxyDebugRetentionHours: config.proxyDebugRetentionHours,
+    proxyDebugMaxBodyBytes: config.proxyDebugMaxBodyBytes,
     routingFallbackUnitCost: config.routingFallbackUnitCost,
     routingWeights: config.routingWeights,
     webhookUrl: config.webhookUrl,
@@ -1007,6 +1081,127 @@ export async function settingsRoutes(app: FastifyInstance) {
       upsertSetting('proxy_session_channel_queue_wait_ms', config.proxySessionChannelQueueWaitMs);
     }
 
+    if (body.proxyDebugTraceEnabled !== undefined) {
+      let nextValue = false;
+      try {
+        nextValue = parseBooleanFlag(body.proxyDebugTraceEnabled, '代理调试追踪开关');
+      } catch (err: any) {
+        return reply.code(400).send({
+          success: false,
+          message: err?.message || '代理调试追踪开关格式无效',
+        });
+      }
+      if (nextValue !== config.proxyDebugTraceEnabled) {
+        changedLabels.push('代理调试追踪');
+      }
+      config.proxyDebugTraceEnabled = nextValue;
+      upsertSetting('proxy_debug_trace_enabled', config.proxyDebugTraceEnabled);
+    }
+
+    if (body.proxyDebugCaptureHeaders !== undefined) {
+      let nextValue = false;
+      try {
+        nextValue = parseBooleanFlag(body.proxyDebugCaptureHeaders, '代理调试请求头采集');
+      } catch (err: any) {
+        return reply.code(400).send({
+          success: false,
+          message: err?.message || '代理调试请求头采集格式无效',
+        });
+      }
+      if (nextValue !== config.proxyDebugCaptureHeaders) {
+        changedLabels.push('代理调试请求头采集');
+      }
+      config.proxyDebugCaptureHeaders = nextValue;
+      upsertSetting('proxy_debug_capture_headers', config.proxyDebugCaptureHeaders);
+    }
+
+    if (body.proxyDebugCaptureBodies !== undefined) {
+      let nextValue = false;
+      try {
+        nextValue = parseBooleanFlag(body.proxyDebugCaptureBodies, '代理调试请求体采集');
+      } catch (err: any) {
+        return reply.code(400).send({
+          success: false,
+          message: err?.message || '代理调试请求体采集格式无效',
+        });
+      }
+      if (nextValue !== config.proxyDebugCaptureBodies) {
+        changedLabels.push('代理调试请求体采集');
+      }
+      config.proxyDebugCaptureBodies = nextValue;
+      upsertSetting('proxy_debug_capture_bodies', config.proxyDebugCaptureBodies);
+    }
+
+    if (body.proxyDebugCaptureStreamChunks !== undefined) {
+      let nextValue = false;
+      try {
+        nextValue = parseBooleanFlag(body.proxyDebugCaptureStreamChunks, '代理调试流式分片采集');
+      } catch (err: any) {
+        return reply.code(400).send({
+          success: false,
+          message: err?.message || '代理调试流式分片采集格式无效',
+        });
+      }
+      if (nextValue !== config.proxyDebugCaptureStreamChunks) {
+        changedLabels.push('代理调试流式分片采集');
+      }
+      config.proxyDebugCaptureStreamChunks = nextValue;
+      upsertSetting('proxy_debug_capture_stream_chunks', config.proxyDebugCaptureStreamChunks);
+    }
+
+    if (body.proxyDebugTargetSessionId !== undefined) {
+      const nextValue = String(body.proxyDebugTargetSessionId || '').trim();
+      if (nextValue !== config.proxyDebugTargetSessionId) {
+        changedLabels.push('代理调试目标会话');
+      }
+      config.proxyDebugTargetSessionId = nextValue;
+      upsertSetting('proxy_debug_target_session_id', config.proxyDebugTargetSessionId);
+    }
+
+    if (body.proxyDebugTargetClientKind !== undefined) {
+      const nextValue = String(body.proxyDebugTargetClientKind || '').trim();
+      if (nextValue !== config.proxyDebugTargetClientKind) {
+        changedLabels.push('代理调试目标客户端');
+      }
+      config.proxyDebugTargetClientKind = nextValue;
+      upsertSetting('proxy_debug_target_client_kind', config.proxyDebugTargetClientKind);
+    }
+
+    if (body.proxyDebugTargetModel !== undefined) {
+      const nextValue = String(body.proxyDebugTargetModel || '').trim();
+      if (nextValue !== config.proxyDebugTargetModel) {
+        changedLabels.push('代理调试目标模型');
+      }
+      config.proxyDebugTargetModel = nextValue;
+      upsertSetting('proxy_debug_target_model', config.proxyDebugTargetModel);
+    }
+
+    if (body.proxyDebugRetentionHours !== undefined) {
+      const retentionHours = Number(body.proxyDebugRetentionHours);
+      if (!Number.isFinite(retentionHours) || retentionHours < 1) {
+        return reply.code(400).send({ success: false, message: '代理调试保留时长必须是大于等于 1 的整数小时' });
+      }
+      const nextValue = Math.trunc(retentionHours);
+      if (nextValue !== config.proxyDebugRetentionHours) {
+        changedLabels.push(`代理调试保留时长（${config.proxyDebugRetentionHours}h -> ${nextValue}h）`);
+      }
+      config.proxyDebugRetentionHours = nextValue;
+      upsertSetting('proxy_debug_retention_hours', config.proxyDebugRetentionHours);
+    }
+
+    if (body.proxyDebugMaxBodyBytes !== undefined) {
+      const maxBodyBytes = Number(body.proxyDebugMaxBodyBytes);
+      if (!Number.isFinite(maxBodyBytes) || maxBodyBytes < 1024) {
+        return reply.code(400).send({ success: false, message: '代理调试抓取体积上限必须是大于等于 1024 的整数字节' });
+      }
+      const nextValue = Math.trunc(maxBodyBytes);
+      if (nextValue !== config.proxyDebugMaxBodyBytes) {
+        changedLabels.push(`代理调试抓取体积上限（${config.proxyDebugMaxBodyBytes}B -> ${nextValue}B）`);
+      }
+      config.proxyDebugMaxBodyBytes = nextValue;
+      upsertSetting('proxy_debug_max_body_bytes', config.proxyDebugMaxBodyBytes);
+    }
+
     if (body.proxyErrorKeywords !== undefined) {
       let nextKeywords: string[] = [];
       try {

src/server/routes/api/stats.proxy-debug.test.ts

@@ -0,0 +1,109 @@
+import Fastify, { type FastifyInstance } from 'fastify';
+import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest';
+import { mkdtempSync } from 'node:fs';
+import { join } from 'node:path';
+import { tmpdir } from 'node:os';
+
+type DbModule = typeof import('../../db/index.js');
+type StoreModule = typeof import('../../services/proxyDebugTraceStore.js');
+
+describe('stats proxy debug api', () => {
+  let app: FastifyInstance;
+  let db: DbModule['db'];
+  let schema: DbModule['schema'];
+  let store: StoreModule;
+  let dataDir = '';
+
+  beforeAll(async () => {
+    dataDir = mkdtempSync(join(tmpdir(), 'metapi-stats-proxy-debug-'));
+    process.env.DATA_DIR = dataDir;
+
+    await import('../../db/migrate.js');
+    const dbModule = await import('../../db/index.js');
+    const storeModule = await import('../../services/proxyDebugTraceStore.js');
+    const statsRoutesModule = await import('./stats.js');
+
+    db = dbModule.db;
+    schema = dbModule.schema;
+    store = storeModule;
+
+    app = Fastify();
+    await app.register(statsRoutesModule.statsRoutes);
+  });
+
+  beforeEach(async () => {
+    await db.delete(schema.proxyDebugAttempts).run();
+    await db.delete(schema.proxyDebugTraces).run();
+  });
+
+  afterAll(async () => {
+    await app.close();
+    const dbModule = await import('../../db/index.js');
+    await dbModule.closeDbConnections();
+    delete process.env.DATA_DIR;
+  });
+
+  it('lists and returns proxy debug traces with attempt details', async () => {
+    const trace = await store.createProxyDebugTrace({
+      downstreamPath: '/v1/chat/completions',
+      clientKind: 'codex',
+      sessionId: 'sess-9',
+      traceHint: 'trace-x',
+      requestedModel: 'gpt-4.1',
+      requestHeaders: { authorization: 'Bearer test' },
+      requestBody: { model: 'gpt-4.1' },
+    });
+    await store.insertProxyDebugAttempt({
+      traceId: trace.id,
+      attemptIndex: 0,
+      endpoint: 'chat',
+      requestPath: '/v1/chat/completions',
+      targetUrl: 'https://example.com/v1/chat/completions',
+      runtimeExecutor: 'default',
+      requestHeaders: { authorization: 'Bearer test' },
+      requestBody: { model: 'gpt-4.1' },
+      responseStatus: 200,
+      responseHeaders: { 'content-type': 'application/json' },
+      responseBody: { id: 'chatcmpl_123' },
+      rawErrorText: null,
+      recoverApplied: false,
+      downgradeDecision: false,
+      downgradeReason: null,
+      memoryWrite: { action: 'success', preferredEndpoint: 'chat' },
+    });
+    await store.finalizeProxyDebugTrace(trace.id, {
+      finalStatus: 'success',
+      finalHttpStatus: 200,
+      finalUpstreamPath: '/v1/chat/completions',
+      finalResponseHeaders: { 'content-type': 'application/json' },
+      finalResponseBody: { id: 'chatcmpl_123' },
+    });
+
+    const listResponse = await app.inject({
+      method: 'GET',
+      url: '/api/stats/proxy-debug/traces?limit=10',
+    });
+    expect(listResponse.statusCode).toBe(200);
+    const listBody = listResponse.json() as { items?: Array<{ id: number; finalStatus: string }> };
+    expect(listBody.items).toHaveLength(1);
+    expect(listBody.items?.[0]?.finalStatus).toBe('success');
+
+    const detailResponse = await app.inject({
+      method: 'GET',
+      url: `/api/stats/proxy-debug/traces/${trace.id}`,
+    });
+    expect(detailResponse.statusCode).toBe(200);
+    const detailBody = detailResponse.json() as {
+      trace?: { requestedModel?: string; sessionId?: string };
+      attempts?: Array<{ endpoint?: string; responseStatus?: number }>;
+    };
+    expect(detailBody.trace).toMatchObject({
+      requestedModel: 'gpt-4.1',
+      sessionId: 'sess-9',
+    });
+    expect(detailBody.attempts?.[0]).toMatchObject({
+      endpoint: 'chat',
+      responseStatus: 200,
+    });
+  });
+});

src/server/routes/api/stats.ts

@@ -14,6 +14,10 @@ import {
   parseProxyLogBillingDetails,
   withProxyLogSelectFields,
 } from '../../services/proxyLogStore.js';
+import {
+  getProxyDebugTraceDetail,
+  listProxyDebugTraces,
+} from '../../services/proxyDebugTraceStore.js';
 import { parseProxyLogMessageMeta } from '../proxy/logPathMeta.js';
 import { requiresManagedAccountTokens } from '../../services/accountExtraConfig.js';
 import { ACCOUNT_TOKEN_VALUE_STATUS_READY } from '../../services/accountTokenService.js';
@@ -820,6 +824,26 @@ export async function statsRoutes(app: FastifyInstance) {
     return mapProxyLogRow(row, { includeBillingDetails: true });
   });
 
+  app.get<{ Querystring: { limit?: string } }>('/api/stats/proxy-debug/traces', async (request) => {
+    const limit = normalizeProxyLogPageSize(request.query.limit);
+    const items = await listProxyDebugTraces({ limit });
+    return { items };
+  });
+
+  app.get<{ Params: { id: string } }>('/api/stats/proxy-debug/traces/:id', async (request, reply) => {
+    const id = Number.parseInt(request.params.id, 10);
+    if (!Number.isFinite(id) || id <= 0) {
+      return reply.code(400).send({ message: 'proxy debug trace id is invalid' });
+    }
+
+    const detail = await getProxyDebugTraceDetail(id);
+    if (!detail) {
+      return reply.code(404).send({ message: 'proxy debug trace not found' });
+    }
+
+    return detail;
+  });
+
   // Models marketplace - refresh upstream models and aggregate.
   app.get<{ Querystring: { refresh?: string; includePricing?: string } }>('/api/models/marketplace', async (request) => {
     const refreshRequested = parseBooleanFlag(request.query.refresh);

src/server/routes/proxy/architecture-boundaries.test.ts

@@ -98,6 +98,12 @@ describe('proxy route architecture boundaries', () => {
     expect(source).not.toContain('shouldInjectDerivedPromptCacheKey');
   });
 
+  it('keeps endpoint runtime snapshot helper out of the route layer', () => {
+    const source = readSource('./upstreamEndpoint.ts');
+    expect(source).not.toContain('function getUpstreamEndpointRuntimeStateSnapshot(');
+    expect(source).not.toContain('export function getUpstreamEndpointRuntimeStateSnapshot(');
+  });
+
   it('keeps gemini runtime closure in transformer-owned helpers', () => {
     const source = readSource('./gemini.ts');
     const surfaceSource = readSource('../../proxy-core/surfaces/geminiSurface.ts');
@@ -179,4 +185,3 @@ describe('proxy route architecture boundaries', () => {
     expect(geminiSource).not.toContain("from '../../transformers/canonical/");
   });
 });
-

src/server/routes/proxy/chat.count-tokens.test.ts

@@ -14,6 +14,11 @@ const resolveProxyUsageWithSelfLogFallbackMock = vi.fn(async ({ usage }: any) =>
   estimatedCostFromQuota: 0,
   recoveredFromSelfLog: false,
 }));
+const startSurfaceProxyDebugTraceMock = vi.fn();
+const safeUpdateSurfaceProxyDebugSelectionMock = vi.fn();
+const safeUpdateSurfaceProxyDebugCandidatesMock = vi.fn();
+const safeInsertSurfaceProxyDebugAttemptMock = vi.fn();
+const safeFinalizeSurfaceProxyDebugTraceMock = vi.fn();
 const dbInsertMock = vi.fn((_arg?: any) => ({
   values: () => ({
     run: () => undefined,
@@ -60,6 +65,19 @@ vi.mock('../../services/proxyUsageFallbackService.js', () => ({
   resolveProxyUsageWithSelfLogFallback: (arg: any) => resolveProxyUsageWithSelfLogFallbackMock(arg),
 }));
 
+vi.mock('../../services/proxyDebugTraceRuntime.js', () => ({
+  startSurfaceProxyDebugTrace: (...args: unknown[]) => startSurfaceProxyDebugTraceMock(...args),
+  safeUpdateSurfaceProxyDebugSelection: (...args: unknown[]) => safeUpdateSurfaceProxyDebugSelectionMock(...args),
+  safeUpdateSurfaceProxyDebugCandidates: (...args: unknown[]) => safeUpdateSurfaceProxyDebugCandidatesMock(...args),
+  safeInsertSurfaceProxyDebugAttempt: (...args: unknown[]) => safeInsertSurfaceProxyDebugAttemptMock(...args),
+  safeFinalizeSurfaceProxyDebugTrace: (...args: unknown[]) => safeFinalizeSurfaceProxyDebugTraceMock(...args),
+  safeUpdateSurfaceProxyDebugAttempt: vi.fn(),
+  reserveSurfaceProxyDebugAttemptBase: () => 0,
+  buildSurfaceProxyDebugResponseHeaders: () => ({}),
+  captureSurfaceProxyDebugSuccessResponseBody: async () => null,
+  parseSurfaceProxyDebugTextPayload: (raw: string) => raw,
+}));
+
 vi.mock('../../services/oauth/quota.js', () => ({
   recordOauthQuotaResetHint: async () => undefined,
 }));
@@ -95,8 +113,28 @@ describe('claude count_tokens proxy route', () => {
     reportProxyAllFailedMock.mockReset();
     reportTokenExpiredMock.mockReset();
     resolveProxyUsageWithSelfLogFallbackMock.mockClear();
+    startSurfaceProxyDebugTraceMock.mockReset();
+    safeUpdateSurfaceProxyDebugSelectionMock.mockReset();
+    safeUpdateSurfaceProxyDebugCandidatesMock.mockReset();
+    safeInsertSurfaceProxyDebugAttemptMock.mockReset();
+    safeFinalizeSurfaceProxyDebugTraceMock.mockReset();
     dbInsertMock.mockClear();
 
+    startSurfaceProxyDebugTraceMock.mockResolvedValue({
+      traceId: 701,
+      options: {
+        enabled: true,
+        captureHeaders: true,
+        captureBodies: true,
+        captureStreamChunks: false,
+        targetSessionId: '',
+        targetClientKind: '',
+        targetModel: '',
+        retentionHours: 24,
+        maxBodyBytes: 262144,
+      },
+    });
+
     selectChannelMock.mockReturnValue({
       channel: { id: 11, routeId: 22 },
       site: { name: 'claude-site', url: 'https://api.anthropic.com', platform: 'claude' },
@@ -146,6 +184,18 @@ describe('claude count_tokens proxy route', () => {
     expect(options.headers['anthropic-version']).toBe('2023-06-01');
     expect(options.headers['anthropic-beta']).toContain('claude-code-20250219');
     expect(options.headers['Accept-Encoding']).toBe('gzip, deflate, br, zstd');
+    expect(startSurfaceProxyDebugTraceMock).toHaveBeenCalledWith(expect.objectContaining({
+      downstreamPath: '/v1/messages/count_tokens',
+      requestedModel: 'claude-opus-4-6',
+    }));
+    expect(safeInsertSurfaceProxyDebugAttemptMock).toHaveBeenCalled();
+    expect(safeFinalizeSurfaceProxyDebugTraceMock).toHaveBeenCalledWith(
+      expect.objectContaining({ traceId: 701 }),
+      expect.objectContaining({
+        finalStatus: 'success',
+        finalUpstreamPath: '/v1/messages/count_tokens?beta=true',
+      }),
+    );
 
     const forwardedBody = JSON.parse(String(options.body));
     expect(forwardedBody.model).toBe('claude-opus-4-6');

src/server/routes/proxy/chat.stream.test.ts

@@ -2,7 +2,7 @@ import { zstdCompressSync } from 'node:zlib';
 import Fastify, { type FastifyInstance } from 'fastify';
 import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
 import { config } from '../../config.js';
-import { resetUpstreamEndpointRuntimeState } from './upstreamEndpoint.js';
+import { resetUpstreamEndpointRuntimeState } from '../../services/upstreamEndpointRuntimeMemory.js';
 
 const fetchMock = vi.fn();
 const selectChannelMock = vi.fn();

src/server/routes/proxy/endpointFlow.ts

@@ -26,6 +26,7 @@ export type EndpointAttemptContext = {
   targetUrl: string;
   response: Awaited<ReturnType<typeof fetch>>;
   rawErrText: string;
+  recoverApplied?: boolean;
 };
 
 export type EndpointAttemptSuccessContext = {
@@ -34,6 +35,7 @@ export type EndpointAttemptSuccessContext = {
   request: BuiltEndpointRequest;
   targetUrl: string;
   response: Awaited<ReturnType<typeof fetch>>;
+  recoverApplied?: boolean;
 };
 
 export type EndpointRecoverResult = {
@@ -125,6 +127,7 @@ export async function executeEndpointFlow(input: ExecuteEndpointFlowInput): Prom
         request,
         targetUrl,
         response,
+        recoverApplied: false,
       }, 'onAttemptSuccess');
       return {
         ok: true,
@@ -141,10 +144,15 @@ export async function executeEndpointFlow(input: ExecuteEndpointFlowInput): Prom
       targetUrl,
       response,
       rawErrText,
+      recoverApplied: false,
     };
 
     if (input.tryRecover) {
       const recovered = await input.tryRecover(baseContext);
+      baseContext.recoverApplied = recovered !== null
+        || baseContext.request !== request
+        || baseContext.response !== response
+        || baseContext.rawErrText !== rawErrText;
       if (recovered?.upstream?.ok) {
         const recoveredRequest = recovered.request ?? baseContext.request;
         const recoveredTargetUrl = recovered.targetUrl ?? (
@@ -158,6 +166,7 @@ export async function executeEndpointFlow(input: ExecuteEndpointFlowInput): Prom
           request: recoveredRequest,
           targetUrl: recoveredTargetUrl,
           response: recovered.upstream,
+          recoverApplied: true,
         }, 'onAttemptSuccess');
         return {
           ok: true,

src/server/routes/proxy/gemini.test.ts

@@ -2,7 +2,7 @@ import Fastify, { type FastifyInstance } from 'fastify';
 import { readFileSync } from 'node:fs';
 import path from 'node:path';
 import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
-import { resetUpstreamEndpointRuntimeState } from './upstreamEndpoint.js';
+import { resetUpstreamEndpointRuntimeState } from '../../services/upstreamEndpointRuntimeMemory.js';
 
 const fetchMock = vi.fn();
 const fetchModelPricingCatalogMock = vi.fn();
@@ -25,6 +25,11 @@ const dbInsertValuesMock = vi.fn((_values?: unknown) => ({
 const dbInsertMock = vi.fn((_table?: unknown) => ({
   values: (values: unknown) => dbInsertValuesMock(values),
 }));
+const startSurfaceProxyDebugTraceMock = vi.fn();
+const safeUpdateSurfaceProxyDebugSelectionMock = vi.fn();
+const safeUpdateSurfaceProxyDebugCandidatesMock = vi.fn();
+const safeInsertSurfaceProxyDebugAttemptMock = vi.fn();
+const safeFinalizeSurfaceProxyDebugTraceMock = vi.fn();
 
 function createDbSelectChain() {
   return {
@@ -107,6 +112,19 @@ vi.mock('../../db/index.js', () => ({
   },
 }));
 
+vi.mock('../../services/proxyDebugTraceRuntime.js', () => ({
+  startSurfaceProxyDebugTrace: (...args: unknown[]) => startSurfaceProxyDebugTraceMock(...args),
+  safeUpdateSurfaceProxyDebugSelection: (...args: unknown[]) => safeUpdateSurfaceProxyDebugSelectionMock(...args),
+  safeUpdateSurfaceProxyDebugCandidates: (...args: unknown[]) => safeUpdateSurfaceProxyDebugCandidatesMock(...args),
+  safeInsertSurfaceProxyDebugAttempt: (...args: unknown[]) => safeInsertSurfaceProxyDebugAttemptMock(...args),
+  safeFinalizeSurfaceProxyDebugTrace: (...args: unknown[]) => safeFinalizeSurfaceProxyDebugTraceMock(...args),
+  safeUpdateSurfaceProxyDebugAttempt: vi.fn(),
+  reserveSurfaceProxyDebugAttemptBase: () => 0,
+  buildSurfaceProxyDebugResponseHeaders: () => ({}),
+  captureSurfaceProxyDebugSuccessResponseBody: async () => null,
+  parseSurfaceProxyDebugTextPayload: (raw: string) => raw,
+}));
+
 function parseSsePayloads(body: string): Array<Record<string, unknown>> {
   return body
     .replace(/\r\n/g, '\n')
@@ -173,6 +191,26 @@ describe('gemini native proxy routes', () => {
     dbInsertValuesMock.mockClear();
     dbSelectAllMock.mockReset();
     dbSelectGetMock.mockReset();
+    startSurfaceProxyDebugTraceMock.mockReset();
+    safeUpdateSurfaceProxyDebugSelectionMock.mockReset();
+    safeUpdateSurfaceProxyDebugCandidatesMock.mockReset();
+    safeInsertSurfaceProxyDebugAttemptMock.mockReset();
+    safeFinalizeSurfaceProxyDebugTraceMock.mockReset();
+
+    startSurfaceProxyDebugTraceMock.mockResolvedValue({
+      traceId: 801,
+      options: {
+        enabled: true,
+        captureHeaders: true,
+        captureBodies: true,
+        captureStreamChunks: false,
+        targetSessionId: '',
+        targetClientKind: '',
+        targetModel: '',
+        retentionHours: 24,
+        maxBodyBytes: 262144,
+      },
+    });
 
     authorizeDownstreamTokenMock.mockResolvedValue({
       ok: true,
@@ -746,6 +784,25 @@ describe('gemini native proxy routes', () => {
       Authorization: 'Bearer openai-access-token',
       'Content-Type': 'application/json',
     });
+    expect(startSurfaceProxyDebugTraceMock).toHaveBeenCalledWith(expect.objectContaining({
+      downstreamPath: '/v1beta/models/gemini-2.5-flash:generateContent',
+      requestedModel: 'gemini-2.5-flash',
+    }));
+    expect(safeUpdateSurfaceProxyDebugSelectionMock).toHaveBeenCalledWith(
+      expect.objectContaining({ traceId: 801 }),
+      expect.objectContaining({
+        selectedChannelId: 41,
+        selectedSitePlatform: 'openai',
+      }),
+    );
+    expect(safeInsertSurfaceProxyDebugAttemptMock).toHaveBeenCalled();
+    expect(safeFinalizeSurfaceProxyDebugTraceMock).toHaveBeenCalledWith(
+      expect.objectContaining({ traceId: 801 }),
+      expect.objectContaining({
+        finalStatus: 'success',
+        finalUpstreamPath: '/v1/chat/completions',
+      }),
+    );
     expect(JSON.parse(String(requestInit.body))).toEqual({
       model: 'gpt-4.1',
       stream: false,

src/server/routes/proxy/upstreamEndpoint.test.ts

@@ -1,4 +1,4 @@
-import { beforeEach, describe, expect, it, vi } from 'vitest';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
 import { config } from '../../config.js';
 
 const fetchModelPricingCatalogMock = vi.fn(async (_arg?: unknown): Promise<any> => null);
@@ -12,14 +12,17 @@ import {
   buildUpstreamEndpointRequest,
   isUnsupportedMediaTypeError,
   isEndpointDowngradeError,
+  resolveUpstreamEndpointCandidates,
+} from './upstreamEndpoint.js';
+import {
   recordUpstreamEndpointFailure,
   recordUpstreamEndpointSuccess,
   resetUpstreamEndpointRuntimeState,
-  resolveUpstreamEndpointCandidates,
+  getUpstreamEndpointRuntimeStateSnapshot,
   boundEndpointRuntimeModelKey,
   MAX_ENDPOINT_RUNTIME_MODEL_KEY_LENGTH,
   MODEL_KEY_HASH_SUFFIX_LENGTH,
-} from './upstreamEndpoint.js';
+} from '../../services/upstreamEndpointRuntimeMemory.js';
 
 const baseContext = {
   site: {
@@ -52,6 +55,9 @@ describe('resolveUpstreamEndpointCandidates', () => {
       filter: [],
     };
   });
+  afterEach(() => {
+    vi.useRealTimers();
+  });
 
   it('uses downstream-aligned endpoint priority for unknown platforms', async () => {
     const openaiOrder = await resolveUpstreamEndpointCandidates(
@@ -236,6 +242,64 @@ describe('resolveUpstreamEndpointCandidates', () => {
     expect(order).toEqual(['chat', 'messages', 'responses']);
   });
 
+  it('does not expose preferred endpoint in snapshot when runtime memory is disabled for multimodal requests', () => {
+    recordUpstreamEndpointSuccess({
+      siteId: baseContext.site.id,
+      endpoint: 'responses',
+      downstreamFormat: 'openai',
+      modelName: 'gpt-5.3',
+      requestCapabilities: {
+        conversationFileSummary: {
+          hasImage: true,
+          hasAudio: false,
+          hasDocument: false,
+          hasRemoteDocumentUrl: false,
+        },
+      },
+    });
+
+    expect(getUpstreamEndpointRuntimeStateSnapshot({
+      siteId: baseContext.site.id,
+      downstreamFormat: 'openai',
+      modelName: 'gpt-5.3',
+      requestCapabilities: {
+        conversationFileSummary: {
+          hasImage: true,
+          hasAudio: false,
+          hasDocument: false,
+          hasRemoteDocumentUrl: false,
+        },
+      },
+    })).toMatchObject({
+      enabled: false,
+      preferredEndpoint: null,
+      blockedEndpoints: [],
+    });
+  });
+
+  it('does not expose expired preferred endpoints in the runtime snapshot', () => {
+    vi.useFakeTimers();
+    vi.setSystemTime(new Date('2026-03-28T00:00:00.000Z'));
+
+    recordUpstreamEndpointSuccess({
+      siteId: baseContext.site.id,
+      endpoint: 'chat',
+      downstreamFormat: 'openai',
+      modelName: 'gpt-5.3',
+    });
+
+    vi.setSystemTime(new Date('2026-03-29T01:00:00.000Z'));
+
+    expect(getUpstreamEndpointRuntimeStateSnapshot({
+      siteId: baseContext.site.id,
+      downstreamFormat: 'openai',
+      modelName: 'gpt-5.3',
+    })).toMatchObject({
+      enabled: true,
+      preferredEndpoint: null,
+    });
+  });
+
   it('does not apply runtime endpoint memory to document attachments', async () => {
     recordUpstreamEndpointSuccess({
       siteId: baseContext.site.id,
@@ -379,12 +443,13 @@ describe('resolveUpstreamEndpointCandidates', () => {
   });
 
   it('does not remember messages fallback success for generic /v1/responses requests', async () => {
-    recordUpstreamEndpointSuccess({
+    const memoryWrite = recordUpstreamEndpointSuccess({
       siteId: baseContext.site.id,
       endpoint: 'messages',
       downstreamFormat: 'responses',
       modelName: 'gpt-5.3',
     });
+    expect(memoryWrite).toBeNull();
 
     const order = await resolveUpstreamEndpointCandidates(
       {
@@ -398,8 +463,23 @@ describe('resolveUpstreamEndpointCandidates', () => {
     expect(order).toEqual(['responses', 'chat', 'messages']);
   });
 
+  it('returns the applied success write when runtime memory stores a preferred endpoint', () => {
+    const memoryWrite = recordUpstreamEndpointSuccess({
+      siteId: baseContext.site.id,
+      endpoint: 'responses',
+      downstreamFormat: 'responses',
+      modelName: 'gpt-5.3',
+    });
+
+    expect(memoryWrite).toMatchObject({
+      action: 'success',
+      endpoint: 'responses',
+      preferredEndpoint: 'responses',
+    });
+  });
+
   it('does not block generic /v1/responses endpoints on transient upstream errors', async () => {
-    recordUpstreamEndpointFailure({
+    const memoryWrite = recordUpstreamEndpointFailure({
       siteId: baseContext.site.id,
       endpoint: 'responses',
       downstreamFormat: 'responses',
@@ -407,6 +487,7 @@ describe('resolveUpstreamEndpointCandidates', () => {
       status: 504,
       errorText: '{"error":{"message":"Gateway time-out","type":"upstream_error"}}',
     });
+    expect(memoryWrite).toBeNull();
 
     const order = await resolveUpstreamEndpointCandidates(
       {
@@ -421,7 +502,7 @@ describe('resolveUpstreamEndpointCandidates', () => {
   });
 
   it('learns a better endpoint from explicit upstream protocol errors', async () => {
-    recordUpstreamEndpointFailure({
+    const memoryWrite = recordUpstreamEndpointFailure({
       siteId: baseContext.site.id,
       endpoint: 'chat',
       downstreamFormat: 'openai',
@@ -429,6 +510,12 @@ describe('resolveUpstreamEndpointCandidates', () => {
       status: 400,
       errorText: 'Unsupported legacy protocol: /v1/chat/completions is not supported. Please use /v1/responses.',
     });
+    expect(memoryWrite).toMatchObject({
+      action: 'failure',
+      endpoint: 'chat',
+      blockedEndpoint: 'chat',
+      preferredEndpoint: 'responses',
+    });
 
     const order = await resolveUpstreamEndpointCandidates(
       {

src/server/routes/proxy/upstreamEndpoint.ts

@@ -1,4 +1,4 @@
-import { randomUUID, createHash } from 'node:crypto';
+import { randomUUID } from 'node:crypto';
 import {
   rankConversationFileEndpoints,
   type ConversationFileInputSummary,
@@ -7,6 +7,10 @@ import { resolveProviderProfile } from '../../proxy-core/providers/registry.js';
 import { config } from '../../config.js';
 import { fetchModelPricingCatalog } from '../../services/modelPricingService.js';
 import { applyPayloadRules } from '../../services/payloadRules.js';
+import {
+  applyUpstreamEndpointRuntimePreference,
+  buildEndpointCapabilityProfile,
+} from '../../services/upstreamEndpointRuntimeMemory.js';
 import type { DownstreamFormat } from '../../transformers/shared/normalized.js';
 import {
   convertOpenAiBodyToResponsesBody as convertOpenAiBodyToResponsesBodyViaTransformer,
@@ -44,23 +48,6 @@ export {
 export type UpstreamEndpoint = 'chat' | 'messages' | 'responses';
 export type EndpointPreference = DownstreamFormat | 'responses';
 
-type EndpointCapabilityProfile = {
-  modelKey: string;
-  preferMessagesForClaudeModel: boolean;
-  hasImageInput: boolean;
-  hasAudioInput: boolean;
-  hasNonImageFileInput: boolean;
-  hasRemoteDocumentUrl: boolean;
-  wantsNativeResponsesReasoning: boolean;
-};
-
-type EndpointRuntimeState = {
-  preferredEndpoint: UpstreamEndpoint | null;
-  preferredUpdatedAtMs: number;
-  lastTouchedAtMs: number;
-  blockedUntilMsByEndpoint: Partial<Record<UpstreamEndpoint, number>>;
-};
-
 type ChannelContext = {
   site: {
     id: number;
@@ -75,26 +62,6 @@ type ChannelContext = {
   };
 };
 
-const ENDPOINT_RUNTIME_PREFERRED_TTL_MS = 24 * 60 * 60 * 1000;
-const ENDPOINT_RUNTIME_BLOCK_TTL_MS = 6 * 60 * 60 * 1000;
-const MAX_ENDPOINT_RUNTIME_STATES = 512;
-export const MAX_ENDPOINT_RUNTIME_MODEL_KEY_LENGTH = 64;
-export const MODEL_KEY_HASH_SUFFIX_LENGTH = 8;
-const endpointRuntimeStates = new Map<string, EndpointRuntimeState>();
-
-export function boundEndpointRuntimeModelKey(value: string): string {
-  if (value.length <= MAX_ENDPOINT_RUNTIME_MODEL_KEY_LENGTH) {
-    return value;
-  }
-
-  const prefix = value.slice(0, MAX_ENDPOINT_RUNTIME_MODEL_KEY_LENGTH);
-  const hash = createHash('sha256')
-    .update(value)
-    .digest('hex')
-    .slice(0, MODEL_KEY_HASH_SUFFIX_LENGTH);
-  return `${prefix}-${hash}`;
-}
-
 function isRecord(value: unknown): value is Record<string, unknown> {
   return !!value && typeof value === 'object';
 }
@@ -103,14 +70,6 @@ function asTrimmedString(value: unknown): string {
   return typeof value === 'string' ? value.trim() : '';
 }
 
-function normalizeEndpointRuntimeModelKey(...values: Array<unknown>): string {
-  for (const value of values) {
-    const normalized = asTrimmedString(value).toLowerCase();
-    if (normalized) return boundEndpointRuntimeModelKey(normalized);
-  }
-  return boundEndpointRuntimeModelKey('unknown-model');
-}
-
 function resolveRequestedModelForPayloadRules(input: {
   modelName: string;
   openaiBody: Record<string, unknown>;
@@ -129,12 +88,6 @@ function normalizePlatformName(platform: unknown): string {
   return asTrimmedString(platform).toLowerCase();
 }
 
-function isClaudeFamilyModel(modelName: string): boolean {
-  const normalized = asTrimmedString(modelName).toLowerCase();
-  if (!normalized) return false;
-  return normalized === 'claude' || normalized.startsWith('claude-') || normalized.includes('claude');
-}
-
 const HOP_BY_HOP_HEADERS = new Set([
   'connection',
   'keep-alive',
@@ -495,277 +448,6 @@ function normalizeEndpointTypes(value: unknown): UpstreamEndpoint[] {
   return Array.from(normalized);
 }
 
-function buildEndpointCapabilityProfile(input?: {
-  modelName?: string;
-  requestedModelHint?: string;
-  requestCapabilities?: {
-    hasNonImageFileInput?: boolean;
-    conversationFileSummary?: ConversationFileInputSummary;
-    wantsNativeResponsesReasoning?: boolean;
-  };
-}): EndpointCapabilityProfile {
-  const conversationFileSummary = input?.requestCapabilities?.conversationFileSummary;
-  return {
-    modelKey: normalizeEndpointRuntimeModelKey(input?.modelName, input?.requestedModelHint),
-    preferMessagesForClaudeModel: (
-      isClaudeFamilyModel(asTrimmedString(input?.modelName))
-      || isClaudeFamilyModel(asTrimmedString(input?.requestedModelHint))
-    ),
-    hasImageInput: conversationFileSummary?.hasImage === true,
-    hasAudioInput: conversationFileSummary?.hasAudio === true,
-    hasNonImageFileInput: (
-      conversationFileSummary?.hasDocument === true
-      || input?.requestCapabilities?.hasNonImageFileInput === true
-    ),
-    hasRemoteDocumentUrl: (
-      conversationFileSummary?.hasRemoteDocumentUrl === true
-    ),
-    wantsNativeResponsesReasoning: input?.requestCapabilities?.wantsNativeResponsesReasoning === true,
-  };
-}
-
-function shouldUseEndpointRuntimeMemory(capabilityProfile: EndpointCapabilityProfile): boolean {
-  // Attachment-capable requests are not protocol-equivalent across chat/messages/responses.
-  // A transient 200 on one endpoint should not bias later multimodal requests onto a lossy path.
-  return (
-    !capabilityProfile.hasImageInput
-    && !capabilityProfile.hasAudioInput
-    && !capabilityProfile.hasNonImageFileInput
-  );
-}
-
-function buildEndpointRuntimeStateKey(input: {
-  siteId: number;
-  downstreamFormat: EndpointPreference;
-  capabilityProfile: EndpointCapabilityProfile;
-}): string {
-  const capabilityProfile = input.capabilityProfile;
-  return [
-    String(input.siteId),
-    input.downstreamFormat,
-    capabilityProfile.modelKey,
-    capabilityProfile.hasNonImageFileInput ? 'files' : 'nofiles',
-    capabilityProfile.hasRemoteDocumentUrl ? 'remoteurl' : 'noremoteurl',
-    capabilityProfile.wantsNativeResponsesReasoning ? 'reasoning' : 'noreasoning',
-  ].join(':');
-}
-
-function getOrCreateEndpointRuntimeState(key: string, nowMs = Date.now()): EndpointRuntimeState {
-  sweepEndpointRuntimeStates(nowMs);
-  const existing = endpointRuntimeStates.get(key);
-  if (existing) {
-    existing.lastTouchedAtMs = nowMs;
-    return existing;
-  }
-
-  const initial: EndpointRuntimeState = {
-    preferredEndpoint: null,
-    preferredUpdatedAtMs: nowMs,
-    lastTouchedAtMs: nowMs,
-    blockedUntilMsByEndpoint: {},
-  };
-  endpointRuntimeStates.set(key, initial);
-  enforceEndpointRuntimeStateLimit();
-  return initial;
-}
-
-function maybeDeleteEndpointRuntimeState(key: string, nowMs = Date.now()): void {
-  const state = endpointRuntimeStates.get(key);
-  if (!state) return;
-
-  const hasActiveBlock = Object.values(state.blockedUntilMsByEndpoint).some((untilMs) => (
-    typeof untilMs === 'number' && untilMs > nowMs
-  ));
-  const preferredFresh = (
-    !!state.preferredEndpoint
-    && (state.preferredUpdatedAtMs + ENDPOINT_RUNTIME_PREFERRED_TTL_MS) > nowMs
-  );
-  if (!hasActiveBlock && !preferredFresh) {
-    endpointRuntimeStates.delete(key);
-  }
-}
-
-function applyEndpointRuntimePreference(
-  candidates: UpstreamEndpoint[],
-  key: string,
-  nowMs = Date.now(),
-): UpstreamEndpoint[] {
-  const state = endpointRuntimeStates.get(key);
-  if (!state || candidates.length <= 1) return candidates;
-  state.lastTouchedAtMs = nowMs;
-
-  const blocked = new Set<UpstreamEndpoint>();
-  for (const endpoint of candidates) {
-    const untilMs = state.blockedUntilMsByEndpoint[endpoint];
-    if (typeof untilMs === 'number' && untilMs > nowMs) {
-      blocked.add(endpoint);
-    }
-  }
-
-  let next = candidates.filter((endpoint) => !blocked.has(endpoint));
-  if (next.length === 0) {
-    next = [...candidates];
-  }
-
-  const preferredFresh = (
-    !!state.preferredEndpoint
-    && (state.preferredUpdatedAtMs + ENDPOINT_RUNTIME_PREFERRED_TTL_MS) > nowMs
-  );
-  if (preferredFresh && state.preferredEndpoint && next.includes(state.preferredEndpoint)) {
-    next = [
-      state.preferredEndpoint,
-      ...next.filter((endpoint) => endpoint !== state.preferredEndpoint),
-    ];
-  }
-
-  maybeDeleteEndpointRuntimeState(key, nowMs);
-  return next;
-}
-
-function sweepEndpointRuntimeStates(nowMs = Date.now()): void {
-  for (const [key, state] of endpointRuntimeStates.entries()) {
-    const hasActiveBlock = Object.values(state.blockedUntilMsByEndpoint).some((untilMs) => (
-      typeof untilMs === 'number' && untilMs > nowMs
-    ));
-    const preferredFresh = (
-      !!state.preferredEndpoint
-      && (state.preferredUpdatedAtMs + ENDPOINT_RUNTIME_PREFERRED_TTL_MS) > nowMs
-    );
-    const recentlyTouched = (state.lastTouchedAtMs + ENDPOINT_RUNTIME_PREFERRED_TTL_MS) > nowMs;
-    if (!hasActiveBlock && !preferredFresh && !recentlyTouched) {
-      endpointRuntimeStates.delete(key);
-    }
-  }
-}
-
-function enforceEndpointRuntimeStateLimit(): void {
-  if (endpointRuntimeStates.size <= MAX_ENDPOINT_RUNTIME_STATES) return;
-
-  const entries = [...endpointRuntimeStates.entries()]
-    .sort((left, right) => left[1].lastTouchedAtMs - right[1].lastTouchedAtMs);
-  const overflowCount = endpointRuntimeStates.size - MAX_ENDPOINT_RUNTIME_STATES;
-  for (const [key] of entries.slice(0, overflowCount)) {
-    endpointRuntimeStates.delete(key);
-  }
-}
-
-function inferSuggestedEndpointFromError(errorText?: string | null): UpstreamEndpoint | null {
-  const text = (errorText || '').toLowerCase();
-  if (!text) return null;
-  if (text.includes('/v1/responses')) return 'responses';
-  if (text.includes('/v1/messages')) return 'messages';
-  if (text.includes('/v1/chat/completions')) return 'chat';
-  return null;
-}
-
-function shouldBlockEndpointByError(status: number, errorText?: string | null): boolean {
-  if (isEndpointDispatchDeniedError(status, errorText)) return true;
-  if (status === 404 || status === 405 || status === 415 || status === 501) return true;
-  if (isUnsupportedMediaTypeError(status, errorText)) return true;
-
-  const text = (errorText || '').toLowerCase();
-  return (
-    text.includes('convert_request_failed')
-    || text.includes('endpoint_not_found')
-    || text.includes('unknown_endpoint')
-    || text.includes('unsupported_endpoint')
-    || text.includes('unsupported_path')
-    || text.includes('not_found_error')
-    || text.includes('unsupported legacy protocol')
-    || text.includes('please use /v1/')
-    || text.includes('does not allow /v1/')
-    || text.includes('unknown endpoint')
-    || text.includes('unsupported endpoint')
-    || text.includes('unsupported path')
-    || text.includes('unrecognized request url')
-    || text.includes('no route matched')
-    || text.includes('does not exist')
-  );
-}
-
-function shouldRememberSuccessfulEndpoint(input: {
-  endpoint: UpstreamEndpoint;
-  downstreamFormat: EndpointPreference;
-}): boolean {
-  if (input.downstreamFormat !== 'responses') return true;
-  return input.endpoint === 'responses';
-}
-
-export function resetUpstreamEndpointRuntimeState(): void {
-  endpointRuntimeStates.clear();
-}
-
-export function recordUpstreamEndpointSuccess(input: {
-  siteId: number;
-  endpoint: UpstreamEndpoint;
-  downstreamFormat: EndpointPreference;
-  modelName?: string;
-  requestedModelHint?: string;
-  requestCapabilities?: {
-    hasNonImageFileInput?: boolean;
-    conversationFileSummary?: ConversationFileInputSummary;
-    wantsNativeResponsesReasoning?: boolean;
-  };
-}): void {
-  const capabilityProfile = buildEndpointCapabilityProfile({
-    modelName: input.modelName,
-    requestedModelHint: input.requestedModelHint,
-    requestCapabilities: input.requestCapabilities,
-  });
-  if (!shouldUseEndpointRuntimeMemory(capabilityProfile)) return;
-  if (!shouldRememberSuccessfulEndpoint(input)) return;
-
-  const nowMs = Date.now();
-  const key = buildEndpointRuntimeStateKey({
-    siteId: input.siteId,
-    downstreamFormat: input.downstreamFormat,
-    capabilityProfile,
-  });
-  const state = getOrCreateEndpointRuntimeState(key, nowMs);
-  state.preferredEndpoint = input.endpoint;
-  state.preferredUpdatedAtMs = nowMs;
-  delete state.blockedUntilMsByEndpoint[input.endpoint];
-}
-
-export function recordUpstreamEndpointFailure(input: {
-  siteId: number;
-  endpoint: UpstreamEndpoint;
-  downstreamFormat: EndpointPreference;
-  status: number;
-  errorText?: string | null;
-  modelName?: string;
-  requestedModelHint?: string;
-  requestCapabilities?: {
-    hasNonImageFileInput?: boolean;
-    conversationFileSummary?: ConversationFileInputSummary;
-    wantsNativeResponsesReasoning?: boolean;
-  };
-}): void {
-  const capabilityProfile = buildEndpointCapabilityProfile({
-    modelName: input.modelName,
-    requestedModelHint: input.requestedModelHint,
-    requestCapabilities: input.requestCapabilities,
-  });
-  if (!shouldUseEndpointRuntimeMemory(capabilityProfile)) return;
-  if (!shouldBlockEndpointByError(input.status, input.errorText)) return;
-
-  const nowMs = Date.now();
-  const key = buildEndpointRuntimeStateKey({
-    siteId: input.siteId,
-    downstreamFormat: input.downstreamFormat,
-    capabilityProfile,
-  });
-  const state = getOrCreateEndpointRuntimeState(key, nowMs);
-  state.blockedUntilMsByEndpoint[input.endpoint] = nowMs + ENDPOINT_RUNTIME_BLOCK_TTL_MS;
-
-  const suggestedEndpoint = inferSuggestedEndpointFromError(input.errorText);
-  if (suggestedEndpoint && suggestedEndpoint !== input.endpoint) {
-    state.preferredEndpoint = suggestedEndpoint;
-    state.preferredUpdatedAtMs = nowMs;
-    delete state.blockedUntilMsByEndpoint[suggestedEndpoint];
-  }
-}
-
 function preferredEndpointOrder(
   downstreamFormat: EndpointPreference,
   sitePlatform?: string,
@@ -848,15 +530,12 @@ export async function resolveUpstreamEndpointCandidates(
   const preferMessagesForClaudeModel = capabilityProfile.preferMessagesForClaudeModel;
   const hasNonImageFileInput = capabilityProfile.hasNonImageFileInput;
   const wantsNativeResponsesReasoning = capabilityProfile.wantsNativeResponsesReasoning;
-  const runtimeStateKey = buildEndpointRuntimeStateKey({
-    siteId: context.site.id,
-    downstreamFormat,
-    capabilityProfile,
-  });
   const applyRuntimePreference = (candidates: UpstreamEndpoint[]) => (
-    shouldUseEndpointRuntimeMemory(capabilityProfile)
-      ? applyEndpointRuntimePreference(candidates, runtimeStateKey)
-      : candidates
+    applyUpstreamEndpointRuntimePreference(candidates, {
+      siteId: context.site.id,
+      downstreamFormat,
+      capabilityProfile,
+    })
   );
   const conversationFileSummary = requestCapabilities?.conversationFileSummary ?? {
     hasImage: false,