[codex] add proxy debug tracing (#299)

* add proxy debug tracing

* fix proxy debug response header typing

* extend proxy debug tracing to side paths

* Refine proxy debug trace review fixes

* Stabilize proxy debug attempt indexing

* Tighten proxy debug trace accuracy

src/server/routes/proxy/architecture-boundaries.test.ts

@@ -98,6 +98,12 @@ describe('proxy route architecture boundaries', () => {
     expect(source).not.toContain('shouldInjectDerivedPromptCacheKey');
   });
 
+  it('keeps endpoint runtime snapshot helper out of the route layer', () => {
+    const source = readSource('./upstreamEndpoint.ts');
+    expect(source).not.toContain('function getUpstreamEndpointRuntimeStateSnapshot(');
+    expect(source).not.toContain('export function getUpstreamEndpointRuntimeStateSnapshot(');
+  });
+
   it('keeps gemini runtime closure in transformer-owned helpers', () => {
     const source = readSource('./gemini.ts');
     const surfaceSource = readSource('../../proxy-core/surfaces/geminiSurface.ts');
@@ -179,4 +185,3 @@ describe('proxy route architecture boundaries', () => {
     expect(geminiSource).not.toContain("from '../../transformers/canonical/");
   });
 });
-

src/server/routes/proxy/chat.count-tokens.test.ts

@@ -14,6 +14,11 @@ const resolveProxyUsageWithSelfLogFallbackMock = vi.fn(async ({ usage }: any) =>
   estimatedCostFromQuota: 0,
   recoveredFromSelfLog: false,
 }));
+const startSurfaceProxyDebugTraceMock = vi.fn();
+const safeUpdateSurfaceProxyDebugSelectionMock = vi.fn();
+const safeUpdateSurfaceProxyDebugCandidatesMock = vi.fn();
+const safeInsertSurfaceProxyDebugAttemptMock = vi.fn();
+const safeFinalizeSurfaceProxyDebugTraceMock = vi.fn();
 const dbInsertMock = vi.fn((_arg?: any) => ({
   values: () => ({
     run: () => undefined,
@@ -60,6 +65,19 @@ vi.mock('../../services/proxyUsageFallbackService.js', () => ({
   resolveProxyUsageWithSelfLogFallback: (arg: any) => resolveProxyUsageWithSelfLogFallbackMock(arg),
 }));
 
+vi.mock('../../services/proxyDebugTraceRuntime.js', () => ({
+  startSurfaceProxyDebugTrace: (...args: unknown[]) => startSurfaceProxyDebugTraceMock(...args),
+  safeUpdateSurfaceProxyDebugSelection: (...args: unknown[]) => safeUpdateSurfaceProxyDebugSelectionMock(...args),
+  safeUpdateSurfaceProxyDebugCandidates: (...args: unknown[]) => safeUpdateSurfaceProxyDebugCandidatesMock(...args),
+  safeInsertSurfaceProxyDebugAttempt: (...args: unknown[]) => safeInsertSurfaceProxyDebugAttemptMock(...args),
+  safeFinalizeSurfaceProxyDebugTrace: (...args: unknown[]) => safeFinalizeSurfaceProxyDebugTraceMock(...args),
+  safeUpdateSurfaceProxyDebugAttempt: vi.fn(),
+  reserveSurfaceProxyDebugAttemptBase: () => 0,
+  buildSurfaceProxyDebugResponseHeaders: () => ({}),
+  captureSurfaceProxyDebugSuccessResponseBody: async () => null,
+  parseSurfaceProxyDebugTextPayload: (raw: string) => raw,
+}));
+
 vi.mock('../../services/oauth/quota.js', () => ({
   recordOauthQuotaResetHint: async () => undefined,
 }));
@@ -95,8 +113,28 @@ describe('claude count_tokens proxy route', () => {
     reportProxyAllFailedMock.mockReset();
     reportTokenExpiredMock.mockReset();
     resolveProxyUsageWithSelfLogFallbackMock.mockClear();
+    startSurfaceProxyDebugTraceMock.mockReset();
+    safeUpdateSurfaceProxyDebugSelectionMock.mockReset();
+    safeUpdateSurfaceProxyDebugCandidatesMock.mockReset();
+    safeInsertSurfaceProxyDebugAttemptMock.mockReset();
+    safeFinalizeSurfaceProxyDebugTraceMock.mockReset();
     dbInsertMock.mockClear();
 
+    startSurfaceProxyDebugTraceMock.mockResolvedValue({
+      traceId: 701,
+      options: {
+        enabled: true,
+        captureHeaders: true,
+        captureBodies: true,
+        captureStreamChunks: false,
+        targetSessionId: '',
+        targetClientKind: '',
+        targetModel: '',
+        retentionHours: 24,
+        maxBodyBytes: 262144,
+      },
+    });
+
     selectChannelMock.mockReturnValue({
       channel: { id: 11, routeId: 22 },
       site: { name: 'claude-site', url: 'https://api.anthropic.com', platform: 'claude' },
@@ -146,6 +184,18 @@ describe('claude count_tokens proxy route', () => {
     expect(options.headers['anthropic-version']).toBe('2023-06-01');
     expect(options.headers['anthropic-beta']).toContain('claude-code-20250219');
     expect(options.headers['Accept-Encoding']).toBe('gzip, deflate, br, zstd');
+    expect(startSurfaceProxyDebugTraceMock).toHaveBeenCalledWith(expect.objectContaining({
+      downstreamPath: '/v1/messages/count_tokens',
+      requestedModel: 'claude-opus-4-6',
+    }));
+    expect(safeInsertSurfaceProxyDebugAttemptMock).toHaveBeenCalled();
+    expect(safeFinalizeSurfaceProxyDebugTraceMock).toHaveBeenCalledWith(
+      expect.objectContaining({ traceId: 701 }),
+      expect.objectContaining({
+        finalStatus: 'success',
+        finalUpstreamPath: '/v1/messages/count_tokens?beta=true',
+      }),
+    );
 
     const forwardedBody = JSON.parse(String(options.body));
     expect(forwardedBody.model).toBe('claude-opus-4-6');

src/server/routes/proxy/chat.stream.test.ts

@@ -2,7 +2,7 @@ import { zstdCompressSync } from 'node:zlib';
 import Fastify, { type FastifyInstance } from 'fastify';
 import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
 import { config } from '../../config.js';
-import { resetUpstreamEndpointRuntimeState } from './upstreamEndpoint.js';
+import { resetUpstreamEndpointRuntimeState } from '../../services/upstreamEndpointRuntimeMemory.js';
 
 const fetchMock = vi.fn();
 const selectChannelMock = vi.fn();

src/server/routes/proxy/endpointFlow.ts

@@ -26,6 +26,7 @@ export type EndpointAttemptContext = {
   targetUrl: string;
   response: Awaited<ReturnType<typeof fetch>>;
   rawErrText: string;
+  recoverApplied?: boolean;
 };
 
 export type EndpointAttemptSuccessContext = {
@@ -34,6 +35,7 @@ export type EndpointAttemptSuccessContext = {
   request: BuiltEndpointRequest;
   targetUrl: string;
   response: Awaited<ReturnType<typeof fetch>>;
+  recoverApplied?: boolean;
 };
 
 export type EndpointRecoverResult = {
@@ -125,6 +127,7 @@ export async function executeEndpointFlow(input: ExecuteEndpointFlowInput): Prom
         request,
         targetUrl,
         response,
+        recoverApplied: false,
       }, 'onAttemptSuccess');
       return {
         ok: true,
@@ -141,10 +144,15 @@ export async function executeEndpointFlow(input: ExecuteEndpointFlowInput): Prom
       targetUrl,
       response,
       rawErrText,
+      recoverApplied: false,
     };
 
     if (input.tryRecover) {
       const recovered = await input.tryRecover(baseContext);
+      baseContext.recoverApplied = recovered !== null
+        || baseContext.request !== request
+        || baseContext.response !== response
+        || baseContext.rawErrText !== rawErrText;
       if (recovered?.upstream?.ok) {
         const recoveredRequest = recovered.request ?? baseContext.request;
         const recoveredTargetUrl = recovered.targetUrl ?? (
@@ -158,6 +166,7 @@ export async function executeEndpointFlow(input: ExecuteEndpointFlowInput): Prom
           request: recoveredRequest,
           targetUrl: recoveredTargetUrl,
           response: recovered.upstream,
+          recoverApplied: true,
         }, 'onAttemptSuccess');
         return {
           ok: true,

src/server/routes/proxy/gemini.test.ts

@@ -2,7 +2,7 @@ import Fastify, { type FastifyInstance } from 'fastify';
 import { readFileSync } from 'node:fs';
 import path from 'node:path';
 import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
-import { resetUpstreamEndpointRuntimeState } from './upstreamEndpoint.js';
+import { resetUpstreamEndpointRuntimeState } from '../../services/upstreamEndpointRuntimeMemory.js';
 
 const fetchMock = vi.fn();
 const fetchModelPricingCatalogMock = vi.fn();
@@ -25,6 +25,11 @@ const dbInsertValuesMock = vi.fn((_values?: unknown) => ({
 const dbInsertMock = vi.fn((_table?: unknown) => ({
   values: (values: unknown) => dbInsertValuesMock(values),
 }));
+const startSurfaceProxyDebugTraceMock = vi.fn();
+const safeUpdateSurfaceProxyDebugSelectionMock = vi.fn();
+const safeUpdateSurfaceProxyDebugCandidatesMock = vi.fn();
+const safeInsertSurfaceProxyDebugAttemptMock = vi.fn();
+const safeFinalizeSurfaceProxyDebugTraceMock = vi.fn();
 
 function createDbSelectChain() {
   return {
@@ -107,6 +112,19 @@ vi.mock('../../db/index.js', () => ({
   },
 }));
 
+vi.mock('../../services/proxyDebugTraceRuntime.js', () => ({
+  startSurfaceProxyDebugTrace: (...args: unknown[]) => startSurfaceProxyDebugTraceMock(...args),
+  safeUpdateSurfaceProxyDebugSelection: (...args: unknown[]) => safeUpdateSurfaceProxyDebugSelectionMock(...args),
+  safeUpdateSurfaceProxyDebugCandidates: (...args: unknown[]) => safeUpdateSurfaceProxyDebugCandidatesMock(...args),
+  safeInsertSurfaceProxyDebugAttempt: (...args: unknown[]) => safeInsertSurfaceProxyDebugAttemptMock(...args),
+  safeFinalizeSurfaceProxyDebugTrace: (...args: unknown[]) => safeFinalizeSurfaceProxyDebugTraceMock(...args),
+  safeUpdateSurfaceProxyDebugAttempt: vi.fn(),
+  reserveSurfaceProxyDebugAttemptBase: () => 0,
+  buildSurfaceProxyDebugResponseHeaders: () => ({}),
+  captureSurfaceProxyDebugSuccessResponseBody: async () => null,
+  parseSurfaceProxyDebugTextPayload: (raw: string) => raw,
+}));
+
 function parseSsePayloads(body: string): Array<Record<string, unknown>> {
   return body
     .replace(/\r\n/g, '\n')
@@ -173,6 +191,26 @@ describe('gemini native proxy routes', () => {
     dbInsertValuesMock.mockClear();
     dbSelectAllMock.mockReset();
     dbSelectGetMock.mockReset();
+    startSurfaceProxyDebugTraceMock.mockReset();
+    safeUpdateSurfaceProxyDebugSelectionMock.mockReset();
+    safeUpdateSurfaceProxyDebugCandidatesMock.mockReset();
+    safeInsertSurfaceProxyDebugAttemptMock.mockReset();
+    safeFinalizeSurfaceProxyDebugTraceMock.mockReset();
+
+    startSurfaceProxyDebugTraceMock.mockResolvedValue({
+      traceId: 801,
+      options: {
+        enabled: true,
+        captureHeaders: true,
+        captureBodies: true,
+        captureStreamChunks: false,
+        targetSessionId: '',
+        targetClientKind: '',
+        targetModel: '',
+        retentionHours: 24,
+        maxBodyBytes: 262144,
+      },
+    });
 
     authorizeDownstreamTokenMock.mockResolvedValue({
       ok: true,
@@ -746,6 +784,25 @@ describe('gemini native proxy routes', () => {
       Authorization: 'Bearer openai-access-token',
       'Content-Type': 'application/json',
     });
+    expect(startSurfaceProxyDebugTraceMock).toHaveBeenCalledWith(expect.objectContaining({
+      downstreamPath: '/v1beta/models/gemini-2.5-flash:generateContent',
+      requestedModel: 'gemini-2.5-flash',
+    }));
+    expect(safeUpdateSurfaceProxyDebugSelectionMock).toHaveBeenCalledWith(
+      expect.objectContaining({ traceId: 801 }),
+      expect.objectContaining({
+        selectedChannelId: 41,
+        selectedSitePlatform: 'openai',
+      }),
+    );
+    expect(safeInsertSurfaceProxyDebugAttemptMock).toHaveBeenCalled();
+    expect(safeFinalizeSurfaceProxyDebugTraceMock).toHaveBeenCalledWith(
+      expect.objectContaining({ traceId: 801 }),
+      expect.objectContaining({
+        finalStatus: 'success',
+        finalUpstreamPath: '/v1/chat/completions',
+      }),
+    );
     expect(JSON.parse(String(requestInit.body))).toEqual({
       model: 'gpt-4.1',
       stream: false,

src/server/routes/proxy/upstreamEndpoint.test.ts

@@ -1,4 +1,4 @@
-import { beforeEach, describe, expect, it, vi } from 'vitest';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
 import { config } from '../../config.js';
 
 const fetchModelPricingCatalogMock = vi.fn(async (_arg?: unknown): Promise<any> => null);
@@ -12,14 +12,17 @@ import {
   buildUpstreamEndpointRequest,
   isUnsupportedMediaTypeError,
   isEndpointDowngradeError,
+  resolveUpstreamEndpointCandidates,
+} from './upstreamEndpoint.js';
+import {
   recordUpstreamEndpointFailure,
   recordUpstreamEndpointSuccess,
   resetUpstreamEndpointRuntimeState,
-  resolveUpstreamEndpointCandidates,
+  getUpstreamEndpointRuntimeStateSnapshot,
   boundEndpointRuntimeModelKey,
   MAX_ENDPOINT_RUNTIME_MODEL_KEY_LENGTH,
   MODEL_KEY_HASH_SUFFIX_LENGTH,
-} from './upstreamEndpoint.js';
+} from '../../services/upstreamEndpointRuntimeMemory.js';
 
 const baseContext = {
   site: {
@@ -52,6 +55,9 @@ describe('resolveUpstreamEndpointCandidates', () => {
       filter: [],
     };
   });
+  afterEach(() => {
+    vi.useRealTimers();
+  });
 
   it('uses downstream-aligned endpoint priority for unknown platforms', async () => {
     const openaiOrder = await resolveUpstreamEndpointCandidates(
@@ -236,6 +242,64 @@ describe('resolveUpstreamEndpointCandidates', () => {
     expect(order).toEqual(['chat', 'messages', 'responses']);
   });
 
+  it('does not expose preferred endpoint in snapshot when runtime memory is disabled for multimodal requests', () => {
+    recordUpstreamEndpointSuccess({
+      siteId: baseContext.site.id,
+      endpoint: 'responses',
+      downstreamFormat: 'openai',
+      modelName: 'gpt-5.3',
+      requestCapabilities: {
+        conversationFileSummary: {
+          hasImage: true,
+          hasAudio: false,
+          hasDocument: false,
+          hasRemoteDocumentUrl: false,
+        },
+      },
+    });
+
+    expect(getUpstreamEndpointRuntimeStateSnapshot({
+      siteId: baseContext.site.id,
+      downstreamFormat: 'openai',
+      modelName: 'gpt-5.3',
+      requestCapabilities: {
+        conversationFileSummary: {
+          hasImage: true,
+          hasAudio: false,
+          hasDocument: false,
+          hasRemoteDocumentUrl: false,
+        },
+      },
+    })).toMatchObject({
+      enabled: false,
+      preferredEndpoint: null,
+      blockedEndpoints: [],
+    });
+  });
+
+  it('does not expose expired preferred endpoints in the runtime snapshot', () => {
+    vi.useFakeTimers();
+    vi.setSystemTime(new Date('2026-03-28T00:00:00.000Z'));
+
+    recordUpstreamEndpointSuccess({
+      siteId: baseContext.site.id,
+      endpoint: 'chat',
+      downstreamFormat: 'openai',
+      modelName: 'gpt-5.3',
+    });
+
+    vi.setSystemTime(new Date('2026-03-29T01:00:00.000Z'));
+
+    expect(getUpstreamEndpointRuntimeStateSnapshot({
+      siteId: baseContext.site.id,
+      downstreamFormat: 'openai',
+      modelName: 'gpt-5.3',
+    })).toMatchObject({
+      enabled: true,
+      preferredEndpoint: null,
+    });
+  });
+
   it('does not apply runtime endpoint memory to document attachments', async () => {
     recordUpstreamEndpointSuccess({
       siteId: baseContext.site.id,
@@ -379,12 +443,13 @@ describe('resolveUpstreamEndpointCandidates', () => {
   });
 
   it('does not remember messages fallback success for generic /v1/responses requests', async () => {
-    recordUpstreamEndpointSuccess({
+    const memoryWrite = recordUpstreamEndpointSuccess({
       siteId: baseContext.site.id,
       endpoint: 'messages',
       downstreamFormat: 'responses',
       modelName: 'gpt-5.3',
     });
+    expect(memoryWrite).toBeNull();
 
     const order = await resolveUpstreamEndpointCandidates(
       {
@@ -398,8 +463,23 @@ describe('resolveUpstreamEndpointCandidates', () => {
     expect(order).toEqual(['responses', 'chat', 'messages']);
   });
 
+  it('returns the applied success write when runtime memory stores a preferred endpoint', () => {
+    const memoryWrite = recordUpstreamEndpointSuccess({
+      siteId: baseContext.site.id,
+      endpoint: 'responses',
+      downstreamFormat: 'responses',
+      modelName: 'gpt-5.3',
+    });
+
+    expect(memoryWrite).toMatchObject({
+      action: 'success',
+      endpoint: 'responses',
+      preferredEndpoint: 'responses',
+    });
+  });
+
   it('does not block generic /v1/responses endpoints on transient upstream errors', async () => {
-    recordUpstreamEndpointFailure({
+    const memoryWrite = recordUpstreamEndpointFailure({
       siteId: baseContext.site.id,
       endpoint: 'responses',
       downstreamFormat: 'responses',
@@ -407,6 +487,7 @@ describe('resolveUpstreamEndpointCandidates', () => {
       status: 504,
       errorText: '{"error":{"message":"Gateway time-out","type":"upstream_error"}}',
     });
+    expect(memoryWrite).toBeNull();
 
     const order = await resolveUpstreamEndpointCandidates(
       {
@@ -421,7 +502,7 @@ describe('resolveUpstreamEndpointCandidates', () => {
   });
 
   it('learns a better endpoint from explicit upstream protocol errors', async () => {
-    recordUpstreamEndpointFailure({
+    const memoryWrite = recordUpstreamEndpointFailure({
       siteId: baseContext.site.id,
       endpoint: 'chat',
       downstreamFormat: 'openai',
@@ -429,6 +510,12 @@ describe('resolveUpstreamEndpointCandidates', () => {
       status: 400,
       errorText: 'Unsupported legacy protocol: /v1/chat/completions is not supported. Please use /v1/responses.',
     });
+    expect(memoryWrite).toMatchObject({
+      action: 'failure',
+      endpoint: 'chat',
+      blockedEndpoint: 'chat',
+      preferredEndpoint: 'responses',
+    });
 
     const order = await resolveUpstreamEndpointCandidates(
       {

src/server/routes/proxy/upstreamEndpoint.ts

@@ -1,4 +1,4 @@
-import { randomUUID, createHash } from 'node:crypto';
+import { randomUUID } from 'node:crypto';
 import {
   rankConversationFileEndpoints,
   type ConversationFileInputSummary,
@@ -7,6 +7,10 @@ import { resolveProviderProfile } from '../../proxy-core/providers/registry.js';
 import { config } from '../../config.js';
 import { fetchModelPricingCatalog } from '../../services/modelPricingService.js';
 import { applyPayloadRules } from '../../services/payloadRules.js';
+import {
+  applyUpstreamEndpointRuntimePreference,
+  buildEndpointCapabilityProfile,
+} from '../../services/upstreamEndpointRuntimeMemory.js';
 import type { DownstreamFormat } from '../../transformers/shared/normalized.js';
 import {
   convertOpenAiBodyToResponsesBody as convertOpenAiBodyToResponsesBodyViaTransformer,
@@ -44,23 +48,6 @@ export {
 export type UpstreamEndpoint = 'chat' | 'messages' | 'responses';
 export type EndpointPreference = DownstreamFormat | 'responses';
 
-type EndpointCapabilityProfile = {
-  modelKey: string;
-  preferMessagesForClaudeModel: boolean;
-  hasImageInput: boolean;
-  hasAudioInput: boolean;
-  hasNonImageFileInput: boolean;
-  hasRemoteDocumentUrl: boolean;
-  wantsNativeResponsesReasoning: boolean;
-};
-
-type EndpointRuntimeState = {
-  preferredEndpoint: UpstreamEndpoint | null;
-  preferredUpdatedAtMs: number;
-  lastTouchedAtMs: number;
-  blockedUntilMsByEndpoint: Partial<Record<UpstreamEndpoint, number>>;
-};
-
 type ChannelContext = {
   site: {
     id: number;
@@ -75,26 +62,6 @@ type ChannelContext = {
   };
 };
 
-const ENDPOINT_RUNTIME_PREFERRED_TTL_MS = 24 * 60 * 60 * 1000;
-const ENDPOINT_RUNTIME_BLOCK_TTL_MS = 6 * 60 * 60 * 1000;
-const MAX_ENDPOINT_RUNTIME_STATES = 512;
-export const MAX_ENDPOINT_RUNTIME_MODEL_KEY_LENGTH = 64;
-export const MODEL_KEY_HASH_SUFFIX_LENGTH = 8;
-const endpointRuntimeStates = new Map<string, EndpointRuntimeState>();
-
-export function boundEndpointRuntimeModelKey(value: string): string {
-  if (value.length <= MAX_ENDPOINT_RUNTIME_MODEL_KEY_LENGTH) {
-    return value;
-  }
-
-  const prefix = value.slice(0, MAX_ENDPOINT_RUNTIME_MODEL_KEY_LENGTH);
-  const hash = createHash('sha256')
-    .update(value)
-    .digest('hex')
-    .slice(0, MODEL_KEY_HASH_SUFFIX_LENGTH);
-  return `${prefix}-${hash}`;
-}
-
 function isRecord(value: unknown): value is Record<string, unknown> {
   return !!value && typeof value === 'object';
 }
@@ -103,14 +70,6 @@ function asTrimmedString(value: unknown): string {
   return typeof value === 'string' ? value.trim() : '';
 }
 
-function normalizeEndpointRuntimeModelKey(...values: Array<unknown>): string {
-  for (const value of values) {
-    const normalized = asTrimmedString(value).toLowerCase();
-    if (normalized) return boundEndpointRuntimeModelKey(normalized);
-  }
-  return boundEndpointRuntimeModelKey('unknown-model');
-}
-
 function resolveRequestedModelForPayloadRules(input: {
   modelName: string;
   openaiBody: Record<string, unknown>;
@@ -129,12 +88,6 @@ function normalizePlatformName(platform: unknown): string {
   return asTrimmedString(platform).toLowerCase();
 }
 
-function isClaudeFamilyModel(modelName: string): boolean {
-  const normalized = asTrimmedString(modelName).toLowerCase();
-  if (!normalized) return false;
-  return normalized === 'claude' || normalized.startsWith('claude-') || normalized.includes('claude');
-}
-
 const HOP_BY_HOP_HEADERS = new Set([
   'connection',
   'keep-alive',
@@ -495,277 +448,6 @@ function normalizeEndpointTypes(value: unknown): UpstreamEndpoint[] {
   return Array.from(normalized);
 }
 
-function buildEndpointCapabilityProfile(input?: {
-  modelName?: string;
-  requestedModelHint?: string;
-  requestCapabilities?: {
-    hasNonImageFileInput?: boolean;
-    conversationFileSummary?: ConversationFileInputSummary;
-    wantsNativeResponsesReasoning?: boolean;
-  };
-}): EndpointCapabilityProfile {
-  const conversationFileSummary = input?.requestCapabilities?.conversationFileSummary;
-  return {
-    modelKey: normalizeEndpointRuntimeModelKey(input?.modelName, input?.requestedModelHint),
-    preferMessagesForClaudeModel: (
-      isClaudeFamilyModel(asTrimmedString(input?.modelName))
-      || isClaudeFamilyModel(asTrimmedString(input?.requestedModelHint))
-    ),
-    hasImageInput: conversationFileSummary?.hasImage === true,
-    hasAudioInput: conversationFileSummary?.hasAudio === true,
-    hasNonImageFileInput: (
-      conversationFileSummary?.hasDocument === true
-      || input?.requestCapabilities?.hasNonImageFileInput === true
-    ),
-    hasRemoteDocumentUrl: (
-      conversationFileSummary?.hasRemoteDocumentUrl === true
-    ),
-    wantsNativeResponsesReasoning: input?.requestCapabilities?.wantsNativeResponsesReasoning === true,
-  };
-}
-
-function shouldUseEndpointRuntimeMemory(capabilityProfile: EndpointCapabilityProfile): boolean {
-  // Attachment-capable requests are not protocol-equivalent across chat/messages/responses.
-  // A transient 200 on one endpoint should not bias later multimodal requests onto a lossy path.
-  return (
-    !capabilityProfile.hasImageInput
-    && !capabilityProfile.hasAudioInput
-    && !capabilityProfile.hasNonImageFileInput
-  );
-}
-
-function buildEndpointRuntimeStateKey(input: {
-  siteId: number;
-  downstreamFormat: EndpointPreference;
-  capabilityProfile: EndpointCapabilityProfile;
-}): string {
-  const capabilityProfile = input.capabilityProfile;
-  return [
-    String(input.siteId),
-    input.downstreamFormat,
-    capabilityProfile.modelKey,
-    capabilityProfile.hasNonImageFileInput ? 'files' : 'nofiles',
-    capabilityProfile.hasRemoteDocumentUrl ? 'remoteurl' : 'noremoteurl',
-    capabilityProfile.wantsNativeResponsesReasoning ? 'reasoning' : 'noreasoning',
-  ].join(':');
-}
-
-function getOrCreateEndpointRuntimeState(key: string, nowMs = Date.now()): EndpointRuntimeState {
-  sweepEndpointRuntimeStates(nowMs);
-  const existing = endpointRuntimeStates.get(key);
-  if (existing) {
-    existing.lastTouchedAtMs = nowMs;
-    return existing;
-  }
-
-  const initial: EndpointRuntimeState = {
-    preferredEndpoint: null,
-    preferredUpdatedAtMs: nowMs,
-    lastTouchedAtMs: nowMs,
-    blockedUntilMsByEndpoint: {},
-  };
-  endpointRuntimeStates.set(key, initial);
-  enforceEndpointRuntimeStateLimit();
-  return initial;
-}
-
-function maybeDeleteEndpointRuntimeState(key: string, nowMs = Date.now()): void {
-  const state = endpointRuntimeStates.get(key);
-  if (!state) return;
-
-  const hasActiveBlock = Object.values(state.blockedUntilMsByEndpoint).some((untilMs) => (
-    typeof untilMs === 'number' && untilMs > nowMs
-  ));
-  const preferredFresh = (
-    !!state.preferredEndpoint
-    && (state.preferredUpdatedAtMs + ENDPOINT_RUNTIME_PREFERRED_TTL_MS) > nowMs
-  );
-  if (!hasActiveBlock && !preferredFresh) {
-    endpointRuntimeStates.delete(key);
-  }
-}
-
-function applyEndpointRuntimePreference(
-  candidates: UpstreamEndpoint[],
-  key: string,
-  nowMs = Date.now(),
-): UpstreamEndpoint[] {
-  const state = endpointRuntimeStates.get(key);
-  if (!state || candidates.length <= 1) return candidates;
-  state.lastTouchedAtMs = nowMs;
-
-  const blocked = new Set<UpstreamEndpoint>();
-  for (const endpoint of candidates) {
-    const untilMs = state.blockedUntilMsByEndpoint[endpoint];
-    if (typeof untilMs === 'number' && untilMs > nowMs) {
-      blocked.add(endpoint);
-    }
-  }
-
-  let next = candidates.filter((endpoint) => !blocked.has(endpoint));
-  if (next.length === 0) {
-    next = [...candidates];
-  }
-
-  const preferredFresh = (
-    !!state.preferredEndpoint
-    && (state.preferredUpdatedAtMs + ENDPOINT_RUNTIME_PREFERRED_TTL_MS) > nowMs
-  );
-  if (preferredFresh && state.preferredEndpoint && next.includes(state.preferredEndpoint)) {
-    next = [
-      state.preferredEndpoint,
-      ...next.filter((endpoint) => endpoint !== state.preferredEndpoint),
-    ];
-  }
-
-  maybeDeleteEndpointRuntimeState(key, nowMs);
-  return next;
-}
-
-function sweepEndpointRuntimeStates(nowMs = Date.now()): void {
-  for (const [key, state] of endpointRuntimeStates.entries()) {
-    const hasActiveBlock = Object.values(state.blockedUntilMsByEndpoint).some((untilMs) => (
-      typeof untilMs === 'number' && untilMs > nowMs
-    ));
-    const preferredFresh = (
-      !!state.preferredEndpoint
-      && (state.preferredUpdatedAtMs + ENDPOINT_RUNTIME_PREFERRED_TTL_MS) > nowMs
-    );
-    const recentlyTouched = (state.lastTouchedAtMs + ENDPOINT_RUNTIME_PREFERRED_TTL_MS) > nowMs;
-    if (!hasActiveBlock && !preferredFresh && !recentlyTouched) {
-      endpointRuntimeStates.delete(key);
-    }
-  }
-}
-
-function enforceEndpointRuntimeStateLimit(): void {
-  if (endpointRuntimeStates.size <= MAX_ENDPOINT_RUNTIME_STATES) return;
-
-  const entries = [...endpointRuntimeStates.entries()]
-    .sort((left, right) => left[1].lastTouchedAtMs - right[1].lastTouchedAtMs);
-  const overflowCount = endpointRuntimeStates.size - MAX_ENDPOINT_RUNTIME_STATES;
-  for (const [key] of entries.slice(0, overflowCount)) {
-    endpointRuntimeStates.delete(key);
-  }
-}
-
-function inferSuggestedEndpointFromError(errorText?: string | null): UpstreamEndpoint | null {
-  const text = (errorText || '').toLowerCase();
-  if (!text) return null;
-  if (text.includes('/v1/responses')) return 'responses';
-  if (text.includes('/v1/messages')) return 'messages';
-  if (text.includes('/v1/chat/completions')) return 'chat';
-  return null;
-}
-
-function shouldBlockEndpointByError(status: number, errorText?: string | null): boolean {
-  if (isEndpointDispatchDeniedError(status, errorText)) return true;
-  if (status === 404 || status === 405 || status === 415 || status === 501) return true;
-  if (isUnsupportedMediaTypeError(status, errorText)) return true;
-
-  const text = (errorText || '').toLowerCase();
-  return (
-    text.includes('convert_request_failed')
-    || text.includes('endpoint_not_found')
-    || text.includes('unknown_endpoint')
-    || text.includes('unsupported_endpoint')
-    || text.includes('unsupported_path')
-    || text.includes('not_found_error')
-    || text.includes('unsupported legacy protocol')
-    || text.includes('please use /v1/')
-    || text.includes('does not allow /v1/')
-    || text.includes('unknown endpoint')
-    || text.includes('unsupported endpoint')
-    || text.includes('unsupported path')
-    || text.includes('unrecognized request url')
-    || text.includes('no route matched')
-    || text.includes('does not exist')
-  );
-}
-
-function shouldRememberSuccessfulEndpoint(input: {
-  endpoint: UpstreamEndpoint;
-  downstreamFormat: EndpointPreference;
-}): boolean {
-  if (input.downstreamFormat !== 'responses') return true;
-  return input.endpoint === 'responses';
-}
-
-export function resetUpstreamEndpointRuntimeState(): void {
-  endpointRuntimeStates.clear();
-}
-
-export function recordUpstreamEndpointSuccess(input: {
-  siteId: number;
-  endpoint: UpstreamEndpoint;
-  downstreamFormat: EndpointPreference;
-  modelName?: string;
-  requestedModelHint?: string;
-  requestCapabilities?: {
-    hasNonImageFileInput?: boolean;
-    conversationFileSummary?: ConversationFileInputSummary;
-    wantsNativeResponsesReasoning?: boolean;
-  };
-}): void {
-  const capabilityProfile = buildEndpointCapabilityProfile({
-    modelName: input.modelName,
-    requestedModelHint: input.requestedModelHint,
-    requestCapabilities: input.requestCapabilities,
-  });
-  if (!shouldUseEndpointRuntimeMemory(capabilityProfile)) return;
-  if (!shouldRememberSuccessfulEndpoint(input)) return;
-
-  const nowMs = Date.now();
-  const key = buildEndpointRuntimeStateKey({
-    siteId: input.siteId,
-    downstreamFormat: input.downstreamFormat,
-    capabilityProfile,
-  });
-  const state = getOrCreateEndpointRuntimeState(key, nowMs);
-  state.preferredEndpoint = input.endpoint;
-  state.preferredUpdatedAtMs = nowMs;
-  delete state.blockedUntilMsByEndpoint[input.endpoint];
-}
-
-export function recordUpstreamEndpointFailure(input: {
-  siteId: number;
-  endpoint: UpstreamEndpoint;
-  downstreamFormat: EndpointPreference;
-  status: number;
-  errorText?: string | null;
-  modelName?: string;
-  requestedModelHint?: string;
-  requestCapabilities?: {
-    hasNonImageFileInput?: boolean;
-    conversationFileSummary?: ConversationFileInputSummary;
-    wantsNativeResponsesReasoning?: boolean;
-  };
-}): void {
-  const capabilityProfile = buildEndpointCapabilityProfile({
-    modelName: input.modelName,
-    requestedModelHint: input.requestedModelHint,
-    requestCapabilities: input.requestCapabilities,
-  });
-  if (!shouldUseEndpointRuntimeMemory(capabilityProfile)) return;
-  if (!shouldBlockEndpointByError(input.status, input.errorText)) return;
-
-  const nowMs = Date.now();
-  const key = buildEndpointRuntimeStateKey({
-    siteId: input.siteId,
-    downstreamFormat: input.downstreamFormat,
-    capabilityProfile,
-  });
-  const state = getOrCreateEndpointRuntimeState(key, nowMs);
-  state.blockedUntilMsByEndpoint[input.endpoint] = nowMs + ENDPOINT_RUNTIME_BLOCK_TTL_MS;
-
-  const suggestedEndpoint = inferSuggestedEndpointFromError(input.errorText);
-  if (suggestedEndpoint && suggestedEndpoint !== input.endpoint) {
-    state.preferredEndpoint = suggestedEndpoint;
-    state.preferredUpdatedAtMs = nowMs;
-    delete state.blockedUntilMsByEndpoint[suggestedEndpoint];
-  }
-}
-
 function preferredEndpointOrder(
   downstreamFormat: EndpointPreference,
   sitePlatform?: string,
@@ -848,15 +530,12 @@ export async function resolveUpstreamEndpointCandidates(
   const preferMessagesForClaudeModel = capabilityProfile.preferMessagesForClaudeModel;
   const hasNonImageFileInput = capabilityProfile.hasNonImageFileInput;
   const wantsNativeResponsesReasoning = capabilityProfile.wantsNativeResponsesReasoning;
-  const runtimeStateKey = buildEndpointRuntimeStateKey({
-    siteId: context.site.id,
-    downstreamFormat,
-    capabilityProfile,
-  });
   const applyRuntimePreference = (candidates: UpstreamEndpoint[]) => (
-    shouldUseEndpointRuntimeMemory(capabilityProfile)
-      ? applyEndpointRuntimePreference(candidates, runtimeStateKey)
-      : candidates
+    applyUpstreamEndpointRuntimePreference(candidates, {
+      siteId: context.site.id,
+      downstreamFormat,
+      capabilityProfile,
+    })
   );
   const conversationFileSummary = requestCapabilities?.conversationFileSummary ?? {
     hasImage: false,