[codex] add proxy debug tracing (#299)

* add proxy debug tracing

* fix proxy debug response header typing

* extend proxy debug tracing to side paths

* Refine proxy debug trace review fixes

* Stabilize proxy debug attempt indexing

* Tighten proxy debug trace accuracy

src/server/services/proxyDebugTraceRuntime.test.ts

@@ -0,0 +1,45 @@
+import { describe, expect, it } from 'vitest';
+import { reserveSurfaceProxyDebugAttemptBase } from './proxyDebugTraceRuntime.js';
+
+describe('reserveSurfaceProxyDebugAttemptBase', () => {
+  it('allocates monotonic attempt bases on the same trace session', () => {
+    const session = {
+      traceId: 801,
+      options: {
+        enabled: true,
+        captureHeaders: true,
+        captureBodies: true,
+        captureStreamChunks: false,
+        targetSessionId: '',
+        targetClientKind: '',
+        targetModel: '',
+        retentionHours: 12,
+        maxBodyBytes: 131072,
+      },
+    };
+
+    expect(reserveSurfaceProxyDebugAttemptBase(session, 3)).toBe(0);
+    expect(reserveSurfaceProxyDebugAttemptBase(session, 2)).toBe(3);
+    expect(reserveSurfaceProxyDebugAttemptBase(session, 4)).toBe(5);
+  });
+
+  it('reserves at least one slot for empty or invalid spans', () => {
+    const session = {
+      traceId: 802,
+      options: {
+        enabled: true,
+        captureHeaders: true,
+        captureBodies: true,
+        captureStreamChunks: false,
+        targetSessionId: '',
+        targetClientKind: '',
+        targetModel: '',
+        retentionHours: 12,
+        maxBodyBytes: 131072,
+      },
+    };
+
+    expect(reserveSurfaceProxyDebugAttemptBase(session, 0)).toBe(0);
+    expect(reserveSurfaceProxyDebugAttemptBase(session, Number.NaN)).toBe(1);
+  });
+});

src/server/services/proxyDebugTraceRuntime.ts

@@ -0,0 +1,177 @@
+import { readRuntimeResponseText } from '../proxy-core/executors/types.js';
+import type {
+  EndpointAttemptSuccessContext,
+} from '../routes/proxy/endpointFlow.js';
+import {
+  finalizeProxyDebugTrace,
+  insertProxyDebugAttempt,
+  normalizeProxyDebugResponseHeaders,
+  startProxyDebugTraceSession,
+  updateProxyDebugAttempt,
+  updateProxyDebugTraceCandidates,
+  updateProxyDebugTraceSelection,
+  type ProxyDebugTraceSession,
+} from './proxyDebugTraceStore.js';
+
+type MutableProxyDebugTraceSession = ProxyDebugTraceSession & {
+  nextAttemptIndex?: number;
+};
+
+function parseDebugTextPayload(rawText: string): unknown {
+  if (!rawText) return null;
+  try {
+    return JSON.parse(rawText);
+  } catch {
+    return rawText;
+  }
+}
+
+export async function startSurfaceProxyDebugTrace(input: {
+  downstreamPath: string;
+  clientKind?: string | null;
+  sessionId?: string | null;
+  traceHint?: string | null;
+  requestedModel?: string | null;
+  downstreamApiKeyId?: number | null;
+  requestHeaders?: Record<string, unknown>;
+  requestBody?: unknown;
+}): Promise<ProxyDebugTraceSession | null> {
+  try {
+    return await startProxyDebugTraceSession(input);
+  } catch (error) {
+    console.warn('[proxy-debug] failed to create trace session', error);
+    return null;
+  }
+}
+
+export async function safeUpdateSurfaceProxyDebugSelection(
+  session: ProxyDebugTraceSession | null,
+  input: Parameters<typeof updateProxyDebugTraceSelection>[1],
+): Promise<void> {
+  if (!session) return;
+  try {
+    await updateProxyDebugTraceSelection(session.traceId, input);
+  } catch (error) {
+    console.warn('[proxy-debug] failed to update selection', error);
+  }
+}
+
+export async function safeUpdateSurfaceProxyDebugCandidates(
+  session: ProxyDebugTraceSession | null,
+  input: Parameters<typeof updateProxyDebugTraceCandidates>[1],
+): Promise<void> {
+  if (!session) return;
+  try {
+    await updateProxyDebugTraceCandidates(session.traceId, input);
+  } catch (error) {
+    console.warn('[proxy-debug] failed to update endpoint candidates', error);
+  }
+}
+
+export async function safeInsertSurfaceProxyDebugAttempt(
+  session: ProxyDebugTraceSession | null,
+  input: Omit<Parameters<typeof insertProxyDebugAttempt>[0], 'traceId'>,
+): Promise<void> {
+  if (!session) return;
+  try {
+    await insertProxyDebugAttempt({
+      ...input,
+      traceId: session.traceId,
+      requestHeaders: session.options.captureHeaders ? input.requestHeaders : null,
+      requestBody: session.options.captureBodies ? input.requestBody : null,
+      responseHeaders: session.options.captureHeaders ? input.responseHeaders : null,
+      responseBody: session.options.captureBodies ? input.responseBody : null,
+      maxBodyBytes: session.options.maxBodyBytes,
+    });
+  } catch (error) {
+    console.warn('[proxy-debug] failed to insert attempt', error);
+  }
+}
+
+export function reserveSurfaceProxyDebugAttemptBase(
+  session: ProxyDebugTraceSession | null,
+  span: number,
+): number {
+  if (!session) return 0;
+
+  const mutableSession = session as MutableProxyDebugTraceSession;
+  const base = mutableSession.nextAttemptIndex ?? 0;
+  const normalizedSpan = Number.isFinite(span)
+    ? Math.max(1, Math.trunc(span))
+    : 1;
+  mutableSession.nextAttemptIndex = base + normalizedSpan;
+  return base;
+}
+
+export async function safeFinalizeSurfaceProxyDebugTrace(
+  session: ProxyDebugTraceSession | null,
+  input: Parameters<typeof finalizeProxyDebugTrace>[1],
+): Promise<void> {
+  if (!session) return;
+  try {
+    await finalizeProxyDebugTrace(session.traceId, {
+      ...input,
+      finalResponseHeaders: session.options.captureHeaders ? input.finalResponseHeaders : null,
+      finalResponseBody: session.options.captureBodies ? input.finalResponseBody : null,
+      maxBodyBytes: session.options.maxBodyBytes,
+    });
+  } catch (error) {
+    console.warn('[proxy-debug] failed to finalize trace', error);
+  }
+}
+
+export async function safeUpdateSurfaceProxyDebugAttempt(
+  session: ProxyDebugTraceSession | null,
+  attemptIndex: number,
+  input: Parameters<typeof updateProxyDebugAttempt>[2],
+): Promise<void> {
+  if (!session) return;
+  try {
+    await updateProxyDebugAttempt(session.traceId, attemptIndex, input);
+  } catch (error) {
+    console.warn('[proxy-debug] failed to update attempt', error);
+  }
+}
+
+export async function captureSurfaceProxyDebugSuccessResponseBody(
+  session: ProxyDebugTraceSession | null,
+  ctx: EndpointAttemptSuccessContext,
+): Promise<unknown> {
+  if (!session?.options.captureBodies) return null;
+
+  const contentType = (ctx.response.headers.get('content-type') || '').toLowerCase();
+  const requestBody = ctx.request.body as Record<string, unknown> | undefined;
+  const isStream = ctx.request.runtime?.stream === true
+    || requestBody?.stream === true
+    || contentType.includes('text/event-stream');
+  if (isStream) return null;
+
+  try {
+    const rawText = await readRuntimeResponseText(ctx.response.clone());
+    return parseDebugTextPayload(rawText);
+  } catch {
+    return null;
+  }
+}
+
+export function buildSurfaceProxyDebugResponseHeaders(
+  response:
+    | Headers
+    | Record<string, unknown>
+    | { headers?: Headers | Record<string, unknown> | null | undefined }
+    | null
+    | undefined,
+): Record<string, unknown> | null {
+  if (!response) return null;
+  if (typeof response === 'object' && 'headers' in response) {
+    const responseHeaders = (response as {
+      headers?: Headers | Record<string, unknown> | null | undefined;
+    }).headers;
+    return normalizeProxyDebugResponseHeaders(responseHeaders ?? null);
+  }
+  return normalizeProxyDebugResponseHeaders(response);
+}
+
+export function parseSurfaceProxyDebugTextPayload(rawText: string): unknown {
+  return parseDebugTextPayload(rawText);
+}

src/server/services/proxyDebugTraceStore.test.ts

@@ -0,0 +1,153 @@
+import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest';
+import { mkdtempSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+
+type DbModule = typeof import('../db/index.js');
+type StoreModule = typeof import('./proxyDebugTraceStore.js');
+
+describe('proxyDebugTraceStore', () => {
+  let db: DbModule['db'];
+  let schema: DbModule['schema'];
+  let store: StoreModule;
+  let dataDir = '';
+
+  beforeAll(async () => {
+    dataDir = mkdtempSync(join(tmpdir(), 'metapi-proxy-debug-traces-'));
+    process.env.DATA_DIR = dataDir;
+
+    await import('../db/migrate.js');
+    const dbModule = await import('../db/index.js');
+    const storeModule = await import('./proxyDebugTraceStore.js');
+    db = dbModule.db;
+    schema = dbModule.schema;
+    store = storeModule;
+  });
+
+  beforeEach(async () => {
+    await db.delete(schema.proxyDebugAttempts).run();
+    await db.delete(schema.proxyDebugTraces).run();
+  });
+
+  afterAll(async () => {
+    const dbModule = await import('../db/index.js');
+    await dbModule.closeDbConnections();
+    delete process.env.DATA_DIR;
+  });
+
+  it('creates, updates, and reads unredacted debug traces and attempts', async () => {
+    const trace = await store.createProxyDebugTrace({
+      downstreamPath: '/v1/responses',
+      clientKind: 'codex',
+      sessionId: 'sess-1',
+      traceHint: 'trace-abc',
+      requestedModel: 'gpt-4o',
+      downstreamApiKeyId: 7,
+      requestHeaders: {
+        authorization: 'Bearer developer-token',
+        'x-client': 'Codex Desktop',
+      },
+      requestBody: {
+        model: 'gpt-4o',
+        input: [{ role: 'user', content: 'hello' }],
+      },
+    });
+
+    await store.updateProxyDebugTraceSelection(trace.id, {
+      stickySessionKey: 'key:7|codex|/v1/responses|gpt-4o|sess-1',
+      stickyHitChannelId: 12,
+      selectedChannelId: 12,
+      selectedRouteId: 99,
+      selectedAccountId: 33,
+      selectedSiteId: 8,
+      selectedSitePlatform: 'codex',
+    });
+
+    await store.updateProxyDebugTraceCandidates(trace.id, {
+      endpointCandidates: ['responses', 'chat'],
+      endpointRuntimeState: {
+        preferredEndpoint: 'responses',
+        blockedEndpoints: [],
+      },
+      decisionSummary: {
+        reason: 'platform default + sticky session',
+      },
+    });
+
+    const attempt = await store.insertProxyDebugAttempt({
+      traceId: trace.id,
+      attemptIndex: 0,
+      endpoint: 'responses',
+      requestPath: '/responses',
+      targetUrl: 'https://chatgpt.com/backend-api/codex/responses',
+      runtimeExecutor: 'codex',
+      requestHeaders: {
+        authorization: 'Bearer developer-token',
+      },
+      requestBody: {
+        model: 'gpt-4o',
+        store: false,
+      },
+      responseStatus: 403,
+      responseHeaders: {
+        'content-type': 'application/json',
+      },
+      responseBody: {
+        error: {
+          message: 'forbidden',
+        },
+      },
+      rawErrorText: '{"error":{"message":"forbidden"}}',
+      recoverApplied: false,
+      downgradeDecision: true,
+      downgradeReason: '[upstream:/responses] forbidden',
+      memoryWrite: {
+        action: 'failure',
+        blockedEndpoint: 'responses',
+      },
+    });
+
+    expect(attempt.id).toBeGreaterThan(0);
+
+    await store.finalizeProxyDebugTrace(trace.id, {
+      finalStatus: 'failed',
+      finalHttpStatus: 503,
+      finalUpstreamPath: '/responses',
+      finalResponseHeaders: {
+        'content-type': 'application/json',
+      },
+      finalResponseBody: {
+        error: {
+          message: 'Channel busy',
+        },
+      },
+    });
+
+    const list = await store.listProxyDebugTraces({ limit: 20 });
+    expect(list).toHaveLength(1);
+    expect(list[0]).toMatchObject({
+      requestedModel: 'gpt-4o',
+      downstreamPath: '/v1/responses',
+      clientKind: 'codex',
+      sessionId: 'sess-1',
+      selectedChannelId: 12,
+      finalStatus: 'failed',
+      finalHttpStatus: 503,
+      finalUpstreamPath: '/responses',
+    });
+
+    const detail = await store.getProxyDebugTraceDetail(trace.id);
+    expect(detail?.trace.requestHeadersJson || '').toContain('Bearer developer-token');
+    expect(detail?.trace.requestBodyJson || '').toContain('"hello"');
+    expect(detail?.trace.finalResponseBodyJson || '').toContain('Channel busy');
+    expect(detail?.attempts).toHaveLength(1);
+    expect(detail?.attempts[0]?.requestHeadersJson || '').toContain('developer-token');
+    expect(detail?.attempts[0]?.responseBodyJson || '').toContain('forbidden');
+    expect(detail?.attempts[0]).toMatchObject({
+      endpoint: 'responses',
+      runtimeExecutor: 'codex',
+      responseStatus: 403,
+      downgradeDecision: true,
+    });
+  });
+});

src/server/services/proxyDebugTraceStore.ts

@@ -0,0 +1,391 @@
+import { and, asc, desc, eq, lt } from 'drizzle-orm';
+import { config } from '../config.js';
+import { db, schema } from '../db/index.js';
+import { formatUtcSqlDateTime } from './localTimeService.js';
+
+type HeadersLike = Headers | Record<string, unknown> | null | undefined;
+
+export type ProxyDebugCaptureOptions = {
+  enabled: boolean;
+  captureHeaders: boolean;
+  captureBodies: boolean;
+  captureStreamChunks: boolean;
+  targetSessionId: string;
+  targetClientKind: string;
+  targetModel: string;
+  retentionHours: number;
+  maxBodyBytes: number;
+};
+
+export type ProxyDebugTraceSession = {
+  traceId: number;
+  options: ProxyDebugCaptureOptions;
+};
+
+let lastPruneAtMs = 0;
+const PRUNE_INTERVAL_MS = 5 * 60 * 1000;
+
+function stringifyDebugValue(value: unknown, maxBytes: number): string | null {
+  if (value == null) return null;
+
+  let text = '';
+  if (typeof value === 'string') {
+    text = value;
+  } else {
+    try {
+      text = JSON.stringify(value, null, 2);
+    } catch {
+      text = String(value);
+    }
+  }
+
+  if (!text) return null;
+  const buffer = Buffer.from(text, 'utf8');
+  if (buffer.length <= maxBytes) {
+    return text;
+  }
+
+  const truncated = buffer.subarray(0, Math.max(0, maxBytes)).toString('utf8');
+  return `${truncated}\n...[truncated ${buffer.length - maxBytes} bytes]`;
+}
+
+function normalizeHeadersValue(value: HeadersLike): Record<string, unknown> | null {
+  if (!value) return null;
+
+  if (typeof Headers !== 'undefined' && value instanceof Headers) {
+    return Object.fromEntries([...value.entries()].sort((left, right) => left[0].localeCompare(right[0])));
+  }
+
+  if (typeof value !== 'object' || Array.isArray(value)) {
+    return null;
+  }
+
+  const entries = Object.entries(value)
+    .filter(([key]) => !!key)
+    .sort((left, right) => left[0].localeCompare(right[0]));
+  return Object.fromEntries(entries);
+}
+
+function normalizeLimit(rawLimit: number | undefined): number {
+  const parsed = Number(rawLimit || 20);
+  if (!Number.isFinite(parsed)) return 20;
+  return Math.max(1, Math.min(200, Math.trunc(parsed)));
+}
+
+function asNormalizedText(value: unknown): string {
+  return typeof value === 'string' ? value.trim().toLowerCase() : '';
+}
+
+function getCaptureOptions(): ProxyDebugCaptureOptions {
+  return {
+    enabled: config.proxyDebugTraceEnabled,
+    captureHeaders: config.proxyDebugCaptureHeaders,
+    captureBodies: config.proxyDebugCaptureBodies,
+    captureStreamChunks: config.proxyDebugCaptureStreamChunks,
+    targetSessionId: (config.proxyDebugTargetSessionId || '').trim(),
+    targetClientKind: (config.proxyDebugTargetClientKind || '').trim(),
+    targetModel: (config.proxyDebugTargetModel || '').trim(),
+    retentionHours: Math.max(1, Math.trunc(config.proxyDebugRetentionHours || 24)),
+    maxBodyBytes: Math.max(1024, Math.trunc(config.proxyDebugMaxBodyBytes || 262_144)),
+  };
+}
+
+function serializeHeaders(value: HeadersLike, maxBytes: number): string | null {
+  return stringifyDebugValue(normalizeHeadersValue(value), maxBytes);
+}
+
+export function normalizeProxyDebugResponseHeaders(value: HeadersLike): Record<string, unknown> | null {
+  return normalizeHeadersValue(value);
+}
+
+export function getProxyDebugCaptureOptions(): ProxyDebugCaptureOptions {
+  return getCaptureOptions();
+}
+
+export function shouldTraceProxyDebugRequest(input: {
+  clientKind?: string | null;
+  sessionId?: string | null;
+  requestedModel?: string | null;
+}, options = getCaptureOptions()): boolean {
+  if (!options.enabled) return false;
+
+  const targetSessionId = options.targetSessionId;
+  if (targetSessionId && (input.sessionId || '').trim() !== targetSessionId) {
+    return false;
+  }
+
+  const targetClientKind = asNormalizedText(options.targetClientKind);
+  if (targetClientKind && asNormalizedText(input.clientKind) !== targetClientKind) {
+    return false;
+  }
+
+  const targetModel = asNormalizedText(options.targetModel);
+  if (targetModel && asNormalizedText(input.requestedModel) !== targetModel) {
+    return false;
+  }
+
+  return true;
+}
+
+export async function deleteExpiredProxyDebugTraces(retentionHours = config.proxyDebugRetentionHours): Promise<number> {
+  const normalizedRetentionHours = Math.max(1, Math.trunc(Number(retentionHours || 24)));
+  const cutoff = formatUtcSqlDateTime(new Date(Date.now() - (normalizedRetentionHours * 60 * 60 * 1000)));
+  const result = await db
+    .delete(schema.proxyDebugTraces)
+    .where(lt(schema.proxyDebugTraces.createdAt, cutoff))
+    .run();
+  return Number(result.changes || 0);
+}
+
+async function pruneProxyDebugTracesIfNeeded(nowMs = Date.now(), retentionHours = config.proxyDebugRetentionHours): Promise<void> {
+  if (nowMs - lastPruneAtMs < PRUNE_INTERVAL_MS) return;
+  lastPruneAtMs = nowMs;
+  try {
+    await deleteExpiredProxyDebugTraces(retentionHours);
+  } catch (error) {
+    console.warn('[proxy-debug] failed to prune expired traces', error);
+  }
+}
+
+export async function createProxyDebugTrace(input: {
+  downstreamPath: string;
+  clientKind?: string | null;
+  sessionId?: string | null;
+  traceHint?: string | null;
+  requestedModel?: string | null;
+  downstreamApiKeyId?: number | null;
+  requestHeaders?: HeadersLike;
+  requestBody?: unknown;
+  maxBodyBytes?: number;
+}) {
+  const now = formatUtcSqlDateTime(new Date());
+  const maxBodyBytes = Math.max(1024, Math.trunc(input.maxBodyBytes || config.proxyDebugMaxBodyBytes || 262_144));
+  await pruneProxyDebugTracesIfNeeded(Date.now(), config.proxyDebugRetentionHours);
+
+  const inserted = await db.insert(schema.proxyDebugTraces).values({
+    downstreamPath: input.downstreamPath,
+    clientKind: input.clientKind ?? null,
+    sessionId: input.sessionId ?? null,
+    traceHint: input.traceHint ?? null,
+    requestedModel: input.requestedModel ?? null,
+    downstreamApiKeyId: input.downstreamApiKeyId ?? null,
+    requestHeadersJson: serializeHeaders(input.requestHeaders, maxBodyBytes),
+    requestBodyJson: stringifyDebugValue(input.requestBody, maxBodyBytes),
+    createdAt: now,
+    updatedAt: now,
+  }).run();
+
+  return {
+    id: Number(inserted.lastInsertRowid || 0),
+    createdAt: now,
+  };
+}
+
+export async function startProxyDebugTraceSession(input: {
+  downstreamPath: string;
+  clientKind?: string | null;
+  sessionId?: string | null;
+  traceHint?: string | null;
+  requestedModel?: string | null;
+  downstreamApiKeyId?: number | null;
+  requestHeaders?: HeadersLike;
+  requestBody?: unknown;
+}): Promise<ProxyDebugTraceSession | null> {
+  const options = getCaptureOptions();
+  if (!shouldTraceProxyDebugRequest(input, options)) {
+    return null;
+  }
+
+  const trace = await createProxyDebugTrace({
+    downstreamPath: input.downstreamPath,
+    clientKind: input.clientKind ?? null,
+    sessionId: input.sessionId ?? null,
+    traceHint: input.traceHint ?? null,
+    requestedModel: input.requestedModel ?? null,
+    downstreamApiKeyId: input.downstreamApiKeyId ?? null,
+    requestHeaders: options.captureHeaders ? input.requestHeaders : null,
+    requestBody: options.captureBodies ? input.requestBody : null,
+    maxBodyBytes: options.maxBodyBytes,
+  });
+
+  return {
+    traceId: trace.id,
+    options,
+  };
+}
+
+export async function updateProxyDebugTraceSelection(traceId: number, input: {
+  stickySessionKey?: string | null;
+  stickyHitChannelId?: number | null;
+  selectedChannelId?: number | null;
+  selectedRouteId?: number | null;
+  selectedAccountId?: number | null;
+  selectedSiteId?: number | null;
+  selectedSitePlatform?: string | null;
+}) {
+  const now = formatUtcSqlDateTime(new Date());
+  await db.update(schema.proxyDebugTraces).set({
+    stickySessionKey: input.stickySessionKey ?? null,
+    stickyHitChannelId: input.stickyHitChannelId ?? null,
+    selectedChannelId: input.selectedChannelId ?? null,
+    selectedRouteId: input.selectedRouteId ?? null,
+    selectedAccountId: input.selectedAccountId ?? null,
+    selectedSiteId: input.selectedSiteId ?? null,
+    selectedSitePlatform: input.selectedSitePlatform ?? null,
+    updatedAt: now,
+  }).where(eq(schema.proxyDebugTraces.id, traceId)).run();
+}
+
+export async function updateProxyDebugTraceCandidates(traceId: number, input: {
+  endpointCandidates?: unknown;
+  endpointRuntimeState?: unknown;
+  decisionSummary?: unknown;
+}) {
+  const now = formatUtcSqlDateTime(new Date());
+  await db.update(schema.proxyDebugTraces).set({
+    endpointCandidatesJson: stringifyDebugValue(input.endpointCandidates, config.proxyDebugMaxBodyBytes),
+    endpointRuntimeStateJson: stringifyDebugValue(input.endpointRuntimeState, config.proxyDebugMaxBodyBytes),
+    decisionSummaryJson: stringifyDebugValue(input.decisionSummary, config.proxyDebugMaxBodyBytes),
+    updatedAt: now,
+  }).where(eq(schema.proxyDebugTraces.id, traceId)).run();
+}
+
+export async function insertProxyDebugAttempt(input: {
+  traceId: number;
+  attemptIndex: number;
+  endpoint: string;
+  requestPath: string;
+  targetUrl: string;
+  runtimeExecutor?: string | null;
+  requestHeaders?: HeadersLike;
+  requestBody?: unknown;
+  responseStatus?: number | null;
+  responseHeaders?: HeadersLike;
+  responseBody?: unknown;
+  rawErrorText?: string | null;
+  recoverApplied?: boolean;
+  downgradeDecision?: boolean;
+  downgradeReason?: string | null;
+  memoryWrite?: unknown;
+  maxBodyBytes?: number;
+}) {
+  const now = formatUtcSqlDateTime(new Date());
+  const maxBodyBytes = Math.max(1024, Math.trunc(input.maxBodyBytes || config.proxyDebugMaxBodyBytes || 262_144));
+  const inserted = await db.insert(schema.proxyDebugAttempts).values({
+    traceId: input.traceId,
+    attemptIndex: input.attemptIndex,
+    endpoint: input.endpoint,
+    requestPath: input.requestPath,
+    targetUrl: input.targetUrl,
+    runtimeExecutor: input.runtimeExecutor ?? null,
+    requestHeadersJson: serializeHeaders(input.requestHeaders, maxBodyBytes),
+    requestBodyJson: stringifyDebugValue(input.requestBody, maxBodyBytes),
+    responseStatus: input.responseStatus ?? null,
+    responseHeadersJson: serializeHeaders(input.responseHeaders, maxBodyBytes),
+    responseBodyJson: stringifyDebugValue(input.responseBody, maxBodyBytes),
+    rawErrorText: input.rawErrorText ?? null,
+    recoverApplied: input.recoverApplied === true,
+    downgradeDecision: input.downgradeDecision === true,
+    downgradeReason: input.downgradeReason ?? null,
+    memoryWriteJson: stringifyDebugValue(input.memoryWrite, maxBodyBytes),
+    createdAt: now,
+  }).run();
+
+  return {
+    id: Number(inserted.lastInsertRowid || 0),
+    createdAt: now,
+  };
+}
+
+export async function updateProxyDebugAttempt(traceId: number, attemptIndex: number, input: {
+  downgradeDecision?: boolean;
+  downgradeReason?: string | null;
+  rawErrorText?: string | null;
+}) {
+  await db.update(schema.proxyDebugAttempts).set({
+    ...(input.downgradeDecision !== undefined ? { downgradeDecision: input.downgradeDecision } : {}),
+    ...(input.downgradeReason !== undefined ? { downgradeReason: input.downgradeReason } : {}),
+    ...(input.rawErrorText !== undefined ? { rawErrorText: input.rawErrorText } : {}),
+  }).where(and(
+    eq(schema.proxyDebugAttempts.traceId, traceId),
+    eq(schema.proxyDebugAttempts.attemptIndex, attemptIndex),
+  )).run();
+}
+
+export async function finalizeProxyDebugTrace(traceId: number, input: {
+  finalStatus?: string | null;
+  finalHttpStatus?: number | null;
+  finalUpstreamPath?: string | null;
+  finalResponseHeaders?: HeadersLike;
+  finalResponseBody?: unknown;
+  maxBodyBytes?: number;
+}) {
+  const now = formatUtcSqlDateTime(new Date());
+  const maxBodyBytes = Math.max(1024, Math.trunc(input.maxBodyBytes || config.proxyDebugMaxBodyBytes || 262_144));
+  await db.update(schema.proxyDebugTraces).set({
+    finalStatus: input.finalStatus ?? null,
+    finalHttpStatus: input.finalHttpStatus ?? null,
+    finalUpstreamPath: input.finalUpstreamPath ?? null,
+    finalResponseHeadersJson: serializeHeaders(input.finalResponseHeaders, maxBodyBytes),
+    finalResponseBodyJson: stringifyDebugValue(input.finalResponseBody, maxBodyBytes),
+    updatedAt: now,
+  }).where(eq(schema.proxyDebugTraces.id, traceId)).run();
+}
+
+export async function listProxyDebugTraces(input: { limit?: number }) {
+  const limit = normalizeLimit(input.limit);
+  return await db.select({
+    id: schema.proxyDebugTraces.id,
+    createdAt: schema.proxyDebugTraces.createdAt,
+    downstreamPath: schema.proxyDebugTraces.downstreamPath,
+    clientKind: schema.proxyDebugTraces.clientKind,
+    sessionId: schema.proxyDebugTraces.sessionId,
+    requestedModel: schema.proxyDebugTraces.requestedModel,
+    selectedChannelId: schema.proxyDebugTraces.selectedChannelId,
+    finalStatus: schema.proxyDebugTraces.finalStatus,
+    finalHttpStatus: schema.proxyDebugTraces.finalHttpStatus,
+    finalUpstreamPath: schema.proxyDebugTraces.finalUpstreamPath,
+  }).from(schema.proxyDebugTraces)
+    .orderBy(desc(schema.proxyDebugTraces.createdAt), desc(schema.proxyDebugTraces.id))
+    .limit(limit)
+    .all();
+}
+
+export async function getProxyDebugTraceDetail(traceId: number) {
+  const trace = await db.select().from(schema.proxyDebugTraces)
+    .where(eq(schema.proxyDebugTraces.id, traceId))
+    .get();
+  if (!trace) return null;
+
+  const attempts = await db.select().from(schema.proxyDebugAttempts)
+    .where(eq(schema.proxyDebugAttempts.traceId, traceId))
+    .orderBy(asc(schema.proxyDebugAttempts.attemptIndex), asc(schema.proxyDebugAttempts.id))
+    .all();
+
+  return {
+    trace,
+    attempts,
+  };
+}
+
+export async function findLatestProxyDebugTrace(input: {
+  sessionId?: string | null;
+  clientKind?: string | null;
+  requestedModel?: string | null;
+}) {
+  const conditions = [
+    input.sessionId ? eq(schema.proxyDebugTraces.sessionId, input.sessionId) : null,
+    input.clientKind ? eq(schema.proxyDebugTraces.clientKind, input.clientKind) : null,
+    input.requestedModel ? eq(schema.proxyDebugTraces.requestedModel, input.requestedModel) : null,
+  ].filter((condition): condition is NonNullable<typeof condition> => condition !== null);
+
+  let query = db.select().from(schema.proxyDebugTraces);
+  if (conditions.length === 1) {
+    query = query.where(conditions[0]) as typeof query;
+  } else if (conditions.length > 1) {
+    query = query.where(and(...conditions)) as typeof query;
+  }
+  return await query
+    .orderBy(desc(schema.proxyDebugTraces.createdAt), desc(schema.proxyDebugTraces.id))
+    .get();
+}

src/server/services/upstreamEndpointRuntimeMemory.ts

@@ -0,0 +1,430 @@
+import { createHash } from 'node:crypto';
+import type { ConversationFileInputSummary } from '../proxy-core/capabilities/conversationFileCapabilities.js';
+import type { DownstreamFormat } from '../transformers/shared/normalized.js';
+import {
+  isEndpointDispatchDeniedError,
+  isUnsupportedMediaTypeError,
+} from '../transformers/shared/endpointCompatibility.js';
+
+export type UpstreamEndpointRuntimeEndpoint = 'chat' | 'messages' | 'responses';
+export type UpstreamEndpointRuntimePreference = DownstreamFormat | 'responses';
+export type UpstreamEndpointRuntimeMemoryWrite =
+  | {
+    action: 'success';
+    endpoint: UpstreamEndpointRuntimeEndpoint;
+    preferredEndpoint: UpstreamEndpointRuntimeEndpoint;
+    stateKey: string;
+    timestampMs: number;
+  }
+  | {
+    action: 'failure';
+    endpoint: UpstreamEndpointRuntimeEndpoint;
+    blockedEndpoint: UpstreamEndpointRuntimeEndpoint;
+    preferredEndpoint: UpstreamEndpointRuntimeEndpoint | null;
+    stateKey: string;
+    timestampMs: number;
+  };
+
+export type EndpointCapabilityProfile = {
+  modelKey: string;
+  preferMessagesForClaudeModel: boolean;
+  hasImageInput: boolean;
+  hasAudioInput: boolean;
+  hasNonImageFileInput: boolean;
+  hasRemoteDocumentUrl: boolean;
+  wantsNativeResponsesReasoning: boolean;
+};
+
+type EndpointRuntimeState = {
+  preferredEndpoint: UpstreamEndpointRuntimeEndpoint | null;
+  preferredUpdatedAtMs: number;
+  lastTouchedAtMs: number;
+  blockedUntilMsByEndpoint: Partial<Record<UpstreamEndpointRuntimeEndpoint, number>>;
+};
+
+const ENDPOINT_RUNTIME_PREFERRED_TTL_MS = 24 * 60 * 60 * 1000;
+const ENDPOINT_RUNTIME_BLOCK_TTL_MS = 6 * 60 * 60 * 1000;
+const MAX_ENDPOINT_RUNTIME_STATES = 512;
+export const MAX_ENDPOINT_RUNTIME_MODEL_KEY_LENGTH = 64;
+export const MODEL_KEY_HASH_SUFFIX_LENGTH = 8;
+
+const endpointRuntimeStates = new Map<string, EndpointRuntimeState>();
+
+function asTrimmedString(value: unknown): string {
+  return typeof value === 'string' ? value.trim() : '';
+}
+
+function isClaudeFamilyModel(modelName: string): boolean {
+  const normalized = asTrimmedString(modelName).toLowerCase();
+  if (!normalized) return false;
+  return normalized === 'claude' || normalized.startsWith('claude-') || normalized.includes('claude');
+}
+
+export function boundEndpointRuntimeModelKey(value: string): string {
+  if (value.length <= MAX_ENDPOINT_RUNTIME_MODEL_KEY_LENGTH) {
+    return value;
+  }
+
+  const prefix = value.slice(0, MAX_ENDPOINT_RUNTIME_MODEL_KEY_LENGTH);
+  const hash = createHash('sha256')
+    .update(value)
+    .digest('hex')
+    .slice(0, MODEL_KEY_HASH_SUFFIX_LENGTH);
+  return `${prefix}-${hash}`;
+}
+
+function normalizeEndpointRuntimeModelKey(...values: Array<unknown>): string {
+  for (const value of values) {
+    const normalized = asTrimmedString(value).toLowerCase();
+    if (normalized) return boundEndpointRuntimeModelKey(normalized);
+  }
+  return boundEndpointRuntimeModelKey('unknown-model');
+}
+
+export function buildEndpointCapabilityProfile(input?: {
+  modelName?: string;
+  requestedModelHint?: string;
+  requestCapabilities?: {
+    hasNonImageFileInput?: boolean;
+    conversationFileSummary?: ConversationFileInputSummary;
+    wantsNativeResponsesReasoning?: boolean;
+  };
+}): EndpointCapabilityProfile {
+  const conversationFileSummary = input?.requestCapabilities?.conversationFileSummary;
+  return {
+    modelKey: normalizeEndpointRuntimeModelKey(input?.modelName, input?.requestedModelHint),
+    preferMessagesForClaudeModel: (
+      isClaudeFamilyModel(asTrimmedString(input?.modelName))
+      || isClaudeFamilyModel(asTrimmedString(input?.requestedModelHint))
+    ),
+    hasImageInput: conversationFileSummary?.hasImage === true,
+    hasAudioInput: conversationFileSummary?.hasAudio === true,
+    hasNonImageFileInput: (
+      conversationFileSummary?.hasDocument === true
+      || input?.requestCapabilities?.hasNonImageFileInput === true
+    ),
+    hasRemoteDocumentUrl: (
+      conversationFileSummary?.hasRemoteDocumentUrl === true
+    ),
+    wantsNativeResponsesReasoning: input?.requestCapabilities?.wantsNativeResponsesReasoning === true,
+  };
+}
+
+function shouldUseEndpointRuntimeMemory(capabilityProfile: EndpointCapabilityProfile): boolean {
+  return (
+    !capabilityProfile.hasImageInput
+    && !capabilityProfile.hasAudioInput
+    && !capabilityProfile.hasNonImageFileInput
+  );
+}
+
+function buildEndpointRuntimeStateKey(input: {
+  siteId: number;
+  downstreamFormat: UpstreamEndpointRuntimePreference;
+  capabilityProfile: EndpointCapabilityProfile;
+}): string {
+  const capabilityProfile = input.capabilityProfile;
+  return [
+    String(input.siteId),
+    input.downstreamFormat,
+    capabilityProfile.modelKey,
+    capabilityProfile.hasNonImageFileInput ? 'files' : 'nofiles',
+    capabilityProfile.hasRemoteDocumentUrl ? 'remoteurl' : 'noremoteurl',
+    capabilityProfile.wantsNativeResponsesReasoning ? 'reasoning' : 'noreasoning',
+  ].join(':');
+}
+
+function sweepEndpointRuntimeStates(nowMs = Date.now()): void {
+  for (const [key, state] of endpointRuntimeStates.entries()) {
+    const hasActiveBlock = Object.values(state.blockedUntilMsByEndpoint).some((untilMs) => (
+      typeof untilMs === 'number' && untilMs > nowMs
+    ));
+    const preferredFresh = (
+      !!state.preferredEndpoint
+      && (state.preferredUpdatedAtMs + ENDPOINT_RUNTIME_PREFERRED_TTL_MS) > nowMs
+    );
+    const recentlyTouched = (state.lastTouchedAtMs + ENDPOINT_RUNTIME_PREFERRED_TTL_MS) > nowMs;
+    if (!hasActiveBlock && !preferredFresh && !recentlyTouched) {
+      endpointRuntimeStates.delete(key);
+    }
+  }
+}
+
+function enforceEndpointRuntimeStateLimit(): void {
+  if (endpointRuntimeStates.size <= MAX_ENDPOINT_RUNTIME_STATES) return;
+
+  const entries = [...endpointRuntimeStates.entries()]
+    .sort((left, right) => left[1].lastTouchedAtMs - right[1].lastTouchedAtMs);
+  const overflowCount = endpointRuntimeStates.size - MAX_ENDPOINT_RUNTIME_STATES;
+  for (const [key] of entries.slice(0, overflowCount)) {
+    endpointRuntimeStates.delete(key);
+  }
+}
+
+function getOrCreateEndpointRuntimeState(key: string, nowMs = Date.now()): EndpointRuntimeState {
+  sweepEndpointRuntimeStates(nowMs);
+  const existing = endpointRuntimeStates.get(key);
+  if (existing) {
+    existing.lastTouchedAtMs = nowMs;
+    return existing;
+  }
+
+  const initial: EndpointRuntimeState = {
+    preferredEndpoint: null,
+    preferredUpdatedAtMs: nowMs,
+    lastTouchedAtMs: nowMs,
+    blockedUntilMsByEndpoint: {},
+  };
+  endpointRuntimeStates.set(key, initial);
+  enforceEndpointRuntimeStateLimit();
+  return initial;
+}
+
+function maybeDeleteEndpointRuntimeState(key: string, nowMs = Date.now()): void {
+  const state = endpointRuntimeStates.get(key);
+  if (!state) return;
+
+  const hasActiveBlock = Object.values(state.blockedUntilMsByEndpoint).some((untilMs) => (
+    typeof untilMs === 'number' && untilMs > nowMs
+  ));
+  const preferredFresh = (
+    !!state.preferredEndpoint
+    && (state.preferredUpdatedAtMs + ENDPOINT_RUNTIME_PREFERRED_TTL_MS) > nowMs
+  );
+  if (!hasActiveBlock && !preferredFresh) {
+    endpointRuntimeStates.delete(key);
+  }
+}
+
+function inferSuggestedEndpointFromError(errorText?: string | null): UpstreamEndpointRuntimeEndpoint | null {
+  const text = (errorText || '').toLowerCase();
+  if (!text) return null;
+  if (text.includes('/v1/responses')) return 'responses';
+  if (text.includes('/v1/messages')) return 'messages';
+  if (text.includes('/v1/chat/completions')) return 'chat';
+  return null;
+}
+
+function shouldBlockEndpointByError(status: number, errorText?: string | null): boolean {
+  if (isEndpointDispatchDeniedError(status, errorText)) return true;
+  if (status === 404 || status === 405 || status === 415 || status === 501) return true;
+  if (isUnsupportedMediaTypeError(status, errorText)) return true;
+
+  const text = (errorText || '').toLowerCase();
+  return (
+    text.includes('convert_request_failed')
+    || text.includes('endpoint_not_found')
+    || text.includes('unknown_endpoint')
+    || text.includes('unsupported_endpoint')
+    || text.includes('unsupported_path')
+    || text.includes('not_found_error')
+    || text.includes('unsupported legacy protocol')
+    || text.includes('please use /v1/')
+    || text.includes('does not allow /v1/')
+    || text.includes('unknown endpoint')
+    || text.includes('unsupported endpoint')
+    || text.includes('unsupported path')
+    || text.includes('unrecognized request url')
+    || text.includes('no route matched')
+    || text.includes('does not exist')
+  );
+}
+
+function shouldRememberSuccessfulEndpoint(input: {
+  endpoint: UpstreamEndpointRuntimeEndpoint;
+  downstreamFormat: UpstreamEndpointRuntimePreference;
+}): boolean {
+  if (input.downstreamFormat !== 'responses') return true;
+  return input.endpoint === 'responses';
+}
+
+export function getUpstreamEndpointRuntimeStateSnapshot(input: {
+  siteId: number;
+  downstreamFormat: UpstreamEndpointRuntimePreference;
+  modelName?: string;
+  requestedModelHint?: string;
+  requestCapabilities?: {
+    hasNonImageFileInput?: boolean;
+    conversationFileSummary?: ConversationFileInputSummary;
+    wantsNativeResponsesReasoning?: boolean;
+  };
+}) {
+  const capabilityProfile = buildEndpointCapabilityProfile({
+    modelName: input.modelName,
+    requestedModelHint: input.requestedModelHint,
+    requestCapabilities: input.requestCapabilities,
+  });
+  const enabled = shouldUseEndpointRuntimeMemory(capabilityProfile);
+  const stateKey = buildEndpointRuntimeStateKey({
+    siteId: input.siteId,
+    downstreamFormat: input.downstreamFormat,
+    capabilityProfile,
+  });
+  const nowMs = Date.now();
+  const state = endpointRuntimeStates.get(stateKey);
+  const preferredEndpoint = (
+    enabled
+    && state?.preferredEndpoint
+    && (state.preferredUpdatedAtMs + ENDPOINT_RUNTIME_PREFERRED_TTL_MS) > nowMs
+    && !(
+      typeof state.blockedUntilMsByEndpoint[state.preferredEndpoint] === 'number'
+      && (state.blockedUntilMsByEndpoint[state.preferredEndpoint] as number) > nowMs
+    )
+  ) ? state.preferredEndpoint : null;
+
+  return {
+    enabled,
+    stateKey,
+    preferredEndpoint,
+    blockedEndpoints: enabled
+      ? (['chat', 'messages', 'responses'] as UpstreamEndpointRuntimeEndpoint[]).filter((endpoint) => {
+        const untilMs = state?.blockedUntilMsByEndpoint[endpoint];
+        return typeof untilMs === 'number' && untilMs > nowMs;
+      })
+      : [],
+  };
+}
+
+export function applyUpstreamEndpointRuntimePreference(
+  candidates: UpstreamEndpointRuntimeEndpoint[],
+  input: {
+    siteId: number;
+    downstreamFormat: UpstreamEndpointRuntimePreference;
+    capabilityProfile: EndpointCapabilityProfile;
+  },
+  nowMs = Date.now(),
+): UpstreamEndpointRuntimeEndpoint[] {
+  if (!shouldUseEndpointRuntimeMemory(input.capabilityProfile)) {
+    return candidates;
+  }
+
+  const key = buildEndpointRuntimeStateKey({
+    siteId: input.siteId,
+    downstreamFormat: input.downstreamFormat,
+    capabilityProfile: input.capabilityProfile,
+  });
+  const state = endpointRuntimeStates.get(key);
+  if (!state || candidates.length <= 1) return candidates;
+  state.lastTouchedAtMs = nowMs;
+
+  const blocked = new Set<UpstreamEndpointRuntimeEndpoint>();
+  for (const endpoint of candidates) {
+    const untilMs = state.blockedUntilMsByEndpoint[endpoint];
+    if (typeof untilMs === 'number' && untilMs > nowMs) {
+      blocked.add(endpoint);
+    }
+  }
+
+  let next = candidates.filter((endpoint) => !blocked.has(endpoint));
+  if (next.length === 0) {
+    next = [...candidates];
+  }
+
+  const preferredFresh = (
+    !!state.preferredEndpoint
+    && (state.preferredUpdatedAtMs + ENDPOINT_RUNTIME_PREFERRED_TTL_MS) > nowMs
+  );
+  if (preferredFresh && state.preferredEndpoint && next.includes(state.preferredEndpoint)) {
+    next = [
+      state.preferredEndpoint,
+      ...next.filter((endpoint) => endpoint !== state.preferredEndpoint),
+    ];
+  }
+
+  maybeDeleteEndpointRuntimeState(key, nowMs);
+  return next;
+}
+
+export function resetUpstreamEndpointRuntimeState(): void {
+  endpointRuntimeStates.clear();
+}
+
+export function recordUpstreamEndpointSuccess(input: {
+  siteId: number;
+  endpoint: UpstreamEndpointRuntimeEndpoint;
+  downstreamFormat: UpstreamEndpointRuntimePreference;
+  modelName?: string;
+  requestedModelHint?: string;
+  requestCapabilities?: {
+    hasNonImageFileInput?: boolean;
+    conversationFileSummary?: ConversationFileInputSummary;
+    wantsNativeResponsesReasoning?: boolean;
+  };
+}): UpstreamEndpointRuntimeMemoryWrite | null {
+  const capabilityProfile = buildEndpointCapabilityProfile({
+    modelName: input.modelName,
+    requestedModelHint: input.requestedModelHint,
+    requestCapabilities: input.requestCapabilities,
+  });
+  if (!shouldUseEndpointRuntimeMemory(capabilityProfile)) return null;
+  if (!shouldRememberSuccessfulEndpoint(input)) return null;
+
+  const nowMs = Date.now();
+  const key = buildEndpointRuntimeStateKey({
+    siteId: input.siteId,
+    downstreamFormat: input.downstreamFormat,
+    capabilityProfile,
+  });
+  const state = getOrCreateEndpointRuntimeState(key, nowMs);
+  state.preferredEndpoint = input.endpoint;
+  state.preferredUpdatedAtMs = nowMs;
+  delete state.blockedUntilMsByEndpoint[input.endpoint];
+  return {
+    action: 'success',
+    endpoint: input.endpoint,
+    preferredEndpoint: input.endpoint,
+    stateKey: key,
+    timestampMs: nowMs,
+  };
+}
+
+export function recordUpstreamEndpointFailure(input: {
+  siteId: number;
+  endpoint: UpstreamEndpointRuntimeEndpoint;
+  downstreamFormat: UpstreamEndpointRuntimePreference;
+  status: number;
+  errorText?: string | null;
+  modelName?: string;
+  requestedModelHint?: string;
+  requestCapabilities?: {
+    hasNonImageFileInput?: boolean;
+    conversationFileSummary?: ConversationFileInputSummary;
+    wantsNativeResponsesReasoning?: boolean;
+  };
+}): UpstreamEndpointRuntimeMemoryWrite | null {
+  const capabilityProfile = buildEndpointCapabilityProfile({
+    modelName: input.modelName,
+    requestedModelHint: input.requestedModelHint,
+    requestCapabilities: input.requestCapabilities,
+  });
+  if (!shouldUseEndpointRuntimeMemory(capabilityProfile)) return null;
+  if (!shouldBlockEndpointByError(input.status, input.errorText)) return null;
+
+  const nowMs = Date.now();
+  const key = buildEndpointRuntimeStateKey({
+    siteId: input.siteId,
+    downstreamFormat: input.downstreamFormat,
+    capabilityProfile,
+  });
+  const state = getOrCreateEndpointRuntimeState(key, nowMs);
+  state.blockedUntilMsByEndpoint[input.endpoint] = nowMs + ENDPOINT_RUNTIME_BLOCK_TTL_MS;
+
+  const suggestedEndpoint = inferSuggestedEndpointFromError(input.errorText);
+  if (suggestedEndpoint && suggestedEndpoint !== input.endpoint) {
+    state.preferredEndpoint = suggestedEndpoint;
+    state.preferredUpdatedAtMs = nowMs;
+    delete state.blockedUntilMsByEndpoint[suggestedEndpoint];
+  }
+  return {
+    action: 'failure',
+    endpoint: input.endpoint,
+    blockedEndpoint: input.endpoint,
+    preferredEndpoint: (
+      suggestedEndpoint && suggestedEndpoint !== input.endpoint
+        ? suggestedEndpoint
+        : null
+    ),
+    stateKey: key,
+    timestampMs: nowMs,
+  };
+}