[codex] add proxy debug tracing (#299)

* add proxy debug tracing * fix proxy debug response header typing * extend proxy debug tracing to side paths * Refine proxy debug trace review fixes * Stabilize proxy debug attempt indexing * Tighten proxy debug trace accuracy
2026-03-28 20:05:48 +08:00
parent 3bf3cd1622
commit 4d4684a39b
35 changed files with 6669 additions and 495 deletions
@@ -55,6 +55,15 @@ describe('settings and auth events', () => {
    config.codexUpstreamWebsocketEnabled = false;
    config.proxySessionChannelConcurrencyLimit = 2;
    config.proxySessionChannelQueueWaitMs = 1500;
+    (config as any).proxyDebugTraceEnabled = false;
+    (config as any).proxyDebugCaptureHeaders = true;
+    (config as any).proxyDebugCaptureBodies = false;
+    (config as any).proxyDebugCaptureStreamChunks = false;
+    (config as any).proxyDebugTargetSessionId = '';
+    (config as any).proxyDebugTargetClientKind = '';
+    (config as any).proxyDebugTargetModel = '';
+    (config as any).proxyDebugRetentionHours = 24;
+    (config as any).proxyDebugMaxBodyBytes = 262144;
    config.routingFallbackUnitCost = 1;
    (config as any).telegramEnabled = false;
    (config as any).telegramApiBaseUrl = 'https://api.telegram.org';
@@ -146,6 +155,67 @@ describe('settings and auth events', () => {
    expect(savedQueueWait?.value).toBe(JSON.stringify(4200));
  });

+  it('persists proxy debug runtime settings from runtime settings', async () => {
+    const updateResponse = await app.inject({
+      method: 'PUT',
+      url: '/api/settings/runtime',
+      payload: {
+        proxyDebugTraceEnabled: true,
+        proxyDebugCaptureHeaders: true,
+        proxyDebugCaptureBodies: true,
+        proxyDebugCaptureStreamChunks: true,
+        proxyDebugTargetSessionId: 'sess-debug-1',
+        proxyDebugTargetClientKind: 'codex',
+        proxyDebugTargetModel: 'gpt-4o',
+        proxyDebugRetentionHours: 12,
+        proxyDebugMaxBodyBytes: 131072,
+      },
+    });
+
+    expect(updateResponse.statusCode).toBe(200);
+    const updated = updateResponse.json() as {
+      proxyDebugTraceEnabled?: boolean;
+      proxyDebugCaptureHeaders?: boolean;
+      proxyDebugCaptureBodies?: boolean;
+      proxyDebugCaptureStreamChunks?: boolean;
+      proxyDebugTargetSessionId?: string;
+      proxyDebugTargetClientKind?: string;
+      proxyDebugTargetModel?: string;
+      proxyDebugRetentionHours?: number;
+      proxyDebugMaxBodyBytes?: number;
+    };
+    expect(updated).toMatchObject({
+      proxyDebugTraceEnabled: true,
+      proxyDebugCaptureHeaders: true,
+      proxyDebugCaptureBodies: true,
+      proxyDebugCaptureStreamChunks: true,
+      proxyDebugTargetSessionId: 'sess-debug-1',
+      proxyDebugTargetClientKind: 'codex',
+      proxyDebugTargetModel: 'gpt-4o',
+      proxyDebugRetentionHours: 12,
+      proxyDebugMaxBodyBytes: 131072,
+    });
+
+    const savedEnabled = await db.select().from(schema.settings).where(eq(schema.settings.key, 'proxy_debug_trace_enabled')).get();
+    const savedHeaders = await db.select().from(schema.settings).where(eq(schema.settings.key, 'proxy_debug_capture_headers')).get();
+    const savedBodies = await db.select().from(schema.settings).where(eq(schema.settings.key, 'proxy_debug_capture_bodies')).get();
+    const savedStreamChunks = await db.select().from(schema.settings).where(eq(schema.settings.key, 'proxy_debug_capture_stream_chunks')).get();
+    const savedTargetSessionId = await db.select().from(schema.settings).where(eq(schema.settings.key, 'proxy_debug_target_session_id')).get();
+    const savedTargetClientKind = await db.select().from(schema.settings).where(eq(schema.settings.key, 'proxy_debug_target_client_kind')).get();
+    const savedTargetModel = await db.select().from(schema.settings).where(eq(schema.settings.key, 'proxy_debug_target_model')).get();
+    const savedRetentionHours = await db.select().from(schema.settings).where(eq(schema.settings.key, 'proxy_debug_retention_hours')).get();
+    const savedMaxBodyBytes = await db.select().from(schema.settings).where(eq(schema.settings.key, 'proxy_debug_max_body_bytes')).get();
+    expect(savedEnabled?.value).toBe(JSON.stringify(true));
+    expect(savedHeaders?.value).toBe(JSON.stringify(true));
+    expect(savedBodies?.value).toBe(JSON.stringify(true));
+    expect(savedStreamChunks?.value).toBe(JSON.stringify(true));
+    expect(savedTargetSessionId?.value).toBe(JSON.stringify('sess-debug-1'));
+    expect(savedTargetClientKind?.value).toBe(JSON.stringify('codex'));
+    expect(savedTargetModel?.value).toBe(JSON.stringify('gpt-4o'));
+    expect(savedRetentionHours?.value).toBe(JSON.stringify(12));
+    expect(savedMaxBodyBytes?.value).toBe(JSON.stringify(131072));
+  });
+
  it('returns current recognized admin IP in runtime settings response', async () => {
    const response = await app.inject({
      method: 'GET',
@@ -41,6 +41,15 @@ interface RuntimeSettingsBody {
  codexUpstreamWebsocketEnabled?: boolean;
  proxySessionChannelConcurrencyLimit?: number;
  proxySessionChannelQueueWaitMs?: number;
+  proxyDebugTraceEnabled?: boolean;
+  proxyDebugCaptureHeaders?: boolean;
+  proxyDebugCaptureBodies?: boolean;
+  proxyDebugCaptureStreamChunks?: boolean;
+  proxyDebugTargetSessionId?: string;
+  proxyDebugTargetClientKind?: string;
+  proxyDebugTargetModel?: string;
+  proxyDebugRetentionHours?: number;
+  proxyDebugMaxBodyBytes?: number;
  checkinCron?: string;
  checkinScheduleMode?: 'cron' | 'interval';
  checkinIntervalHours?: number;
@@ -409,6 +418,62 @@ function applyImportedSettingToRuntime(key: string, value: unknown) {
      config.proxySessionChannelQueueWaitMs = Math.trunc(queueWaitMs);
      return;
    }
+    case 'proxy_debug_trace_enabled': {
+      try {
+        config.proxyDebugTraceEnabled = parseBooleanFlag(value, '代理调试追踪开关');
+      } catch {
+        return;
+      }
+      return;
+    }
+    case 'proxy_debug_capture_headers': {
+      try {
+        config.proxyDebugCaptureHeaders = parseBooleanFlag(value, '代理调试请求头采集');
+      } catch {
+        return;
+      }
+      return;
+    }
+    case 'proxy_debug_capture_bodies': {
+      try {
+        config.proxyDebugCaptureBodies = parseBooleanFlag(value, '代理调试请求体采集');
+      } catch {
+        return;
+      }
+      return;
+    }
+    case 'proxy_debug_capture_stream_chunks': {
+      try {
+        config.proxyDebugCaptureStreamChunks = parseBooleanFlag(value, '代理调试流式分片采集');
+      } catch {
+        return;
+      }
+      return;
+    }
+    case 'proxy_debug_target_session_id': {
+      config.proxyDebugTargetSessionId = typeof value === 'string' ? value.trim() : '';
+      return;
+    }
+    case 'proxy_debug_target_client_kind': {
+      config.proxyDebugTargetClientKind = typeof value === 'string' ? value.trim() : '';
+      return;
+    }
+    case 'proxy_debug_target_model': {
+      config.proxyDebugTargetModel = typeof value === 'string' ? value.trim() : '';
+      return;
+    }
+    case 'proxy_debug_retention_hours': {
+      const retentionHours = Number(value);
+      if (!Number.isFinite(retentionHours) || retentionHours < 1) return;
+      config.proxyDebugRetentionHours = Math.trunc(retentionHours);
+      return;
+    }
+    case 'proxy_debug_max_body_bytes': {
+      const maxBodyBytes = Number(value);
+      if (!Number.isFinite(maxBodyBytes) || maxBodyBytes < 1024) return;
+      config.proxyDebugMaxBodyBytes = Math.trunc(maxBodyBytes);
+      return;
+    }
    case 'proxy_empty_content_fail_enabled': {
      try {
        config.proxyEmptyContentFailEnabled = parseBooleanFlag(value, '空内容判定失败开关');
@@ -580,6 +645,15 @@ function getRuntimeSettingsResponse(currentAdminIp = '') {
    codexUpstreamWebsocketEnabled: config.codexUpstreamWebsocketEnabled,
    proxySessionChannelConcurrencyLimit: config.proxySessionChannelConcurrencyLimit,
    proxySessionChannelQueueWaitMs: config.proxySessionChannelQueueWaitMs,
+    proxyDebugTraceEnabled: config.proxyDebugTraceEnabled,
+    proxyDebugCaptureHeaders: config.proxyDebugCaptureHeaders,
+    proxyDebugCaptureBodies: config.proxyDebugCaptureBodies,
+    proxyDebugCaptureStreamChunks: config.proxyDebugCaptureStreamChunks,
+    proxyDebugTargetSessionId: config.proxyDebugTargetSessionId,
+    proxyDebugTargetClientKind: config.proxyDebugTargetClientKind,
+    proxyDebugTargetModel: config.proxyDebugTargetModel,
+    proxyDebugRetentionHours: config.proxyDebugRetentionHours,
+    proxyDebugMaxBodyBytes: config.proxyDebugMaxBodyBytes,
    routingFallbackUnitCost: config.routingFallbackUnitCost,
    routingWeights: config.routingWeights,
    webhookUrl: config.webhookUrl,
@@ -1007,6 +1081,127 @@ export async function settingsRoutes(app: FastifyInstance) {
      upsertSetting('proxy_session_channel_queue_wait_ms', config.proxySessionChannelQueueWaitMs);
    }

+    if (body.proxyDebugTraceEnabled !== undefined) {
+      let nextValue = false;
+      try {
+        nextValue = parseBooleanFlag(body.proxyDebugTraceEnabled, '代理调试追踪开关');
+      } catch (err: any) {
+        return reply.code(400).send({
+          success: false,
+          message: err?.message || '代理调试追踪开关格式无效',
+        });
+      }
+      if (nextValue !== config.proxyDebugTraceEnabled) {
+        changedLabels.push('代理调试追踪');
+      }
+      config.proxyDebugTraceEnabled = nextValue;
+      upsertSetting('proxy_debug_trace_enabled', config.proxyDebugTraceEnabled);
+    }
+
+    if (body.proxyDebugCaptureHeaders !== undefined) {
+      let nextValue = false;
+      try {
+        nextValue = parseBooleanFlag(body.proxyDebugCaptureHeaders, '代理调试请求头采集');
+      } catch (err: any) {
+        return reply.code(400).send({
+          success: false,
+          message: err?.message || '代理调试请求头采集格式无效',
+        });
+      }
+      if (nextValue !== config.proxyDebugCaptureHeaders) {
+        changedLabels.push('代理调试请求头采集');
+      }
+      config.proxyDebugCaptureHeaders = nextValue;
+      upsertSetting('proxy_debug_capture_headers', config.proxyDebugCaptureHeaders);
+    }
+
+    if (body.proxyDebugCaptureBodies !== undefined) {
+      let nextValue = false;
+      try {
+        nextValue = parseBooleanFlag(body.proxyDebugCaptureBodies, '代理调试请求体采集');
+      } catch (err: any) {
+        return reply.code(400).send({
+          success: false,
+          message: err?.message || '代理调试请求体采集格式无效',
+        });
+      }
+      if (nextValue !== config.proxyDebugCaptureBodies) {
+        changedLabels.push('代理调试请求体采集');
+      }
+      config.proxyDebugCaptureBodies = nextValue;
+      upsertSetting('proxy_debug_capture_bodies', config.proxyDebugCaptureBodies);
+    }
+
+    if (body.proxyDebugCaptureStreamChunks !== undefined) {
+      let nextValue = false;
+      try {
+        nextValue = parseBooleanFlag(body.proxyDebugCaptureStreamChunks, '代理调试流式分片采集');
+      } catch (err: any) {
+        return reply.code(400).send({
+          success: false,
+          message: err?.message || '代理调试流式分片采集格式无效',
+        });
+      }
+      if (nextValue !== config.proxyDebugCaptureStreamChunks) {
+        changedLabels.push('代理调试流式分片采集');
+      }
+      config.proxyDebugCaptureStreamChunks = nextValue;
+      upsertSetting('proxy_debug_capture_stream_chunks', config.proxyDebugCaptureStreamChunks);
+    }
+
+    if (body.proxyDebugTargetSessionId !== undefined) {
+      const nextValue = String(body.proxyDebugTargetSessionId || '').trim();
+      if (nextValue !== config.proxyDebugTargetSessionId) {
+        changedLabels.push('代理调试目标会话');
+      }
+      config.proxyDebugTargetSessionId = nextValue;
+      upsertSetting('proxy_debug_target_session_id', config.proxyDebugTargetSessionId);
+    }
+
+    if (body.proxyDebugTargetClientKind !== undefined) {
+      const nextValue = String(body.proxyDebugTargetClientKind || '').trim();
+      if (nextValue !== config.proxyDebugTargetClientKind) {
+        changedLabels.push('代理调试目标客户端');
+      }
+      config.proxyDebugTargetClientKind = nextValue;
+      upsertSetting('proxy_debug_target_client_kind', config.proxyDebugTargetClientKind);
+    }
+
+    if (body.proxyDebugTargetModel !== undefined) {
+      const nextValue = String(body.proxyDebugTargetModel || '').trim();
+      if (nextValue !== config.proxyDebugTargetModel) {
+        changedLabels.push('代理调试目标模型');
+      }
+      config.proxyDebugTargetModel = nextValue;
+      upsertSetting('proxy_debug_target_model', config.proxyDebugTargetModel);
+    }
+
+    if (body.proxyDebugRetentionHours !== undefined) {
+      const retentionHours = Number(body.proxyDebugRetentionHours);
+      if (!Number.isFinite(retentionHours) || retentionHours < 1) {
+        return reply.code(400).send({ success: false, message: '代理调试保留时长必须是大于等于 1 的整数小时' });
+      }
+      const nextValue = Math.trunc(retentionHours);
+      if (nextValue !== config.proxyDebugRetentionHours) {
+        changedLabels.push(`代理调试保留时长（${config.proxyDebugRetentionHours}h -> ${nextValue}h）`);
+      }
+      config.proxyDebugRetentionHours = nextValue;
+      upsertSetting('proxy_debug_retention_hours', config.proxyDebugRetentionHours);
+    }
+
+    if (body.proxyDebugMaxBodyBytes !== undefined) {
+      const maxBodyBytes = Number(body.proxyDebugMaxBodyBytes);
+      if (!Number.isFinite(maxBodyBytes) || maxBodyBytes < 1024) {
+        return reply.code(400).send({ success: false, message: '代理调试抓取体积上限必须是大于等于 1024 的整数字节' });
+      }
+      const nextValue = Math.trunc(maxBodyBytes);
+      if (nextValue !== config.proxyDebugMaxBodyBytes) {
+        changedLabels.push(`代理调试抓取体积上限（${config.proxyDebugMaxBodyBytes}B -> ${nextValue}B）`);
+      }
+      config.proxyDebugMaxBodyBytes = nextValue;
+      upsertSetting('proxy_debug_max_body_bytes', config.proxyDebugMaxBodyBytes);
+    }
+
    if (body.proxyErrorKeywords !== undefined) {
      let nextKeywords: string[] = [];
      try {
@@ -0,0 +1,109 @@
+import Fastify, { type FastifyInstance } from 'fastify';
+import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest';
+import { mkdtempSync } from 'node:fs';
+import { join } from 'node:path';
+import { tmpdir } from 'node:os';
+
+type DbModule = typeof import('../../db/index.js');
+type StoreModule = typeof import('../../services/proxyDebugTraceStore.js');
+
+describe('stats proxy debug api', () => {
+  let app: FastifyInstance;
+  let db: DbModule['db'];
+  let schema: DbModule['schema'];
+  let store: StoreModule;
+  let dataDir = '';
+
+  beforeAll(async () => {
+    dataDir = mkdtempSync(join(tmpdir(), 'metapi-stats-proxy-debug-'));
+    process.env.DATA_DIR = dataDir;
+
+    await import('../../db/migrate.js');
+    const dbModule = await import('../../db/index.js');
+    const storeModule = await import('../../services/proxyDebugTraceStore.js');
+    const statsRoutesModule = await import('./stats.js');
+
+    db = dbModule.db;
+    schema = dbModule.schema;
+    store = storeModule;
+
+    app = Fastify();
+    await app.register(statsRoutesModule.statsRoutes);
+  });
+
+  beforeEach(async () => {
+    await db.delete(schema.proxyDebugAttempts).run();
+    await db.delete(schema.proxyDebugTraces).run();
+  });
+
+  afterAll(async () => {
+    await app.close();
+    const dbModule = await import('../../db/index.js');
+    await dbModule.closeDbConnections();
+    delete process.env.DATA_DIR;
+  });
+
+  it('lists and returns proxy debug traces with attempt details', async () => {
+    const trace = await store.createProxyDebugTrace({
+      downstreamPath: '/v1/chat/completions',
+      clientKind: 'codex',
+      sessionId: 'sess-9',
+      traceHint: 'trace-x',
+      requestedModel: 'gpt-4.1',
+      requestHeaders: { authorization: 'Bearer test' },
+      requestBody: { model: 'gpt-4.1' },
+    });
+    await store.insertProxyDebugAttempt({
+      traceId: trace.id,
+      attemptIndex: 0,
+      endpoint: 'chat',
+      requestPath: '/v1/chat/completions',
+      targetUrl: 'https://example.com/v1/chat/completions',
+      runtimeExecutor: 'default',
+      requestHeaders: { authorization: 'Bearer test' },
+      requestBody: { model: 'gpt-4.1' },
+      responseStatus: 200,
+      responseHeaders: { 'content-type': 'application/json' },
+      responseBody: { id: 'chatcmpl_123' },
+      rawErrorText: null,
+      recoverApplied: false,
+      downgradeDecision: false,
+      downgradeReason: null,
+      memoryWrite: { action: 'success', preferredEndpoint: 'chat' },
+    });
+    await store.finalizeProxyDebugTrace(trace.id, {
+      finalStatus: 'success',
+      finalHttpStatus: 200,
+      finalUpstreamPath: '/v1/chat/completions',
+      finalResponseHeaders: { 'content-type': 'application/json' },
+      finalResponseBody: { id: 'chatcmpl_123' },
+    });
+
+    const listResponse = await app.inject({
+      method: 'GET',
+      url: '/api/stats/proxy-debug/traces?limit=10',
+    });
+    expect(listResponse.statusCode).toBe(200);
+    const listBody = listResponse.json() as { items?: Array<{ id: number; finalStatus: string }> };
+    expect(listBody.items).toHaveLength(1);
+    expect(listBody.items?.[0]?.finalStatus).toBe('success');
+
+    const detailResponse = await app.inject({
+      method: 'GET',
+      url: `/api/stats/proxy-debug/traces/${trace.id}`,
+    });
+    expect(detailResponse.statusCode).toBe(200);
+    const detailBody = detailResponse.json() as {
+      trace?: { requestedModel?: string; sessionId?: string };
+      attempts?: Array<{ endpoint?: string; responseStatus?: number }>;
+    };
+    expect(detailBody.trace).toMatchObject({
+      requestedModel: 'gpt-4.1',
+      sessionId: 'sess-9',
+    });
+    expect(detailBody.attempts?.[0]).toMatchObject({
+      endpoint: 'chat',
+      responseStatus: 200,
+    });
+  });
+});
@@ -14,6 +14,10 @@ import {
  parseProxyLogBillingDetails,
  withProxyLogSelectFields,
 } from '../../services/proxyLogStore.js';
+import {
+  getProxyDebugTraceDetail,
+  listProxyDebugTraces,
+} from '../../services/proxyDebugTraceStore.js';
 import { parseProxyLogMessageMeta } from '../proxy/logPathMeta.js';
 import { requiresManagedAccountTokens } from '../../services/accountExtraConfig.js';
 import { ACCOUNT_TOKEN_VALUE_STATUS_READY } from '../../services/accountTokenService.js';
@@ -820,6 +824,26 @@ export async function statsRoutes(app: FastifyInstance) {
    return mapProxyLogRow(row, { includeBillingDetails: true });
  });

+  app.get<{ Querystring: { limit?: string } }>('/api/stats/proxy-debug/traces', async (request) => {
+    const limit = normalizeProxyLogPageSize(request.query.limit);
+    const items = await listProxyDebugTraces({ limit });
+    return { items };
+  });
+
+  app.get<{ Params: { id: string } }>('/api/stats/proxy-debug/traces/:id', async (request, reply) => {
+    const id = Number.parseInt(request.params.id, 10);
+    if (!Number.isFinite(id) || id <= 0) {
+      return reply.code(400).send({ message: 'proxy debug trace id is invalid' });
+    }
+
+    const detail = await getProxyDebugTraceDetail(id);
+    if (!detail) {
+      return reply.code(404).send({ message: 'proxy debug trace not found' });
+    }
+
+    return detail;
+  });
+
  // Models marketplace - refresh upstream models and aggregate.
  app.get<{ Querystring: { refresh?: string; includePricing?: string } }>('/api/models/marketplace', async (request) => {
    const refreshRequested = parseBooleanFlag(request.query.refresh);