[codex] add proxy debug tracing (#299)

* add proxy debug tracing

* fix proxy debug response header typing

* extend proxy debug tracing to side paths

* Refine proxy debug trace review fixes

* Stabilize proxy debug attempt indexing

* Tighten proxy debug trace accuracy

src/server/db/generated/mysql.bootstrap.sql

@@ -5,6 +5,8 @@ CREATE TABLE IF NOT EXISTS `checkin_logs` (`id` INT AUTO_INCREMENT NOT NULL PRIM
 CREATE TABLE IF NOT EXISTS `downstream_api_keys` (`id` INT AUTO_INCREMENT NOT NULL PRIMARY KEY, `name` TEXT NOT NULL, `key` TEXT NOT NULL, `description` TEXT, `enabled` BOOLEAN DEFAULT true, `expires_at` VARCHAR(191), `max_cost` DOUBLE, `used_cost` DOUBLE DEFAULT 0, `max_requests` INT, `used_requests` INT DEFAULT 0, `supported_models` JSON, `allowed_route_ids` JSON, `site_weight_multipliers` JSON, `last_used_at` VARCHAR(191), `created_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), '%Y-%m-%d %H:%i:%s')), `updated_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), '%Y-%m-%d %H:%i:%s')), `group_name` TEXT, `tags` TEXT);
 CREATE TABLE IF NOT EXISTS `events` (`id` INT AUTO_INCREMENT NOT NULL PRIMARY KEY, `type` TEXT NOT NULL, `title` TEXT NOT NULL, `message` TEXT, `level` VARCHAR(191) NOT NULL DEFAULT 'info', `read` BOOLEAN DEFAULT false, `related_id` INT, `related_type` TEXT, `created_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), '%Y-%m-%d %H:%i:%s')));
 CREATE TABLE IF NOT EXISTS `model_availability` (`id` INT AUTO_INCREMENT NOT NULL PRIMARY KEY, `account_id` INT NOT NULL, `model_name` TEXT NOT NULL, `available` BOOLEAN, `latency_ms` INT, `checked_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), '%Y-%m-%d %H:%i:%s')), `is_manual` BOOLEAN DEFAULT false, FOREIGN KEY (`account_id`) REFERENCES `accounts`(`id`) ON DELETE CASCADE);
+CREATE TABLE IF NOT EXISTS `proxy_debug_traces` (`id` INT AUTO_INCREMENT NOT NULL PRIMARY KEY, `downstream_path` TEXT NOT NULL, `client_kind` TEXT, `session_id` TEXT, `trace_hint` TEXT, `requested_model` TEXT, `downstream_api_key_id` INT, `request_headers_json` JSON, `request_body_json` JSON, `sticky_session_key` TEXT, `sticky_hit_channel_id` INT, `selected_channel_id` INT, `selected_route_id` INT, `selected_account_id` INT, `selected_site_id` INT, `selected_site_platform` TEXT, `endpoint_candidates_json` JSON, `endpoint_runtime_state_json` JSON, `decision_summary_json` JSON, `final_status` TEXT, `final_http_status` INT, `final_upstream_path` TEXT, `final_response_headers_json` JSON, `final_response_body_json` JSON, `created_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), '%Y-%m-%d %H:%i:%s')), `updated_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), '%Y-%m-%d %H:%i:%s')));
+CREATE TABLE IF NOT EXISTS `proxy_debug_attempts` (`id` INT AUTO_INCREMENT NOT NULL PRIMARY KEY, `trace_id` INT NOT NULL, `attempt_index` INT NOT NULL, `endpoint` TEXT NOT NULL, `request_path` TEXT NOT NULL, `target_url` TEXT NOT NULL, `runtime_executor` TEXT, `request_headers_json` JSON, `request_body_json` JSON, `response_status` INT, `response_headers_json` JSON, `response_body_json` JSON, `raw_error_text` TEXT, `recover_applied` BOOLEAN DEFAULT false, `downgrade_decision` BOOLEAN DEFAULT false, `downgrade_reason` TEXT, `memory_write_json` JSON, `created_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), '%Y-%m-%d %H:%i:%s')), FOREIGN KEY (`trace_id`) REFERENCES `proxy_debug_traces`(`id`) ON DELETE CASCADE);
 CREATE TABLE IF NOT EXISTS `proxy_files` (`id` INT AUTO_INCREMENT NOT NULL PRIMARY KEY, `public_id` TEXT NOT NULL, `owner_type` TEXT NOT NULL, `owner_id` TEXT NOT NULL, `filename` TEXT NOT NULL, `mime_type` TEXT NOT NULL, `purpose` TEXT, `byte_size` INT NOT NULL, `sha256` TEXT NOT NULL, `content_base64` TEXT NOT NULL, `created_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), '%Y-%m-%d %H:%i:%s')), `updated_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), '%Y-%m-%d %H:%i:%s')), `deleted_at` VARCHAR(191));
 CREATE TABLE IF NOT EXISTS `proxy_logs` (`id` INT AUTO_INCREMENT NOT NULL PRIMARY KEY, `route_id` INT, `channel_id` INT, `account_id` INT, `model_requested` TEXT, `model_actual` TEXT, `status` TEXT, `http_status` INT, `latency_ms` INT, `prompt_tokens` INT, `completion_tokens` INT, `total_tokens` INT, `estimated_cost` DOUBLE, `error_message` TEXT, `retry_count` INT DEFAULT 0, `created_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), '%Y-%m-%d %H:%i:%s')), `billing_details` JSON, `downstream_api_key_id` INT, `client_family` TEXT, `client_app_id` TEXT, `client_app_name` TEXT, `client_confidence` TEXT);
 CREATE TABLE IF NOT EXISTS `proxy_video_tasks` (`id` INT AUTO_INCREMENT NOT NULL PRIMARY KEY, `public_id` TEXT NOT NULL, `upstream_video_id` TEXT NOT NULL, `site_url` TEXT NOT NULL, `token_value` TEXT NOT NULL, `requested_model` TEXT, `actual_model` TEXT, `channel_id` INT, `account_id` INT, `status_snapshot` JSON, `upstream_response_meta` JSON, `last_upstream_status` INT, `last_polled_at` VARCHAR(191), `created_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), '%Y-%m-%d %H:%i:%s')), `updated_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), '%Y-%m-%d %H:%i:%s')));
@@ -17,6 +19,7 @@ CREATE TABLE IF NOT EXISTS `site_disabled_models` (`id` INT AUTO_INCREMENT NOT N
 CREATE TABLE IF NOT EXISTS `token_model_availability` (`id` INT AUTO_INCREMENT NOT NULL PRIMARY KEY, `token_id` INT NOT NULL, `model_name` TEXT NOT NULL, `available` BOOLEAN, `latency_ms` INT, `checked_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), '%Y-%m-%d %H:%i:%s')), FOREIGN KEY (`token_id`) REFERENCES `account_tokens`(`id`) ON DELETE CASCADE);
 CREATE UNIQUE INDEX `downstream_api_keys_key_unique` ON `downstream_api_keys` (`key`(191));
 CREATE UNIQUE INDEX `model_availability_account_model_unique` ON `model_availability` (`account_id`, `model_name`(191));
+CREATE UNIQUE INDEX `proxy_debug_attempts_trace_attempt_unique` ON `proxy_debug_attempts` (`trace_id`, `attempt_index`);
 CREATE UNIQUE INDEX `proxy_files_public_id_unique` ON `proxy_files` (`public_id`(191));
 CREATE UNIQUE INDEX `proxy_video_tasks_public_id_unique` ON `proxy_video_tasks` (`public_id`(191));
 CREATE UNIQUE INDEX `route_group_sources_group_source_unique` ON `route_group_sources` (`group_route_id`, `source_route_id`);
@@ -43,6 +46,11 @@ CREATE INDEX `events_read_created_at_idx` ON `events` (`read`, `created_at`);
 CREATE INDEX `events_type_created_at_idx` ON `events` (`type`(191), `created_at`);
 CREATE INDEX `model_availability_account_available_idx` ON `model_availability` (`account_id`, `available`);
 CREATE INDEX `model_availability_model_name_idx` ON `model_availability` (`model_name`(191));
+CREATE INDEX `proxy_debug_attempts_trace_created_at_idx` ON `proxy_debug_attempts` (`trace_id`, `created_at`);
+CREATE INDEX `proxy_debug_traces_created_at_idx` ON `proxy_debug_traces` (`created_at`);
+CREATE INDEX `proxy_debug_traces_final_status_created_at_idx` ON `proxy_debug_traces` (`final_status`(191), `created_at`);
+CREATE INDEX `proxy_debug_traces_model_created_at_idx` ON `proxy_debug_traces` (`requested_model`(191), `created_at`);
+CREATE INDEX `proxy_debug_traces_session_created_at_idx` ON `proxy_debug_traces` (`session_id`(191), `created_at`);
 CREATE INDEX `proxy_files_owner_lookup_idx` ON `proxy_files` (`owner_type`(191), `owner_id`(191), `deleted_at`);
 CREATE INDEX `proxy_logs_account_created_at_idx` ON `proxy_logs` (`account_id`, `created_at`);
 CREATE INDEX `proxy_logs_client_app_id_created_at_idx` ON `proxy_logs` (`client_app_id`(191), `created_at`);

src/server/db/generated/mysql.upgrade.sql

@@ -1 +1,8 @@
--- no schema changes detected for mysql
+CREATE TABLE IF NOT EXISTS `proxy_debug_traces` (`id` INT AUTO_INCREMENT NOT NULL PRIMARY KEY, `downstream_path` TEXT NOT NULL, `client_kind` TEXT, `session_id` TEXT, `trace_hint` TEXT, `requested_model` TEXT, `downstream_api_key_id` INT, `request_headers_json` JSON, `request_body_json` JSON, `sticky_session_key` TEXT, `sticky_hit_channel_id` INT, `selected_channel_id` INT, `selected_route_id` INT, `selected_account_id` INT, `selected_site_id` INT, `selected_site_platform` TEXT, `endpoint_candidates_json` JSON, `endpoint_runtime_state_json` JSON, `decision_summary_json` JSON, `final_status` TEXT, `final_http_status` INT, `final_upstream_path` TEXT, `final_response_headers_json` JSON, `final_response_body_json` JSON, `created_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), '%Y-%m-%d %H:%i:%s')), `updated_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), '%Y-%m-%d %H:%i:%s')));
+CREATE TABLE IF NOT EXISTS `proxy_debug_attempts` (`id` INT AUTO_INCREMENT NOT NULL PRIMARY KEY, `trace_id` INT NOT NULL, `attempt_index` INT NOT NULL, `endpoint` TEXT NOT NULL, `request_path` TEXT NOT NULL, `target_url` TEXT NOT NULL, `runtime_executor` TEXT, `request_headers_json` JSON, `request_body_json` JSON, `response_status` INT, `response_headers_json` JSON, `response_body_json` JSON, `raw_error_text` TEXT, `recover_applied` BOOLEAN DEFAULT false, `downgrade_decision` BOOLEAN DEFAULT false, `downgrade_reason` TEXT, `memory_write_json` JSON, `created_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), '%Y-%m-%d %H:%i:%s')), FOREIGN KEY (`trace_id`) REFERENCES `proxy_debug_traces`(`id`) ON DELETE CASCADE);
+CREATE UNIQUE INDEX `proxy_debug_attempts_trace_attempt_unique` ON `proxy_debug_attempts` (`trace_id`, `attempt_index`);
+CREATE INDEX `proxy_debug_attempts_trace_created_at_idx` ON `proxy_debug_attempts` (`trace_id`, `created_at`);
+CREATE INDEX `proxy_debug_traces_created_at_idx` ON `proxy_debug_traces` (`created_at`);
+CREATE INDEX `proxy_debug_traces_final_status_created_at_idx` ON `proxy_debug_traces` (`final_status`(191), `created_at`);
+CREATE INDEX `proxy_debug_traces_model_created_at_idx` ON `proxy_debug_traces` (`requested_model`(191), `created_at`);
+CREATE INDEX `proxy_debug_traces_session_created_at_idx` ON `proxy_debug_traces` (`session_id`(191), `created_at`);

src/server/db/generated/postgres.bootstrap.sql

@@ -5,6 +5,8 @@ CREATE TABLE IF NOT EXISTS "checkin_logs" ("id" INTEGER GENERATED BY DEFAULT AS
 CREATE TABLE IF NOT EXISTS "downstream_api_keys" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL PRIMARY KEY, "name" TEXT NOT NULL, "key" TEXT NOT NULL, "description" TEXT, "enabled" BOOLEAN DEFAULT true, "expires_at" TEXT, "max_cost" DOUBLE PRECISION, "used_cost" DOUBLE PRECISION DEFAULT 0, "max_requests" INTEGER, "used_requests" INTEGER DEFAULT 0, "supported_models" JSONB, "allowed_route_ids" JSONB, "site_weight_multipliers" JSONB, "last_used_at" TEXT, "created_at" TEXT DEFAULT to_char(timezone('UTC', CURRENT_TIMESTAMP), 'YYYY-MM-DD HH24:MI:SS'), "updated_at" TEXT DEFAULT to_char(timezone('UTC', CURRENT_TIMESTAMP), 'YYYY-MM-DD HH24:MI:SS'), "group_name" TEXT, "tags" TEXT);
 CREATE TABLE IF NOT EXISTS "events" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL PRIMARY KEY, "type" TEXT NOT NULL, "title" TEXT NOT NULL, "message" TEXT, "level" TEXT NOT NULL DEFAULT 'info', "read" BOOLEAN DEFAULT false, "related_id" INTEGER, "related_type" TEXT, "created_at" TEXT DEFAULT to_char(timezone('UTC', CURRENT_TIMESTAMP), 'YYYY-MM-DD HH24:MI:SS'));
 CREATE TABLE IF NOT EXISTS "model_availability" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL PRIMARY KEY, "account_id" INTEGER NOT NULL, "model_name" TEXT NOT NULL, "available" BOOLEAN, "latency_ms" INTEGER, "checked_at" TEXT DEFAULT to_char(timezone('UTC', CURRENT_TIMESTAMP), 'YYYY-MM-DD HH24:MI:SS'), "is_manual" BOOLEAN DEFAULT false, FOREIGN KEY ("account_id") REFERENCES "accounts"("id") ON DELETE CASCADE);
+CREATE TABLE IF NOT EXISTS "proxy_debug_traces" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL PRIMARY KEY, "downstream_path" TEXT NOT NULL, "client_kind" TEXT, "session_id" TEXT, "trace_hint" TEXT, "requested_model" TEXT, "downstream_api_key_id" INTEGER, "request_headers_json" JSONB, "request_body_json" JSONB, "sticky_session_key" TEXT, "sticky_hit_channel_id" INTEGER, "selected_channel_id" INTEGER, "selected_route_id" INTEGER, "selected_account_id" INTEGER, "selected_site_id" INTEGER, "selected_site_platform" TEXT, "endpoint_candidates_json" JSONB, "endpoint_runtime_state_json" JSONB, "decision_summary_json" JSONB, "final_status" TEXT, "final_http_status" INTEGER, "final_upstream_path" TEXT, "final_response_headers_json" JSONB, "final_response_body_json" JSONB, "created_at" TEXT DEFAULT to_char(timezone('UTC', CURRENT_TIMESTAMP), 'YYYY-MM-DD HH24:MI:SS'), "updated_at" TEXT DEFAULT to_char(timezone('UTC', CURRENT_TIMESTAMP), 'YYYY-MM-DD HH24:MI:SS'));
+CREATE TABLE IF NOT EXISTS "proxy_debug_attempts" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL PRIMARY KEY, "trace_id" INTEGER NOT NULL, "attempt_index" INTEGER NOT NULL, "endpoint" TEXT NOT NULL, "request_path" TEXT NOT NULL, "target_url" TEXT NOT NULL, "runtime_executor" TEXT, "request_headers_json" JSONB, "request_body_json" JSONB, "response_status" INTEGER, "response_headers_json" JSONB, "response_body_json" JSONB, "raw_error_text" TEXT, "recover_applied" BOOLEAN DEFAULT false, "downgrade_decision" BOOLEAN DEFAULT false, "downgrade_reason" TEXT, "memory_write_json" JSONB, "created_at" TEXT DEFAULT to_char(timezone('UTC', CURRENT_TIMESTAMP), 'YYYY-MM-DD HH24:MI:SS'), FOREIGN KEY ("trace_id") REFERENCES "proxy_debug_traces"("id") ON DELETE CASCADE);
 CREATE TABLE IF NOT EXISTS "proxy_files" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL PRIMARY KEY, "public_id" TEXT NOT NULL, "owner_type" TEXT NOT NULL, "owner_id" TEXT NOT NULL, "filename" TEXT NOT NULL, "mime_type" TEXT NOT NULL, "purpose" TEXT, "byte_size" INTEGER NOT NULL, "sha256" TEXT NOT NULL, "content_base64" TEXT NOT NULL, "created_at" TEXT DEFAULT to_char(timezone('UTC', CURRENT_TIMESTAMP), 'YYYY-MM-DD HH24:MI:SS'), "updated_at" TEXT DEFAULT to_char(timezone('UTC', CURRENT_TIMESTAMP), 'YYYY-MM-DD HH24:MI:SS'), "deleted_at" TEXT);
 CREATE TABLE IF NOT EXISTS "proxy_logs" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL PRIMARY KEY, "route_id" INTEGER, "channel_id" INTEGER, "account_id" INTEGER, "model_requested" TEXT, "model_actual" TEXT, "status" TEXT, "http_status" INTEGER, "latency_ms" INTEGER, "prompt_tokens" INTEGER, "completion_tokens" INTEGER, "total_tokens" INTEGER, "estimated_cost" DOUBLE PRECISION, "error_message" TEXT, "retry_count" INTEGER DEFAULT 0, "created_at" TEXT DEFAULT to_char(timezone('UTC', CURRENT_TIMESTAMP), 'YYYY-MM-DD HH24:MI:SS'), "billing_details" JSONB, "downstream_api_key_id" INTEGER, "client_family" TEXT, "client_app_id" TEXT, "client_app_name" TEXT, "client_confidence" TEXT);
 CREATE TABLE IF NOT EXISTS "proxy_video_tasks" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL PRIMARY KEY, "public_id" TEXT NOT NULL, "upstream_video_id" TEXT NOT NULL, "site_url" TEXT NOT NULL, "token_value" TEXT NOT NULL, "requested_model" TEXT, "actual_model" TEXT, "channel_id" INTEGER, "account_id" INTEGER, "status_snapshot" JSONB, "upstream_response_meta" JSONB, "last_upstream_status" INTEGER, "last_polled_at" TEXT, "created_at" TEXT DEFAULT to_char(timezone('UTC', CURRENT_TIMESTAMP), 'YYYY-MM-DD HH24:MI:SS'), "updated_at" TEXT DEFAULT to_char(timezone('UTC', CURRENT_TIMESTAMP), 'YYYY-MM-DD HH24:MI:SS'));
@@ -17,6 +19,7 @@ CREATE TABLE IF NOT EXISTS "site_disabled_models" ("id" INTEGER GENERATED BY DEF
 CREATE TABLE IF NOT EXISTS "token_model_availability" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL PRIMARY KEY, "token_id" INTEGER NOT NULL, "model_name" TEXT NOT NULL, "available" BOOLEAN, "latency_ms" INTEGER, "checked_at" TEXT DEFAULT to_char(timezone('UTC', CURRENT_TIMESTAMP), 'YYYY-MM-DD HH24:MI:SS'), FOREIGN KEY ("token_id") REFERENCES "account_tokens"("id") ON DELETE CASCADE);
 CREATE UNIQUE INDEX "downstream_api_keys_key_unique" ON "downstream_api_keys" ("key");
 CREATE UNIQUE INDEX "model_availability_account_model_unique" ON "model_availability" ("account_id", "model_name");
+CREATE UNIQUE INDEX "proxy_debug_attempts_trace_attempt_unique" ON "proxy_debug_attempts" ("trace_id", "attempt_index");
 CREATE UNIQUE INDEX "proxy_files_public_id_unique" ON "proxy_files" ("public_id");
 CREATE UNIQUE INDEX "proxy_video_tasks_public_id_unique" ON "proxy_video_tasks" ("public_id");
 CREATE UNIQUE INDEX "route_group_sources_group_source_unique" ON "route_group_sources" ("group_route_id", "source_route_id");
@@ -43,6 +46,11 @@ CREATE INDEX "events_read_created_at_idx" ON "events" ("read", "created_at");
 CREATE INDEX "events_type_created_at_idx" ON "events" ("type", "created_at");
 CREATE INDEX "model_availability_account_available_idx" ON "model_availability" ("account_id", "available");
 CREATE INDEX "model_availability_model_name_idx" ON "model_availability" ("model_name");
+CREATE INDEX "proxy_debug_attempts_trace_created_at_idx" ON "proxy_debug_attempts" ("trace_id", "created_at");
+CREATE INDEX "proxy_debug_traces_created_at_idx" ON "proxy_debug_traces" ("created_at");
+CREATE INDEX "proxy_debug_traces_final_status_created_at_idx" ON "proxy_debug_traces" ("final_status", "created_at");
+CREATE INDEX "proxy_debug_traces_model_created_at_idx" ON "proxy_debug_traces" ("requested_model", "created_at");
+CREATE INDEX "proxy_debug_traces_session_created_at_idx" ON "proxy_debug_traces" ("session_id", "created_at");
 CREATE INDEX "proxy_files_owner_lookup_idx" ON "proxy_files" ("owner_type", "owner_id", "deleted_at");
 CREATE INDEX "proxy_logs_account_created_at_idx" ON "proxy_logs" ("account_id", "created_at");
 CREATE INDEX "proxy_logs_client_app_id_created_at_idx" ON "proxy_logs" ("client_app_id", "created_at");

src/server/db/generated/postgres.upgrade.sql

@@ -1 +1,8 @@
--- no schema changes detected for postgres
+CREATE TABLE IF NOT EXISTS "proxy_debug_traces" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL PRIMARY KEY, "downstream_path" TEXT NOT NULL, "client_kind" TEXT, "session_id" TEXT, "trace_hint" TEXT, "requested_model" TEXT, "downstream_api_key_id" INTEGER, "request_headers_json" JSONB, "request_body_json" JSONB, "sticky_session_key" TEXT, "sticky_hit_channel_id" INTEGER, "selected_channel_id" INTEGER, "selected_route_id" INTEGER, "selected_account_id" INTEGER, "selected_site_id" INTEGER, "selected_site_platform" TEXT, "endpoint_candidates_json" JSONB, "endpoint_runtime_state_json" JSONB, "decision_summary_json" JSONB, "final_status" TEXT, "final_http_status" INTEGER, "final_upstream_path" TEXT, "final_response_headers_json" JSONB, "final_response_body_json" JSONB, "created_at" TEXT DEFAULT to_char(timezone('UTC', CURRENT_TIMESTAMP), 'YYYY-MM-DD HH24:MI:SS'), "updated_at" TEXT DEFAULT to_char(timezone('UTC', CURRENT_TIMESTAMP), 'YYYY-MM-DD HH24:MI:SS'));
+CREATE TABLE IF NOT EXISTS "proxy_debug_attempts" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL PRIMARY KEY, "trace_id" INTEGER NOT NULL, "attempt_index" INTEGER NOT NULL, "endpoint" TEXT NOT NULL, "request_path" TEXT NOT NULL, "target_url" TEXT NOT NULL, "runtime_executor" TEXT, "request_headers_json" JSONB, "request_body_json" JSONB, "response_status" INTEGER, "response_headers_json" JSONB, "response_body_json" JSONB, "raw_error_text" TEXT, "recover_applied" BOOLEAN DEFAULT false, "downgrade_decision" BOOLEAN DEFAULT false, "downgrade_reason" TEXT, "memory_write_json" JSONB, "created_at" TEXT DEFAULT to_char(timezone('UTC', CURRENT_TIMESTAMP), 'YYYY-MM-DD HH24:MI:SS'), FOREIGN KEY ("trace_id") REFERENCES "proxy_debug_traces"("id") ON DELETE CASCADE);
+CREATE UNIQUE INDEX "proxy_debug_attempts_trace_attempt_unique" ON "proxy_debug_attempts" ("trace_id", "attempt_index");
+CREATE INDEX "proxy_debug_attempts_trace_created_at_idx" ON "proxy_debug_attempts" ("trace_id", "created_at");
+CREATE INDEX "proxy_debug_traces_created_at_idx" ON "proxy_debug_traces" ("created_at");
+CREATE INDEX "proxy_debug_traces_final_status_created_at_idx" ON "proxy_debug_traces" ("final_status", "created_at");
+CREATE INDEX "proxy_debug_traces_model_created_at_idx" ON "proxy_debug_traces" ("requested_model", "created_at");
+CREATE INDEX "proxy_debug_traces_session_created_at_idx" ON "proxy_debug_traces" ("session_id", "created_at");

src/server/db/generated/schemaContract.json

@@ -462,6 +462,278 @@
         }
       }
     },
+    "proxy_debug_attempts": {
+      "columns": {
+        "id": {
+          "logicalType": "integer",
+          "notNull": true,
+          "defaultValue": null,
+          "primaryKey": true
+        },
+        "trace_id": {
+          "logicalType": "integer",
+          "notNull": true,
+          "defaultValue": null,
+          "primaryKey": false
+        },
+        "attempt_index": {
+          "logicalType": "integer",
+          "notNull": true,
+          "defaultValue": null,
+          "primaryKey": false
+        },
+        "endpoint": {
+          "logicalType": "text",
+          "notNull": true,
+          "defaultValue": null,
+          "primaryKey": false
+        },
+        "request_path": {
+          "logicalType": "text",
+          "notNull": true,
+          "defaultValue": null,
+          "primaryKey": false
+        },
+        "target_url": {
+          "logicalType": "text",
+          "notNull": true,
+          "defaultValue": null,
+          "primaryKey": false
+        },
+        "runtime_executor": {
+          "logicalType": "text",
+          "notNull": false,
+          "defaultValue": null,
+          "primaryKey": false
+        },
+        "request_headers_json": {
+          "logicalType": "json",
+          "notNull": false,
+          "defaultValue": null,
+          "primaryKey": false
+        },
+        "request_body_json": {
+          "logicalType": "json",
+          "notNull": false,
+          "defaultValue": null,
+          "primaryKey": false
+        },
+        "response_status": {
+          "logicalType": "integer",
+          "notNull": false,
+          "defaultValue": null,
+          "primaryKey": false
+        },
+        "response_headers_json": {
+          "logicalType": "json",
+          "notNull": false,
+          "defaultValue": null,
+          "primaryKey": false
+        },
+        "response_body_json": {
+          "logicalType": "json",
+          "notNull": false,
+          "defaultValue": null,
+          "primaryKey": false
+        },
+        "raw_error_text": {
+          "logicalType": "text",
+          "notNull": false,
+          "defaultValue": null,
+          "primaryKey": false
+        },
+        "recover_applied": {
+          "logicalType": "boolean",
+          "notNull": false,
+          "defaultValue": "false",
+          "primaryKey": false
+        },
+        "downgrade_decision": {
+          "logicalType": "boolean",
+          "notNull": false,
+          "defaultValue": "false",
+          "primaryKey": false
+        },
+        "downgrade_reason": {
+          "logicalType": "text",
+          "notNull": false,
+          "defaultValue": null,
+          "primaryKey": false
+        },
+        "memory_write_json": {
+          "logicalType": "json",
+          "notNull": false,
+          "defaultValue": null,
+          "primaryKey": false
+        },
+        "created_at": {
+          "logicalType": "datetime",
+          "notNull": false,
+          "defaultValue": "datetime('now')",
+          "primaryKey": false
+        }
+      }
+    },
+    "proxy_debug_traces": {
+      "columns": {
+        "id": {
+          "logicalType": "integer",
+          "notNull": true,
+          "defaultValue": null,
+          "primaryKey": true
+        },
+        "downstream_path": {
+          "logicalType": "text",
+          "notNull": true,
+          "defaultValue": null,
+          "primaryKey": false
+        },
+        "client_kind": {
+          "logicalType": "text",
+          "notNull": false,
+          "defaultValue": null,
+          "primaryKey": false
+        },
+        "session_id": {
+          "logicalType": "text",
+          "notNull": false,
+          "defaultValue": null,
+          "primaryKey": false
+        },
+        "trace_hint": {
+          "logicalType": "text",
+          "notNull": false,
+          "defaultValue": null,
+          "primaryKey": false
+        },
+        "requested_model": {
+          "logicalType": "text",
+          "notNull": false,
+          "defaultValue": null,
+          "primaryKey": false
+        },
+        "downstream_api_key_id": {
+          "logicalType": "integer",
+          "notNull": false,
+          "defaultValue": null,
+          "primaryKey": false
+        },
+        "request_headers_json": {
+          "logicalType": "json",
+          "notNull": false,
+          "defaultValue": null,
+          "primaryKey": false
+        },
+        "request_body_json": {
+          "logicalType": "json",
+          "notNull": false,
+          "defaultValue": null,
+          "primaryKey": false
+        },
+        "sticky_session_key": {
+          "logicalType": "text",
+          "notNull": false,
+          "defaultValue": null,
+          "primaryKey": false
+        },
+        "sticky_hit_channel_id": {
+          "logicalType": "integer",
+          "notNull": false,
+          "defaultValue": null,
+          "primaryKey": false
+        },
+        "selected_channel_id": {
+          "logicalType": "integer",
+          "notNull": false,
+          "defaultValue": null,
+          "primaryKey": false
+        },
+        "selected_route_id": {
+          "logicalType": "integer",
+          "notNull": false,
+          "defaultValue": null,
+          "primaryKey": false
+        },
+        "selected_account_id": {
+          "logicalType": "integer",
+          "notNull": false,
+          "defaultValue": null,
+          "primaryKey": false
+        },
+        "selected_site_id": {
+          "logicalType": "integer",
+          "notNull": false,
+          "defaultValue": null,
+          "primaryKey": false
+        },
+        "selected_site_platform": {
+          "logicalType": "text",
+          "notNull": false,
+          "defaultValue": null,
+          "primaryKey": false
+        },
+        "endpoint_candidates_json": {
+          "logicalType": "json",
+          "notNull": false,
+          "defaultValue": null,
+          "primaryKey": false
+        },
+        "endpoint_runtime_state_json": {
+          "logicalType": "json",
+          "notNull": false,
+          "defaultValue": null,
+          "primaryKey": false
+        },
+        "decision_summary_json": {
+          "logicalType": "json",
+          "notNull": false,
+          "defaultValue": null,
+          "primaryKey": false
+        },
+        "final_status": {
+          "logicalType": "text",
+          "notNull": false,
+          "defaultValue": null,
+          "primaryKey": false
+        },
+        "final_http_status": {
+          "logicalType": "integer",
+          "notNull": false,
+          "defaultValue": null,
+          "primaryKey": false
+        },
+        "final_upstream_path": {
+          "logicalType": "text",
+          "notNull": false,
+          "defaultValue": null,
+          "primaryKey": false
+        },
+        "final_response_headers_json": {
+          "logicalType": "json",
+          "notNull": false,
+          "defaultValue": null,
+          "primaryKey": false
+        },
+        "final_response_body_json": {
+          "logicalType": "json",
+          "notNull": false,
+          "defaultValue": null,
+          "primaryKey": false
+        },
+        "created_at": {
+          "logicalType": "datetime",
+          "notNull": false,
+          "defaultValue": "datetime('now')",
+          "primaryKey": false
+        },
+        "updated_at": {
+          "logicalType": "datetime",
+          "notNull": false,
+          "defaultValue": "datetime('now')",
+          "primaryKey": false
+        }
+      }
+    },
     "proxy_files": {
       "columns": {
         "id": {
@@ -1453,6 +1725,59 @@
       ],
       "unique": false
     },
+    {
+      "name": "proxy_debug_attempts_trace_attempt_unique",
+      "table": "proxy_debug_attempts",
+      "columns": [
+        "trace_id",
+        "attempt_index"
+      ],
+      "unique": true
+    },
+    {
+      "name": "proxy_debug_attempts_trace_created_at_idx",
+      "table": "proxy_debug_attempts",
+      "columns": [
+        "trace_id",
+        "created_at"
+      ],
+      "unique": false
+    },
+    {
+      "name": "proxy_debug_traces_created_at_idx",
+      "table": "proxy_debug_traces",
+      "columns": [
+        "created_at"
+      ],
+      "unique": false
+    },
+    {
+      "name": "proxy_debug_traces_final_status_created_at_idx",
+      "table": "proxy_debug_traces",
+      "columns": [
+        "final_status",
+        "created_at"
+      ],
+      "unique": false
+    },
+    {
+      "name": "proxy_debug_traces_model_created_at_idx",
+      "table": "proxy_debug_traces",
+      "columns": [
+        "requested_model",
+        "created_at"
+      ],
+      "unique": false
+    },
+    {
+      "name": "proxy_debug_traces_session_created_at_idx",
+      "table": "proxy_debug_traces",
+      "columns": [
+        "session_id",
+        "created_at"
+      ],
+      "unique": false
+    },
     {
       "name": "proxy_files_owner_lookup_idx",
       "table": "proxy_files",
@@ -1743,6 +2068,14 @@
         "model_name"
       ]
     },
+    {
+      "name": "proxy_debug_attempts_trace_attempt_unique",
+      "table": "proxy_debug_attempts",
+      "columns": [
+        "trace_id",
+        "attempt_index"
+      ]
+    },
     {
       "name": "proxy_files_public_id_unique",
       "table": "proxy_files",
@@ -1843,6 +2176,17 @@
       ],
       "onDelete": "CASCADE"
     },
+    {
+      "table": "proxy_debug_attempts",
+      "columns": [
+        "trace_id"
+      ],
+      "referencedTable": "proxy_debug_traces",
+      "referencedColumns": [
+        "id"
+      ],
+      "onDelete": "CASCADE"
+    },
     {
       "table": "route_channels",
       "columns": [

src/server/db/index.ts

@@ -34,6 +34,8 @@ const TABLES_WITH_NUMERIC_ID = new Set([
   'route_group_sources',
   'route_channels',
   'proxy_logs',
+  'proxy_debug_traces',
+  'proxy_debug_attempts',
   'proxy_video_tasks',
   'proxy_files',
   'downstream_api_keys',

src/server/db/schema.ts

@@ -210,6 +210,64 @@ export const proxyLogs = sqliteTable('proxy_logs', {
   clientFamilyCreatedIdx: index('proxy_logs_client_family_created_at_idx').on(table.clientFamily, table.createdAt),
 }));
 
+export const proxyDebugTraces = sqliteTable('proxy_debug_traces', {
+  id: integer('id').primaryKey({ autoIncrement: true }),
+  downstreamPath: text('downstream_path').notNull(),
+  clientKind: text('client_kind'),
+  sessionId: text('session_id'),
+  traceHint: text('trace_hint'),
+  requestedModel: text('requested_model'),
+  downstreamApiKeyId: integer('downstream_api_key_id'),
+  requestHeadersJson: text('request_headers_json'),
+  requestBodyJson: text('request_body_json'),
+  stickySessionKey: text('sticky_session_key'),
+  stickyHitChannelId: integer('sticky_hit_channel_id'),
+  selectedChannelId: integer('selected_channel_id'),
+  selectedRouteId: integer('selected_route_id'),
+  selectedAccountId: integer('selected_account_id'),
+  selectedSiteId: integer('selected_site_id'),
+  selectedSitePlatform: text('selected_site_platform'),
+  endpointCandidatesJson: text('endpoint_candidates_json'),
+  endpointRuntimeStateJson: text('endpoint_runtime_state_json'),
+  decisionSummaryJson: text('decision_summary_json'),
+  finalStatus: text('final_status'),
+  finalHttpStatus: integer('final_http_status'),
+  finalUpstreamPath: text('final_upstream_path'),
+  finalResponseHeadersJson: text('final_response_headers_json'),
+  finalResponseBodyJson: text('final_response_body_json'),
+  createdAt: text('created_at').default(sql`(datetime('now'))`),
+  updatedAt: text('updated_at').default(sql`(datetime('now'))`),
+}, (table) => ({
+  createdAtIdx: index('proxy_debug_traces_created_at_idx').on(table.createdAt),
+  sessionCreatedIdx: index('proxy_debug_traces_session_created_at_idx').on(table.sessionId, table.createdAt),
+  modelCreatedIdx: index('proxy_debug_traces_model_created_at_idx').on(table.requestedModel, table.createdAt),
+  finalStatusCreatedIdx: index('proxy_debug_traces_final_status_created_at_idx').on(table.finalStatus, table.createdAt),
+}));
+
+export const proxyDebugAttempts = sqliteTable('proxy_debug_attempts', {
+  id: integer('id').primaryKey({ autoIncrement: true }),
+  traceId: integer('trace_id').notNull().references(() => proxyDebugTraces.id, { onDelete: 'cascade' }),
+  attemptIndex: integer('attempt_index').notNull(),
+  endpoint: text('endpoint').notNull(),
+  requestPath: text('request_path').notNull(),
+  targetUrl: text('target_url').notNull(),
+  runtimeExecutor: text('runtime_executor'),
+  requestHeadersJson: text('request_headers_json'),
+  requestBodyJson: text('request_body_json'),
+  responseStatus: integer('response_status'),
+  responseHeadersJson: text('response_headers_json'),
+  responseBodyJson: text('response_body_json'),
+  rawErrorText: text('raw_error_text'),
+  recoverApplied: integer('recover_applied', { mode: 'boolean' }).default(false),
+  downgradeDecision: integer('downgrade_decision', { mode: 'boolean' }).default(false),
+  downgradeReason: text('downgrade_reason'),
+  memoryWriteJson: text('memory_write_json'),
+  createdAt: text('created_at').default(sql`(datetime('now'))`),
+}, (table) => ({
+  traceAttemptUnique: uniqueIndex('proxy_debug_attempts_trace_attempt_unique').on(table.traceId, table.attemptIndex),
+  traceCreatedIdx: index('proxy_debug_attempts_trace_created_at_idx').on(table.traceId, table.createdAt),
+}));
+
 export const proxyVideoTasks = sqliteTable('proxy_video_tasks', {
   id: integer('id').primaryKey({ autoIncrement: true }),
   publicId: text('public_id').notNull(),