[codex] add proxy debug tracing (#299)
* add proxy debug tracing * fix proxy debug response header typing * extend proxy debug tracing to side paths * Refine proxy debug trace review fixes * Stabilize proxy debug attempt indexing * Tighten proxy debug trace accuracy
This commit is contained in:
Cita
@@ -0,0 +1,57 @@
|
||||
CREATE TABLE `proxy_debug_traces` (
|
||||
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||
`downstream_path` text NOT NULL,
|
||||
`client_kind` text,
|
||||
`session_id` text,
|
||||
`trace_hint` text,
|
||||
`requested_model` text,
|
||||
`downstream_api_key_id` integer,
|
||||
`request_headers_json` text,
|
||||
`request_body_json` text,
|
||||
`sticky_session_key` text,
|
||||
`sticky_hit_channel_id` integer,
|
||||
`selected_channel_id` integer,
|
||||
`selected_route_id` integer,
|
||||
`selected_account_id` integer,
|
||||
`selected_site_id` integer,
|
||||
`selected_site_platform` text,
|
||||
`endpoint_candidates_json` text,
|
||||
`endpoint_runtime_state_json` text,
|
||||
`decision_summary_json` text,
|
||||
`final_status` text,
|
||||
`final_http_status` integer,
|
||||
`final_upstream_path` text,
|
||||
`final_response_headers_json` text,
|
||||
`final_response_body_json` text,
|
||||
`created_at` text DEFAULT (datetime('now')),
|
||||
`updated_at` text DEFAULT (datetime('now'))
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE INDEX `proxy_debug_traces_created_at_idx` ON `proxy_debug_traces` (`created_at`);--> statement-breakpoint
|
||||
CREATE INDEX `proxy_debug_traces_session_created_at_idx` ON `proxy_debug_traces` (`session_id`,`created_at`);--> statement-breakpoint
|
||||
CREATE INDEX `proxy_debug_traces_model_created_at_idx` ON `proxy_debug_traces` (`requested_model`,`created_at`);--> statement-breakpoint
|
||||
CREATE INDEX `proxy_debug_traces_final_status_created_at_idx` ON `proxy_debug_traces` (`final_status`,`created_at`);--> statement-breakpoint
|
||||
CREATE TABLE `proxy_debug_attempts` (
|
||||
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||
`trace_id` integer NOT NULL,
|
||||
`attempt_index` integer NOT NULL,
|
||||
`endpoint` text NOT NULL,
|
||||
`request_path` text NOT NULL,
|
||||
`target_url` text NOT NULL,
|
||||
`runtime_executor` text,
|
||||
`request_headers_json` text,
|
||||
`request_body_json` text,
|
||||
`response_status` integer,
|
||||
`response_headers_json` text,
|
||||
`response_body_json` text,
|
||||
`raw_error_text` text,
|
||||
`recover_applied` integer DEFAULT false,
|
||||
`downgrade_decision` integer DEFAULT false,
|
||||
`downgrade_reason` text,
|
||||
`memory_write_json` text,
|
||||
`created_at` text DEFAULT (datetime('now')),
|
||||
FOREIGN KEY (`trace_id`) REFERENCES `proxy_debug_traces`(`id`) ON UPDATE no action ON DELETE cascade
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX `proxy_debug_attempts_trace_attempt_unique` ON `proxy_debug_attempts` (`trace_id`,`attempt_index`);--> statement-breakpoint
|
||||
CREATE INDEX `proxy_debug_attempts_trace_created_at_idx` ON `proxy_debug_attempts` (`trace_id`,`created_at`);
|
||||
File diff suppressed because it is too large
Load Diff
@@ -113,6 +113,13 @@
|
||||
"when": 1773979652606,
|
||||
"tag": "0016_proxy_logs_client_fields",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 16,
|
||||
"version": "6",
|
||||
"when": 1774692431588,
|
||||
"tag": "0017_proxy_debug_traces",
|
||||
"breakpoints": true
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -116,6 +116,15 @@ export function buildConfig(env: NodeJS.ProcessEnv) {
|
||||
proxySessionChannelLeaseTtlMs: Math.max(5_000, Math.trunc(parseNumber(env.PROXY_SESSION_CHANNEL_LEASE_TTL_MS, 90_000))),
|
||||
proxySessionChannelLeaseKeepaliveMs: Math.max(1_000, Math.trunc(parseNumber(env.PROXY_SESSION_CHANNEL_LEASE_KEEPALIVE_MS, 15_000))),
|
||||
codexUpstreamWebsocketEnabled: parseBoolean(env.CODEX_UPSTREAM_WEBSOCKET_ENABLED, false),
|
||||
proxyDebugTraceEnabled: parseBoolean(env.PROXY_DEBUG_TRACE_ENABLED, false),
|
||||
proxyDebugCaptureHeaders: parseBoolean(env.PROXY_DEBUG_CAPTURE_HEADERS, true),
|
||||
proxyDebugCaptureBodies: parseBoolean(env.PROXY_DEBUG_CAPTURE_BODIES, false),
|
||||
proxyDebugCaptureStreamChunks: parseBoolean(env.PROXY_DEBUG_CAPTURE_STREAM_CHUNKS, false),
|
||||
proxyDebugTargetSessionId: (env.PROXY_DEBUG_TARGET_SESSION_ID || '').trim(),
|
||||
proxyDebugTargetClientKind: (env.PROXY_DEBUG_TARGET_CLIENT_KIND || '').trim(),
|
||||
proxyDebugTargetModel: (env.PROXY_DEBUG_TARGET_MODEL || '').trim(),
|
||||
proxyDebugRetentionHours: Math.max(1, Math.trunc(parseNumber(env.PROXY_DEBUG_RETENTION_HOURS, 24))),
|
||||
proxyDebugMaxBodyBytes: Math.max(1024, Math.trunc(parseNumber(env.PROXY_DEBUG_MAX_BODY_BYTES, 262_144))),
|
||||
proxyLogRetentionDays: Math.max(0, Math.trunc(parseNumber(env.PROXY_LOG_RETENTION_DAYS, 30))),
|
||||
proxyLogRetentionPruneIntervalMinutes: Math.max(1, Math.trunc(parseNumber(env.PROXY_LOG_RETENTION_PRUNE_INTERVAL_MINUTES, 30))),
|
||||
proxyFileRetentionDays: Math.max(0, Math.trunc(parseNumber(env.PROXY_FILE_RETENTION_DAYS, 30))),
|
||||
|
||||
@@ -5,6 +5,8 @@ CREATE TABLE IF NOT EXISTS `checkin_logs` (`id` INT AUTO_INCREMENT NOT NULL PRIM
|
||||
CREATE TABLE IF NOT EXISTS `downstream_api_keys` (`id` INT AUTO_INCREMENT NOT NULL PRIMARY KEY, `name` TEXT NOT NULL, `key` TEXT NOT NULL, `description` TEXT, `enabled` BOOLEAN DEFAULT true, `expires_at` VARCHAR(191), `max_cost` DOUBLE, `used_cost` DOUBLE DEFAULT 0, `max_requests` INT, `used_requests` INT DEFAULT 0, `supported_models` JSON, `allowed_route_ids` JSON, `site_weight_multipliers` JSON, `last_used_at` VARCHAR(191), `created_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), '%Y-%m-%d %H:%i:%s')), `updated_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), '%Y-%m-%d %H:%i:%s')), `group_name` TEXT, `tags` TEXT);
|
||||
CREATE TABLE IF NOT EXISTS `events` (`id` INT AUTO_INCREMENT NOT NULL PRIMARY KEY, `type` TEXT NOT NULL, `title` TEXT NOT NULL, `message` TEXT, `level` VARCHAR(191) NOT NULL DEFAULT 'info', `read` BOOLEAN DEFAULT false, `related_id` INT, `related_type` TEXT, `created_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), '%Y-%m-%d %H:%i:%s')));
|
||||
CREATE TABLE IF NOT EXISTS `model_availability` (`id` INT AUTO_INCREMENT NOT NULL PRIMARY KEY, `account_id` INT NOT NULL, `model_name` TEXT NOT NULL, `available` BOOLEAN, `latency_ms` INT, `checked_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), '%Y-%m-%d %H:%i:%s')), `is_manual` BOOLEAN DEFAULT false, FOREIGN KEY (`account_id`) REFERENCES `accounts`(`id`) ON DELETE CASCADE);
|
||||
CREATE TABLE IF NOT EXISTS `proxy_debug_traces` (`id` INT AUTO_INCREMENT NOT NULL PRIMARY KEY, `downstream_path` TEXT NOT NULL, `client_kind` TEXT, `session_id` TEXT, `trace_hint` TEXT, `requested_model` TEXT, `downstream_api_key_id` INT, `request_headers_json` JSON, `request_body_json` JSON, `sticky_session_key` TEXT, `sticky_hit_channel_id` INT, `selected_channel_id` INT, `selected_route_id` INT, `selected_account_id` INT, `selected_site_id` INT, `selected_site_platform` TEXT, `endpoint_candidates_json` JSON, `endpoint_runtime_state_json` JSON, `decision_summary_json` JSON, `final_status` TEXT, `final_http_status` INT, `final_upstream_path` TEXT, `final_response_headers_json` JSON, `final_response_body_json` JSON, `created_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), '%Y-%m-%d %H:%i:%s')), `updated_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), '%Y-%m-%d %H:%i:%s')));
|
||||
CREATE TABLE IF NOT EXISTS `proxy_debug_attempts` (`id` INT AUTO_INCREMENT NOT NULL PRIMARY KEY, `trace_id` INT NOT NULL, `attempt_index` INT NOT NULL, `endpoint` TEXT NOT NULL, `request_path` TEXT NOT NULL, `target_url` TEXT NOT NULL, `runtime_executor` TEXT, `request_headers_json` JSON, `request_body_json` JSON, `response_status` INT, `response_headers_json` JSON, `response_body_json` JSON, `raw_error_text` TEXT, `recover_applied` BOOLEAN DEFAULT false, `downgrade_decision` BOOLEAN DEFAULT false, `downgrade_reason` TEXT, `memory_write_json` JSON, `created_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), '%Y-%m-%d %H:%i:%s')), FOREIGN KEY (`trace_id`) REFERENCES `proxy_debug_traces`(`id`) ON DELETE CASCADE);
|
||||
CREATE TABLE IF NOT EXISTS `proxy_files` (`id` INT AUTO_INCREMENT NOT NULL PRIMARY KEY, `public_id` TEXT NOT NULL, `owner_type` TEXT NOT NULL, `owner_id` TEXT NOT NULL, `filename` TEXT NOT NULL, `mime_type` TEXT NOT NULL, `purpose` TEXT, `byte_size` INT NOT NULL, `sha256` TEXT NOT NULL, `content_base64` TEXT NOT NULL, `created_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), '%Y-%m-%d %H:%i:%s')), `updated_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), '%Y-%m-%d %H:%i:%s')), `deleted_at` VARCHAR(191));
|
||||
CREATE TABLE IF NOT EXISTS `proxy_logs` (`id` INT AUTO_INCREMENT NOT NULL PRIMARY KEY, `route_id` INT, `channel_id` INT, `account_id` INT, `model_requested` TEXT, `model_actual` TEXT, `status` TEXT, `http_status` INT, `latency_ms` INT, `prompt_tokens` INT, `completion_tokens` INT, `total_tokens` INT, `estimated_cost` DOUBLE, `error_message` TEXT, `retry_count` INT DEFAULT 0, `created_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), '%Y-%m-%d %H:%i:%s')), `billing_details` JSON, `downstream_api_key_id` INT, `client_family` TEXT, `client_app_id` TEXT, `client_app_name` TEXT, `client_confidence` TEXT);
|
||||
CREATE TABLE IF NOT EXISTS `proxy_video_tasks` (`id` INT AUTO_INCREMENT NOT NULL PRIMARY KEY, `public_id` TEXT NOT NULL, `upstream_video_id` TEXT NOT NULL, `site_url` TEXT NOT NULL, `token_value` TEXT NOT NULL, `requested_model` TEXT, `actual_model` TEXT, `channel_id` INT, `account_id` INT, `status_snapshot` JSON, `upstream_response_meta` JSON, `last_upstream_status` INT, `last_polled_at` VARCHAR(191), `created_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), '%Y-%m-%d %H:%i:%s')), `updated_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), '%Y-%m-%d %H:%i:%s')));
|
||||
@@ -17,6 +19,7 @@ CREATE TABLE IF NOT EXISTS `site_disabled_models` (`id` INT AUTO_INCREMENT NOT N
|
||||
CREATE TABLE IF NOT EXISTS `token_model_availability` (`id` INT AUTO_INCREMENT NOT NULL PRIMARY KEY, `token_id` INT NOT NULL, `model_name` TEXT NOT NULL, `available` BOOLEAN, `latency_ms` INT, `checked_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), '%Y-%m-%d %H:%i:%s')), FOREIGN KEY (`token_id`) REFERENCES `account_tokens`(`id`) ON DELETE CASCADE);
|
||||
CREATE UNIQUE INDEX `downstream_api_keys_key_unique` ON `downstream_api_keys` (`key`(191));
|
||||
CREATE UNIQUE INDEX `model_availability_account_model_unique` ON `model_availability` (`account_id`, `model_name`(191));
|
||||
CREATE UNIQUE INDEX `proxy_debug_attempts_trace_attempt_unique` ON `proxy_debug_attempts` (`trace_id`, `attempt_index`);
|
||||
CREATE UNIQUE INDEX `proxy_files_public_id_unique` ON `proxy_files` (`public_id`(191));
|
||||
CREATE UNIQUE INDEX `proxy_video_tasks_public_id_unique` ON `proxy_video_tasks` (`public_id`(191));
|
||||
CREATE UNIQUE INDEX `route_group_sources_group_source_unique` ON `route_group_sources` (`group_route_id`, `source_route_id`);
|
||||
@@ -43,6 +46,11 @@ CREATE INDEX `events_read_created_at_idx` ON `events` (`read`, `created_at`);
|
||||
CREATE INDEX `events_type_created_at_idx` ON `events` (`type`(191), `created_at`);
|
||||
CREATE INDEX `model_availability_account_available_idx` ON `model_availability` (`account_id`, `available`);
|
||||
CREATE INDEX `model_availability_model_name_idx` ON `model_availability` (`model_name`(191));
|
||||
CREATE INDEX `proxy_debug_attempts_trace_created_at_idx` ON `proxy_debug_attempts` (`trace_id`, `created_at`);
|
||||
CREATE INDEX `proxy_debug_traces_created_at_idx` ON `proxy_debug_traces` (`created_at`);
|
||||
CREATE INDEX `proxy_debug_traces_final_status_created_at_idx` ON `proxy_debug_traces` (`final_status`(191), `created_at`);
|
||||
CREATE INDEX `proxy_debug_traces_model_created_at_idx` ON `proxy_debug_traces` (`requested_model`(191), `created_at`);
|
||||
CREATE INDEX `proxy_debug_traces_session_created_at_idx` ON `proxy_debug_traces` (`session_id`(191), `created_at`);
|
||||
CREATE INDEX `proxy_files_owner_lookup_idx` ON `proxy_files` (`owner_type`(191), `owner_id`(191), `deleted_at`);
|
||||
CREATE INDEX `proxy_logs_account_created_at_idx` ON `proxy_logs` (`account_id`, `created_at`);
|
||||
CREATE INDEX `proxy_logs_client_app_id_created_at_idx` ON `proxy_logs` (`client_app_id`(191), `created_at`);
|
||||
|
||||
@@ -1 +1,8 @@
|
||||
-- no schema changes detected for mysql
|
||||
CREATE TABLE IF NOT EXISTS `proxy_debug_traces` (`id` INT AUTO_INCREMENT NOT NULL PRIMARY KEY, `downstream_path` TEXT NOT NULL, `client_kind` TEXT, `session_id` TEXT, `trace_hint` TEXT, `requested_model` TEXT, `downstream_api_key_id` INT, `request_headers_json` JSON, `request_body_json` JSON, `sticky_session_key` TEXT, `sticky_hit_channel_id` INT, `selected_channel_id` INT, `selected_route_id` INT, `selected_account_id` INT, `selected_site_id` INT, `selected_site_platform` TEXT, `endpoint_candidates_json` JSON, `endpoint_runtime_state_json` JSON, `decision_summary_json` JSON, `final_status` TEXT, `final_http_status` INT, `final_upstream_path` TEXT, `final_response_headers_json` JSON, `final_response_body_json` JSON, `created_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), '%Y-%m-%d %H:%i:%s')), `updated_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), '%Y-%m-%d %H:%i:%s')));
|
||||
CREATE TABLE IF NOT EXISTS `proxy_debug_attempts` (`id` INT AUTO_INCREMENT NOT NULL PRIMARY KEY, `trace_id` INT NOT NULL, `attempt_index` INT NOT NULL, `endpoint` TEXT NOT NULL, `request_path` TEXT NOT NULL, `target_url` TEXT NOT NULL, `runtime_executor` TEXT, `request_headers_json` JSON, `request_body_json` JSON, `response_status` INT, `response_headers_json` JSON, `response_body_json` JSON, `raw_error_text` TEXT, `recover_applied` BOOLEAN DEFAULT false, `downgrade_decision` BOOLEAN DEFAULT false, `downgrade_reason` TEXT, `memory_write_json` JSON, `created_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), '%Y-%m-%d %H:%i:%s')), FOREIGN KEY (`trace_id`) REFERENCES `proxy_debug_traces`(`id`) ON DELETE CASCADE);
|
||||
CREATE UNIQUE INDEX `proxy_debug_attempts_trace_attempt_unique` ON `proxy_debug_attempts` (`trace_id`, `attempt_index`);
|
||||
CREATE INDEX `proxy_debug_attempts_trace_created_at_idx` ON `proxy_debug_attempts` (`trace_id`, `created_at`);
|
||||
CREATE INDEX `proxy_debug_traces_created_at_idx` ON `proxy_debug_traces` (`created_at`);
|
||||
CREATE INDEX `proxy_debug_traces_final_status_created_at_idx` ON `proxy_debug_traces` (`final_status`(191), `created_at`);
|
||||
CREATE INDEX `proxy_debug_traces_model_created_at_idx` ON `proxy_debug_traces` (`requested_model`(191), `created_at`);
|
||||
CREATE INDEX `proxy_debug_traces_session_created_at_idx` ON `proxy_debug_traces` (`session_id`(191), `created_at`);
|
||||
|
||||
@@ -5,6 +5,8 @@ CREATE TABLE IF NOT EXISTS "checkin_logs" ("id" INTEGER GENERATED BY DEFAULT AS
|
||||
CREATE TABLE IF NOT EXISTS "downstream_api_keys" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL PRIMARY KEY, "name" TEXT NOT NULL, "key" TEXT NOT NULL, "description" TEXT, "enabled" BOOLEAN DEFAULT true, "expires_at" TEXT, "max_cost" DOUBLE PRECISION, "used_cost" DOUBLE PRECISION DEFAULT 0, "max_requests" INTEGER, "used_requests" INTEGER DEFAULT 0, "supported_models" JSONB, "allowed_route_ids" JSONB, "site_weight_multipliers" JSONB, "last_used_at" TEXT, "created_at" TEXT DEFAULT to_char(timezone('UTC', CURRENT_TIMESTAMP), 'YYYY-MM-DD HH24:MI:SS'), "updated_at" TEXT DEFAULT to_char(timezone('UTC', CURRENT_TIMESTAMP), 'YYYY-MM-DD HH24:MI:SS'), "group_name" TEXT, "tags" TEXT);
|
||||
CREATE TABLE IF NOT EXISTS "events" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL PRIMARY KEY, "type" TEXT NOT NULL, "title" TEXT NOT NULL, "message" TEXT, "level" TEXT NOT NULL DEFAULT 'info', "read" BOOLEAN DEFAULT false, "related_id" INTEGER, "related_type" TEXT, "created_at" TEXT DEFAULT to_char(timezone('UTC', CURRENT_TIMESTAMP), 'YYYY-MM-DD HH24:MI:SS'));
|
||||
CREATE TABLE IF NOT EXISTS "model_availability" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL PRIMARY KEY, "account_id" INTEGER NOT NULL, "model_name" TEXT NOT NULL, "available" BOOLEAN, "latency_ms" INTEGER, "checked_at" TEXT DEFAULT to_char(timezone('UTC', CURRENT_TIMESTAMP), 'YYYY-MM-DD HH24:MI:SS'), "is_manual" BOOLEAN DEFAULT false, FOREIGN KEY ("account_id") REFERENCES "accounts"("id") ON DELETE CASCADE);
|
||||
CREATE TABLE IF NOT EXISTS "proxy_debug_traces" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL PRIMARY KEY, "downstream_path" TEXT NOT NULL, "client_kind" TEXT, "session_id" TEXT, "trace_hint" TEXT, "requested_model" TEXT, "downstream_api_key_id" INTEGER, "request_headers_json" JSONB, "request_body_json" JSONB, "sticky_session_key" TEXT, "sticky_hit_channel_id" INTEGER, "selected_channel_id" INTEGER, "selected_route_id" INTEGER, "selected_account_id" INTEGER, "selected_site_id" INTEGER, "selected_site_platform" TEXT, "endpoint_candidates_json" JSONB, "endpoint_runtime_state_json" JSONB, "decision_summary_json" JSONB, "final_status" TEXT, "final_http_status" INTEGER, "final_upstream_path" TEXT, "final_response_headers_json" JSONB, "final_response_body_json" JSONB, "created_at" TEXT DEFAULT to_char(timezone('UTC', CURRENT_TIMESTAMP), 'YYYY-MM-DD HH24:MI:SS'), "updated_at" TEXT DEFAULT to_char(timezone('UTC', CURRENT_TIMESTAMP), 'YYYY-MM-DD HH24:MI:SS'));
|
||||
CREATE TABLE IF NOT EXISTS "proxy_debug_attempts" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL PRIMARY KEY, "trace_id" INTEGER NOT NULL, "attempt_index" INTEGER NOT NULL, "endpoint" TEXT NOT NULL, "request_path" TEXT NOT NULL, "target_url" TEXT NOT NULL, "runtime_executor" TEXT, "request_headers_json" JSONB, "request_body_json" JSONB, "response_status" INTEGER, "response_headers_json" JSONB, "response_body_json" JSONB, "raw_error_text" TEXT, "recover_applied" BOOLEAN DEFAULT false, "downgrade_decision" BOOLEAN DEFAULT false, "downgrade_reason" TEXT, "memory_write_json" JSONB, "created_at" TEXT DEFAULT to_char(timezone('UTC', CURRENT_TIMESTAMP), 'YYYY-MM-DD HH24:MI:SS'), FOREIGN KEY ("trace_id") REFERENCES "proxy_debug_traces"("id") ON DELETE CASCADE);
|
||||
CREATE TABLE IF NOT EXISTS "proxy_files" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL PRIMARY KEY, "public_id" TEXT NOT NULL, "owner_type" TEXT NOT NULL, "owner_id" TEXT NOT NULL, "filename" TEXT NOT NULL, "mime_type" TEXT NOT NULL, "purpose" TEXT, "byte_size" INTEGER NOT NULL, "sha256" TEXT NOT NULL, "content_base64" TEXT NOT NULL, "created_at" TEXT DEFAULT to_char(timezone('UTC', CURRENT_TIMESTAMP), 'YYYY-MM-DD HH24:MI:SS'), "updated_at" TEXT DEFAULT to_char(timezone('UTC', CURRENT_TIMESTAMP), 'YYYY-MM-DD HH24:MI:SS'), "deleted_at" TEXT);
|
||||
CREATE TABLE IF NOT EXISTS "proxy_logs" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL PRIMARY KEY, "route_id" INTEGER, "channel_id" INTEGER, "account_id" INTEGER, "model_requested" TEXT, "model_actual" TEXT, "status" TEXT, "http_status" INTEGER, "latency_ms" INTEGER, "prompt_tokens" INTEGER, "completion_tokens" INTEGER, "total_tokens" INTEGER, "estimated_cost" DOUBLE PRECISION, "error_message" TEXT, "retry_count" INTEGER DEFAULT 0, "created_at" TEXT DEFAULT to_char(timezone('UTC', CURRENT_TIMESTAMP), 'YYYY-MM-DD HH24:MI:SS'), "billing_details" JSONB, "downstream_api_key_id" INTEGER, "client_family" TEXT, "client_app_id" TEXT, "client_app_name" TEXT, "client_confidence" TEXT);
|
||||
CREATE TABLE IF NOT EXISTS "proxy_video_tasks" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL PRIMARY KEY, "public_id" TEXT NOT NULL, "upstream_video_id" TEXT NOT NULL, "site_url" TEXT NOT NULL, "token_value" TEXT NOT NULL, "requested_model" TEXT, "actual_model" TEXT, "channel_id" INTEGER, "account_id" INTEGER, "status_snapshot" JSONB, "upstream_response_meta" JSONB, "last_upstream_status" INTEGER, "last_polled_at" TEXT, "created_at" TEXT DEFAULT to_char(timezone('UTC', CURRENT_TIMESTAMP), 'YYYY-MM-DD HH24:MI:SS'), "updated_at" TEXT DEFAULT to_char(timezone('UTC', CURRENT_TIMESTAMP), 'YYYY-MM-DD HH24:MI:SS'));
|
||||
@@ -17,6 +19,7 @@ CREATE TABLE IF NOT EXISTS "site_disabled_models" ("id" INTEGER GENERATED BY DEF
|
||||
CREATE TABLE IF NOT EXISTS "token_model_availability" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL PRIMARY KEY, "token_id" INTEGER NOT NULL, "model_name" TEXT NOT NULL, "available" BOOLEAN, "latency_ms" INTEGER, "checked_at" TEXT DEFAULT to_char(timezone('UTC', CURRENT_TIMESTAMP), 'YYYY-MM-DD HH24:MI:SS'), FOREIGN KEY ("token_id") REFERENCES "account_tokens"("id") ON DELETE CASCADE);
|
||||
CREATE UNIQUE INDEX "downstream_api_keys_key_unique" ON "downstream_api_keys" ("key");
|
||||
CREATE UNIQUE INDEX "model_availability_account_model_unique" ON "model_availability" ("account_id", "model_name");
|
||||
CREATE UNIQUE INDEX "proxy_debug_attempts_trace_attempt_unique" ON "proxy_debug_attempts" ("trace_id", "attempt_index");
|
||||
CREATE UNIQUE INDEX "proxy_files_public_id_unique" ON "proxy_files" ("public_id");
|
||||
CREATE UNIQUE INDEX "proxy_video_tasks_public_id_unique" ON "proxy_video_tasks" ("public_id");
|
||||
CREATE UNIQUE INDEX "route_group_sources_group_source_unique" ON "route_group_sources" ("group_route_id", "source_route_id");
|
||||
@@ -43,6 +46,11 @@ CREATE INDEX "events_read_created_at_idx" ON "events" ("read", "created_at");
|
||||
CREATE INDEX "events_type_created_at_idx" ON "events" ("type", "created_at");
|
||||
CREATE INDEX "model_availability_account_available_idx" ON "model_availability" ("account_id", "available");
|
||||
CREATE INDEX "model_availability_model_name_idx" ON "model_availability" ("model_name");
|
||||
CREATE INDEX "proxy_debug_attempts_trace_created_at_idx" ON "proxy_debug_attempts" ("trace_id", "created_at");
|
||||
CREATE INDEX "proxy_debug_traces_created_at_idx" ON "proxy_debug_traces" ("created_at");
|
||||
CREATE INDEX "proxy_debug_traces_final_status_created_at_idx" ON "proxy_debug_traces" ("final_status", "created_at");
|
||||
CREATE INDEX "proxy_debug_traces_model_created_at_idx" ON "proxy_debug_traces" ("requested_model", "created_at");
|
||||
CREATE INDEX "proxy_debug_traces_session_created_at_idx" ON "proxy_debug_traces" ("session_id", "created_at");
|
||||
CREATE INDEX "proxy_files_owner_lookup_idx" ON "proxy_files" ("owner_type", "owner_id", "deleted_at");
|
||||
CREATE INDEX "proxy_logs_account_created_at_idx" ON "proxy_logs" ("account_id", "created_at");
|
||||
CREATE INDEX "proxy_logs_client_app_id_created_at_idx" ON "proxy_logs" ("client_app_id", "created_at");
|
||||
|
||||
@@ -1 +1,8 @@
|
||||
-- no schema changes detected for postgres
|
||||
CREATE TABLE IF NOT EXISTS "proxy_debug_traces" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL PRIMARY KEY, "downstream_path" TEXT NOT NULL, "client_kind" TEXT, "session_id" TEXT, "trace_hint" TEXT, "requested_model" TEXT, "downstream_api_key_id" INTEGER, "request_headers_json" JSONB, "request_body_json" JSONB, "sticky_session_key" TEXT, "sticky_hit_channel_id" INTEGER, "selected_channel_id" INTEGER, "selected_route_id" INTEGER, "selected_account_id" INTEGER, "selected_site_id" INTEGER, "selected_site_platform" TEXT, "endpoint_candidates_json" JSONB, "endpoint_runtime_state_json" JSONB, "decision_summary_json" JSONB, "final_status" TEXT, "final_http_status" INTEGER, "final_upstream_path" TEXT, "final_response_headers_json" JSONB, "final_response_body_json" JSONB, "created_at" TEXT DEFAULT to_char(timezone('UTC', CURRENT_TIMESTAMP), 'YYYY-MM-DD HH24:MI:SS'), "updated_at" TEXT DEFAULT to_char(timezone('UTC', CURRENT_TIMESTAMP), 'YYYY-MM-DD HH24:MI:SS'));
|
||||
CREATE TABLE IF NOT EXISTS "proxy_debug_attempts" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL PRIMARY KEY, "trace_id" INTEGER NOT NULL, "attempt_index" INTEGER NOT NULL, "endpoint" TEXT NOT NULL, "request_path" TEXT NOT NULL, "target_url" TEXT NOT NULL, "runtime_executor" TEXT, "request_headers_json" JSONB, "request_body_json" JSONB, "response_status" INTEGER, "response_headers_json" JSONB, "response_body_json" JSONB, "raw_error_text" TEXT, "recover_applied" BOOLEAN DEFAULT false, "downgrade_decision" BOOLEAN DEFAULT false, "downgrade_reason" TEXT, "memory_write_json" JSONB, "created_at" TEXT DEFAULT to_char(timezone('UTC', CURRENT_TIMESTAMP), 'YYYY-MM-DD HH24:MI:SS'), FOREIGN KEY ("trace_id") REFERENCES "proxy_debug_traces"("id") ON DELETE CASCADE);
|
||||
CREATE UNIQUE INDEX "proxy_debug_attempts_trace_attempt_unique" ON "proxy_debug_attempts" ("trace_id", "attempt_index");
|
||||
CREATE INDEX "proxy_debug_attempts_trace_created_at_idx" ON "proxy_debug_attempts" ("trace_id", "created_at");
|
||||
CREATE INDEX "proxy_debug_traces_created_at_idx" ON "proxy_debug_traces" ("created_at");
|
||||
CREATE INDEX "proxy_debug_traces_final_status_created_at_idx" ON "proxy_debug_traces" ("final_status", "created_at");
|
||||
CREATE INDEX "proxy_debug_traces_model_created_at_idx" ON "proxy_debug_traces" ("requested_model", "created_at");
|
||||
CREATE INDEX "proxy_debug_traces_session_created_at_idx" ON "proxy_debug_traces" ("session_id", "created_at");
|
||||
|
||||
@@ -462,6 +462,278 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"proxy_debug_attempts": {
|
||||
"columns": {
|
||||
"id": {
|
||||
"logicalType": "integer",
|
||||
"notNull": true,
|
||||
"defaultValue": null,
|
||||
"primaryKey": true
|
||||
},
|
||||
"trace_id": {
|
||||
"logicalType": "integer",
|
||||
"notNull": true,
|
||||
"defaultValue": null,
|
||||
"primaryKey": false
|
||||
},
|
||||
"attempt_index": {
|
||||
"logicalType": "integer",
|
||||
"notNull": true,
|
||||
"defaultValue": null,
|
||||
"primaryKey": false
|
||||
},
|
||||
"endpoint": {
|
||||
"logicalType": "text",
|
||||
"notNull": true,
|
||||
"defaultValue": null,
|
||||
"primaryKey": false
|
||||
},
|
||||
"request_path": {
|
||||
"logicalType": "text",
|
||||
"notNull": true,
|
||||
"defaultValue": null,
|
||||
"primaryKey": false
|
||||
},
|
||||
"target_url": {
|
||||
"logicalType": "text",
|
||||
"notNull": true,
|
||||
"defaultValue": null,
|
||||
"primaryKey": false
|
||||
},
|
||||
"runtime_executor": {
|
||||
"logicalType": "text",
|
||||
"notNull": false,
|
||||
"defaultValue": null,
|
||||
"primaryKey": false
|
||||
},
|
||||
"request_headers_json": {
|
||||
"logicalType": "json",
|
||||
"notNull": false,
|
||||
"defaultValue": null,
|
||||
"primaryKey": false
|
||||
},
|
||||
"request_body_json": {
|
||||
"logicalType": "json",
|
||||
"notNull": false,
|
||||
"defaultValue": null,
|
||||
"primaryKey": false
|
||||
},
|
||||
"response_status": {
|
||||
"logicalType": "integer",
|
||||
"notNull": false,
|
||||
"defaultValue": null,
|
||||
"primaryKey": false
|
||||
},
|
||||
"response_headers_json": {
|
||||
"logicalType": "json",
|
||||
"notNull": false,
|
||||
"defaultValue": null,
|
||||
"primaryKey": false
|
||||
},
|
||||
"response_body_json": {
|
||||
"logicalType": "json",
|
||||
"notNull": false,
|
||||
"defaultValue": null,
|
||||
"primaryKey": false
|
||||
},
|
||||
"raw_error_text": {
|
||||
"logicalType": "text",
|
||||
"notNull": false,
|
||||
"defaultValue": null,
|
||||
"primaryKey": false
|
||||
},
|
||||
"recover_applied": {
|
||||
"logicalType": "boolean",
|
||||
"notNull": false,
|
||||
"defaultValue": "false",
|
||||
"primaryKey": false
|
||||
},
|
||||
"downgrade_decision": {
|
||||
"logicalType": "boolean",
|
||||
"notNull": false,
|
||||
"defaultValue": "false",
|
||||
"primaryKey": false
|
||||
},
|
||||
"downgrade_reason": {
|
||||
"logicalType": "text",
|
||||
"notNull": false,
|
||||
"defaultValue": null,
|
||||
"primaryKey": false
|
||||
},
|
||||
"memory_write_json": {
|
||||
"logicalType": "json",
|
||||
"notNull": false,
|
||||
"defaultValue": null,
|
||||
"primaryKey": false
|
||||
},
|
||||
"created_at": {
|
||||
"logicalType": "datetime",
|
||||
"notNull": false,
|
||||
"defaultValue": "datetime('now')",
|
||||
"primaryKey": false
|
||||
}
|
||||
}
|
||||
},
|
||||
"proxy_debug_traces": {
|
||||
"columns": {
|
||||
"id": {
|
||||
"logicalType": "integer",
|
||||
"notNull": true,
|
||||
"defaultValue": null,
|
||||
"primaryKey": true
|
||||
},
|
||||
"downstream_path": {
|
||||
"logicalType": "text",
|
||||
"notNull": true,
|
||||
"defaultValue": null,
|
||||
"primaryKey": false
|
||||
},
|
||||
"client_kind": {
|
||||
"logicalType": "text",
|
||||
"notNull": false,
|
||||
"defaultValue": null,
|
||||
"primaryKey": false
|
||||
},
|
||||
"session_id": {
|
||||
"logicalType": "text",
|
||||
"notNull": false,
|
||||
"defaultValue": null,
|
||||
"primaryKey": false
|
||||
},
|
||||
"trace_hint": {
|
||||
"logicalType": "text",
|
||||
"notNull": false,
|
||||
"defaultValue": null,
|
||||
"primaryKey": false
|
||||
},
|
||||
"requested_model": {
|
||||
"logicalType": "text",
|
||||
"notNull": false,
|
||||
"defaultValue": null,
|
||||
"primaryKey": false
|
||||
},
|
||||
"downstream_api_key_id": {
|
||||
"logicalType": "integer",
|
||||
"notNull": false,
|
||||
"defaultValue": null,
|
||||
"primaryKey": false
|
||||
},
|
||||
"request_headers_json": {
|
||||
"logicalType": "json",
|
||||
"notNull": false,
|
||||
"defaultValue": null,
|
||||
"primaryKey": false
|
||||
},
|
||||
"request_body_json": {
|
||||
"logicalType": "json",
|
||||
"notNull": false,
|
||||
"defaultValue": null,
|
||||
"primaryKey": false
|
||||
},
|
||||
"sticky_session_key": {
|
||||
"logicalType": "text",
|
||||
"notNull": false,
|
||||
"defaultValue": null,
|
||||
"primaryKey": false
|
||||
},
|
||||
"sticky_hit_channel_id": {
|
||||
"logicalType": "integer",
|
||||
"notNull": false,
|
||||
"defaultValue": null,
|
||||
"primaryKey": false
|
||||
},
|
||||
"selected_channel_id": {
|
||||
"logicalType": "integer",
|
||||
"notNull": false,
|
||||
"defaultValue": null,
|
||||
"primaryKey": false
|
||||
},
|
||||
"selected_route_id": {
|
||||
"logicalType": "integer",
|
||||
"notNull": false,
|
||||
"defaultValue": null,
|
||||
"primaryKey": false
|
||||
},
|
||||
"selected_account_id": {
|
||||
"logicalType": "integer",
|
||||
"notNull": false,
|
||||
"defaultValue": null,
|
||||
"primaryKey": false
|
||||
},
|
||||
"selected_site_id": {
|
||||
"logicalType": "integer",
|
||||
"notNull": false,
|
||||
"defaultValue": null,
|
||||
"primaryKey": false
|
||||
},
|
||||
"selected_site_platform": {
|
||||
"logicalType": "text",
|
||||
"notNull": false,
|
||||
"defaultValue": null,
|
||||
"primaryKey": false
|
||||
},
|
||||
"endpoint_candidates_json": {
|
||||
"logicalType": "json",
|
||||
"notNull": false,
|
||||
"defaultValue": null,
|
||||
"primaryKey": false
|
||||
},
|
||||
"endpoint_runtime_state_json": {
|
||||
"logicalType": "json",
|
||||
"notNull": false,
|
||||
"defaultValue": null,
|
||||
"primaryKey": false
|
||||
},
|
||||
"decision_summary_json": {
|
||||
"logicalType": "json",
|
||||
"notNull": false,
|
||||
"defaultValue": null,
|
||||
"primaryKey": false
|
||||
},
|
||||
"final_status": {
|
||||
"logicalType": "text",
|
||||
"notNull": false,
|
||||
"defaultValue": null,
|
||||
"primaryKey": false
|
||||
},
|
||||
"final_http_status": {
|
||||
"logicalType": "integer",
|
||||
"notNull": false,
|
||||
"defaultValue": null,
|
||||
"primaryKey": false
|
||||
},
|
||||
"final_upstream_path": {
|
||||
"logicalType": "text",
|
||||
"notNull": false,
|
||||
"defaultValue": null,
|
||||
"primaryKey": false
|
||||
},
|
||||
"final_response_headers_json": {
|
||||
"logicalType": "json",
|
||||
"notNull": false,
|
||||
"defaultValue": null,
|
||||
"primaryKey": false
|
||||
},
|
||||
"final_response_body_json": {
|
||||
"logicalType": "json",
|
||||
"notNull": false,
|
||||
"defaultValue": null,
|
||||
"primaryKey": false
|
||||
},
|
||||
"created_at": {
|
||||
"logicalType": "datetime",
|
||||
"notNull": false,
|
||||
"defaultValue": "datetime('now')",
|
||||
"primaryKey": false
|
||||
},
|
||||
"updated_at": {
|
||||
"logicalType": "datetime",
|
||||
"notNull": false,
|
||||
"defaultValue": "datetime('now')",
|
||||
"primaryKey": false
|
||||
}
|
||||
}
|
||||
},
|
||||
"proxy_files": {
|
||||
"columns": {
|
||||
"id": {
|
||||
@@ -1453,6 +1725,59 @@
|
||||
],
|
||||
"unique": false
|
||||
},
|
||||
{
|
||||
"name": "proxy_debug_attempts_trace_attempt_unique",
|
||||
"table": "proxy_debug_attempts",
|
||||
"columns": [
|
||||
"trace_id",
|
||||
"attempt_index"
|
||||
],
|
||||
"unique": true
|
||||
},
|
||||
{
|
||||
"name": "proxy_debug_attempts_trace_created_at_idx",
|
||||
"table": "proxy_debug_attempts",
|
||||
"columns": [
|
||||
"trace_id",
|
||||
"created_at"
|
||||
],
|
||||
"unique": false
|
||||
},
|
||||
{
|
||||
"name": "proxy_debug_traces_created_at_idx",
|
||||
"table": "proxy_debug_traces",
|
||||
"columns": [
|
||||
"created_at"
|
||||
],
|
||||
"unique": false
|
||||
},
|
||||
{
|
||||
"name": "proxy_debug_traces_final_status_created_at_idx",
|
||||
"table": "proxy_debug_traces",
|
||||
"columns": [
|
||||
"final_status",
|
||||
"created_at"
|
||||
],
|
||||
"unique": false
|
||||
},
|
||||
{
|
||||
"name": "proxy_debug_traces_model_created_at_idx",
|
||||
"table": "proxy_debug_traces",
|
||||
"columns": [
|
||||
"requested_model",
|
||||
"created_at"
|
||||
],
|
||||
"unique": false
|
||||
},
|
||||
{
|
||||
"name": "proxy_debug_traces_session_created_at_idx",
|
||||
"table": "proxy_debug_traces",
|
||||
"columns": [
|
||||
"session_id",
|
||||
"created_at"
|
||||
],
|
||||
"unique": false
|
||||
},
|
||||
{
|
||||
"name": "proxy_files_owner_lookup_idx",
|
||||
"table": "proxy_files",
|
||||
@@ -1743,6 +2068,14 @@
|
||||
"model_name"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "proxy_debug_attempts_trace_attempt_unique",
|
||||
"table": "proxy_debug_attempts",
|
||||
"columns": [
|
||||
"trace_id",
|
||||
"attempt_index"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "proxy_files_public_id_unique",
|
||||
"table": "proxy_files",
|
||||
@@ -1843,6 +2176,17 @@
|
||||
],
|
||||
"onDelete": "CASCADE"
|
||||
},
|
||||
{
|
||||
"table": "proxy_debug_attempts",
|
||||
"columns": [
|
||||
"trace_id"
|
||||
],
|
||||
"referencedTable": "proxy_debug_traces",
|
||||
"referencedColumns": [
|
||||
"id"
|
||||
],
|
||||
"onDelete": "CASCADE"
|
||||
},
|
||||
{
|
||||
"table": "route_channels",
|
||||
"columns": [
|
||||
|
||||
@@ -34,6 +34,8 @@ const TABLES_WITH_NUMERIC_ID = new Set([
|
||||
'route_group_sources',
|
||||
'route_channels',
|
||||
'proxy_logs',
|
||||
'proxy_debug_traces',
|
||||
'proxy_debug_attempts',
|
||||
'proxy_video_tasks',
|
||||
'proxy_files',
|
||||
'downstream_api_keys',
|
||||
|
||||
@@ -210,6 +210,64 @@ export const proxyLogs = sqliteTable('proxy_logs', {
|
||||
clientFamilyCreatedIdx: index('proxy_logs_client_family_created_at_idx').on(table.clientFamily, table.createdAt),
|
||||
}));
|
||||
|
||||
export const proxyDebugTraces = sqliteTable('proxy_debug_traces', {
|
||||
id: integer('id').primaryKey({ autoIncrement: true }),
|
||||
downstreamPath: text('downstream_path').notNull(),
|
||||
clientKind: text('client_kind'),
|
||||
sessionId: text('session_id'),
|
||||
traceHint: text('trace_hint'),
|
||||
requestedModel: text('requested_model'),
|
||||
downstreamApiKeyId: integer('downstream_api_key_id'),
|
||||
requestHeadersJson: text('request_headers_json'),
|
||||
requestBodyJson: text('request_body_json'),
|
||||
stickySessionKey: text('sticky_session_key'),
|
||||
stickyHitChannelId: integer('sticky_hit_channel_id'),
|
||||
selectedChannelId: integer('selected_channel_id'),
|
||||
selectedRouteId: integer('selected_route_id'),
|
||||
selectedAccountId: integer('selected_account_id'),
|
||||
selectedSiteId: integer('selected_site_id'),
|
||||
selectedSitePlatform: text('selected_site_platform'),
|
||||
endpointCandidatesJson: text('endpoint_candidates_json'),
|
||||
endpointRuntimeStateJson: text('endpoint_runtime_state_json'),
|
||||
decisionSummaryJson: text('decision_summary_json'),
|
||||
finalStatus: text('final_status'),
|
||||
finalHttpStatus: integer('final_http_status'),
|
||||
finalUpstreamPath: text('final_upstream_path'),
|
||||
finalResponseHeadersJson: text('final_response_headers_json'),
|
||||
finalResponseBodyJson: text('final_response_body_json'),
|
||||
createdAt: text('created_at').default(sql`(datetime('now'))`),
|
||||
updatedAt: text('updated_at').default(sql`(datetime('now'))`),
|
||||
}, (table) => ({
|
||||
createdAtIdx: index('proxy_debug_traces_created_at_idx').on(table.createdAt),
|
||||
sessionCreatedIdx: index('proxy_debug_traces_session_created_at_idx').on(table.sessionId, table.createdAt),
|
||||
modelCreatedIdx: index('proxy_debug_traces_model_created_at_idx').on(table.requestedModel, table.createdAt),
|
||||
finalStatusCreatedIdx: index('proxy_debug_traces_final_status_created_at_idx').on(table.finalStatus, table.createdAt),
|
||||
}));
|
||||
|
||||
export const proxyDebugAttempts = sqliteTable('proxy_debug_attempts', {
|
||||
id: integer('id').primaryKey({ autoIncrement: true }),
|
||||
traceId: integer('trace_id').notNull().references(() => proxyDebugTraces.id, { onDelete: 'cascade' }),
|
||||
attemptIndex: integer('attempt_index').notNull(),
|
||||
endpoint: text('endpoint').notNull(),
|
||||
requestPath: text('request_path').notNull(),
|
||||
targetUrl: text('target_url').notNull(),
|
||||
runtimeExecutor: text('runtime_executor'),
|
||||
requestHeadersJson: text('request_headers_json'),
|
||||
requestBodyJson: text('request_body_json'),
|
||||
responseStatus: integer('response_status'),
|
||||
responseHeadersJson: text('response_headers_json'),
|
||||
responseBodyJson: text('response_body_json'),
|
||||
rawErrorText: text('raw_error_text'),
|
||||
recoverApplied: integer('recover_applied', { mode: 'boolean' }).default(false),
|
||||
downgradeDecision: integer('downgrade_decision', { mode: 'boolean' }).default(false),
|
||||
downgradeReason: text('downgrade_reason'),
|
||||
memoryWriteJson: text('memory_write_json'),
|
||||
createdAt: text('created_at').default(sql`(datetime('now'))`),
|
||||
}, (table) => ({
|
||||
traceAttemptUnique: uniqueIndex('proxy_debug_attempts_trace_attempt_unique').on(table.traceId, table.attemptIndex),
|
||||
traceCreatedIdx: index('proxy_debug_attempts_trace_created_at_idx').on(table.traceId, table.createdAt),
|
||||
}));
|
||||
|
||||
export const proxyVideoTasks = sqliteTable('proxy_video_tasks', {
|
||||
id: integer('id').primaryKey({ autoIncrement: true }),
|
||||
publicId: text('public_id').notNull(),
|
||||
|
||||
@@ -227,6 +227,51 @@ function applyRuntimeSettings(settingsMap: Map<string, string>) {
|
||||
config.proxySessionChannelQueueWaitMs = Math.trunc(proxySessionChannelQueueWaitMs);
|
||||
}
|
||||
|
||||
const proxyDebugTraceEnabled = parseSettingFromMap<boolean>(settingsMap, 'proxy_debug_trace_enabled');
|
||||
if (typeof proxyDebugTraceEnabled === 'boolean') {
|
||||
config.proxyDebugTraceEnabled = proxyDebugTraceEnabled;
|
||||
}
|
||||
|
||||
const proxyDebugCaptureHeaders = parseSettingFromMap<boolean>(settingsMap, 'proxy_debug_capture_headers');
|
||||
if (typeof proxyDebugCaptureHeaders === 'boolean') {
|
||||
config.proxyDebugCaptureHeaders = proxyDebugCaptureHeaders;
|
||||
}
|
||||
|
||||
const proxyDebugCaptureBodies = parseSettingFromMap<boolean>(settingsMap, 'proxy_debug_capture_bodies');
|
||||
if (typeof proxyDebugCaptureBodies === 'boolean') {
|
||||
config.proxyDebugCaptureBodies = proxyDebugCaptureBodies;
|
||||
}
|
||||
|
||||
const proxyDebugCaptureStreamChunks = parseSettingFromMap<boolean>(settingsMap, 'proxy_debug_capture_stream_chunks');
|
||||
if (typeof proxyDebugCaptureStreamChunks === 'boolean') {
|
||||
config.proxyDebugCaptureStreamChunks = proxyDebugCaptureStreamChunks;
|
||||
}
|
||||
|
||||
const proxyDebugTargetSessionId = parseSettingFromMap<string>(settingsMap, 'proxy_debug_target_session_id');
|
||||
if (typeof proxyDebugTargetSessionId === 'string') {
|
||||
config.proxyDebugTargetSessionId = proxyDebugTargetSessionId.trim();
|
||||
}
|
||||
|
||||
const proxyDebugTargetClientKind = parseSettingFromMap<string>(settingsMap, 'proxy_debug_target_client_kind');
|
||||
if (typeof proxyDebugTargetClientKind === 'string') {
|
||||
config.proxyDebugTargetClientKind = proxyDebugTargetClientKind.trim();
|
||||
}
|
||||
|
||||
const proxyDebugTargetModel = parseSettingFromMap<string>(settingsMap, 'proxy_debug_target_model');
|
||||
if (typeof proxyDebugTargetModel === 'string') {
|
||||
config.proxyDebugTargetModel = proxyDebugTargetModel.trim();
|
||||
}
|
||||
|
||||
const proxyDebugRetentionHours = parseSettingFromMap<number>(settingsMap, 'proxy_debug_retention_hours');
|
||||
if (typeof proxyDebugRetentionHours === 'number' && Number.isFinite(proxyDebugRetentionHours) && proxyDebugRetentionHours >= 1) {
|
||||
config.proxyDebugRetentionHours = Math.trunc(proxyDebugRetentionHours);
|
||||
}
|
||||
|
||||
const proxyDebugMaxBodyBytes = parseSettingFromMap<number>(settingsMap, 'proxy_debug_max_body_bytes');
|
||||
if (typeof proxyDebugMaxBodyBytes === 'number' && Number.isFinite(proxyDebugMaxBodyBytes) && proxyDebugMaxBodyBytes >= 1024) {
|
||||
config.proxyDebugMaxBodyBytes = Math.trunc(proxyDebugMaxBodyBytes);
|
||||
}
|
||||
|
||||
const routingWeights = parseSettingFromMap<Partial<typeof config.routingWeights>>(settingsMap, 'routing_weights');
|
||||
if (routingWeights && typeof routingWeights === 'object') {
|
||||
config.routingWeights = {
|
||||
|
||||
@@ -7,10 +7,13 @@ import { type DownstreamFormat } from '../../transformers/shared/normalized.js';
|
||||
import {
|
||||
buildClaudeCountTokensUpstreamRequest,
|
||||
buildUpstreamEndpointRequest,
|
||||
recordUpstreamEndpointFailure,
|
||||
recordUpstreamEndpointSuccess,
|
||||
resolveUpstreamEndpointCandidates,
|
||||
} from '../../routes/proxy/upstreamEndpoint.js';
|
||||
import {
|
||||
getUpstreamEndpointRuntimeStateSnapshot,
|
||||
recordUpstreamEndpointFailure,
|
||||
recordUpstreamEndpointSuccess,
|
||||
} from '../../services/upstreamEndpointRuntimeMemory.js';
|
||||
import {
|
||||
ensureModelAllowedForDownstreamKey,
|
||||
getDownstreamRoutingPolicy,
|
||||
@@ -51,10 +54,23 @@ import {
|
||||
clearSurfaceStickyChannel,
|
||||
createSurfaceFailureToolkit,
|
||||
createSurfaceDispatchRequest,
|
||||
getSurfaceStickyPreferredChannelId,
|
||||
recordSurfaceSuccess,
|
||||
selectSurfaceChannelForAttempt,
|
||||
trySurfaceOauthRefreshRecovery,
|
||||
} from './sharedSurface.js';
|
||||
import {
|
||||
buildSurfaceProxyDebugResponseHeaders,
|
||||
captureSurfaceProxyDebugSuccessResponseBody,
|
||||
parseSurfaceProxyDebugTextPayload,
|
||||
reserveSurfaceProxyDebugAttemptBase,
|
||||
safeFinalizeSurfaceProxyDebugTrace,
|
||||
safeInsertSurfaceProxyDebugAttempt,
|
||||
safeUpdateSurfaceProxyDebugAttempt,
|
||||
safeUpdateSurfaceProxyDebugCandidates,
|
||||
safeUpdateSurfaceProxyDebugSelection,
|
||||
startSurfaceProxyDebugTrace,
|
||||
} from '../../services/proxyDebugTraceRuntime.js';
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return !!value && typeof value === 'object' && !Array.isArray(value);
|
||||
@@ -138,11 +154,44 @@ export async function handleChatSurfaceRequest(
|
||||
downstreamPath,
|
||||
downstreamApiKeyId,
|
||||
});
|
||||
const debugTrace = await startSurfaceProxyDebugTrace({
|
||||
downstreamPath,
|
||||
clientKind: clientContext.clientKind,
|
||||
sessionId: clientContext.sessionId || null,
|
||||
traceHint: clientContext.traceHint || null,
|
||||
requestedModel,
|
||||
downstreamApiKeyId,
|
||||
requestHeaders: request.headers as Record<string, unknown>,
|
||||
requestBody: request.body,
|
||||
});
|
||||
const finalizeDebugFailure = async (status: number, payload: unknown, upstreamPath: string | null = null) => {
|
||||
await safeFinalizeSurfaceProxyDebugTrace(debugTrace, {
|
||||
finalStatus: 'failed',
|
||||
finalHttpStatus: status,
|
||||
finalUpstreamPath: upstreamPath,
|
||||
finalResponseHeaders: {
|
||||
'content-type': 'application/json',
|
||||
},
|
||||
finalResponseBody: payload,
|
||||
});
|
||||
};
|
||||
const finalizeDebugSuccess = async (status: number, upstreamPath: string | null, responseHeaders: unknown, responseBody: unknown) => {
|
||||
await safeFinalizeSurfaceProxyDebugTrace(debugTrace, {
|
||||
finalStatus: 'success',
|
||||
finalHttpStatus: status,
|
||||
finalUpstreamPath: upstreamPath,
|
||||
finalResponseHeaders: responseHeaders as Record<string, unknown> | null,
|
||||
finalResponseBody: responseBody,
|
||||
});
|
||||
};
|
||||
|
||||
const excludeChannelIds: number[] = [];
|
||||
let retryCount = 0;
|
||||
|
||||
while (retryCount <= maxRetries) {
|
||||
const stickyPreferredChannelId = retryCount === 0
|
||||
? getSurfaceStickyPreferredChannelId(stickySessionKey)
|
||||
: null;
|
||||
const selected = await selectSurfaceChannelForAttempt({
|
||||
requestedModel,
|
||||
downstreamPolicy,
|
||||
@@ -156,12 +205,29 @@ export async function handleChatSurfaceRequest(
|
||||
model: requestedModel,
|
||||
reason: 'No available channels after retries',
|
||||
});
|
||||
const payload = {
|
||||
error: { message: 'No available channels for this model', type: 'server_error' as const },
|
||||
};
|
||||
await finalizeDebugFailure(503, payload, null);
|
||||
return reply.code(503).send({
|
||||
error: { message: 'No available channels for this model', type: 'server_error' },
|
||||
});
|
||||
}
|
||||
|
||||
excludeChannelIds.push(selected.channel.id);
|
||||
await safeUpdateSurfaceProxyDebugSelection(debugTrace, {
|
||||
stickySessionKey,
|
||||
stickyHitChannelId: (
|
||||
stickyPreferredChannelId && stickyPreferredChannelId === selected.channel.id
|
||||
? stickyPreferredChannelId
|
||||
: null
|
||||
),
|
||||
selectedChannelId: selected.channel.id,
|
||||
selectedRouteId: selected.channel.routeId ?? null,
|
||||
selectedAccountId: selected.account.id,
|
||||
selectedSiteId: selected.site.id,
|
||||
selectedSitePlatform: selected.site.platform,
|
||||
});
|
||||
|
||||
const modelName = selected.actualModel || requestedModel;
|
||||
const oauth = getOauthInfoFromAccount(selected.account);
|
||||
@@ -194,6 +260,18 @@ export async function handleChatSurfaceRequest(
|
||||
conversationFileSummary,
|
||||
},
|
||||
};
|
||||
await safeUpdateSurfaceProxyDebugCandidates(debugTrace, {
|
||||
endpointCandidates,
|
||||
endpointRuntimeState: getUpstreamEndpointRuntimeStateSnapshot(endpointRuntimeContext),
|
||||
decisionSummary: {
|
||||
retryCount,
|
||||
downstreamFormat,
|
||||
stickySessionKey,
|
||||
stickyPreferredChannelId,
|
||||
oauthProvider: oauth?.provider || null,
|
||||
isCodexSite,
|
||||
},
|
||||
});
|
||||
const buildProviderHeaders = () => (
|
||||
buildOauthProviderHeaders({
|
||||
account: selected.account,
|
||||
@@ -286,6 +364,12 @@ export async function handleChatSurfaceRequest(
|
||||
retryCount += 1;
|
||||
continue;
|
||||
}
|
||||
await finalizeDebugFailure(503, {
|
||||
error: {
|
||||
message: busyMessage,
|
||||
type: 'server_error',
|
||||
},
|
||||
});
|
||||
return reply.code(503).send({
|
||||
error: {
|
||||
message: busyMessage,
|
||||
@@ -296,28 +380,69 @@ export async function handleChatSurfaceRequest(
|
||||
const channelLease = leaseResult.lease;
|
||||
|
||||
try {
|
||||
const debugAttemptBase = reserveSurfaceProxyDebugAttemptBase(debugTrace, endpointCandidates.length);
|
||||
const endpointResult = await executeEndpointFlow({
|
||||
siteUrl: selected.site.url,
|
||||
endpointCandidates,
|
||||
buildRequest: (endpoint) => buildEndpointRequest(endpoint),
|
||||
dispatchRequest,
|
||||
tryRecover,
|
||||
onAttemptFailure: (ctx) => {
|
||||
recordUpstreamEndpointFailure({
|
||||
onAttemptFailure: async (ctx) => {
|
||||
const memoryWrite = recordUpstreamEndpointFailure({
|
||||
...endpointRuntimeContext,
|
||||
endpoint: ctx.request.endpoint,
|
||||
status: ctx.response.status,
|
||||
errorText: ctx.rawErrText,
|
||||
});
|
||||
await safeInsertSurfaceProxyDebugAttempt(debugTrace, {
|
||||
attemptIndex: debugAttemptBase + ctx.endpointIndex,
|
||||
endpoint: ctx.request.endpoint,
|
||||
requestPath: ctx.request.path,
|
||||
targetUrl: ctx.targetUrl,
|
||||
runtimeExecutor: ctx.request.runtime?.executor || 'default',
|
||||
requestHeaders: ctx.request.headers,
|
||||
requestBody: ctx.request.body,
|
||||
responseStatus: ctx.response.status,
|
||||
responseHeaders: buildSurfaceProxyDebugResponseHeaders(ctx.response),
|
||||
responseBody: parseSurfaceProxyDebugTextPayload(ctx.rawErrText),
|
||||
rawErrorText: ctx.rawErrText,
|
||||
recoverApplied: ctx.recoverApplied === true,
|
||||
downgradeDecision: false,
|
||||
downgradeReason: null,
|
||||
memoryWrite,
|
||||
});
|
||||
},
|
||||
onAttemptSuccess: (ctx) => {
|
||||
recordUpstreamEndpointSuccess({
|
||||
onAttemptSuccess: async (ctx) => {
|
||||
const memoryWrite = recordUpstreamEndpointSuccess({
|
||||
...endpointRuntimeContext,
|
||||
endpoint: ctx.request.endpoint,
|
||||
});
|
||||
const responseBody = await captureSurfaceProxyDebugSuccessResponseBody(debugTrace, ctx);
|
||||
await safeInsertSurfaceProxyDebugAttempt(debugTrace, {
|
||||
attemptIndex: debugAttemptBase + ctx.endpointIndex,
|
||||
endpoint: ctx.request.endpoint,
|
||||
requestPath: ctx.request.path,
|
||||
targetUrl: ctx.targetUrl,
|
||||
runtimeExecutor: ctx.request.runtime?.executor || 'default',
|
||||
requestHeaders: ctx.request.headers,
|
||||
requestBody: ctx.request.body,
|
||||
responseStatus: ctx.response.status,
|
||||
responseHeaders: buildSurfaceProxyDebugResponseHeaders(ctx.response),
|
||||
responseBody,
|
||||
rawErrorText: null,
|
||||
recoverApplied: ctx.recoverApplied === true,
|
||||
downgradeDecision: false,
|
||||
downgradeReason: null,
|
||||
memoryWrite,
|
||||
});
|
||||
},
|
||||
shouldDowngrade: endpointStrategy.shouldDowngrade,
|
||||
onDowngrade: (ctx) => {
|
||||
onDowngrade: async (ctx) => {
|
||||
await safeUpdateSurfaceProxyDebugAttempt(debugTrace, debugAttemptBase + ctx.endpointIndex, {
|
||||
downgradeDecision: true,
|
||||
downgradeReason: ctx.errText,
|
||||
rawErrorText: ctx.rawErrText,
|
||||
});
|
||||
return failureToolkit.log({
|
||||
selected,
|
||||
modelRequested: requestedModel,
|
||||
@@ -349,6 +474,11 @@ export async function handleChatSurfaceRequest(
|
||||
retryCount += 1;
|
||||
continue;
|
||||
}
|
||||
await finalizeDebugFailure(
|
||||
failureOutcome.status,
|
||||
failureOutcome.payload,
|
||||
null,
|
||||
);
|
||||
return reply.code(failureOutcome.status).send(failureOutcome.payload);
|
||||
}
|
||||
|
||||
@@ -422,8 +552,25 @@ export async function handleChatSurfaceRequest(
|
||||
totalTokens: parsedUsage.totalTokens,
|
||||
upstreamPath: successfulUpstreamPath,
|
||||
});
|
||||
await finalizeDebugFailure(502, {
|
||||
error: {
|
||||
message: streamResult.errorMessage,
|
||||
type: 'stream_error',
|
||||
},
|
||||
}, successfulUpstreamPath);
|
||||
return;
|
||||
}
|
||||
await finalizeDebugSuccess(
|
||||
200,
|
||||
successfulUpstreamPath,
|
||||
buildSurfaceProxyDebugResponseHeaders(upstream),
|
||||
debugTrace?.options.captureStreamChunks
|
||||
? fallbackText
|
||||
: {
|
||||
stream: true,
|
||||
usage: parsedUsage,
|
||||
},
|
||||
);
|
||||
bindSurfaceStickyChannel({
|
||||
stickySessionKey,
|
||||
selected,
|
||||
@@ -463,6 +610,11 @@ export async function handleChatSurfaceRequest(
|
||||
retryCount += 1;
|
||||
continue;
|
||||
}
|
||||
await finalizeDebugFailure(
|
||||
failureOutcome.status,
|
||||
failureOutcome.payload,
|
||||
successfulUpstreamPath,
|
||||
);
|
||||
return reply.code(failureOutcome.status).send(failureOutcome.payload);
|
||||
}
|
||||
|
||||
@@ -486,8 +638,25 @@ export async function handleChatSurfaceRequest(
|
||||
upstreamPath: successfulUpstreamPath,
|
||||
runtimeFailureStatus: 502,
|
||||
});
|
||||
await finalizeDebugFailure(502, {
|
||||
error: {
|
||||
message: streamResult.errorMessage,
|
||||
type: 'stream_error',
|
||||
},
|
||||
}, successfulUpstreamPath);
|
||||
return;
|
||||
}
|
||||
await finalizeDebugSuccess(
|
||||
200,
|
||||
successfulUpstreamPath,
|
||||
buildSurfaceProxyDebugResponseHeaders(upstream),
|
||||
debugTrace?.options.captureStreamChunks
|
||||
? fallbackText
|
||||
: {
|
||||
stream: true,
|
||||
usage: parsedUsage,
|
||||
},
|
||||
);
|
||||
bindSurfaceStickyChannel({
|
||||
stickySessionKey,
|
||||
selected,
|
||||
@@ -538,6 +707,12 @@ export async function handleChatSurfaceRequest(
|
||||
upstreamPath: successfulUpstreamPath,
|
||||
runtimeFailureStatus: 502,
|
||||
});
|
||||
await finalizeDebugFailure(502, {
|
||||
error: {
|
||||
message: streamResult.errorMessage,
|
||||
type: 'stream_error',
|
||||
},
|
||||
}, successfulUpstreamPath);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -565,6 +740,17 @@ export async function handleChatSurfaceRequest(
|
||||
errorLabel: '[proxy/chat] failed to record success metrics',
|
||||
},
|
||||
});
|
||||
await finalizeDebugSuccess(
|
||||
200,
|
||||
successfulUpstreamPath,
|
||||
buildSurfaceProxyDebugResponseHeaders(upstream),
|
||||
debugTrace?.options.captureStreamChunks
|
||||
? rawText
|
||||
: {
|
||||
stream: true,
|
||||
usage: parsedUsage,
|
||||
},
|
||||
);
|
||||
bindSurfaceStickyChannel({
|
||||
stickySessionKey,
|
||||
selected,
|
||||
@@ -620,6 +806,11 @@ export async function handleChatSurfaceRequest(
|
||||
retryCount += 1;
|
||||
continue;
|
||||
}
|
||||
await finalizeDebugFailure(
|
||||
failureOutcome.status,
|
||||
failureOutcome.payload,
|
||||
successfulUpstreamPath,
|
||||
);
|
||||
return reply.code(failureOutcome.status).send(failureOutcome.payload);
|
||||
}
|
||||
const normalizedFinal = downstreamTransformer.transformFinalResponse(upstreamData, modelName, rawText);
|
||||
@@ -642,6 +833,12 @@ export async function handleChatSurfaceRequest(
|
||||
errorLabel: '[proxy/chat] failed to record success metrics',
|
||||
},
|
||||
});
|
||||
await finalizeDebugSuccess(
|
||||
upstream.status,
|
||||
successfulUpstreamPath,
|
||||
buildSurfaceProxyDebugResponseHeaders(upstream),
|
||||
downstreamResponse,
|
||||
);
|
||||
bindSurfaceStickyChannel({
|
||||
stickySessionKey,
|
||||
selected,
|
||||
@@ -665,6 +862,11 @@ export async function handleChatSurfaceRequest(
|
||||
retryCount += 1;
|
||||
continue;
|
||||
}
|
||||
await finalizeDebugFailure(
|
||||
failureOutcome.status,
|
||||
failureOutcome.payload,
|
||||
null,
|
||||
);
|
||||
return reply.code(failureOutcome.status).send(failureOutcome.payload);
|
||||
} finally {
|
||||
channelLease.release();
|
||||
@@ -742,10 +944,43 @@ export async function handleClaudeCountTokensSurfaceRequest(
|
||||
downstreamPath,
|
||||
downstreamApiKeyId,
|
||||
});
|
||||
const debugTrace = await startSurfaceProxyDebugTrace({
|
||||
downstreamPath,
|
||||
clientKind: clientContext.clientKind,
|
||||
sessionId: clientContext.sessionId || null,
|
||||
traceHint: clientContext.traceHint || null,
|
||||
requestedModel,
|
||||
downstreamApiKeyId,
|
||||
requestHeaders: request.headers as Record<string, unknown>,
|
||||
requestBody: rawBody,
|
||||
});
|
||||
const finalizeDebugFailure = async (status: number, payload: unknown, upstreamPath: string | null = null) => {
|
||||
await safeFinalizeSurfaceProxyDebugTrace(debugTrace, {
|
||||
finalStatus: 'failed',
|
||||
finalHttpStatus: status,
|
||||
finalUpstreamPath: upstreamPath,
|
||||
finalResponseHeaders: {
|
||||
'content-type': 'application/json',
|
||||
},
|
||||
finalResponseBody: payload,
|
||||
});
|
||||
};
|
||||
const finalizeDebugSuccess = async (status: number, upstreamPath: string | null, responseHeaders: unknown, responseBody: unknown) => {
|
||||
await safeFinalizeSurfaceProxyDebugTrace(debugTrace, {
|
||||
finalStatus: 'success',
|
||||
finalHttpStatus: status,
|
||||
finalUpstreamPath: upstreamPath,
|
||||
finalResponseHeaders: responseHeaders as Record<string, unknown> | null,
|
||||
finalResponseBody: responseBody,
|
||||
});
|
||||
};
|
||||
const excludeChannelIds: number[] = [];
|
||||
let retryCount = 0;
|
||||
|
||||
while (retryCount <= maxRetries) {
|
||||
const stickyPreferredChannelId = retryCount === 0
|
||||
? getSurfaceStickyPreferredChannelId(stickySessionKey)
|
||||
: null;
|
||||
const selected = await selectSurfaceChannelForAttempt({
|
||||
requestedModel,
|
||||
downstreamPolicy,
|
||||
@@ -759,13 +994,35 @@ export async function handleClaudeCountTokensSurfaceRequest(
|
||||
model: requestedModel,
|
||||
reason: 'No available channels after retries',
|
||||
});
|
||||
await finalizeDebugFailure(503, {
|
||||
error: { message: 'No available channels for this model', type: 'server_error' },
|
||||
});
|
||||
return reply.code(503).send({
|
||||
error: { message: 'No available channels for this model', type: 'server_error' },
|
||||
});
|
||||
}
|
||||
|
||||
excludeChannelIds.push(selected.channel.id);
|
||||
await safeUpdateSurfaceProxyDebugSelection(debugTrace, {
|
||||
stickySessionKey,
|
||||
stickyHitChannelId: (
|
||||
stickyPreferredChannelId && stickyPreferredChannelId === selected.channel.id
|
||||
? stickyPreferredChannelId
|
||||
: null
|
||||
),
|
||||
selectedChannelId: selected.channel.id,
|
||||
selectedRouteId: selected.channel.routeId ?? null,
|
||||
selectedAccountId: selected.account.id,
|
||||
selectedSiteId: selected.site.id,
|
||||
selectedSitePlatform: selected.site.platform,
|
||||
});
|
||||
const modelName = selected.actualModel || requestedModel;
|
||||
const endpointRuntimeContext = {
|
||||
siteId: selected.site.id,
|
||||
modelName,
|
||||
downstreamFormat: 'claude' as const,
|
||||
requestedModelHint: requestedModel,
|
||||
};
|
||||
const endpointCandidates = await resolveUpstreamEndpointCandidates(
|
||||
{
|
||||
site: selected.site,
|
||||
@@ -775,11 +1032,27 @@ export async function handleClaudeCountTokensSurfaceRequest(
|
||||
'claude',
|
||||
requestedModel,
|
||||
);
|
||||
await safeUpdateSurfaceProxyDebugCandidates(debugTrace, {
|
||||
endpointCandidates,
|
||||
endpointRuntimeState: getUpstreamEndpointRuntimeStateSnapshot(endpointRuntimeContext),
|
||||
decisionSummary: {
|
||||
retryCount,
|
||||
stickySessionKey,
|
||||
stickyPreferredChannelId,
|
||||
countTokens: true,
|
||||
},
|
||||
});
|
||||
if (!endpointCandidates.includes('messages')) {
|
||||
if (canRetryProxyChannel(retryCount)) {
|
||||
retryCount += 1;
|
||||
continue;
|
||||
}
|
||||
await finalizeDebugFailure(501, {
|
||||
error: {
|
||||
message: 'Claude count_tokens compatibility is not implemented for this upstream',
|
||||
type: 'invalid_request_error',
|
||||
},
|
||||
});
|
||||
return reply.code(501).send({
|
||||
error: {
|
||||
message: 'Claude count_tokens compatibility is not implemented for this upstream',
|
||||
@@ -812,6 +1085,12 @@ export async function handleClaudeCountTokensSurfaceRequest(
|
||||
retryCount += 1;
|
||||
continue;
|
||||
}
|
||||
await finalizeDebugFailure(503, {
|
||||
error: {
|
||||
message: busyMessage,
|
||||
type: 'server_error',
|
||||
},
|
||||
});
|
||||
return reply.code(503).send({
|
||||
error: {
|
||||
message: busyMessage,
|
||||
@@ -846,6 +1125,7 @@ export async function handleClaudeCountTokensSurfaceRequest(
|
||||
accountExtraConfig: selected.account.extraConfig,
|
||||
});
|
||||
let upstream = await dispatchRequest(upstreamRequest);
|
||||
let recoverApplied = false;
|
||||
|
||||
if ((upstream.status === 401 || upstream.status === 403) && oauth) {
|
||||
const recoverContext = {
|
||||
@@ -864,6 +1144,7 @@ export async function handleClaudeCountTokensSurfaceRequest(
|
||||
if (recovered?.upstream?.ok) {
|
||||
upstreamRequest = buildRequest();
|
||||
upstream = recovered.upstream;
|
||||
recoverApplied = true;
|
||||
} else {
|
||||
upstreamRequest = recoverContext.request;
|
||||
upstream = recoverContext.response;
|
||||
@@ -879,6 +1160,23 @@ export async function handleClaudeCountTokensSurfaceRequest(
|
||||
} catch {
|
||||
payload = text;
|
||||
}
|
||||
await safeInsertSurfaceProxyDebugAttempt(debugTrace, {
|
||||
attemptIndex: retryCount,
|
||||
endpoint: upstreamRequest.endpoint,
|
||||
requestPath: upstreamRequest.path,
|
||||
targetUrl: `${selected.site.url}${upstreamRequest.path}`,
|
||||
runtimeExecutor: upstreamRequest.runtime?.executor || 'default',
|
||||
requestHeaders: upstreamRequest.headers,
|
||||
requestBody: upstreamRequest.body,
|
||||
responseStatus: upstream.status,
|
||||
responseHeaders: buildSurfaceProxyDebugResponseHeaders(upstream),
|
||||
responseBody: payload,
|
||||
rawErrorText: upstream.ok ? null : text,
|
||||
recoverApplied,
|
||||
downgradeDecision: false,
|
||||
downgradeReason: null,
|
||||
memoryWrite: null,
|
||||
});
|
||||
|
||||
if (!upstream.ok) {
|
||||
clearSurfaceStickyChannel({
|
||||
@@ -899,6 +1197,7 @@ export async function handleClaudeCountTokensSurfaceRequest(
|
||||
retryCount += 1;
|
||||
continue;
|
||||
}
|
||||
await finalizeDebugFailure(failureOutcome.status, failureOutcome.payload, upstreamRequest.path);
|
||||
return reply.code(failureOutcome.status).send(failureOutcome.payload);
|
||||
}
|
||||
|
||||
@@ -918,6 +1217,12 @@ export async function handleClaudeCountTokensSurfaceRequest(
|
||||
stickySessionKey,
|
||||
selected,
|
||||
});
|
||||
await finalizeDebugSuccess(
|
||||
upstream.status,
|
||||
upstreamRequest.path,
|
||||
buildSurfaceProxyDebugResponseHeaders(upstream),
|
||||
payload,
|
||||
);
|
||||
return reply.code(upstream.status).type(contentType).send(payload);
|
||||
} catch (error: any) {
|
||||
clearSurfaceStickyChannel({
|
||||
@@ -936,6 +1241,7 @@ export async function handleClaudeCountTokensSurfaceRequest(
|
||||
retryCount += 1;
|
||||
continue;
|
||||
}
|
||||
await finalizeDebugFailure(failureOutcome.status, failureOutcome.payload, null);
|
||||
return reply.code(failureOutcome.status).send(failureOutcome.payload);
|
||||
} finally {
|
||||
channelLease.release();
|
||||
|
||||
@@ -17,10 +17,13 @@ import { executeEndpointFlow, type BuiltEndpointRequest } from '../../routes/pro
|
||||
import { composeProxyLogMessage } from '../../routes/proxy/logPathMeta.js';
|
||||
import {
|
||||
buildUpstreamEndpointRequest,
|
||||
recordUpstreamEndpointFailure,
|
||||
recordUpstreamEndpointSuccess,
|
||||
resolveUpstreamEndpointCandidates,
|
||||
} from '../../routes/proxy/upstreamEndpoint.js';
|
||||
import {
|
||||
getUpstreamEndpointRuntimeStateSnapshot,
|
||||
recordUpstreamEndpointFailure,
|
||||
recordUpstreamEndpointSuccess,
|
||||
} from '../../services/upstreamEndpointRuntimeMemory.js';
|
||||
import {
|
||||
geminiGenerateContentTransformer,
|
||||
} from '../../transformers/gemini/generate-content/index.js';
|
||||
@@ -37,6 +40,18 @@ import { insertProxyLog } from '../../services/proxyLogStore.js';
|
||||
import { summarizeConversationFileInputsInOpenAiBody } from '../capabilities/conversationFileCapabilities.js';
|
||||
import { readRuntimeResponseText } from '../executors/types.js';
|
||||
import { canRetryProxyChannel, getProxyMaxChannelRetries } from '../../services/proxyChannelRetry.js';
|
||||
import {
|
||||
buildSurfaceProxyDebugResponseHeaders,
|
||||
captureSurfaceProxyDebugSuccessResponseBody,
|
||||
parseSurfaceProxyDebugTextPayload,
|
||||
reserveSurfaceProxyDebugAttemptBase,
|
||||
safeFinalizeSurfaceProxyDebugTrace,
|
||||
safeInsertSurfaceProxyDebugAttempt,
|
||||
safeUpdateSurfaceProxyDebugAttempt,
|
||||
safeUpdateSurfaceProxyDebugCandidates,
|
||||
safeUpdateSurfaceProxyDebugSelection,
|
||||
startSurfaceProxyDebugTrace,
|
||||
} from '../../services/proxyDebugTraceRuntime.js';
|
||||
const GEMINI_MODEL_PROBES = [
|
||||
'gemini-2.5-flash',
|
||||
'gemini-2.0-flash',
|
||||
@@ -269,6 +284,40 @@ export async function geminiProxyRoute(app: FastifyInstance) {
|
||||
const apiVersion = geminiGenerateContentTransformer.resolveProxyApiVersion(
|
||||
request.params as { geminiApiVersion?: string } | undefined,
|
||||
);
|
||||
const downstreamPath = resolveDownstreamPath(request);
|
||||
const clientContext = detectDownstreamClientContext({
|
||||
downstreamPath,
|
||||
headers: request.headers as Record<string, unknown>,
|
||||
});
|
||||
const debugTrace = await startSurfaceProxyDebugTrace({
|
||||
downstreamPath,
|
||||
clientKind: clientContext.clientKind,
|
||||
sessionId: clientContext.sessionId || null,
|
||||
traceHint: clientContext.traceHint || null,
|
||||
requestedModel: null,
|
||||
requestHeaders: request.headers as Record<string, unknown>,
|
||||
requestBody: null,
|
||||
});
|
||||
const finalizeDebugFailure = async (status: number, payload: unknown, upstreamPath: string | null = null) => {
|
||||
await safeFinalizeSurfaceProxyDebugTrace(debugTrace, {
|
||||
finalStatus: 'failed',
|
||||
finalHttpStatus: status,
|
||||
finalUpstreamPath: upstreamPath,
|
||||
finalResponseHeaders: {
|
||||
'content-type': 'application/json',
|
||||
},
|
||||
finalResponseBody: payload,
|
||||
});
|
||||
};
|
||||
const finalizeDebugSuccess = async (status: number, upstreamPath: string | null, responseHeaders: unknown, responseBody: unknown) => {
|
||||
await safeFinalizeSurfaceProxyDebugTrace(debugTrace, {
|
||||
finalStatus: 'success',
|
||||
finalHttpStatus: status,
|
||||
finalUpstreamPath: upstreamPath,
|
||||
finalResponseHeaders: responseHeaders as Record<string, unknown> | null,
|
||||
finalResponseBody: responseBody,
|
||||
});
|
||||
};
|
||||
const excludeChannelIds: number[] = [];
|
||||
let retryCount = 0;
|
||||
let lastStatus = 503;
|
||||
@@ -280,10 +329,20 @@ export async function geminiProxyRoute(app: FastifyInstance) {
|
||||
? await selectGeminiChannel(request)
|
||||
: await selectNextGeminiProbeChannel(request, excludeChannelIds);
|
||||
if (!selected) {
|
||||
await finalizeDebugFailure(lastStatus, lastText, null);
|
||||
return reply.code(lastStatus).type(lastContentType).send(lastText);
|
||||
}
|
||||
|
||||
excludeChannelIds.push(selected.channel.id);
|
||||
await safeUpdateSurfaceProxyDebugSelection(debugTrace, {
|
||||
stickySessionKey: null,
|
||||
stickyHitChannelId: null,
|
||||
selectedChannelId: selected.channel.id,
|
||||
selectedRouteId: selected.channel.routeId ?? null,
|
||||
selectedAccountId: selected.account.id,
|
||||
selectedSiteId: selected.site.id,
|
||||
selectedSitePlatform: selected.site.platform,
|
||||
});
|
||||
|
||||
try {
|
||||
if (!isDirectGeminiFamilyPlatform(selected.site.platform)) {
|
||||
@@ -292,6 +351,13 @@ export async function geminiProxyRoute(app: FastifyInstance) {
|
||||
await routeRefreshWorkflow.refreshModelsAndRebuildRoutes();
|
||||
models = await readRouteAwareGeminiModels(request);
|
||||
}
|
||||
await safeUpdateSurfaceProxyDebugCandidates(debugTrace, {
|
||||
decisionSummary: {
|
||||
retryCount,
|
||||
pathKind: 'route-aware-gemini-models',
|
||||
},
|
||||
});
|
||||
await finalizeDebugSuccess(200, null, { 'content-type': 'application/json' }, { models });
|
||||
return reply.code(200).send({ models });
|
||||
}
|
||||
|
||||
@@ -300,14 +366,40 @@ export async function geminiProxyRoute(app: FastifyInstance) {
|
||||
{ models: GEMINI_CLI_STATIC_MODELS },
|
||||
request,
|
||||
);
|
||||
await safeUpdateSurfaceProxyDebugCandidates(debugTrace, {
|
||||
decisionSummary: {
|
||||
retryCount,
|
||||
pathKind: 'gemini-cli-static-models',
|
||||
},
|
||||
});
|
||||
await finalizeDebugSuccess(200, null, { 'content-type': 'application/json' }, filtered);
|
||||
return reply.code(200).send(filtered);
|
||||
}
|
||||
|
||||
const targetUrl = geminiGenerateContentTransformer.resolveModelsUrl(selected.site.url, apiVersion, selected.tokenValue);
|
||||
const upstreamPath = `/${apiVersion}/models`;
|
||||
const upstream = await fetch(
|
||||
geminiGenerateContentTransformer.resolveModelsUrl(selected.site.url, apiVersion, selected.tokenValue),
|
||||
targetUrl,
|
||||
{ method: 'GET' },
|
||||
);
|
||||
const text = await readRuntimeResponseText(upstream);
|
||||
await safeInsertSurfaceProxyDebugAttempt(debugTrace, {
|
||||
attemptIndex: retryCount,
|
||||
endpoint: 'gemini-models',
|
||||
requestPath: upstreamPath,
|
||||
targetUrl,
|
||||
runtimeExecutor: 'default',
|
||||
requestHeaders: null,
|
||||
requestBody: null,
|
||||
responseStatus: upstream.status,
|
||||
responseHeaders: buildSurfaceProxyDebugResponseHeaders(upstream),
|
||||
responseBody: parseSurfaceProxyDebugTextPayload(text),
|
||||
rawErrorText: upstream.ok ? null : text,
|
||||
recoverApplied: false,
|
||||
downgradeDecision: false,
|
||||
downgradeReason: null,
|
||||
memoryWrite: null,
|
||||
});
|
||||
if (!upstream.ok) {
|
||||
lastStatus = upstream.status;
|
||||
lastText = text;
|
||||
@@ -320,14 +412,17 @@ export async function geminiProxyRoute(app: FastifyInstance) {
|
||||
retryCount += 1;
|
||||
continue;
|
||||
}
|
||||
await finalizeDebugFailure(lastStatus, parseSurfaceProxyDebugTextPayload(lastText), upstreamPath);
|
||||
return reply.code(lastStatus).type(lastContentType).send(lastText);
|
||||
}
|
||||
|
||||
try {
|
||||
const parsed = JSON.parse(text);
|
||||
const filtered = await filterGeminiListedModelsForPolicy(parsed, request);
|
||||
await finalizeDebugSuccess(upstream.status, upstreamPath, buildSurfaceProxyDebugResponseHeaders(upstream), filtered);
|
||||
return reply.code(upstream.status).send(filtered);
|
||||
} catch {
|
||||
await finalizeDebugSuccess(upstream.status, upstreamPath, buildSurfaceProxyDebugResponseHeaders(upstream), text);
|
||||
return reply.code(upstream.status).type(upstream.headers.get('content-type') || 'application/json').send(text);
|
||||
}
|
||||
} catch (error) {
|
||||
@@ -346,6 +441,7 @@ export async function geminiProxyRoute(app: FastifyInstance) {
|
||||
retryCount += 1;
|
||||
continue;
|
||||
}
|
||||
await finalizeDebugFailure(lastStatus, parseSurfaceProxyDebugTextPayload(lastText), null);
|
||||
return reply.code(lastStatus).type(lastContentType).send(lastText);
|
||||
}
|
||||
}
|
||||
@@ -399,6 +495,35 @@ export async function geminiProxyRoute(app: FastifyInstance) {
|
||||
headers: request.headers as Record<string, unknown>,
|
||||
body: request.body,
|
||||
});
|
||||
const debugTrace = await startSurfaceProxyDebugTrace({
|
||||
downstreamPath,
|
||||
clientKind: clientContext.clientKind,
|
||||
sessionId: clientContext.sessionId || null,
|
||||
traceHint: clientContext.traceHint || null,
|
||||
requestedModel,
|
||||
requestHeaders: request.headers as Record<string, unknown>,
|
||||
requestBody: request.body,
|
||||
});
|
||||
const finalizeDebugFailure = async (status: number, payload: unknown, upstreamPath: string | null = null) => {
|
||||
await safeFinalizeSurfaceProxyDebugTrace(debugTrace, {
|
||||
finalStatus: 'failed',
|
||||
finalHttpStatus: status,
|
||||
finalUpstreamPath: upstreamPath,
|
||||
finalResponseHeaders: {
|
||||
'content-type': 'application/json',
|
||||
},
|
||||
finalResponseBody: payload,
|
||||
});
|
||||
};
|
||||
const finalizeDebugSuccess = async (status: number, upstreamPath: string | null, responseHeaders: unknown, responseBody: unknown) => {
|
||||
await safeFinalizeSurfaceProxyDebugTrace(debugTrace, {
|
||||
finalStatus: 'success',
|
||||
finalHttpStatus: status,
|
||||
finalUpstreamPath: upstreamPath,
|
||||
finalResponseHeaders: responseHeaders as Record<string, unknown> | null,
|
||||
finalResponseBody: responseBody,
|
||||
});
|
||||
};
|
||||
const excludeChannelIds: number[] = [];
|
||||
let retryCount = 0;
|
||||
let lastStatus = 503;
|
||||
@@ -410,10 +535,20 @@ export async function geminiProxyRoute(app: FastifyInstance) {
|
||||
? await tokenRouter.selectChannel(requestedModel, policy)
|
||||
: await tokenRouter.selectNextChannel(requestedModel, excludeChannelIds, policy);
|
||||
if (!selected) {
|
||||
await finalizeDebugFailure(lastStatus, parseSurfaceProxyDebugTextPayload(lastText), null);
|
||||
return reply.code(lastStatus).type(lastContentType).send(lastText);
|
||||
}
|
||||
|
||||
excludeChannelIds.push(selected.channel.id);
|
||||
await safeUpdateSurfaceProxyDebugSelection(debugTrace, {
|
||||
stickySessionKey: null,
|
||||
stickyHitChannelId: null,
|
||||
selectedChannelId: selected.channel.id,
|
||||
selectedRouteId: selected.channel.routeId ?? null,
|
||||
selectedAccountId: selected.account.id,
|
||||
selectedSiteId: selected.site.id,
|
||||
selectedSitePlatform: selected.site.platform,
|
||||
});
|
||||
|
||||
const actualModel = selected.actualModel || requestedModel;
|
||||
const normalizedBody = geminiGenerateContentTransformer.inbound.normalizeRequest(
|
||||
@@ -429,6 +564,16 @@ export async function geminiProxyRoute(app: FastifyInstance) {
|
||||
|
||||
try {
|
||||
if (isDirectGeminiFamily) {
|
||||
await safeUpdateSurfaceProxyDebugCandidates(debugTrace, {
|
||||
decisionSummary: {
|
||||
retryCount,
|
||||
pathKind: 'direct-gemini-family',
|
||||
isGeminiCli,
|
||||
isInternalGemini,
|
||||
isCountTokensAction,
|
||||
isStreamAction,
|
||||
},
|
||||
});
|
||||
if (isGeminiCli && !oauth?.projectId) {
|
||||
lastStatus = 500;
|
||||
lastContentType = 'application/json';
|
||||
@@ -446,6 +591,7 @@ export async function geminiProxyRoute(app: FastifyInstance) {
|
||||
retryCount += 1;
|
||||
continue;
|
||||
}
|
||||
await finalizeDebugFailure(lastStatus, parseSurfaceProxyDebugTextPayload(lastText), upstreamPath || null);
|
||||
return reply.code(lastStatus).type(lastContentType).send(lastText);
|
||||
}
|
||||
|
||||
@@ -458,7 +604,7 @@ export async function geminiProxyRoute(app: FastifyInstance) {
|
||||
: resolveUpstreamPath(apiVersion, actualModelAction);
|
||||
const query = new URLSearchParams(request.query as Record<string, string>).toString();
|
||||
const channelProxyUrl = resolveChannelProxyUrl(selected.site, selected.account.extraConfig);
|
||||
const dispatchSelectedRequest = async () => {
|
||||
const buildDirectDispatchState = () => {
|
||||
const requestBody = isInternalGemini
|
||||
? (
|
||||
isCountTokensAction
|
||||
@@ -492,8 +638,17 @@ export async function geminiProxyRoute(app: FastifyInstance) {
|
||||
selected.tokenValue,
|
||||
query,
|
||||
);
|
||||
const runtimeExecutor = isInternalGemini
|
||||
? (isGeminiCli ? 'gemini-cli' : 'antigravity')
|
||||
: 'default';
|
||||
|
||||
return isInternalGemini
|
||||
return {
|
||||
requestBody,
|
||||
requestHeaders,
|
||||
targetUrl,
|
||||
runtimeExecutor,
|
||||
dispatch: () => (
|
||||
isInternalGemini
|
||||
? dispatchRuntimeRequest({
|
||||
siteUrl: selected.site.url,
|
||||
targetUrl,
|
||||
@@ -522,11 +677,15 @@ export async function geminiProxyRoute(app: FastifyInstance) {
|
||||
method: 'POST',
|
||||
headers: requestHeaders,
|
||||
body: JSON.stringify(requestBody),
|
||||
});
|
||||
})
|
||||
),
|
||||
};
|
||||
};
|
||||
|
||||
let upstream = await dispatchSelectedRequest();
|
||||
let directDispatchState = buildDirectDispatchState();
|
||||
let upstream = await directDispatchState.dispatch();
|
||||
let contentType = upstream.headers.get('content-type') || 'application/json';
|
||||
let recoverApplied = false;
|
||||
if (upstream.status === 401 && oauth) {
|
||||
try {
|
||||
const refreshed = await refreshOauthAccessTokenSingleflight(selected.account.id);
|
||||
@@ -537,8 +696,10 @@ export async function geminiProxyRoute(app: FastifyInstance) {
|
||||
extraConfig: refreshed.extraConfig ?? selected.account.extraConfig,
|
||||
};
|
||||
oauth = getOauthInfoFromAccount(selected.account);
|
||||
upstream = await dispatchSelectedRequest();
|
||||
directDispatchState = buildDirectDispatchState();
|
||||
upstream = await directDispatchState.dispatch();
|
||||
contentType = upstream.headers.get('content-type') || 'application/json';
|
||||
recoverApplied = true;
|
||||
} catch {
|
||||
// Preserve the original 401 response when refresh fails.
|
||||
}
|
||||
@@ -547,6 +708,23 @@ export async function geminiProxyRoute(app: FastifyInstance) {
|
||||
lastStatus = upstream.status;
|
||||
lastContentType = contentType;
|
||||
lastText = await readRuntimeResponseText(upstream);
|
||||
await safeInsertSurfaceProxyDebugAttempt(debugTrace, {
|
||||
attemptIndex: retryCount,
|
||||
endpoint: isInternalGemini ? 'gemini-internal' : 'gemini-native',
|
||||
requestPath: upstreamPath,
|
||||
targetUrl: directDispatchState.targetUrl,
|
||||
runtimeExecutor: directDispatchState.runtimeExecutor,
|
||||
requestHeaders: directDispatchState.requestHeaders,
|
||||
requestBody: directDispatchState.requestBody,
|
||||
responseStatus: upstream.status,
|
||||
responseHeaders: buildSurfaceProxyDebugResponseHeaders(upstream),
|
||||
responseBody: parseSurfaceProxyDebugTextPayload(lastText),
|
||||
rawErrorText: lastText,
|
||||
recoverApplied,
|
||||
downgradeDecision: false,
|
||||
downgradeReason: null,
|
||||
memoryWrite: null,
|
||||
});
|
||||
await tokenRouter.recordFailure?.(selected.channel.id, {
|
||||
status: upstream.status,
|
||||
errorText: lastText,
|
||||
@@ -569,22 +747,25 @@ export async function geminiProxyRoute(app: FastifyInstance) {
|
||||
}
|
||||
|
||||
try {
|
||||
await finalizeDebugFailure(lastStatus, JSON.parse(lastText), upstreamPath);
|
||||
return reply.code(lastStatus).send(JSON.parse(lastText));
|
||||
} catch {
|
||||
await finalizeDebugFailure(lastStatus, lastText, upstreamPath);
|
||||
return reply.code(lastStatus).type(lastContentType).send(lastText);
|
||||
}
|
||||
}
|
||||
|
||||
if (geminiGenerateContentTransformer.stream.isSseContentType(contentType)) {
|
||||
reply.hijack();
|
||||
reply.raw.statusCode = upstream.status;
|
||||
reply.raw.setHeader('Content-Type', contentType || 'text/event-stream');
|
||||
const upstreamReader = upstream.body?.getReader();
|
||||
const reader = isInternalGemini && !isGeminiCliDownstream && upstreamReader
|
||||
? createGeminiCliStreamReader(upstreamReader)
|
||||
: upstreamReader;
|
||||
const captureStreamChunks = debugTrace?.options.captureStreamChunks === true;
|
||||
if (!reader) {
|
||||
const latency = Date.now() - startTime;
|
||||
const responseBody = captureStreamChunks
|
||||
? ''
|
||||
: { stream: true, usage: EMPTY_PROXY_USAGE };
|
||||
await recordGeminiChannelSuccessBestEffort(selected.channel.id, latency, actualModel);
|
||||
await logProxy(
|
||||
selected,
|
||||
@@ -598,18 +779,47 @@ export async function geminiProxyRoute(app: FastifyInstance) {
|
||||
upstreamPath,
|
||||
clientContext,
|
||||
);
|
||||
reply.raw.end();
|
||||
return;
|
||||
await safeInsertSurfaceProxyDebugAttempt(debugTrace, {
|
||||
attemptIndex: retryCount,
|
||||
endpoint: isInternalGemini ? 'gemini-internal' : 'gemini-native',
|
||||
requestPath: upstreamPath,
|
||||
targetUrl: directDispatchState.targetUrl,
|
||||
runtimeExecutor: directDispatchState.runtimeExecutor,
|
||||
requestHeaders: directDispatchState.requestHeaders,
|
||||
requestBody: directDispatchState.requestBody,
|
||||
responseStatus: upstream.status,
|
||||
responseHeaders: buildSurfaceProxyDebugResponseHeaders(upstream),
|
||||
responseBody,
|
||||
rawErrorText: null,
|
||||
recoverApplied,
|
||||
downgradeDecision: false,
|
||||
downgradeReason: null,
|
||||
memoryWrite: null,
|
||||
});
|
||||
await finalizeDebugSuccess(
|
||||
upstream.status,
|
||||
upstreamPath,
|
||||
buildSurfaceProxyDebugResponseHeaders(upstream),
|
||||
responseBody,
|
||||
);
|
||||
return reply.code(upstream.status).type(contentType || 'text/event-stream').send('');
|
||||
}
|
||||
reply.hijack();
|
||||
reply.raw.statusCode = upstream.status;
|
||||
reply.raw.setHeader('Content-Type', contentType || 'text/event-stream');
|
||||
const aggregateState = geminiGenerateContentTransformer.stream.createAggregateState();
|
||||
const decoder = new TextDecoder();
|
||||
let rest = '';
|
||||
let rawStreamText = '';
|
||||
try {
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
if (!value) continue;
|
||||
const chunkText = decoder.decode(value, { stream: true });
|
||||
if (captureStreamChunks) {
|
||||
rawStreamText += chunkText;
|
||||
}
|
||||
const consumed = geminiGenerateContentTransformer.stream.consumeUpstreamSseBuffer(
|
||||
aggregateState,
|
||||
rest + chunkText,
|
||||
@@ -621,6 +831,9 @@ export async function geminiProxyRoute(app: FastifyInstance) {
|
||||
}
|
||||
const tail = decoder.decode();
|
||||
if (tail) {
|
||||
if (captureStreamChunks) {
|
||||
rawStreamText += tail;
|
||||
}
|
||||
const consumed = geminiGenerateContentTransformer.stream.consumeUpstreamSseBuffer(
|
||||
aggregateState,
|
||||
rest + tail,
|
||||
@@ -629,12 +842,11 @@ export async function geminiProxyRoute(app: FastifyInstance) {
|
||||
reply.raw.write(line);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
reader.releaseLock();
|
||||
reply.raw.end();
|
||||
}
|
||||
const parsedUsage = parseProxyUsage(aggregateState);
|
||||
const latency = Date.now() - startTime;
|
||||
const responseBody = captureStreamChunks
|
||||
? rawStreamText
|
||||
: { stream: true, usage: parsedUsage };
|
||||
await recordGeminiChannelSuccessBestEffort(selected.channel.id, latency, actualModel);
|
||||
await logProxy(
|
||||
selected,
|
||||
@@ -651,7 +863,87 @@ export async function geminiProxyRoute(app: FastifyInstance) {
|
||||
parsedUsage.completionTokens,
|
||||
parsedUsage.totalTokens,
|
||||
);
|
||||
await safeInsertSurfaceProxyDebugAttempt(debugTrace, {
|
||||
attemptIndex: retryCount,
|
||||
endpoint: isInternalGemini ? 'gemini-internal' : 'gemini-native',
|
||||
requestPath: upstreamPath,
|
||||
targetUrl: directDispatchState.targetUrl,
|
||||
runtimeExecutor: directDispatchState.runtimeExecutor,
|
||||
requestHeaders: directDispatchState.requestHeaders,
|
||||
requestBody: directDispatchState.requestBody,
|
||||
responseStatus: upstream.status,
|
||||
responseHeaders: buildSurfaceProxyDebugResponseHeaders(upstream),
|
||||
responseBody,
|
||||
rawErrorText: null,
|
||||
recoverApplied,
|
||||
downgradeDecision: false,
|
||||
downgradeReason: null,
|
||||
memoryWrite: null,
|
||||
});
|
||||
await finalizeDebugSuccess(
|
||||
upstream.status,
|
||||
upstreamPath,
|
||||
buildSurfaceProxyDebugResponseHeaders(upstream),
|
||||
responseBody,
|
||||
);
|
||||
return;
|
||||
} catch (error) {
|
||||
const latency = Date.now() - startTime;
|
||||
const errorMessage = error instanceof Error
|
||||
? error.message
|
||||
: 'Gemini upstream stream failed';
|
||||
const parsedUsage = parseProxyUsage(aggregateState);
|
||||
const responseBody = captureStreamChunks
|
||||
? rawStreamText
|
||||
: { stream: true, usage: parsedUsage, error: errorMessage };
|
||||
await tokenRouter.recordFailure?.(selected.channel.id, {
|
||||
status: 502,
|
||||
errorText: errorMessage,
|
||||
});
|
||||
await logProxy(
|
||||
selected,
|
||||
requestedModel,
|
||||
'failed',
|
||||
502,
|
||||
latency,
|
||||
errorMessage,
|
||||
retryCount,
|
||||
downstreamPath,
|
||||
upstreamPath,
|
||||
clientContext,
|
||||
parsedUsage.promptTokens,
|
||||
parsedUsage.completionTokens,
|
||||
parsedUsage.totalTokens,
|
||||
);
|
||||
await safeInsertSurfaceProxyDebugAttempt(debugTrace, {
|
||||
attemptIndex: retryCount,
|
||||
endpoint: isInternalGemini ? 'gemini-internal' : 'gemini-native',
|
||||
requestPath: upstreamPath,
|
||||
targetUrl: directDispatchState.targetUrl,
|
||||
runtimeExecutor: directDispatchState.runtimeExecutor,
|
||||
requestHeaders: directDispatchState.requestHeaders,
|
||||
requestBody: directDispatchState.requestBody,
|
||||
responseStatus: upstream.status,
|
||||
responseHeaders: buildSurfaceProxyDebugResponseHeaders(upstream),
|
||||
responseBody,
|
||||
rawErrorText: errorMessage,
|
||||
recoverApplied,
|
||||
downgradeDecision: false,
|
||||
downgradeReason: null,
|
||||
memoryWrite: null,
|
||||
});
|
||||
await safeFinalizeSurfaceProxyDebugTrace(debugTrace, {
|
||||
finalStatus: 'failed',
|
||||
finalHttpStatus: upstream.status,
|
||||
finalUpstreamPath: upstreamPath,
|
||||
finalResponseHeaders: buildSurfaceProxyDebugResponseHeaders(upstream),
|
||||
finalResponseBody: responseBody,
|
||||
});
|
||||
return;
|
||||
} finally {
|
||||
reader.releaseLock();
|
||||
reply.raw.end();
|
||||
}
|
||||
}
|
||||
|
||||
const text = await readRuntimeResponseText(upstream);
|
||||
@@ -687,6 +979,31 @@ export async function geminiProxyRoute(app: FastifyInstance) {
|
||||
parsedUsage.completionTokens,
|
||||
parsedUsage.totalTokens,
|
||||
);
|
||||
await safeInsertSurfaceProxyDebugAttempt(debugTrace, {
|
||||
attemptIndex: retryCount,
|
||||
endpoint: isInternalGemini ? 'gemini-internal' : 'gemini-native',
|
||||
requestPath: upstreamPath,
|
||||
targetUrl: directDispatchState.targetUrl,
|
||||
runtimeExecutor: directDispatchState.runtimeExecutor,
|
||||
requestHeaders: directDispatchState.requestHeaders,
|
||||
requestBody: directDispatchState.requestBody,
|
||||
responseStatus: upstream.status,
|
||||
responseHeaders: buildSurfaceProxyDebugResponseHeaders(upstream),
|
||||
responseBody: responsePayload,
|
||||
rawErrorText: null,
|
||||
recoverApplied,
|
||||
downgradeDecision: false,
|
||||
downgradeReason: null,
|
||||
memoryWrite: null,
|
||||
});
|
||||
await finalizeDebugSuccess(
|
||||
upstream.status,
|
||||
upstreamPath,
|
||||
buildSurfaceProxyDebugResponseHeaders(upstream),
|
||||
isGeminiCliDownstream && !isCountTokensAction
|
||||
? { response: responsePayload }
|
||||
: responsePayload,
|
||||
);
|
||||
return reply.code(upstream.status).send(
|
||||
isGeminiCliDownstream && !isCountTokensAction
|
||||
? { response: responsePayload }
|
||||
@@ -707,6 +1024,29 @@ export async function geminiProxyRoute(app: FastifyInstance) {
|
||||
upstreamPath,
|
||||
clientContext,
|
||||
);
|
||||
await safeInsertSurfaceProxyDebugAttempt(debugTrace, {
|
||||
attemptIndex: retryCount,
|
||||
endpoint: isInternalGemini ? 'gemini-internal' : 'gemini-native',
|
||||
requestPath: upstreamPath,
|
||||
targetUrl: directDispatchState.targetUrl,
|
||||
runtimeExecutor: directDispatchState.runtimeExecutor,
|
||||
requestHeaders: directDispatchState.requestHeaders,
|
||||
requestBody: directDispatchState.requestBody,
|
||||
responseStatus: upstream.status,
|
||||
responseHeaders: buildSurfaceProxyDebugResponseHeaders(upstream),
|
||||
responseBody: text,
|
||||
rawErrorText: null,
|
||||
recoverApplied,
|
||||
downgradeDecision: false,
|
||||
downgradeReason: null,
|
||||
memoryWrite: null,
|
||||
});
|
||||
await finalizeDebugSuccess(
|
||||
upstream.status,
|
||||
upstreamPath,
|
||||
buildSurfaceProxyDebugResponseHeaders(upstream),
|
||||
text,
|
||||
);
|
||||
return reply.code(upstream.status).type(contentType || 'application/json').send(text);
|
||||
}
|
||||
}
|
||||
@@ -720,6 +1060,13 @@ export async function geminiProxyRoute(app: FastifyInstance) {
|
||||
type: 'invalid_request_error',
|
||||
},
|
||||
});
|
||||
await safeUpdateSurfaceProxyDebugCandidates(debugTrace, {
|
||||
decisionSummary: {
|
||||
retryCount,
|
||||
pathKind: 'gemini-compat-count-tokens-unsupported',
|
||||
},
|
||||
});
|
||||
await finalizeDebugFailure(lastStatus, parseSurfaceProxyDebugTextPayload(lastText), null);
|
||||
return reply.code(lastStatus).type(lastContentType).send(lastText);
|
||||
}
|
||||
|
||||
@@ -753,6 +1100,16 @@ export async function geminiProxyRoute(app: FastifyInstance) {
|
||||
conversationFileSummary,
|
||||
},
|
||||
};
|
||||
await safeUpdateSurfaceProxyDebugCandidates(debugTrace, {
|
||||
endpointCandidates,
|
||||
endpointRuntimeState: getUpstreamEndpointRuntimeStateSnapshot(endpointRuntimeContext),
|
||||
decisionSummary: {
|
||||
retryCount,
|
||||
pathKind: 'gemini-compat-openai',
|
||||
isStreamAction,
|
||||
downstreamProtocol,
|
||||
},
|
||||
});
|
||||
const buildEndpointRequest = (
|
||||
endpoint: 'chat' | 'messages' | 'responses',
|
||||
requestOptions: { forceNormalizeClaudeBody?: boolean } = {},
|
||||
@@ -809,27 +1166,70 @@ export async function geminiProxyRoute(app: FastifyInstance) {
|
||||
),
|
||||
dispatchRequest,
|
||||
});
|
||||
const debugAttemptBase = reserveSurfaceProxyDebugAttemptBase(debugTrace, endpointCandidates.length);
|
||||
const endpointResult = await executeEndpointFlow({
|
||||
siteUrl: selected.site.url,
|
||||
endpointCandidates,
|
||||
buildRequest: (endpoint) => buildEndpointRequest(endpoint),
|
||||
dispatchRequest,
|
||||
tryRecover: endpointStrategy.tryRecover,
|
||||
onAttemptFailure: (ctx) => {
|
||||
recordUpstreamEndpointFailure({
|
||||
onAttemptFailure: async (ctx) => {
|
||||
const memoryWrite = recordUpstreamEndpointFailure({
|
||||
...endpointRuntimeContext,
|
||||
endpoint: ctx.request.endpoint,
|
||||
status: ctx.response.status,
|
||||
errorText: ctx.rawErrText,
|
||||
});
|
||||
await safeInsertSurfaceProxyDebugAttempt(debugTrace, {
|
||||
attemptIndex: debugAttemptBase + ctx.endpointIndex,
|
||||
endpoint: ctx.request.endpoint,
|
||||
requestPath: ctx.request.path,
|
||||
targetUrl: ctx.targetUrl,
|
||||
runtimeExecutor: ctx.request.runtime?.executor || 'default',
|
||||
requestHeaders: ctx.request.headers,
|
||||
requestBody: ctx.request.body,
|
||||
responseStatus: ctx.response.status,
|
||||
responseHeaders: buildSurfaceProxyDebugResponseHeaders(ctx.response),
|
||||
responseBody: parseSurfaceProxyDebugTextPayload(ctx.rawErrText),
|
||||
rawErrorText: ctx.rawErrText,
|
||||
recoverApplied: ctx.recoverApplied === true,
|
||||
downgradeDecision: false,
|
||||
downgradeReason: null,
|
||||
memoryWrite,
|
||||
});
|
||||
},
|
||||
onAttemptSuccess: (ctx) => {
|
||||
recordUpstreamEndpointSuccess({
|
||||
onAttemptSuccess: async (ctx) => {
|
||||
const memoryWrite = recordUpstreamEndpointSuccess({
|
||||
...endpointRuntimeContext,
|
||||
endpoint: ctx.request.endpoint,
|
||||
});
|
||||
const responseBody = await captureSurfaceProxyDebugSuccessResponseBody(debugTrace, ctx);
|
||||
await safeInsertSurfaceProxyDebugAttempt(debugTrace, {
|
||||
attemptIndex: debugAttemptBase + ctx.endpointIndex,
|
||||
endpoint: ctx.request.endpoint,
|
||||
requestPath: ctx.request.path,
|
||||
targetUrl: ctx.targetUrl,
|
||||
runtimeExecutor: ctx.request.runtime?.executor || 'default',
|
||||
requestHeaders: ctx.request.headers,
|
||||
requestBody: ctx.request.body,
|
||||
responseStatus: ctx.response.status,
|
||||
responseHeaders: buildSurfaceProxyDebugResponseHeaders(ctx.response),
|
||||
responseBody,
|
||||
rawErrorText: null,
|
||||
recoverApplied: ctx.recoverApplied === true,
|
||||
downgradeDecision: false,
|
||||
downgradeReason: null,
|
||||
memoryWrite,
|
||||
});
|
||||
},
|
||||
shouldDowngrade: endpointStrategy.shouldDowngrade,
|
||||
onDowngrade: async (ctx) => {
|
||||
await safeUpdateSurfaceProxyDebugAttempt(debugTrace, debugAttemptBase + ctx.endpointIndex, {
|
||||
downgradeDecision: true,
|
||||
downgradeReason: ctx.errText,
|
||||
rawErrorText: ctx.rawErrText,
|
||||
});
|
||||
},
|
||||
});
|
||||
if (!endpointResult.ok) {
|
||||
lastStatus = endpointResult.status;
|
||||
@@ -860,6 +1260,9 @@ export async function geminiProxyRoute(app: FastifyInstance) {
|
||||
retryCount += 1;
|
||||
continue;
|
||||
}
|
||||
await finalizeDebugFailure(lastStatus, JSON.parse(lastText), null).catch(async () => {
|
||||
await finalizeDebugFailure(lastStatus, parseSurfaceProxyDebugTextPayload(lastText), null);
|
||||
});
|
||||
return reply.code(lastStatus).type(lastContentType).send(lastText);
|
||||
}
|
||||
|
||||
@@ -900,6 +1303,12 @@ export async function geminiProxyRoute(app: FastifyInstance) {
|
||||
const downstreamPayload = isGeminiCliDownstream
|
||||
? { response: geminiResponse }
|
||||
: geminiResponse;
|
||||
await finalizeDebugSuccess(
|
||||
upstream.status,
|
||||
upstreamPath,
|
||||
buildSurfaceProxyDebugResponseHeaders(upstream),
|
||||
downstreamPayload,
|
||||
);
|
||||
if (wantsSseEnvelope) {
|
||||
// Some compatibility upstreams finish stream requests with a final JSON payload.
|
||||
// Preserve Gemini streaming UX by wrapping that terminal payload back into one SSE event.
|
||||
@@ -937,6 +1346,7 @@ export async function geminiProxyRoute(app: FastifyInstance) {
|
||||
retryCount += 1;
|
||||
continue;
|
||||
}
|
||||
await finalizeDebugFailure(lastStatus, parseSurfaceProxyDebugTextPayload(lastText), upstreamPath || null);
|
||||
return reply.code(lastStatus).type(lastContentType).send(lastText);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,10 +5,13 @@ import { mergeProxyUsage, parseProxyUsage } from '../../services/proxyUsageParse
|
||||
import { openAiResponsesTransformer } from '../../transformers/openai/responses/index.js';
|
||||
import {
|
||||
buildUpstreamEndpointRequest,
|
||||
recordUpstreamEndpointFailure,
|
||||
recordUpstreamEndpointSuccess,
|
||||
resolveUpstreamEndpointCandidates,
|
||||
} from '../../routes/proxy/upstreamEndpoint.js';
|
||||
import {
|
||||
getUpstreamEndpointRuntimeStateSnapshot,
|
||||
recordUpstreamEndpointFailure,
|
||||
recordUpstreamEndpointSuccess,
|
||||
} from '../../services/upstreamEndpointRuntimeMemory.js';
|
||||
import { ensureModelAllowedForDownstreamKey, getDownstreamRoutingPolicy, recordDownstreamCostUsage } from '../../routes/proxy/downstreamPolicy.js';
|
||||
import { executeEndpointFlow, type BuiltEndpointRequest } from '../../routes/proxy/endpointFlow.js';
|
||||
import { detectProxyFailure } from '../../routes/proxy/proxyFailureJudge.js';
|
||||
@@ -49,10 +52,23 @@ import {
|
||||
clearSurfaceStickyChannel,
|
||||
createSurfaceFailureToolkit,
|
||||
createSurfaceDispatchRequest,
|
||||
getSurfaceStickyPreferredChannelId,
|
||||
recordSurfaceSuccess,
|
||||
selectSurfaceChannelForAttempt,
|
||||
trySurfaceOauthRefreshRecovery,
|
||||
} from './sharedSurface.js';
|
||||
import {
|
||||
buildSurfaceProxyDebugResponseHeaders,
|
||||
captureSurfaceProxyDebugSuccessResponseBody,
|
||||
parseSurfaceProxyDebugTextPayload,
|
||||
reserveSurfaceProxyDebugAttemptBase,
|
||||
safeFinalizeSurfaceProxyDebugTrace,
|
||||
safeInsertSurfaceProxyDebugAttempt,
|
||||
safeUpdateSurfaceProxyDebugAttempt,
|
||||
safeUpdateSurfaceProxyDebugCandidates,
|
||||
safeUpdateSurfaceProxyDebugSelection,
|
||||
startSurfaceProxyDebugTrace,
|
||||
} from '../../services/proxyDebugTraceRuntime.js';
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return !!value && typeof value === 'object';
|
||||
@@ -213,10 +229,43 @@ export async function handleOpenAiResponsesSurfaceRequest(
|
||||
downstreamPath,
|
||||
downstreamApiKeyId,
|
||||
});
|
||||
const debugTrace = await startSurfaceProxyDebugTrace({
|
||||
downstreamPath,
|
||||
clientKind: clientContext.clientKind,
|
||||
sessionId: clientContext.sessionId || null,
|
||||
traceHint: clientContext.traceHint || null,
|
||||
requestedModel,
|
||||
downstreamApiKeyId,
|
||||
requestHeaders: request.headers as Record<string, unknown>,
|
||||
requestBody: request.body,
|
||||
});
|
||||
const finalizeDebugFailure = async (status: number, payload: unknown, upstreamPath: string | null = null) => {
|
||||
await safeFinalizeSurfaceProxyDebugTrace(debugTrace, {
|
||||
finalStatus: 'failed',
|
||||
finalHttpStatus: status,
|
||||
finalUpstreamPath: upstreamPath,
|
||||
finalResponseHeaders: {
|
||||
'content-type': 'application/json',
|
||||
},
|
||||
finalResponseBody: payload,
|
||||
});
|
||||
};
|
||||
const finalizeDebugSuccess = async (status: number, upstreamPath: string | null, responseHeaders: unknown, responseBody: unknown) => {
|
||||
await safeFinalizeSurfaceProxyDebugTrace(debugTrace, {
|
||||
finalStatus: 'success',
|
||||
finalHttpStatus: status,
|
||||
finalUpstreamPath: upstreamPath,
|
||||
finalResponseHeaders: responseHeaders as Record<string, unknown> | null,
|
||||
finalResponseBody: responseBody,
|
||||
});
|
||||
};
|
||||
const excludeChannelIds: number[] = [];
|
||||
let retryCount = 0;
|
||||
|
||||
while (retryCount <= maxRetries) {
|
||||
const stickyPreferredChannelId = retryCount === 0
|
||||
? getSurfaceStickyPreferredChannelId(stickySessionKey)
|
||||
: null;
|
||||
const selected = await selectSurfaceChannelForAttempt({
|
||||
requestedModel,
|
||||
downstreamPolicy,
|
||||
@@ -230,12 +279,29 @@ export async function handleOpenAiResponsesSurfaceRequest(
|
||||
model: requestedModel,
|
||||
reason: 'No available channels after retries',
|
||||
});
|
||||
const payload = {
|
||||
error: { message: 'No available channels for this model', type: 'server_error' as const },
|
||||
};
|
||||
await finalizeDebugFailure(503, payload, null);
|
||||
return reply.code(503).send({
|
||||
error: { message: 'No available channels for this model', type: 'server_error' },
|
||||
});
|
||||
}
|
||||
|
||||
excludeChannelIds.push(selected.channel.id);
|
||||
await safeUpdateSurfaceProxyDebugSelection(debugTrace, {
|
||||
stickySessionKey,
|
||||
stickyHitChannelId: (
|
||||
stickyPreferredChannelId && stickyPreferredChannelId === selected.channel.id
|
||||
? stickyPreferredChannelId
|
||||
: null
|
||||
),
|
||||
selectedChannelId: selected.channel.id,
|
||||
selectedRouteId: selected.channel.routeId ?? null,
|
||||
selectedAccountId: selected.account.id,
|
||||
selectedSiteId: selected.site.id,
|
||||
selectedSitePlatform: selected.site.platform,
|
||||
});
|
||||
|
||||
const modelName = selected.actualModel || requestedModel;
|
||||
const oauth = getOauthInfoFromAccount(selected.account);
|
||||
@@ -299,6 +365,20 @@ export async function handleOpenAiResponsesSurfaceRequest(
|
||||
wantsNativeResponsesReasoning: prefersNativeResponsesReasoning,
|
||||
},
|
||||
};
|
||||
await safeUpdateSurfaceProxyDebugCandidates(debugTrace, {
|
||||
endpointCandidates,
|
||||
endpointRuntimeState: getUpstreamEndpointRuntimeStateSnapshot(endpointRuntimeContext),
|
||||
decisionSummary: {
|
||||
retryCount,
|
||||
downstreamFormat: 'responses',
|
||||
stickySessionKey,
|
||||
stickyPreferredChannelId,
|
||||
oauthProvider: oauth?.provider || null,
|
||||
isCodexSite,
|
||||
requiresNativeResponsesFileUrl,
|
||||
isCompactRequest,
|
||||
},
|
||||
});
|
||||
const buildProviderHeaders = () => (
|
||||
buildOauthProviderHeaders({
|
||||
account: selected.account,
|
||||
@@ -402,6 +482,12 @@ export async function handleOpenAiResponsesSurfaceRequest(
|
||||
if (retryCount <= maxRetries) {
|
||||
continue;
|
||||
}
|
||||
await finalizeDebugFailure(503, {
|
||||
error: {
|
||||
message: busyMessage,
|
||||
type: 'server_error',
|
||||
},
|
||||
});
|
||||
return reply.code(503).send({
|
||||
error: {
|
||||
message: busyMessage,
|
||||
@@ -412,28 +498,69 @@ export async function handleOpenAiResponsesSurfaceRequest(
|
||||
const channelLease = leaseResult.lease;
|
||||
|
||||
try {
|
||||
const debugAttemptBase = reserveSurfaceProxyDebugAttemptBase(debugTrace, endpointCandidates.length);
|
||||
const endpointResult = await executeEndpointFlow({
|
||||
siteUrl: selected.site.url,
|
||||
endpointCandidates,
|
||||
buildRequest: (endpoint) => buildEndpointRequest(endpoint),
|
||||
dispatchRequest,
|
||||
tryRecover,
|
||||
onAttemptFailure: (ctx) => {
|
||||
recordUpstreamEndpointFailure({
|
||||
onAttemptFailure: async (ctx) => {
|
||||
const memoryWrite = recordUpstreamEndpointFailure({
|
||||
...endpointRuntimeContext,
|
||||
endpoint: ctx.request.endpoint,
|
||||
status: ctx.response.status,
|
||||
errorText: ctx.rawErrText,
|
||||
});
|
||||
await safeInsertSurfaceProxyDebugAttempt(debugTrace, {
|
||||
attemptIndex: debugAttemptBase + ctx.endpointIndex,
|
||||
endpoint: ctx.request.endpoint,
|
||||
requestPath: ctx.request.path,
|
||||
targetUrl: ctx.targetUrl,
|
||||
runtimeExecutor: ctx.request.runtime?.executor || 'default',
|
||||
requestHeaders: ctx.request.headers,
|
||||
requestBody: ctx.request.body,
|
||||
responseStatus: ctx.response.status,
|
||||
responseHeaders: buildSurfaceProxyDebugResponseHeaders(ctx.response),
|
||||
responseBody: parseSurfaceProxyDebugTextPayload(ctx.rawErrText),
|
||||
rawErrorText: ctx.rawErrText,
|
||||
recoverApplied: ctx.recoverApplied === true,
|
||||
downgradeDecision: false,
|
||||
downgradeReason: null,
|
||||
memoryWrite,
|
||||
});
|
||||
},
|
||||
onAttemptSuccess: (ctx) => {
|
||||
recordUpstreamEndpointSuccess({
|
||||
onAttemptSuccess: async (ctx) => {
|
||||
const memoryWrite = recordUpstreamEndpointSuccess({
|
||||
...endpointRuntimeContext,
|
||||
endpoint: ctx.request.endpoint,
|
||||
});
|
||||
const responseBody = await captureSurfaceProxyDebugSuccessResponseBody(debugTrace, ctx);
|
||||
await safeInsertSurfaceProxyDebugAttempt(debugTrace, {
|
||||
attemptIndex: debugAttemptBase + ctx.endpointIndex,
|
||||
endpoint: ctx.request.endpoint,
|
||||
requestPath: ctx.request.path,
|
||||
targetUrl: ctx.targetUrl,
|
||||
runtimeExecutor: ctx.request.runtime?.executor || 'default',
|
||||
requestHeaders: ctx.request.headers,
|
||||
requestBody: ctx.request.body,
|
||||
responseStatus: ctx.response.status,
|
||||
responseHeaders: buildSurfaceProxyDebugResponseHeaders(ctx.response),
|
||||
responseBody,
|
||||
rawErrorText: null,
|
||||
recoverApplied: ctx.recoverApplied === true,
|
||||
downgradeDecision: false,
|
||||
downgradeReason: null,
|
||||
memoryWrite,
|
||||
});
|
||||
},
|
||||
shouldDowngrade: endpointStrategy.shouldDowngrade,
|
||||
onDowngrade: (ctx) => {
|
||||
onDowngrade: async (ctx) => {
|
||||
await safeUpdateSurfaceProxyDebugAttempt(debugTrace, debugAttemptBase + ctx.endpointIndex, {
|
||||
downgradeDecision: true,
|
||||
downgradeReason: ctx.errText,
|
||||
rawErrorText: ctx.rawErrText,
|
||||
});
|
||||
return failureToolkit.log({
|
||||
selected,
|
||||
modelRequested: requestedModel,
|
||||
@@ -465,12 +592,17 @@ export async function handleOpenAiResponsesSurfaceRequest(
|
||||
retryCount += 1;
|
||||
continue;
|
||||
}
|
||||
await finalizeDebugFailure(
|
||||
failureOutcome.status,
|
||||
failureOutcome.payload,
|
||||
null,
|
||||
);
|
||||
return reply.code(failureOutcome.status).send(failureOutcome.payload);
|
||||
}
|
||||
|
||||
const upstream = endpointResult.upstream;
|
||||
const successfulUpstreamPath = endpointResult.upstreamPath;
|
||||
const finalizeStreamSuccess = async (parsedUsage: UsageSummary, latency: number) => {
|
||||
const finalizeStreamSuccess = async (parsedUsage: UsageSummary, latency: number, streamDebugBody: unknown) => {
|
||||
try {
|
||||
await recordSurfaceSuccess({
|
||||
selected,
|
||||
@@ -492,6 +624,12 @@ export async function handleOpenAiResponsesSurfaceRequest(
|
||||
} catch (error) {
|
||||
console.error('[responses] post-stream success logging failed:', error);
|
||||
}
|
||||
await finalizeDebugSuccess(
|
||||
200,
|
||||
successfulUpstreamPath,
|
||||
buildSurfaceProxyDebugResponseHeaders(upstream),
|
||||
streamDebugBody,
|
||||
);
|
||||
};
|
||||
|
||||
if (isStream) {
|
||||
@@ -557,10 +695,20 @@ export async function handleOpenAiResponsesSurfaceRequest(
|
||||
totalTokens: parsedUsage.totalTokens,
|
||||
upstreamPath: successfulUpstreamPath,
|
||||
});
|
||||
await finalizeDebugFailure(502, {
|
||||
error: {
|
||||
message: streamResult.errorMessage,
|
||||
type: 'stream_error',
|
||||
},
|
||||
}, successfulUpstreamPath);
|
||||
return;
|
||||
}
|
||||
|
||||
await finalizeStreamSuccess(parsedUsage, latency);
|
||||
await finalizeStreamSuccess(
|
||||
parsedUsage,
|
||||
latency,
|
||||
debugTrace?.options.captureStreamChunks ? rawText : { stream: true, usage: parsedUsage },
|
||||
);
|
||||
bindSurfaceStickyChannel({
|
||||
stickySessionKey,
|
||||
selected,
|
||||
@@ -601,6 +749,11 @@ export async function handleOpenAiResponsesSurfaceRequest(
|
||||
retryCount += 1;
|
||||
continue;
|
||||
}
|
||||
await finalizeDebugFailure(
|
||||
failureOutcome.status,
|
||||
failureOutcome.payload,
|
||||
successfulUpstreamPath,
|
||||
);
|
||||
return reply.code(failureOutcome.status).send(failureOutcome.payload);
|
||||
}
|
||||
|
||||
@@ -624,10 +777,20 @@ export async function handleOpenAiResponsesSurfaceRequest(
|
||||
upstreamPath: successfulUpstreamPath,
|
||||
runtimeFailureStatus: 502,
|
||||
});
|
||||
await finalizeDebugFailure(502, {
|
||||
error: {
|
||||
message: streamResult.errorMessage,
|
||||
type: 'stream_error',
|
||||
},
|
||||
}, successfulUpstreamPath);
|
||||
return;
|
||||
}
|
||||
|
||||
await finalizeStreamSuccess(parsedUsage, latency);
|
||||
await finalizeStreamSuccess(
|
||||
parsedUsage,
|
||||
latency,
|
||||
debugTrace?.options.captureStreamChunks ? rawText : upstreamData,
|
||||
);
|
||||
bindSurfaceStickyChannel({
|
||||
stickySessionKey,
|
||||
selected,
|
||||
@@ -660,7 +823,11 @@ export async function handleOpenAiResponsesSurfaceRequest(
|
||||
]);
|
||||
reply.raw.end();
|
||||
const latency = Date.now() - startTime;
|
||||
await finalizeStreamSuccess(parsedUsage, latency);
|
||||
await finalizeStreamSuccess(
|
||||
parsedUsage,
|
||||
latency,
|
||||
debugTrace?.options.captureStreamChunks ? rawText : collectedPayload,
|
||||
);
|
||||
return;
|
||||
} catch {
|
||||
// Fall through to the generic stream session for response.failed/error terminals.
|
||||
@@ -685,10 +852,20 @@ export async function handleOpenAiResponsesSurfaceRequest(
|
||||
upstreamPath: successfulUpstreamPath,
|
||||
runtimeFailureStatus: 502,
|
||||
});
|
||||
await finalizeDebugFailure(502, {
|
||||
error: {
|
||||
message: streamResult.errorMessage,
|
||||
type: 'stream_error',
|
||||
},
|
||||
}, successfulUpstreamPath);
|
||||
return;
|
||||
}
|
||||
|
||||
await finalizeStreamSuccess(parsedUsage, latency);
|
||||
await finalizeStreamSuccess(
|
||||
parsedUsage,
|
||||
latency,
|
||||
debugTrace?.options.captureStreamChunks ? rawText : { stream: true, usage: parsedUsage },
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -740,6 +917,12 @@ export async function handleOpenAiResponsesSurfaceRequest(
|
||||
upstreamPath: successfulUpstreamPath,
|
||||
runtimeFailureStatus: 502,
|
||||
});
|
||||
await finalizeDebugFailure(502, {
|
||||
error: {
|
||||
message: streamResult.errorMessage,
|
||||
type: 'stream_error',
|
||||
},
|
||||
}, successfulUpstreamPath);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -748,7 +931,11 @@ export async function handleOpenAiResponsesSurfaceRequest(
|
||||
// response or retry on another channel. Responses stream failures are
|
||||
// handled in-band by the proxy stream session.
|
||||
|
||||
await finalizeStreamSuccess(parsedUsage, latency);
|
||||
await finalizeStreamSuccess(
|
||||
parsedUsage,
|
||||
latency,
|
||||
debugTrace?.options.captureStreamChunks ? rawText : { stream: true, usage: parsedUsage },
|
||||
);
|
||||
bindSurfaceStickyChannel({
|
||||
stickySessionKey,
|
||||
selected,
|
||||
@@ -809,6 +996,11 @@ export async function handleOpenAiResponsesSurfaceRequest(
|
||||
retryCount += 1;
|
||||
continue;
|
||||
}
|
||||
await finalizeDebugFailure(
|
||||
failureOutcome.status,
|
||||
failureOutcome.payload,
|
||||
successfulUpstreamPath,
|
||||
);
|
||||
return reply.code(failureOutcome.status).send(failureOutcome.payload);
|
||||
}
|
||||
const normalized = openAiResponsesTransformer.transformFinalResponse(
|
||||
@@ -843,6 +1035,12 @@ export async function handleOpenAiResponsesSurfaceRequest(
|
||||
} catch (error) {
|
||||
console.error('[responses] post-response success logging failed:', error);
|
||||
}
|
||||
await finalizeDebugSuccess(
|
||||
upstream.status,
|
||||
successfulUpstreamPath,
|
||||
buildSurfaceProxyDebugResponseHeaders(upstream),
|
||||
downstreamData,
|
||||
);
|
||||
bindSurfaceStickyChannel({
|
||||
stickySessionKey,
|
||||
selected,
|
||||
@@ -865,6 +1063,11 @@ export async function handleOpenAiResponsesSurfaceRequest(
|
||||
retryCount += 1;
|
||||
continue;
|
||||
}
|
||||
await finalizeDebugFailure(
|
||||
failureOutcome.status,
|
||||
failureOutcome.payload,
|
||||
null,
|
||||
);
|
||||
return reply.code(failureOutcome.status).send(failureOutcome.payload);
|
||||
} finally {
|
||||
channelLease.release();
|
||||
|
||||
@@ -161,6 +161,11 @@ export function buildSurfaceStickySessionKey(input: {
|
||||
});
|
||||
}
|
||||
|
||||
export function getSurfaceStickyPreferredChannelId(stickySessionKey?: string | null): number | null {
|
||||
if (!stickySessionKey) return null;
|
||||
return proxyChannelCoordinator.getStickyChannelId(stickySessionKey) ?? null;
|
||||
}
|
||||
|
||||
export function bindSurfaceStickyChannel(input: {
|
||||
stickySessionKey?: string | null;
|
||||
selected: {
|
||||
|
||||
@@ -55,6 +55,15 @@ describe('settings and auth events', () => {
|
||||
config.codexUpstreamWebsocketEnabled = false;
|
||||
config.proxySessionChannelConcurrencyLimit = 2;
|
||||
config.proxySessionChannelQueueWaitMs = 1500;
|
||||
(config as any).proxyDebugTraceEnabled = false;
|
||||
(config as any).proxyDebugCaptureHeaders = true;
|
||||
(config as any).proxyDebugCaptureBodies = false;
|
||||
(config as any).proxyDebugCaptureStreamChunks = false;
|
||||
(config as any).proxyDebugTargetSessionId = '';
|
||||
(config as any).proxyDebugTargetClientKind = '';
|
||||
(config as any).proxyDebugTargetModel = '';
|
||||
(config as any).proxyDebugRetentionHours = 24;
|
||||
(config as any).proxyDebugMaxBodyBytes = 262144;
|
||||
config.routingFallbackUnitCost = 1;
|
||||
(config as any).telegramEnabled = false;
|
||||
(config as any).telegramApiBaseUrl = 'https://api.telegram.org';
|
||||
@@ -146,6 +155,67 @@ describe('settings and auth events', () => {
|
||||
expect(savedQueueWait?.value).toBe(JSON.stringify(4200));
|
||||
});
|
||||
|
||||
it('persists proxy debug runtime settings from runtime settings', async () => {
|
||||
const updateResponse = await app.inject({
|
||||
method: 'PUT',
|
||||
url: '/api/settings/runtime',
|
||||
payload: {
|
||||
proxyDebugTraceEnabled: true,
|
||||
proxyDebugCaptureHeaders: true,
|
||||
proxyDebugCaptureBodies: true,
|
||||
proxyDebugCaptureStreamChunks: true,
|
||||
proxyDebugTargetSessionId: 'sess-debug-1',
|
||||
proxyDebugTargetClientKind: 'codex',
|
||||
proxyDebugTargetModel: 'gpt-4o',
|
||||
proxyDebugRetentionHours: 12,
|
||||
proxyDebugMaxBodyBytes: 131072,
|
||||
},
|
||||
});
|
||||
|
||||
expect(updateResponse.statusCode).toBe(200);
|
||||
const updated = updateResponse.json() as {
|
||||
proxyDebugTraceEnabled?: boolean;
|
||||
proxyDebugCaptureHeaders?: boolean;
|
||||
proxyDebugCaptureBodies?: boolean;
|
||||
proxyDebugCaptureStreamChunks?: boolean;
|
||||
proxyDebugTargetSessionId?: string;
|
||||
proxyDebugTargetClientKind?: string;
|
||||
proxyDebugTargetModel?: string;
|
||||
proxyDebugRetentionHours?: number;
|
||||
proxyDebugMaxBodyBytes?: number;
|
||||
};
|
||||
expect(updated).toMatchObject({
|
||||
proxyDebugTraceEnabled: true,
|
||||
proxyDebugCaptureHeaders: true,
|
||||
proxyDebugCaptureBodies: true,
|
||||
proxyDebugCaptureStreamChunks: true,
|
||||
proxyDebugTargetSessionId: 'sess-debug-1',
|
||||
proxyDebugTargetClientKind: 'codex',
|
||||
proxyDebugTargetModel: 'gpt-4o',
|
||||
proxyDebugRetentionHours: 12,
|
||||
proxyDebugMaxBodyBytes: 131072,
|
||||
});
|
||||
|
||||
const savedEnabled = await db.select().from(schema.settings).where(eq(schema.settings.key, 'proxy_debug_trace_enabled')).get();
|
||||
const savedHeaders = await db.select().from(schema.settings).where(eq(schema.settings.key, 'proxy_debug_capture_headers')).get();
|
||||
const savedBodies = await db.select().from(schema.settings).where(eq(schema.settings.key, 'proxy_debug_capture_bodies')).get();
|
||||
const savedStreamChunks = await db.select().from(schema.settings).where(eq(schema.settings.key, 'proxy_debug_capture_stream_chunks')).get();
|
||||
const savedTargetSessionId = await db.select().from(schema.settings).where(eq(schema.settings.key, 'proxy_debug_target_session_id')).get();
|
||||
const savedTargetClientKind = await db.select().from(schema.settings).where(eq(schema.settings.key, 'proxy_debug_target_client_kind')).get();
|
||||
const savedTargetModel = await db.select().from(schema.settings).where(eq(schema.settings.key, 'proxy_debug_target_model')).get();
|
||||
const savedRetentionHours = await db.select().from(schema.settings).where(eq(schema.settings.key, 'proxy_debug_retention_hours')).get();
|
||||
const savedMaxBodyBytes = await db.select().from(schema.settings).where(eq(schema.settings.key, 'proxy_debug_max_body_bytes')).get();
|
||||
expect(savedEnabled?.value).toBe(JSON.stringify(true));
|
||||
expect(savedHeaders?.value).toBe(JSON.stringify(true));
|
||||
expect(savedBodies?.value).toBe(JSON.stringify(true));
|
||||
expect(savedStreamChunks?.value).toBe(JSON.stringify(true));
|
||||
expect(savedTargetSessionId?.value).toBe(JSON.stringify('sess-debug-1'));
|
||||
expect(savedTargetClientKind?.value).toBe(JSON.stringify('codex'));
|
||||
expect(savedTargetModel?.value).toBe(JSON.stringify('gpt-4o'));
|
||||
expect(savedRetentionHours?.value).toBe(JSON.stringify(12));
|
||||
expect(savedMaxBodyBytes?.value).toBe(JSON.stringify(131072));
|
||||
});
|
||||
|
||||
it('returns current recognized admin IP in runtime settings response', async () => {
|
||||
const response = await app.inject({
|
||||
method: 'GET',
|
||||
|
||||
@@ -41,6 +41,15 @@ interface RuntimeSettingsBody {
|
||||
codexUpstreamWebsocketEnabled?: boolean;
|
||||
proxySessionChannelConcurrencyLimit?: number;
|
||||
proxySessionChannelQueueWaitMs?: number;
|
||||
proxyDebugTraceEnabled?: boolean;
|
||||
proxyDebugCaptureHeaders?: boolean;
|
||||
proxyDebugCaptureBodies?: boolean;
|
||||
proxyDebugCaptureStreamChunks?: boolean;
|
||||
proxyDebugTargetSessionId?: string;
|
||||
proxyDebugTargetClientKind?: string;
|
||||
proxyDebugTargetModel?: string;
|
||||
proxyDebugRetentionHours?: number;
|
||||
proxyDebugMaxBodyBytes?: number;
|
||||
checkinCron?: string;
|
||||
checkinScheduleMode?: 'cron' | 'interval';
|
||||
checkinIntervalHours?: number;
|
||||
@@ -409,6 +418,62 @@ function applyImportedSettingToRuntime(key: string, value: unknown) {
|
||||
config.proxySessionChannelQueueWaitMs = Math.trunc(queueWaitMs);
|
||||
return;
|
||||
}
|
||||
case 'proxy_debug_trace_enabled': {
|
||||
try {
|
||||
config.proxyDebugTraceEnabled = parseBooleanFlag(value, '代理调试追踪开关');
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
return;
|
||||
}
|
||||
case 'proxy_debug_capture_headers': {
|
||||
try {
|
||||
config.proxyDebugCaptureHeaders = parseBooleanFlag(value, '代理调试请求头采集');
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
return;
|
||||
}
|
||||
case 'proxy_debug_capture_bodies': {
|
||||
try {
|
||||
config.proxyDebugCaptureBodies = parseBooleanFlag(value, '代理调试请求体采集');
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
return;
|
||||
}
|
||||
case 'proxy_debug_capture_stream_chunks': {
|
||||
try {
|
||||
config.proxyDebugCaptureStreamChunks = parseBooleanFlag(value, '代理调试流式分片采集');
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
return;
|
||||
}
|
||||
case 'proxy_debug_target_session_id': {
|
||||
config.proxyDebugTargetSessionId = typeof value === 'string' ? value.trim() : '';
|
||||
return;
|
||||
}
|
||||
case 'proxy_debug_target_client_kind': {
|
||||
config.proxyDebugTargetClientKind = typeof value === 'string' ? value.trim() : '';
|
||||
return;
|
||||
}
|
||||
case 'proxy_debug_target_model': {
|
||||
config.proxyDebugTargetModel = typeof value === 'string' ? value.trim() : '';
|
||||
return;
|
||||
}
|
||||
case 'proxy_debug_retention_hours': {
|
||||
const retentionHours = Number(value);
|
||||
if (!Number.isFinite(retentionHours) || retentionHours < 1) return;
|
||||
config.proxyDebugRetentionHours = Math.trunc(retentionHours);
|
||||
return;
|
||||
}
|
||||
case 'proxy_debug_max_body_bytes': {
|
||||
const maxBodyBytes = Number(value);
|
||||
if (!Number.isFinite(maxBodyBytes) || maxBodyBytes < 1024) return;
|
||||
config.proxyDebugMaxBodyBytes = Math.trunc(maxBodyBytes);
|
||||
return;
|
||||
}
|
||||
case 'proxy_empty_content_fail_enabled': {
|
||||
try {
|
||||
config.proxyEmptyContentFailEnabled = parseBooleanFlag(value, '空内容判定失败开关');
|
||||
@@ -580,6 +645,15 @@ function getRuntimeSettingsResponse(currentAdminIp = '') {
|
||||
codexUpstreamWebsocketEnabled: config.codexUpstreamWebsocketEnabled,
|
||||
proxySessionChannelConcurrencyLimit: config.proxySessionChannelConcurrencyLimit,
|
||||
proxySessionChannelQueueWaitMs: config.proxySessionChannelQueueWaitMs,
|
||||
proxyDebugTraceEnabled: config.proxyDebugTraceEnabled,
|
||||
proxyDebugCaptureHeaders: config.proxyDebugCaptureHeaders,
|
||||
proxyDebugCaptureBodies: config.proxyDebugCaptureBodies,
|
||||
proxyDebugCaptureStreamChunks: config.proxyDebugCaptureStreamChunks,
|
||||
proxyDebugTargetSessionId: config.proxyDebugTargetSessionId,
|
||||
proxyDebugTargetClientKind: config.proxyDebugTargetClientKind,
|
||||
proxyDebugTargetModel: config.proxyDebugTargetModel,
|
||||
proxyDebugRetentionHours: config.proxyDebugRetentionHours,
|
||||
proxyDebugMaxBodyBytes: config.proxyDebugMaxBodyBytes,
|
||||
routingFallbackUnitCost: config.routingFallbackUnitCost,
|
||||
routingWeights: config.routingWeights,
|
||||
webhookUrl: config.webhookUrl,
|
||||
@@ -1007,6 +1081,127 @@ export async function settingsRoutes(app: FastifyInstance) {
|
||||
upsertSetting('proxy_session_channel_queue_wait_ms', config.proxySessionChannelQueueWaitMs);
|
||||
}
|
||||
|
||||
if (body.proxyDebugTraceEnabled !== undefined) {
|
||||
let nextValue = false;
|
||||
try {
|
||||
nextValue = parseBooleanFlag(body.proxyDebugTraceEnabled, '代理调试追踪开关');
|
||||
} catch (err: any) {
|
||||
return reply.code(400).send({
|
||||
success: false,
|
||||
message: err?.message || '代理调试追踪开关格式无效',
|
||||
});
|
||||
}
|
||||
if (nextValue !== config.proxyDebugTraceEnabled) {
|
||||
changedLabels.push('代理调试追踪');
|
||||
}
|
||||
config.proxyDebugTraceEnabled = nextValue;
|
||||
upsertSetting('proxy_debug_trace_enabled', config.proxyDebugTraceEnabled);
|
||||
}
|
||||
|
||||
if (body.proxyDebugCaptureHeaders !== undefined) {
|
||||
let nextValue = false;
|
||||
try {
|
||||
nextValue = parseBooleanFlag(body.proxyDebugCaptureHeaders, '代理调试请求头采集');
|
||||
} catch (err: any) {
|
||||
return reply.code(400).send({
|
||||
success: false,
|
||||
message: err?.message || '代理调试请求头采集格式无效',
|
||||
});
|
||||
}
|
||||
if (nextValue !== config.proxyDebugCaptureHeaders) {
|
||||
changedLabels.push('代理调试请求头采集');
|
||||
}
|
||||
config.proxyDebugCaptureHeaders = nextValue;
|
||||
upsertSetting('proxy_debug_capture_headers', config.proxyDebugCaptureHeaders);
|
||||
}
|
||||
|
||||
if (body.proxyDebugCaptureBodies !== undefined) {
|
||||
let nextValue = false;
|
||||
try {
|
||||
nextValue = parseBooleanFlag(body.proxyDebugCaptureBodies, '代理调试请求体采集');
|
||||
} catch (err: any) {
|
||||
return reply.code(400).send({
|
||||
success: false,
|
||||
message: err?.message || '代理调试请求体采集格式无效',
|
||||
});
|
||||
}
|
||||
if (nextValue !== config.proxyDebugCaptureBodies) {
|
||||
changedLabels.push('代理调试请求体采集');
|
||||
}
|
||||
config.proxyDebugCaptureBodies = nextValue;
|
||||
upsertSetting('proxy_debug_capture_bodies', config.proxyDebugCaptureBodies);
|
||||
}
|
||||
|
||||
if (body.proxyDebugCaptureStreamChunks !== undefined) {
|
||||
let nextValue = false;
|
||||
try {
|
||||
nextValue = parseBooleanFlag(body.proxyDebugCaptureStreamChunks, '代理调试流式分片采集');
|
||||
} catch (err: any) {
|
||||
return reply.code(400).send({
|
||||
success: false,
|
||||
message: err?.message || '代理调试流式分片采集格式无效',
|
||||
});
|
||||
}
|
||||
if (nextValue !== config.proxyDebugCaptureStreamChunks) {
|
||||
changedLabels.push('代理调试流式分片采集');
|
||||
}
|
||||
config.proxyDebugCaptureStreamChunks = nextValue;
|
||||
upsertSetting('proxy_debug_capture_stream_chunks', config.proxyDebugCaptureStreamChunks);
|
||||
}
|
||||
|
||||
if (body.proxyDebugTargetSessionId !== undefined) {
|
||||
const nextValue = String(body.proxyDebugTargetSessionId || '').trim();
|
||||
if (nextValue !== config.proxyDebugTargetSessionId) {
|
||||
changedLabels.push('代理调试目标会话');
|
||||
}
|
||||
config.proxyDebugTargetSessionId = nextValue;
|
||||
upsertSetting('proxy_debug_target_session_id', config.proxyDebugTargetSessionId);
|
||||
}
|
||||
|
||||
if (body.proxyDebugTargetClientKind !== undefined) {
|
||||
const nextValue = String(body.proxyDebugTargetClientKind || '').trim();
|
||||
if (nextValue !== config.proxyDebugTargetClientKind) {
|
||||
changedLabels.push('代理调试目标客户端');
|
||||
}
|
||||
config.proxyDebugTargetClientKind = nextValue;
|
||||
upsertSetting('proxy_debug_target_client_kind', config.proxyDebugTargetClientKind);
|
||||
}
|
||||
|
||||
if (body.proxyDebugTargetModel !== undefined) {
|
||||
const nextValue = String(body.proxyDebugTargetModel || '').trim();
|
||||
if (nextValue !== config.proxyDebugTargetModel) {
|
||||
changedLabels.push('代理调试目标模型');
|
||||
}
|
||||
config.proxyDebugTargetModel = nextValue;
|
||||
upsertSetting('proxy_debug_target_model', config.proxyDebugTargetModel);
|
||||
}
|
||||
|
||||
if (body.proxyDebugRetentionHours !== undefined) {
|
||||
const retentionHours = Number(body.proxyDebugRetentionHours);
|
||||
if (!Number.isFinite(retentionHours) || retentionHours < 1) {
|
||||
return reply.code(400).send({ success: false, message: '代理调试保留时长必须是大于等于 1 的整数小时' });
|
||||
}
|
||||
const nextValue = Math.trunc(retentionHours);
|
||||
if (nextValue !== config.proxyDebugRetentionHours) {
|
||||
changedLabels.push(`代理调试保留时长（${config.proxyDebugRetentionHours}h -> ${nextValue}h）`);
|
||||
}
|
||||
config.proxyDebugRetentionHours = nextValue;
|
||||
upsertSetting('proxy_debug_retention_hours', config.proxyDebugRetentionHours);
|
||||
}
|
||||
|
||||
if (body.proxyDebugMaxBodyBytes !== undefined) {
|
||||
const maxBodyBytes = Number(body.proxyDebugMaxBodyBytes);
|
||||
if (!Number.isFinite(maxBodyBytes) || maxBodyBytes < 1024) {
|
||||
return reply.code(400).send({ success: false, message: '代理调试抓取体积上限必须是大于等于 1024 的整数字节' });
|
||||
}
|
||||
const nextValue = Math.trunc(maxBodyBytes);
|
||||
if (nextValue !== config.proxyDebugMaxBodyBytes) {
|
||||
changedLabels.push(`代理调试抓取体积上限（${config.proxyDebugMaxBodyBytes}B -> ${nextValue}B）`);
|
||||
}
|
||||
config.proxyDebugMaxBodyBytes = nextValue;
|
||||
upsertSetting('proxy_debug_max_body_bytes', config.proxyDebugMaxBodyBytes);
|
||||
}
|
||||
|
||||
if (body.proxyErrorKeywords !== undefined) {
|
||||
let nextKeywords: string[] = [];
|
||||
try {
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
import Fastify, { type FastifyInstance } from 'fastify';
|
||||
import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest';
|
||||
import { mkdtempSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
import { tmpdir } from 'node:os';
|
||||
|
||||
type DbModule = typeof import('../../db/index.js');
|
||||
type StoreModule = typeof import('../../services/proxyDebugTraceStore.js');
|
||||
|
||||
describe('stats proxy debug api', () => {
|
||||
let app: FastifyInstance;
|
||||
let db: DbModule['db'];
|
||||
let schema: DbModule['schema'];
|
||||
let store: StoreModule;
|
||||
let dataDir = '';
|
||||
|
||||
beforeAll(async () => {
|
||||
dataDir = mkdtempSync(join(tmpdir(), 'metapi-stats-proxy-debug-'));
|
||||
process.env.DATA_DIR = dataDir;
|
||||
|
||||
await import('../../db/migrate.js');
|
||||
const dbModule = await import('../../db/index.js');
|
||||
const storeModule = await import('../../services/proxyDebugTraceStore.js');
|
||||
const statsRoutesModule = await import('./stats.js');
|
||||
|
||||
db = dbModule.db;
|
||||
schema = dbModule.schema;
|
||||
store = storeModule;
|
||||
|
||||
app = Fastify();
|
||||
await app.register(statsRoutesModule.statsRoutes);
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
await db.delete(schema.proxyDebugAttempts).run();
|
||||
await db.delete(schema.proxyDebugTraces).run();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await app.close();
|
||||
const dbModule = await import('../../db/index.js');
|
||||
await dbModule.closeDbConnections();
|
||||
delete process.env.DATA_DIR;
|
||||
});
|
||||
|
||||
it('lists and returns proxy debug traces with attempt details', async () => {
|
||||
const trace = await store.createProxyDebugTrace({
|
||||
downstreamPath: '/v1/chat/completions',
|
||||
clientKind: 'codex',
|
||||
sessionId: 'sess-9',
|
||||
traceHint: 'trace-x',
|
||||
requestedModel: 'gpt-4.1',
|
||||
requestHeaders: { authorization: 'Bearer test' },
|
||||
requestBody: { model: 'gpt-4.1' },
|
||||
});
|
||||
await store.insertProxyDebugAttempt({
|
||||
traceId: trace.id,
|
||||
attemptIndex: 0,
|
||||
endpoint: 'chat',
|
||||
requestPath: '/v1/chat/completions',
|
||||
targetUrl: 'https://example.com/v1/chat/completions',
|
||||
runtimeExecutor: 'default',
|
||||
requestHeaders: { authorization: 'Bearer test' },
|
||||
requestBody: { model: 'gpt-4.1' },
|
||||
responseStatus: 200,
|
||||
responseHeaders: { 'content-type': 'application/json' },
|
||||
responseBody: { id: 'chatcmpl_123' },
|
||||
rawErrorText: null,
|
||||
recoverApplied: false,
|
||||
downgradeDecision: false,
|
||||
downgradeReason: null,
|
||||
memoryWrite: { action: 'success', preferredEndpoint: 'chat' },
|
||||
});
|
||||
await store.finalizeProxyDebugTrace(trace.id, {
|
||||
finalStatus: 'success',
|
||||
finalHttpStatus: 200,
|
||||
finalUpstreamPath: '/v1/chat/completions',
|
||||
finalResponseHeaders: { 'content-type': 'application/json' },
|
||||
finalResponseBody: { id: 'chatcmpl_123' },
|
||||
});
|
||||
|
||||
const listResponse = await app.inject({
|
||||
method: 'GET',
|
||||
url: '/api/stats/proxy-debug/traces?limit=10',
|
||||
});
|
||||
expect(listResponse.statusCode).toBe(200);
|
||||
const listBody = listResponse.json() as { items?: Array<{ id: number; finalStatus: string }> };
|
||||
expect(listBody.items).toHaveLength(1);
|
||||
expect(listBody.items?.[0]?.finalStatus).toBe('success');
|
||||
|
||||
const detailResponse = await app.inject({
|
||||
method: 'GET',
|
||||
url: `/api/stats/proxy-debug/traces/${trace.id}`,
|
||||
});
|
||||
expect(detailResponse.statusCode).toBe(200);
|
||||
const detailBody = detailResponse.json() as {
|
||||
trace?: { requestedModel?: string; sessionId?: string };
|
||||
attempts?: Array<{ endpoint?: string; responseStatus?: number }>;
|
||||
};
|
||||
expect(detailBody.trace).toMatchObject({
|
||||
requestedModel: 'gpt-4.1',
|
||||
sessionId: 'sess-9',
|
||||
});
|
||||
expect(detailBody.attempts?.[0]).toMatchObject({
|
||||
endpoint: 'chat',
|
||||
responseStatus: 200,
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -14,6 +14,10 @@ import {
|
||||
parseProxyLogBillingDetails,
|
||||
withProxyLogSelectFields,
|
||||
} from '../../services/proxyLogStore.js';
|
||||
import {
|
||||
getProxyDebugTraceDetail,
|
||||
listProxyDebugTraces,
|
||||
} from '../../services/proxyDebugTraceStore.js';
|
||||
import { parseProxyLogMessageMeta } from '../proxy/logPathMeta.js';
|
||||
import { requiresManagedAccountTokens } from '../../services/accountExtraConfig.js';
|
||||
import { ACCOUNT_TOKEN_VALUE_STATUS_READY } from '../../services/accountTokenService.js';
|
||||
@@ -820,6 +824,26 @@ export async function statsRoutes(app: FastifyInstance) {
|
||||
return mapProxyLogRow(row, { includeBillingDetails: true });
|
||||
});
|
||||
|
||||
app.get<{ Querystring: { limit?: string } }>('/api/stats/proxy-debug/traces', async (request) => {
|
||||
const limit = normalizeProxyLogPageSize(request.query.limit);
|
||||
const items = await listProxyDebugTraces({ limit });
|
||||
return { items };
|
||||
});
|
||||
|
||||
app.get<{ Params: { id: string } }>('/api/stats/proxy-debug/traces/:id', async (request, reply) => {
|
||||
const id = Number.parseInt(request.params.id, 10);
|
||||
if (!Number.isFinite(id) || id <= 0) {
|
||||
return reply.code(400).send({ message: 'proxy debug trace id is invalid' });
|
||||
}
|
||||
|
||||
const detail = await getProxyDebugTraceDetail(id);
|
||||
if (!detail) {
|
||||
return reply.code(404).send({ message: 'proxy debug trace not found' });
|
||||
}
|
||||
|
||||
return detail;
|
||||
});
|
||||
|
||||
// Models marketplace - refresh upstream models and aggregate.
|
||||
app.get<{ Querystring: { refresh?: string; includePricing?: string } }>('/api/models/marketplace', async (request) => {
|
||||
const refreshRequested = parseBooleanFlag(request.query.refresh);
|
||||
|
||||
@@ -98,6 +98,12 @@ describe('proxy route architecture boundaries', () => {
|
||||
expect(source).not.toContain('shouldInjectDerivedPromptCacheKey');
|
||||
});
|
||||
|
||||
it('keeps endpoint runtime snapshot helper out of the route layer', () => {
|
||||
const source = readSource('./upstreamEndpoint.ts');
|
||||
expect(source).not.toContain('function getUpstreamEndpointRuntimeStateSnapshot(');
|
||||
expect(source).not.toContain('export function getUpstreamEndpointRuntimeStateSnapshot(');
|
||||
});
|
||||
|
||||
it('keeps gemini runtime closure in transformer-owned helpers', () => {
|
||||
const source = readSource('./gemini.ts');
|
||||
const surfaceSource = readSource('../../proxy-core/surfaces/geminiSurface.ts');
|
||||
@@ -179,4 +185,3 @@ describe('proxy route architecture boundaries', () => {
|
||||
expect(geminiSource).not.toContain("from '../../transformers/canonical/");
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -14,6 +14,11 @@ const resolveProxyUsageWithSelfLogFallbackMock = vi.fn(async ({ usage }: any) =>
|
||||
estimatedCostFromQuota: 0,
|
||||
recoveredFromSelfLog: false,
|
||||
}));
|
||||
const startSurfaceProxyDebugTraceMock = vi.fn();
|
||||
const safeUpdateSurfaceProxyDebugSelectionMock = vi.fn();
|
||||
const safeUpdateSurfaceProxyDebugCandidatesMock = vi.fn();
|
||||
const safeInsertSurfaceProxyDebugAttemptMock = vi.fn();
|
||||
const safeFinalizeSurfaceProxyDebugTraceMock = vi.fn();
|
||||
const dbInsertMock = vi.fn((_arg?: any) => ({
|
||||
values: () => ({
|
||||
run: () => undefined,
|
||||
@@ -60,6 +65,19 @@ vi.mock('../../services/proxyUsageFallbackService.js', () => ({
|
||||
resolveProxyUsageWithSelfLogFallback: (arg: any) => resolveProxyUsageWithSelfLogFallbackMock(arg),
|
||||
}));
|
||||
|
||||
vi.mock('../../services/proxyDebugTraceRuntime.js', () => ({
|
||||
startSurfaceProxyDebugTrace: (...args: unknown[]) => startSurfaceProxyDebugTraceMock(...args),
|
||||
safeUpdateSurfaceProxyDebugSelection: (...args: unknown[]) => safeUpdateSurfaceProxyDebugSelectionMock(...args),
|
||||
safeUpdateSurfaceProxyDebugCandidates: (...args: unknown[]) => safeUpdateSurfaceProxyDebugCandidatesMock(...args),
|
||||
safeInsertSurfaceProxyDebugAttempt: (...args: unknown[]) => safeInsertSurfaceProxyDebugAttemptMock(...args),
|
||||
safeFinalizeSurfaceProxyDebugTrace: (...args: unknown[]) => safeFinalizeSurfaceProxyDebugTraceMock(...args),
|
||||
safeUpdateSurfaceProxyDebugAttempt: vi.fn(),
|
||||
reserveSurfaceProxyDebugAttemptBase: () => 0,
|
||||
buildSurfaceProxyDebugResponseHeaders: () => ({}),
|
||||
captureSurfaceProxyDebugSuccessResponseBody: async () => null,
|
||||
parseSurfaceProxyDebugTextPayload: (raw: string) => raw,
|
||||
}));
|
||||
|
||||
vi.mock('../../services/oauth/quota.js', () => ({
|
||||
recordOauthQuotaResetHint: async () => undefined,
|
||||
}));
|
||||
@@ -95,8 +113,28 @@ describe('claude count_tokens proxy route', () => {
|
||||
reportProxyAllFailedMock.mockReset();
|
||||
reportTokenExpiredMock.mockReset();
|
||||
resolveProxyUsageWithSelfLogFallbackMock.mockClear();
|
||||
startSurfaceProxyDebugTraceMock.mockReset();
|
||||
safeUpdateSurfaceProxyDebugSelectionMock.mockReset();
|
||||
safeUpdateSurfaceProxyDebugCandidatesMock.mockReset();
|
||||
safeInsertSurfaceProxyDebugAttemptMock.mockReset();
|
||||
safeFinalizeSurfaceProxyDebugTraceMock.mockReset();
|
||||
dbInsertMock.mockClear();
|
||||
|
||||
startSurfaceProxyDebugTraceMock.mockResolvedValue({
|
||||
traceId: 701,
|
||||
options: {
|
||||
enabled: true,
|
||||
captureHeaders: true,
|
||||
captureBodies: true,
|
||||
captureStreamChunks: false,
|
||||
targetSessionId: '',
|
||||
targetClientKind: '',
|
||||
targetModel: '',
|
||||
retentionHours: 24,
|
||||
maxBodyBytes: 262144,
|
||||
},
|
||||
});
|
||||
|
||||
selectChannelMock.mockReturnValue({
|
||||
channel: { id: 11, routeId: 22 },
|
||||
site: { name: 'claude-site', url: 'https://api.anthropic.com', platform: 'claude' },
|
||||
@@ -146,6 +184,18 @@ describe('claude count_tokens proxy route', () => {
|
||||
expect(options.headers['anthropic-version']).toBe('2023-06-01');
|
||||
expect(options.headers['anthropic-beta']).toContain('claude-code-20250219');
|
||||
expect(options.headers['Accept-Encoding']).toBe('gzip, deflate, br, zstd');
|
||||
expect(startSurfaceProxyDebugTraceMock).toHaveBeenCalledWith(expect.objectContaining({
|
||||
downstreamPath: '/v1/messages/count_tokens',
|
||||
requestedModel: 'claude-opus-4-6',
|
||||
}));
|
||||
expect(safeInsertSurfaceProxyDebugAttemptMock).toHaveBeenCalled();
|
||||
expect(safeFinalizeSurfaceProxyDebugTraceMock).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ traceId: 701 }),
|
||||
expect.objectContaining({
|
||||
finalStatus: 'success',
|
||||
finalUpstreamPath: '/v1/messages/count_tokens?beta=true',
|
||||
}),
|
||||
);
|
||||
|
||||
const forwardedBody = JSON.parse(String(options.body));
|
||||
expect(forwardedBody.model).toBe('claude-opus-4-6');
|
||||
|
||||
@@ -2,7 +2,7 @@ import { zstdCompressSync } from 'node:zlib';
|
||||
import Fastify, { type FastifyInstance } from 'fastify';
|
||||
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { config } from '../../config.js';
|
||||
import { resetUpstreamEndpointRuntimeState } from './upstreamEndpoint.js';
|
||||
import { resetUpstreamEndpointRuntimeState } from '../../services/upstreamEndpointRuntimeMemory.js';
|
||||
|
||||
const fetchMock = vi.fn();
|
||||
const selectChannelMock = vi.fn();
|
||||
|
||||
@@ -26,6 +26,7 @@ export type EndpointAttemptContext = {
|
||||
targetUrl: string;
|
||||
response: Awaited<ReturnType<typeof fetch>>;
|
||||
rawErrText: string;
|
||||
recoverApplied?: boolean;
|
||||
};
|
||||
|
||||
export type EndpointAttemptSuccessContext = {
|
||||
@@ -34,6 +35,7 @@ export type EndpointAttemptSuccessContext = {
|
||||
request: BuiltEndpointRequest;
|
||||
targetUrl: string;
|
||||
response: Awaited<ReturnType<typeof fetch>>;
|
||||
recoverApplied?: boolean;
|
||||
};
|
||||
|
||||
export type EndpointRecoverResult = {
|
||||
@@ -125,6 +127,7 @@ export async function executeEndpointFlow(input: ExecuteEndpointFlowInput): Prom
|
||||
request,
|
||||
targetUrl,
|
||||
response,
|
||||
recoverApplied: false,
|
||||
}, 'onAttemptSuccess');
|
||||
return {
|
||||
ok: true,
|
||||
@@ -141,10 +144,15 @@ export async function executeEndpointFlow(input: ExecuteEndpointFlowInput): Prom
|
||||
targetUrl,
|
||||
response,
|
||||
rawErrText,
|
||||
recoverApplied: false,
|
||||
};
|
||||
|
||||
if (input.tryRecover) {
|
||||
const recovered = await input.tryRecover(baseContext);
|
||||
baseContext.recoverApplied = recovered !== null
|
||||
|| baseContext.request !== request
|
||||
|| baseContext.response !== response
|
||||
|| baseContext.rawErrText !== rawErrText;
|
||||
if (recovered?.upstream?.ok) {
|
||||
const recoveredRequest = recovered.request ?? baseContext.request;
|
||||
const recoveredTargetUrl = recovered.targetUrl ?? (
|
||||
@@ -158,6 +166,7 @@ export async function executeEndpointFlow(input: ExecuteEndpointFlowInput): Prom
|
||||
request: recoveredRequest,
|
||||
targetUrl: recoveredTargetUrl,
|
||||
response: recovered.upstream,
|
||||
recoverApplied: true,
|
||||
}, 'onAttemptSuccess');
|
||||
return {
|
||||
ok: true,
|
||||
|
||||
@@ -2,7 +2,7 @@ import Fastify, { type FastifyInstance } from 'fastify';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { resetUpstreamEndpointRuntimeState } from './upstreamEndpoint.js';
|
||||
import { resetUpstreamEndpointRuntimeState } from '../../services/upstreamEndpointRuntimeMemory.js';
|
||||
|
||||
const fetchMock = vi.fn();
|
||||
const fetchModelPricingCatalogMock = vi.fn();
|
||||
@@ -25,6 +25,11 @@ const dbInsertValuesMock = vi.fn((_values?: unknown) => ({
|
||||
const dbInsertMock = vi.fn((_table?: unknown) => ({
|
||||
values: (values: unknown) => dbInsertValuesMock(values),
|
||||
}));
|
||||
const startSurfaceProxyDebugTraceMock = vi.fn();
|
||||
const safeUpdateSurfaceProxyDebugSelectionMock = vi.fn();
|
||||
const safeUpdateSurfaceProxyDebugCandidatesMock = vi.fn();
|
||||
const safeInsertSurfaceProxyDebugAttemptMock = vi.fn();
|
||||
const safeFinalizeSurfaceProxyDebugTraceMock = vi.fn();
|
||||
|
||||
function createDbSelectChain() {
|
||||
return {
|
||||
@@ -107,6 +112,19 @@ vi.mock('../../db/index.js', () => ({
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock('../../services/proxyDebugTraceRuntime.js', () => ({
|
||||
startSurfaceProxyDebugTrace: (...args: unknown[]) => startSurfaceProxyDebugTraceMock(...args),
|
||||
safeUpdateSurfaceProxyDebugSelection: (...args: unknown[]) => safeUpdateSurfaceProxyDebugSelectionMock(...args),
|
||||
safeUpdateSurfaceProxyDebugCandidates: (...args: unknown[]) => safeUpdateSurfaceProxyDebugCandidatesMock(...args),
|
||||
safeInsertSurfaceProxyDebugAttempt: (...args: unknown[]) => safeInsertSurfaceProxyDebugAttemptMock(...args),
|
||||
safeFinalizeSurfaceProxyDebugTrace: (...args: unknown[]) => safeFinalizeSurfaceProxyDebugTraceMock(...args),
|
||||
safeUpdateSurfaceProxyDebugAttempt: vi.fn(),
|
||||
reserveSurfaceProxyDebugAttemptBase: () => 0,
|
||||
buildSurfaceProxyDebugResponseHeaders: () => ({}),
|
||||
captureSurfaceProxyDebugSuccessResponseBody: async () => null,
|
||||
parseSurfaceProxyDebugTextPayload: (raw: string) => raw,
|
||||
}));
|
||||
|
||||
function parseSsePayloads(body: string): Array<Record<string, unknown>> {
|
||||
return body
|
||||
.replace(/\r\n/g, '\n')
|
||||
@@ -173,6 +191,26 @@ describe('gemini native proxy routes', () => {
|
||||
dbInsertValuesMock.mockClear();
|
||||
dbSelectAllMock.mockReset();
|
||||
dbSelectGetMock.mockReset();
|
||||
startSurfaceProxyDebugTraceMock.mockReset();
|
||||
safeUpdateSurfaceProxyDebugSelectionMock.mockReset();
|
||||
safeUpdateSurfaceProxyDebugCandidatesMock.mockReset();
|
||||
safeInsertSurfaceProxyDebugAttemptMock.mockReset();
|
||||
safeFinalizeSurfaceProxyDebugTraceMock.mockReset();
|
||||
|
||||
startSurfaceProxyDebugTraceMock.mockResolvedValue({
|
||||
traceId: 801,
|
||||
options: {
|
||||
enabled: true,
|
||||
captureHeaders: true,
|
||||
captureBodies: true,
|
||||
captureStreamChunks: false,
|
||||
targetSessionId: '',
|
||||
targetClientKind: '',
|
||||
targetModel: '',
|
||||
retentionHours: 24,
|
||||
maxBodyBytes: 262144,
|
||||
},
|
||||
});
|
||||
|
||||
authorizeDownstreamTokenMock.mockResolvedValue({
|
||||
ok: true,
|
||||
@@ -746,6 +784,25 @@ describe('gemini native proxy routes', () => {
|
||||
Authorization: 'Bearer openai-access-token',
|
||||
'Content-Type': 'application/json',
|
||||
});
|
||||
expect(startSurfaceProxyDebugTraceMock).toHaveBeenCalledWith(expect.objectContaining({
|
||||
downstreamPath: '/v1beta/models/gemini-2.5-flash:generateContent',
|
||||
requestedModel: 'gemini-2.5-flash',
|
||||
}));
|
||||
expect(safeUpdateSurfaceProxyDebugSelectionMock).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ traceId: 801 }),
|
||||
expect.objectContaining({
|
||||
selectedChannelId: 41,
|
||||
selectedSitePlatform: 'openai',
|
||||
}),
|
||||
);
|
||||
expect(safeInsertSurfaceProxyDebugAttemptMock).toHaveBeenCalled();
|
||||
expect(safeFinalizeSurfaceProxyDebugTraceMock).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ traceId: 801 }),
|
||||
expect.objectContaining({
|
||||
finalStatus: 'success',
|
||||
finalUpstreamPath: '/v1/chat/completions',
|
||||
}),
|
||||
);
|
||||
expect(JSON.parse(String(requestInit.body))).toEqual({
|
||||
model: 'gpt-4.1',
|
||||
stream: false,
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { config } from '../../config.js';
|
||||
|
||||
const fetchModelPricingCatalogMock = vi.fn(async (_arg?: unknown): Promise<any> => null);
|
||||
@@ -12,14 +12,17 @@ import {
|
||||
buildUpstreamEndpointRequest,
|
||||
isUnsupportedMediaTypeError,
|
||||
isEndpointDowngradeError,
|
||||
resolveUpstreamEndpointCandidates,
|
||||
} from './upstreamEndpoint.js';
|
||||
import {
|
||||
recordUpstreamEndpointFailure,
|
||||
recordUpstreamEndpointSuccess,
|
||||
resetUpstreamEndpointRuntimeState,
|
||||
resolveUpstreamEndpointCandidates,
|
||||
getUpstreamEndpointRuntimeStateSnapshot,
|
||||
boundEndpointRuntimeModelKey,
|
||||
MAX_ENDPOINT_RUNTIME_MODEL_KEY_LENGTH,
|
||||
MODEL_KEY_HASH_SUFFIX_LENGTH,
|
||||
} from './upstreamEndpoint.js';
|
||||
} from '../../services/upstreamEndpointRuntimeMemory.js';
|
||||
|
||||
const baseContext = {
|
||||
site: {
|
||||
@@ -52,6 +55,9 @@ describe('resolveUpstreamEndpointCandidates', () => {
|
||||
filter: [],
|
||||
};
|
||||
});
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it('uses downstream-aligned endpoint priority for unknown platforms', async () => {
|
||||
const openaiOrder = await resolveUpstreamEndpointCandidates(
|
||||
@@ -236,6 +242,64 @@ describe('resolveUpstreamEndpointCandidates', () => {
|
||||
expect(order).toEqual(['chat', 'messages', 'responses']);
|
||||
});
|
||||
|
||||
it('does not expose preferred endpoint in snapshot when runtime memory is disabled for multimodal requests', () => {
|
||||
recordUpstreamEndpointSuccess({
|
||||
siteId: baseContext.site.id,
|
||||
endpoint: 'responses',
|
||||
downstreamFormat: 'openai',
|
||||
modelName: 'gpt-5.3',
|
||||
requestCapabilities: {
|
||||
conversationFileSummary: {
|
||||
hasImage: true,
|
||||
hasAudio: false,
|
||||
hasDocument: false,
|
||||
hasRemoteDocumentUrl: false,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(getUpstreamEndpointRuntimeStateSnapshot({
|
||||
siteId: baseContext.site.id,
|
||||
downstreamFormat: 'openai',
|
||||
modelName: 'gpt-5.3',
|
||||
requestCapabilities: {
|
||||
conversationFileSummary: {
|
||||
hasImage: true,
|
||||
hasAudio: false,
|
||||
hasDocument: false,
|
||||
hasRemoteDocumentUrl: false,
|
||||
},
|
||||
},
|
||||
})).toMatchObject({
|
||||
enabled: false,
|
||||
preferredEndpoint: null,
|
||||
blockedEndpoints: [],
|
||||
});
|
||||
});
|
||||
|
||||
it('does not expose expired preferred endpoints in the runtime snapshot', () => {
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(new Date('2026-03-28T00:00:00.000Z'));
|
||||
|
||||
recordUpstreamEndpointSuccess({
|
||||
siteId: baseContext.site.id,
|
||||
endpoint: 'chat',
|
||||
downstreamFormat: 'openai',
|
||||
modelName: 'gpt-5.3',
|
||||
});
|
||||
|
||||
vi.setSystemTime(new Date('2026-03-29T01:00:00.000Z'));
|
||||
|
||||
expect(getUpstreamEndpointRuntimeStateSnapshot({
|
||||
siteId: baseContext.site.id,
|
||||
downstreamFormat: 'openai',
|
||||
modelName: 'gpt-5.3',
|
||||
})).toMatchObject({
|
||||
enabled: true,
|
||||
preferredEndpoint: null,
|
||||
});
|
||||
});
|
||||
|
||||
it('does not apply runtime endpoint memory to document attachments', async () => {
|
||||
recordUpstreamEndpointSuccess({
|
||||
siteId: baseContext.site.id,
|
||||
@@ -379,12 +443,13 @@ describe('resolveUpstreamEndpointCandidates', () => {
|
||||
});
|
||||
|
||||
it('does not remember messages fallback success for generic /v1/responses requests', async () => {
|
||||
recordUpstreamEndpointSuccess({
|
||||
const memoryWrite = recordUpstreamEndpointSuccess({
|
||||
siteId: baseContext.site.id,
|
||||
endpoint: 'messages',
|
||||
downstreamFormat: 'responses',
|
||||
modelName: 'gpt-5.3',
|
||||
});
|
||||
expect(memoryWrite).toBeNull();
|
||||
|
||||
const order = await resolveUpstreamEndpointCandidates(
|
||||
{
|
||||
@@ -398,8 +463,23 @@ describe('resolveUpstreamEndpointCandidates', () => {
|
||||
expect(order).toEqual(['responses', 'chat', 'messages']);
|
||||
});
|
||||
|
||||
it('returns the applied success write when runtime memory stores a preferred endpoint', () => {
|
||||
const memoryWrite = recordUpstreamEndpointSuccess({
|
||||
siteId: baseContext.site.id,
|
||||
endpoint: 'responses',
|
||||
downstreamFormat: 'responses',
|
||||
modelName: 'gpt-5.3',
|
||||
});
|
||||
|
||||
expect(memoryWrite).toMatchObject({
|
||||
action: 'success',
|
||||
endpoint: 'responses',
|
||||
preferredEndpoint: 'responses',
|
||||
});
|
||||
});
|
||||
|
||||
it('does not block generic /v1/responses endpoints on transient upstream errors', async () => {
|
||||
recordUpstreamEndpointFailure({
|
||||
const memoryWrite = recordUpstreamEndpointFailure({
|
||||
siteId: baseContext.site.id,
|
||||
endpoint: 'responses',
|
||||
downstreamFormat: 'responses',
|
||||
@@ -407,6 +487,7 @@ describe('resolveUpstreamEndpointCandidates', () => {
|
||||
status: 504,
|
||||
errorText: '{"error":{"message":"Gateway time-out","type":"upstream_error"}}',
|
||||
});
|
||||
expect(memoryWrite).toBeNull();
|
||||
|
||||
const order = await resolveUpstreamEndpointCandidates(
|
||||
{
|
||||
@@ -421,7 +502,7 @@ describe('resolveUpstreamEndpointCandidates', () => {
|
||||
});
|
||||
|
||||
it('learns a better endpoint from explicit upstream protocol errors', async () => {
|
||||
recordUpstreamEndpointFailure({
|
||||
const memoryWrite = recordUpstreamEndpointFailure({
|
||||
siteId: baseContext.site.id,
|
||||
endpoint: 'chat',
|
||||
downstreamFormat: 'openai',
|
||||
@@ -429,6 +510,12 @@ describe('resolveUpstreamEndpointCandidates', () => {
|
||||
status: 400,
|
||||
errorText: 'Unsupported legacy protocol: /v1/chat/completions is not supported. Please use /v1/responses.',
|
||||
});
|
||||
expect(memoryWrite).toMatchObject({
|
||||
action: 'failure',
|
||||
endpoint: 'chat',
|
||||
blockedEndpoint: 'chat',
|
||||
preferredEndpoint: 'responses',
|
||||
});
|
||||
|
||||
const order = await resolveUpstreamEndpointCandidates(
|
||||
{
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { randomUUID, createHash } from 'node:crypto';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import {
|
||||
rankConversationFileEndpoints,
|
||||
type ConversationFileInputSummary,
|
||||
@@ -7,6 +7,10 @@ import { resolveProviderProfile } from '../../proxy-core/providers/registry.js';
|
||||
import { config } from '../../config.js';
|
||||
import { fetchModelPricingCatalog } from '../../services/modelPricingService.js';
|
||||
import { applyPayloadRules } from '../../services/payloadRules.js';
|
||||
import {
|
||||
applyUpstreamEndpointRuntimePreference,
|
||||
buildEndpointCapabilityProfile,
|
||||
} from '../../services/upstreamEndpointRuntimeMemory.js';
|
||||
import type { DownstreamFormat } from '../../transformers/shared/normalized.js';
|
||||
import {
|
||||
convertOpenAiBodyToResponsesBody as convertOpenAiBodyToResponsesBodyViaTransformer,
|
||||
@@ -44,23 +48,6 @@ export {
|
||||
export type UpstreamEndpoint = 'chat' | 'messages' | 'responses';
|
||||
export type EndpointPreference = DownstreamFormat | 'responses';
|
||||
|
||||
type EndpointCapabilityProfile = {
|
||||
modelKey: string;
|
||||
preferMessagesForClaudeModel: boolean;
|
||||
hasImageInput: boolean;
|
||||
hasAudioInput: boolean;
|
||||
hasNonImageFileInput: boolean;
|
||||
hasRemoteDocumentUrl: boolean;
|
||||
wantsNativeResponsesReasoning: boolean;
|
||||
};
|
||||
|
||||
type EndpointRuntimeState = {
|
||||
preferredEndpoint: UpstreamEndpoint | null;
|
||||
preferredUpdatedAtMs: number;
|
||||
lastTouchedAtMs: number;
|
||||
blockedUntilMsByEndpoint: Partial<Record<UpstreamEndpoint, number>>;
|
||||
};
|
||||
|
||||
type ChannelContext = {
|
||||
site: {
|
||||
id: number;
|
||||
@@ -75,26 +62,6 @@ type ChannelContext = {
|
||||
};
|
||||
};
|
||||
|
||||
const ENDPOINT_RUNTIME_PREFERRED_TTL_MS = 24 * 60 * 60 * 1000;
|
||||
const ENDPOINT_RUNTIME_BLOCK_TTL_MS = 6 * 60 * 60 * 1000;
|
||||
const MAX_ENDPOINT_RUNTIME_STATES = 512;
|
||||
export const MAX_ENDPOINT_RUNTIME_MODEL_KEY_LENGTH = 64;
|
||||
export const MODEL_KEY_HASH_SUFFIX_LENGTH = 8;
|
||||
const endpointRuntimeStates = new Map<string, EndpointRuntimeState>();
|
||||
|
||||
export function boundEndpointRuntimeModelKey(value: string): string {
|
||||
if (value.length <= MAX_ENDPOINT_RUNTIME_MODEL_KEY_LENGTH) {
|
||||
return value;
|
||||
}
|
||||
|
||||
const prefix = value.slice(0, MAX_ENDPOINT_RUNTIME_MODEL_KEY_LENGTH);
|
||||
const hash = createHash('sha256')
|
||||
.update(value)
|
||||
.digest('hex')
|
||||
.slice(0, MODEL_KEY_HASH_SUFFIX_LENGTH);
|
||||
return `${prefix}-${hash}`;
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return !!value && typeof value === 'object';
|
||||
}
|
||||
@@ -103,14 +70,6 @@ function asTrimmedString(value: unknown): string {
|
||||
return typeof value === 'string' ? value.trim() : '';
|
||||
}
|
||||
|
||||
function normalizeEndpointRuntimeModelKey(...values: Array<unknown>): string {
|
||||
for (const value of values) {
|
||||
const normalized = asTrimmedString(value).toLowerCase();
|
||||
if (normalized) return boundEndpointRuntimeModelKey(normalized);
|
||||
}
|
||||
return boundEndpointRuntimeModelKey('unknown-model');
|
||||
}
|
||||
|
||||
function resolveRequestedModelForPayloadRules(input: {
|
||||
modelName: string;
|
||||
openaiBody: Record<string, unknown>;
|
||||
@@ -129,12 +88,6 @@ function normalizePlatformName(platform: unknown): string {
|
||||
return asTrimmedString(platform).toLowerCase();
|
||||
}
|
||||
|
||||
function isClaudeFamilyModel(modelName: string): boolean {
|
||||
const normalized = asTrimmedString(modelName).toLowerCase();
|
||||
if (!normalized) return false;
|
||||
return normalized === 'claude' || normalized.startsWith('claude-') || normalized.includes('claude');
|
||||
}
|
||||
|
||||
const HOP_BY_HOP_HEADERS = new Set([
|
||||
'connection',
|
||||
'keep-alive',
|
||||
@@ -495,277 +448,6 @@ function normalizeEndpointTypes(value: unknown): UpstreamEndpoint[] {
|
||||
return Array.from(normalized);
|
||||
}
|
||||
|
||||
function buildEndpointCapabilityProfile(input?: {
|
||||
modelName?: string;
|
||||
requestedModelHint?: string;
|
||||
requestCapabilities?: {
|
||||
hasNonImageFileInput?: boolean;
|
||||
conversationFileSummary?: ConversationFileInputSummary;
|
||||
wantsNativeResponsesReasoning?: boolean;
|
||||
};
|
||||
}): EndpointCapabilityProfile {
|
||||
const conversationFileSummary = input?.requestCapabilities?.conversationFileSummary;
|
||||
return {
|
||||
modelKey: normalizeEndpointRuntimeModelKey(input?.modelName, input?.requestedModelHint),
|
||||
preferMessagesForClaudeModel: (
|
||||
isClaudeFamilyModel(asTrimmedString(input?.modelName))
|
||||
|| isClaudeFamilyModel(asTrimmedString(input?.requestedModelHint))
|
||||
),
|
||||
hasImageInput: conversationFileSummary?.hasImage === true,
|
||||
hasAudioInput: conversationFileSummary?.hasAudio === true,
|
||||
hasNonImageFileInput: (
|
||||
conversationFileSummary?.hasDocument === true
|
||||
|| input?.requestCapabilities?.hasNonImageFileInput === true
|
||||
),
|
||||
hasRemoteDocumentUrl: (
|
||||
conversationFileSummary?.hasRemoteDocumentUrl === true
|
||||
),
|
||||
wantsNativeResponsesReasoning: input?.requestCapabilities?.wantsNativeResponsesReasoning === true,
|
||||
};
|
||||
}
|
||||
|
||||
function shouldUseEndpointRuntimeMemory(capabilityProfile: EndpointCapabilityProfile): boolean {
|
||||
// Attachment-capable requests are not protocol-equivalent across chat/messages/responses.
|
||||
// A transient 200 on one endpoint should not bias later multimodal requests onto a lossy path.
|
||||
return (
|
||||
!capabilityProfile.hasImageInput
|
||||
&& !capabilityProfile.hasAudioInput
|
||||
&& !capabilityProfile.hasNonImageFileInput
|
||||
);
|
||||
}
|
||||
|
||||
function buildEndpointRuntimeStateKey(input: {
|
||||
siteId: number;
|
||||
downstreamFormat: EndpointPreference;
|
||||
capabilityProfile: EndpointCapabilityProfile;
|
||||
}): string {
|
||||
const capabilityProfile = input.capabilityProfile;
|
||||
return [
|
||||
String(input.siteId),
|
||||
input.downstreamFormat,
|
||||
capabilityProfile.modelKey,
|
||||
capabilityProfile.hasNonImageFileInput ? 'files' : 'nofiles',
|
||||
capabilityProfile.hasRemoteDocumentUrl ? 'remoteurl' : 'noremoteurl',
|
||||
capabilityProfile.wantsNativeResponsesReasoning ? 'reasoning' : 'noreasoning',
|
||||
].join(':');
|
||||
}
|
||||
|
||||
function getOrCreateEndpointRuntimeState(key: string, nowMs = Date.now()): EndpointRuntimeState {
|
||||
sweepEndpointRuntimeStates(nowMs);
|
||||
const existing = endpointRuntimeStates.get(key);
|
||||
if (existing) {
|
||||
existing.lastTouchedAtMs = nowMs;
|
||||
return existing;
|
||||
}
|
||||
|
||||
const initial: EndpointRuntimeState = {
|
||||
preferredEndpoint: null,
|
||||
preferredUpdatedAtMs: nowMs,
|
||||
lastTouchedAtMs: nowMs,
|
||||
blockedUntilMsByEndpoint: {},
|
||||
};
|
||||
endpointRuntimeStates.set(key, initial);
|
||||
enforceEndpointRuntimeStateLimit();
|
||||
return initial;
|
||||
}
|
||||
|
||||
function maybeDeleteEndpointRuntimeState(key: string, nowMs = Date.now()): void {
|
||||
const state = endpointRuntimeStates.get(key);
|
||||
if (!state) return;
|
||||
|
||||
const hasActiveBlock = Object.values(state.blockedUntilMsByEndpoint).some((untilMs) => (
|
||||
typeof untilMs === 'number' && untilMs > nowMs
|
||||
));
|
||||
const preferredFresh = (
|
||||
!!state.preferredEndpoint
|
||||
&& (state.preferredUpdatedAtMs + ENDPOINT_RUNTIME_PREFERRED_TTL_MS) > nowMs
|
||||
);
|
||||
if (!hasActiveBlock && !preferredFresh) {
|
||||
endpointRuntimeStates.delete(key);
|
||||
}
|
||||
}
|
||||
|
||||
function applyEndpointRuntimePreference(
|
||||
candidates: UpstreamEndpoint[],
|
||||
key: string,
|
||||
nowMs = Date.now(),
|
||||
): UpstreamEndpoint[] {
|
||||
const state = endpointRuntimeStates.get(key);
|
||||
if (!state || candidates.length <= 1) return candidates;
|
||||
state.lastTouchedAtMs = nowMs;
|
||||
|
||||
const blocked = new Set<UpstreamEndpoint>();
|
||||
for (const endpoint of candidates) {
|
||||
const untilMs = state.blockedUntilMsByEndpoint[endpoint];
|
||||
if (typeof untilMs === 'number' && untilMs > nowMs) {
|
||||
blocked.add(endpoint);
|
||||
}
|
||||
}
|
||||
|
||||
let next = candidates.filter((endpoint) => !blocked.has(endpoint));
|
||||
if (next.length === 0) {
|
||||
next = [...candidates];
|
||||
}
|
||||
|
||||
const preferredFresh = (
|
||||
!!state.preferredEndpoint
|
||||
&& (state.preferredUpdatedAtMs + ENDPOINT_RUNTIME_PREFERRED_TTL_MS) > nowMs
|
||||
);
|
||||
if (preferredFresh && state.preferredEndpoint && next.includes(state.preferredEndpoint)) {
|
||||
next = [
|
||||
state.preferredEndpoint,
|
||||
...next.filter((endpoint) => endpoint !== state.preferredEndpoint),
|
||||
];
|
||||
}
|
||||
|
||||
maybeDeleteEndpointRuntimeState(key, nowMs);
|
||||
return next;
|
||||
}
|
||||
|
||||
function sweepEndpointRuntimeStates(nowMs = Date.now()): void {
|
||||
for (const [key, state] of endpointRuntimeStates.entries()) {
|
||||
const hasActiveBlock = Object.values(state.blockedUntilMsByEndpoint).some((untilMs) => (
|
||||
typeof untilMs === 'number' && untilMs > nowMs
|
||||
));
|
||||
const preferredFresh = (
|
||||
!!state.preferredEndpoint
|
||||
&& (state.preferredUpdatedAtMs + ENDPOINT_RUNTIME_PREFERRED_TTL_MS) > nowMs
|
||||
);
|
||||
const recentlyTouched = (state.lastTouchedAtMs + ENDPOINT_RUNTIME_PREFERRED_TTL_MS) > nowMs;
|
||||
if (!hasActiveBlock && !preferredFresh && !recentlyTouched) {
|
||||
endpointRuntimeStates.delete(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function enforceEndpointRuntimeStateLimit(): void {
|
||||
if (endpointRuntimeStates.size <= MAX_ENDPOINT_RUNTIME_STATES) return;
|
||||
|
||||
const entries = [...endpointRuntimeStates.entries()]
|
||||
.sort((left, right) => left[1].lastTouchedAtMs - right[1].lastTouchedAtMs);
|
||||
const overflowCount = endpointRuntimeStates.size - MAX_ENDPOINT_RUNTIME_STATES;
|
||||
for (const [key] of entries.slice(0, overflowCount)) {
|
||||
endpointRuntimeStates.delete(key);
|
||||
}
|
||||
}
|
||||
|
||||
function inferSuggestedEndpointFromError(errorText?: string | null): UpstreamEndpoint | null {
|
||||
const text = (errorText || '').toLowerCase();
|
||||
if (!text) return null;
|
||||
if (text.includes('/v1/responses')) return 'responses';
|
||||
if (text.includes('/v1/messages')) return 'messages';
|
||||
if (text.includes('/v1/chat/completions')) return 'chat';
|
||||
return null;
|
||||
}
|
||||
|
||||
function shouldBlockEndpointByError(status: number, errorText?: string | null): boolean {
|
||||
if (isEndpointDispatchDeniedError(status, errorText)) return true;
|
||||
if (status === 404 || status === 405 || status === 415 || status === 501) return true;
|
||||
if (isUnsupportedMediaTypeError(status, errorText)) return true;
|
||||
|
||||
const text = (errorText || '').toLowerCase();
|
||||
return (
|
||||
text.includes('convert_request_failed')
|
||||
|| text.includes('endpoint_not_found')
|
||||
|| text.includes('unknown_endpoint')
|
||||
|| text.includes('unsupported_endpoint')
|
||||
|| text.includes('unsupported_path')
|
||||
|| text.includes('not_found_error')
|
||||
|| text.includes('unsupported legacy protocol')
|
||||
|| text.includes('please use /v1/')
|
||||
|| text.includes('does not allow /v1/')
|
||||
|| text.includes('unknown endpoint')
|
||||
|| text.includes('unsupported endpoint')
|
||||
|| text.includes('unsupported path')
|
||||
|| text.includes('unrecognized request url')
|
||||
|| text.includes('no route matched')
|
||||
|| text.includes('does not exist')
|
||||
);
|
||||
}
|
||||
|
||||
function shouldRememberSuccessfulEndpoint(input: {
|
||||
endpoint: UpstreamEndpoint;
|
||||
downstreamFormat: EndpointPreference;
|
||||
}): boolean {
|
||||
if (input.downstreamFormat !== 'responses') return true;
|
||||
return input.endpoint === 'responses';
|
||||
}
|
||||
|
||||
export function resetUpstreamEndpointRuntimeState(): void {
|
||||
endpointRuntimeStates.clear();
|
||||
}
|
||||
|
||||
export function recordUpstreamEndpointSuccess(input: {
|
||||
siteId: number;
|
||||
endpoint: UpstreamEndpoint;
|
||||
downstreamFormat: EndpointPreference;
|
||||
modelName?: string;
|
||||
requestedModelHint?: string;
|
||||
requestCapabilities?: {
|
||||
hasNonImageFileInput?: boolean;
|
||||
conversationFileSummary?: ConversationFileInputSummary;
|
||||
wantsNativeResponsesReasoning?: boolean;
|
||||
};
|
||||
}): void {
|
||||
const capabilityProfile = buildEndpointCapabilityProfile({
|
||||
modelName: input.modelName,
|
||||
requestedModelHint: input.requestedModelHint,
|
||||
requestCapabilities: input.requestCapabilities,
|
||||
});
|
||||
if (!shouldUseEndpointRuntimeMemory(capabilityProfile)) return;
|
||||
if (!shouldRememberSuccessfulEndpoint(input)) return;
|
||||
|
||||
const nowMs = Date.now();
|
||||
const key = buildEndpointRuntimeStateKey({
|
||||
siteId: input.siteId,
|
||||
downstreamFormat: input.downstreamFormat,
|
||||
capabilityProfile,
|
||||
});
|
||||
const state = getOrCreateEndpointRuntimeState(key, nowMs);
|
||||
state.preferredEndpoint = input.endpoint;
|
||||
state.preferredUpdatedAtMs = nowMs;
|
||||
delete state.blockedUntilMsByEndpoint[input.endpoint];
|
||||
}
|
||||
|
||||
export function recordUpstreamEndpointFailure(input: {
|
||||
siteId: number;
|
||||
endpoint: UpstreamEndpoint;
|
||||
downstreamFormat: EndpointPreference;
|
||||
status: number;
|
||||
errorText?: string | null;
|
||||
modelName?: string;
|
||||
requestedModelHint?: string;
|
||||
requestCapabilities?: {
|
||||
hasNonImageFileInput?: boolean;
|
||||
conversationFileSummary?: ConversationFileInputSummary;
|
||||
wantsNativeResponsesReasoning?: boolean;
|
||||
};
|
||||
}): void {
|
||||
const capabilityProfile = buildEndpointCapabilityProfile({
|
||||
modelName: input.modelName,
|
||||
requestedModelHint: input.requestedModelHint,
|
||||
requestCapabilities: input.requestCapabilities,
|
||||
});
|
||||
if (!shouldUseEndpointRuntimeMemory(capabilityProfile)) return;
|
||||
if (!shouldBlockEndpointByError(input.status, input.errorText)) return;
|
||||
|
||||
const nowMs = Date.now();
|
||||
const key = buildEndpointRuntimeStateKey({
|
||||
siteId: input.siteId,
|
||||
downstreamFormat: input.downstreamFormat,
|
||||
capabilityProfile,
|
||||
});
|
||||
const state = getOrCreateEndpointRuntimeState(key, nowMs);
|
||||
state.blockedUntilMsByEndpoint[input.endpoint] = nowMs + ENDPOINT_RUNTIME_BLOCK_TTL_MS;
|
||||
|
||||
const suggestedEndpoint = inferSuggestedEndpointFromError(input.errorText);
|
||||
if (suggestedEndpoint && suggestedEndpoint !== input.endpoint) {
|
||||
state.preferredEndpoint = suggestedEndpoint;
|
||||
state.preferredUpdatedAtMs = nowMs;
|
||||
delete state.blockedUntilMsByEndpoint[suggestedEndpoint];
|
||||
}
|
||||
}
|
||||
|
||||
function preferredEndpointOrder(
|
||||
downstreamFormat: EndpointPreference,
|
||||
sitePlatform?: string,
|
||||
@@ -848,15 +530,12 @@ export async function resolveUpstreamEndpointCandidates(
|
||||
const preferMessagesForClaudeModel = capabilityProfile.preferMessagesForClaudeModel;
|
||||
const hasNonImageFileInput = capabilityProfile.hasNonImageFileInput;
|
||||
const wantsNativeResponsesReasoning = capabilityProfile.wantsNativeResponsesReasoning;
|
||||
const runtimeStateKey = buildEndpointRuntimeStateKey({
|
||||
const applyRuntimePreference = (candidates: UpstreamEndpoint[]) => (
|
||||
applyUpstreamEndpointRuntimePreference(candidates, {
|
||||
siteId: context.site.id,
|
||||
downstreamFormat,
|
||||
capabilityProfile,
|
||||
});
|
||||
const applyRuntimePreference = (candidates: UpstreamEndpoint[]) => (
|
||||
shouldUseEndpointRuntimeMemory(capabilityProfile)
|
||||
? applyEndpointRuntimePreference(candidates, runtimeStateKey)
|
||||
: candidates
|
||||
})
|
||||
);
|
||||
const conversationFileSummary = requestCapabilities?.conversationFileSummary ?? {
|
||||
hasImage: false,
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { reserveSurfaceProxyDebugAttemptBase } from './proxyDebugTraceRuntime.js';
|
||||
|
||||
describe('reserveSurfaceProxyDebugAttemptBase', () => {
|
||||
it('allocates monotonic attempt bases on the same trace session', () => {
|
||||
const session = {
|
||||
traceId: 801,
|
||||
options: {
|
||||
enabled: true,
|
||||
captureHeaders: true,
|
||||
captureBodies: true,
|
||||
captureStreamChunks: false,
|
||||
targetSessionId: '',
|
||||
targetClientKind: '',
|
||||
targetModel: '',
|
||||
retentionHours: 12,
|
||||
maxBodyBytes: 131072,
|
||||
},
|
||||
};
|
||||
|
||||
expect(reserveSurfaceProxyDebugAttemptBase(session, 3)).toBe(0);
|
||||
expect(reserveSurfaceProxyDebugAttemptBase(session, 2)).toBe(3);
|
||||
expect(reserveSurfaceProxyDebugAttemptBase(session, 4)).toBe(5);
|
||||
});
|
||||
|
||||
it('reserves at least one slot for empty or invalid spans', () => {
|
||||
const session = {
|
||||
traceId: 802,
|
||||
options: {
|
||||
enabled: true,
|
||||
captureHeaders: true,
|
||||
captureBodies: true,
|
||||
captureStreamChunks: false,
|
||||
targetSessionId: '',
|
||||
targetClientKind: '',
|
||||
targetModel: '',
|
||||
retentionHours: 12,
|
||||
maxBodyBytes: 131072,
|
||||
},
|
||||
};
|
||||
|
||||
expect(reserveSurfaceProxyDebugAttemptBase(session, 0)).toBe(0);
|
||||
expect(reserveSurfaceProxyDebugAttemptBase(session, Number.NaN)).toBe(1);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,177 @@
|
||||
import { readRuntimeResponseText } from '../proxy-core/executors/types.js';
|
||||
import type {
|
||||
EndpointAttemptSuccessContext,
|
||||
} from '../routes/proxy/endpointFlow.js';
|
||||
import {
|
||||
finalizeProxyDebugTrace,
|
||||
insertProxyDebugAttempt,
|
||||
normalizeProxyDebugResponseHeaders,
|
||||
startProxyDebugTraceSession,
|
||||
updateProxyDebugAttempt,
|
||||
updateProxyDebugTraceCandidates,
|
||||
updateProxyDebugTraceSelection,
|
||||
type ProxyDebugTraceSession,
|
||||
} from './proxyDebugTraceStore.js';
|
||||
|
||||
type MutableProxyDebugTraceSession = ProxyDebugTraceSession & {
|
||||
nextAttemptIndex?: number;
|
||||
};
|
||||
|
||||
function parseDebugTextPayload(rawText: string): unknown {
|
||||
if (!rawText) return null;
|
||||
try {
|
||||
return JSON.parse(rawText);
|
||||
} catch {
|
||||
return rawText;
|
||||
}
|
||||
}
|
||||
|
||||
export async function startSurfaceProxyDebugTrace(input: {
|
||||
downstreamPath: string;
|
||||
clientKind?: string | null;
|
||||
sessionId?: string | null;
|
||||
traceHint?: string | null;
|
||||
requestedModel?: string | null;
|
||||
downstreamApiKeyId?: number | null;
|
||||
requestHeaders?: Record<string, unknown>;
|
||||
requestBody?: unknown;
|
||||
}): Promise<ProxyDebugTraceSession | null> {
|
||||
try {
|
||||
return await startProxyDebugTraceSession(input);
|
||||
} catch (error) {
|
||||
console.warn('[proxy-debug] failed to create trace session', error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export async function safeUpdateSurfaceProxyDebugSelection(
|
||||
session: ProxyDebugTraceSession | null,
|
||||
input: Parameters<typeof updateProxyDebugTraceSelection>[1],
|
||||
): Promise<void> {
|
||||
if (!session) return;
|
||||
try {
|
||||
await updateProxyDebugTraceSelection(session.traceId, input);
|
||||
} catch (error) {
|
||||
console.warn('[proxy-debug] failed to update selection', error);
|
||||
}
|
||||
}
|
||||
|
||||
export async function safeUpdateSurfaceProxyDebugCandidates(
|
||||
session: ProxyDebugTraceSession | null,
|
||||
input: Parameters<typeof updateProxyDebugTraceCandidates>[1],
|
||||
): Promise<void> {
|
||||
if (!session) return;
|
||||
try {
|
||||
await updateProxyDebugTraceCandidates(session.traceId, input);
|
||||
} catch (error) {
|
||||
console.warn('[proxy-debug] failed to update endpoint candidates', error);
|
||||
}
|
||||
}
|
||||
|
||||
export async function safeInsertSurfaceProxyDebugAttempt(
|
||||
session: ProxyDebugTraceSession | null,
|
||||
input: Omit<Parameters<typeof insertProxyDebugAttempt>[0], 'traceId'>,
|
||||
): Promise<void> {
|
||||
if (!session) return;
|
||||
try {
|
||||
await insertProxyDebugAttempt({
|
||||
...input,
|
||||
traceId: session.traceId,
|
||||
requestHeaders: session.options.captureHeaders ? input.requestHeaders : null,
|
||||
requestBody: session.options.captureBodies ? input.requestBody : null,
|
||||
responseHeaders: session.options.captureHeaders ? input.responseHeaders : null,
|
||||
responseBody: session.options.captureBodies ? input.responseBody : null,
|
||||
maxBodyBytes: session.options.maxBodyBytes,
|
||||
});
|
||||
} catch (error) {
|
||||
console.warn('[proxy-debug] failed to insert attempt', error);
|
||||
}
|
||||
}
|
||||
|
||||
export function reserveSurfaceProxyDebugAttemptBase(
|
||||
session: ProxyDebugTraceSession | null,
|
||||
span: number,
|
||||
): number {
|
||||
if (!session) return 0;
|
||||
|
||||
const mutableSession = session as MutableProxyDebugTraceSession;
|
||||
const base = mutableSession.nextAttemptIndex ?? 0;
|
||||
const normalizedSpan = Number.isFinite(span)
|
||||
? Math.max(1, Math.trunc(span))
|
||||
: 1;
|
||||
mutableSession.nextAttemptIndex = base + normalizedSpan;
|
||||
return base;
|
||||
}
|
||||
|
||||
export async function safeFinalizeSurfaceProxyDebugTrace(
|
||||
session: ProxyDebugTraceSession | null,
|
||||
input: Parameters<typeof finalizeProxyDebugTrace>[1],
|
||||
): Promise<void> {
|
||||
if (!session) return;
|
||||
try {
|
||||
await finalizeProxyDebugTrace(session.traceId, {
|
||||
...input,
|
||||
finalResponseHeaders: session.options.captureHeaders ? input.finalResponseHeaders : null,
|
||||
finalResponseBody: session.options.captureBodies ? input.finalResponseBody : null,
|
||||
maxBodyBytes: session.options.maxBodyBytes,
|
||||
});
|
||||
} catch (error) {
|
||||
console.warn('[proxy-debug] failed to finalize trace', error);
|
||||
}
|
||||
}
|
||||
|
||||
export async function safeUpdateSurfaceProxyDebugAttempt(
|
||||
session: ProxyDebugTraceSession | null,
|
||||
attemptIndex: number,
|
||||
input: Parameters<typeof updateProxyDebugAttempt>[2],
|
||||
): Promise<void> {
|
||||
if (!session) return;
|
||||
try {
|
||||
await updateProxyDebugAttempt(session.traceId, attemptIndex, input);
|
||||
} catch (error) {
|
||||
console.warn('[proxy-debug] failed to update attempt', error);
|
||||
}
|
||||
}
|
||||
|
||||
export async function captureSurfaceProxyDebugSuccessResponseBody(
|
||||
session: ProxyDebugTraceSession | null,
|
||||
ctx: EndpointAttemptSuccessContext,
|
||||
): Promise<unknown> {
|
||||
if (!session?.options.captureBodies) return null;
|
||||
|
||||
const contentType = (ctx.response.headers.get('content-type') || '').toLowerCase();
|
||||
const requestBody = ctx.request.body as Record<string, unknown> | undefined;
|
||||
const isStream = ctx.request.runtime?.stream === true
|
||||
|| requestBody?.stream === true
|
||||
|| contentType.includes('text/event-stream');
|
||||
if (isStream) return null;
|
||||
|
||||
try {
|
||||
const rawText = await readRuntimeResponseText(ctx.response.clone());
|
||||
return parseDebugTextPayload(rawText);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export function buildSurfaceProxyDebugResponseHeaders(
|
||||
response:
|
||||
| Headers
|
||||
| Record<string, unknown>
|
||||
| { headers?: Headers | Record<string, unknown> | null | undefined }
|
||||
| null
|
||||
| undefined,
|
||||
): Record<string, unknown> | null {
|
||||
if (!response) return null;
|
||||
if (typeof response === 'object' && 'headers' in response) {
|
||||
const responseHeaders = (response as {
|
||||
headers?: Headers | Record<string, unknown> | null | undefined;
|
||||
}).headers;
|
||||
return normalizeProxyDebugResponseHeaders(responseHeaders ?? null);
|
||||
}
|
||||
return normalizeProxyDebugResponseHeaders(response);
|
||||
}
|
||||
|
||||
export function parseSurfaceProxyDebugTextPayload(rawText: string): unknown {
|
||||
return parseDebugTextPayload(rawText);
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest';
|
||||
import { mkdtempSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
|
||||
type DbModule = typeof import('../db/index.js');
|
||||
type StoreModule = typeof import('./proxyDebugTraceStore.js');
|
||||
|
||||
describe('proxyDebugTraceStore', () => {
|
||||
let db: DbModule['db'];
|
||||
let schema: DbModule['schema'];
|
||||
let store: StoreModule;
|
||||
let dataDir = '';
|
||||
|
||||
beforeAll(async () => {
|
||||
dataDir = mkdtempSync(join(tmpdir(), 'metapi-proxy-debug-traces-'));
|
||||
process.env.DATA_DIR = dataDir;
|
||||
|
||||
await import('../db/migrate.js');
|
||||
const dbModule = await import('../db/index.js');
|
||||
const storeModule = await import('./proxyDebugTraceStore.js');
|
||||
db = dbModule.db;
|
||||
schema = dbModule.schema;
|
||||
store = storeModule;
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
await db.delete(schema.proxyDebugAttempts).run();
|
||||
await db.delete(schema.proxyDebugTraces).run();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
const dbModule = await import('../db/index.js');
|
||||
await dbModule.closeDbConnections();
|
||||
delete process.env.DATA_DIR;
|
||||
});
|
||||
|
||||
it('creates, updates, and reads unredacted debug traces and attempts', async () => {
|
||||
const trace = await store.createProxyDebugTrace({
|
||||
downstreamPath: '/v1/responses',
|
||||
clientKind: 'codex',
|
||||
sessionId: 'sess-1',
|
||||
traceHint: 'trace-abc',
|
||||
requestedModel: 'gpt-4o',
|
||||
downstreamApiKeyId: 7,
|
||||
requestHeaders: {
|
||||
authorization: 'Bearer developer-token',
|
||||
'x-client': 'Codex Desktop',
|
||||
},
|
||||
requestBody: {
|
||||
model: 'gpt-4o',
|
||||
input: [{ role: 'user', content: 'hello' }],
|
||||
},
|
||||
});
|
||||
|
||||
await store.updateProxyDebugTraceSelection(trace.id, {
|
||||
stickySessionKey: 'key:7|codex|/v1/responses|gpt-4o|sess-1',
|
||||
stickyHitChannelId: 12,
|
||||
selectedChannelId: 12,
|
||||
selectedRouteId: 99,
|
||||
selectedAccountId: 33,
|
||||
selectedSiteId: 8,
|
||||
selectedSitePlatform: 'codex',
|
||||
});
|
||||
|
||||
await store.updateProxyDebugTraceCandidates(trace.id, {
|
||||
endpointCandidates: ['responses', 'chat'],
|
||||
endpointRuntimeState: {
|
||||
preferredEndpoint: 'responses',
|
||||
blockedEndpoints: [],
|
||||
},
|
||||
decisionSummary: {
|
||||
reason: 'platform default + sticky session',
|
||||
},
|
||||
});
|
||||
|
||||
const attempt = await store.insertProxyDebugAttempt({
|
||||
traceId: trace.id,
|
||||
attemptIndex: 0,
|
||||
endpoint: 'responses',
|
||||
requestPath: '/responses',
|
||||
targetUrl: 'https://chatgpt.com/backend-api/codex/responses',
|
||||
runtimeExecutor: 'codex',
|
||||
requestHeaders: {
|
||||
authorization: 'Bearer developer-token',
|
||||
},
|
||||
requestBody: {
|
||||
model: 'gpt-4o',
|
||||
store: false,
|
||||
},
|
||||
responseStatus: 403,
|
||||
responseHeaders: {
|
||||
'content-type': 'application/json',
|
||||
},
|
||||
responseBody: {
|
||||
error: {
|
||||
message: 'forbidden',
|
||||
},
|
||||
},
|
||||
rawErrorText: '{"error":{"message":"forbidden"}}',
|
||||
recoverApplied: false,
|
||||
downgradeDecision: true,
|
||||
downgradeReason: '[upstream:/responses] forbidden',
|
||||
memoryWrite: {
|
||||
action: 'failure',
|
||||
blockedEndpoint: 'responses',
|
||||
},
|
||||
});
|
||||
|
||||
expect(attempt.id).toBeGreaterThan(0);
|
||||
|
||||
await store.finalizeProxyDebugTrace(trace.id, {
|
||||
finalStatus: 'failed',
|
||||
finalHttpStatus: 503,
|
||||
finalUpstreamPath: '/responses',
|
||||
finalResponseHeaders: {
|
||||
'content-type': 'application/json',
|
||||
},
|
||||
finalResponseBody: {
|
||||
error: {
|
||||
message: 'Channel busy',
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const list = await store.listProxyDebugTraces({ limit: 20 });
|
||||
expect(list).toHaveLength(1);
|
||||
expect(list[0]).toMatchObject({
|
||||
requestedModel: 'gpt-4o',
|
||||
downstreamPath: '/v1/responses',
|
||||
clientKind: 'codex',
|
||||
sessionId: 'sess-1',
|
||||
selectedChannelId: 12,
|
||||
finalStatus: 'failed',
|
||||
finalHttpStatus: 503,
|
||||
finalUpstreamPath: '/responses',
|
||||
});
|
||||
|
||||
const detail = await store.getProxyDebugTraceDetail(trace.id);
|
||||
expect(detail?.trace.requestHeadersJson || '').toContain('Bearer developer-token');
|
||||
expect(detail?.trace.requestBodyJson || '').toContain('"hello"');
|
||||
expect(detail?.trace.finalResponseBodyJson || '').toContain('Channel busy');
|
||||
expect(detail?.attempts).toHaveLength(1);
|
||||
expect(detail?.attempts[0]?.requestHeadersJson || '').toContain('developer-token');
|
||||
expect(detail?.attempts[0]?.responseBodyJson || '').toContain('forbidden');
|
||||
expect(detail?.attempts[0]).toMatchObject({
|
||||
endpoint: 'responses',
|
||||
runtimeExecutor: 'codex',
|
||||
responseStatus: 403,
|
||||
downgradeDecision: true,
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,391 @@
|
||||
import { and, asc, desc, eq, lt } from 'drizzle-orm';
|
||||
import { config } from '../config.js';
|
||||
import { db, schema } from '../db/index.js';
|
||||
import { formatUtcSqlDateTime } from './localTimeService.js';
|
||||
|
||||
type HeadersLike = Headers | Record<string, unknown> | null | undefined;
|
||||
|
||||
export type ProxyDebugCaptureOptions = {
|
||||
enabled: boolean;
|
||||
captureHeaders: boolean;
|
||||
captureBodies: boolean;
|
||||
captureStreamChunks: boolean;
|
||||
targetSessionId: string;
|
||||
targetClientKind: string;
|
||||
targetModel: string;
|
||||
retentionHours: number;
|
||||
maxBodyBytes: number;
|
||||
};
|
||||
|
||||
export type ProxyDebugTraceSession = {
|
||||
traceId: number;
|
||||
options: ProxyDebugCaptureOptions;
|
||||
};
|
||||
|
||||
let lastPruneAtMs = 0;
|
||||
const PRUNE_INTERVAL_MS = 5 * 60 * 1000;
|
||||
|
||||
function stringifyDebugValue(value: unknown, maxBytes: number): string | null {
|
||||
if (value == null) return null;
|
||||
|
||||
let text = '';
|
||||
if (typeof value === 'string') {
|
||||
text = value;
|
||||
} else {
|
||||
try {
|
||||
text = JSON.stringify(value, null, 2);
|
||||
} catch {
|
||||
text = String(value);
|
||||
}
|
||||
}
|
||||
|
||||
if (!text) return null;
|
||||
const buffer = Buffer.from(text, 'utf8');
|
||||
if (buffer.length <= maxBytes) {
|
||||
return text;
|
||||
}
|
||||
|
||||
const truncated = buffer.subarray(0, Math.max(0, maxBytes)).toString('utf8');
|
||||
return `${truncated}\n...[truncated ${buffer.length - maxBytes} bytes]`;
|
||||
}
|
||||
|
||||
function normalizeHeadersValue(value: HeadersLike): Record<string, unknown> | null {
|
||||
if (!value) return null;
|
||||
|
||||
if (typeof Headers !== 'undefined' && value instanceof Headers) {
|
||||
return Object.fromEntries([...value.entries()].sort((left, right) => left[0].localeCompare(right[0])));
|
||||
}
|
||||
|
||||
if (typeof value !== 'object' || Array.isArray(value)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const entries = Object.entries(value)
|
||||
.filter(([key]) => !!key)
|
||||
.sort((left, right) => left[0].localeCompare(right[0]));
|
||||
return Object.fromEntries(entries);
|
||||
}
|
||||
|
||||
function normalizeLimit(rawLimit: number | undefined): number {
|
||||
const parsed = Number(rawLimit || 20);
|
||||
if (!Number.isFinite(parsed)) return 20;
|
||||
return Math.max(1, Math.min(200, Math.trunc(parsed)));
|
||||
}
|
||||
|
||||
function asNormalizedText(value: unknown): string {
|
||||
return typeof value === 'string' ? value.trim().toLowerCase() : '';
|
||||
}
|
||||
|
||||
function getCaptureOptions(): ProxyDebugCaptureOptions {
|
||||
return {
|
||||
enabled: config.proxyDebugTraceEnabled,
|
||||
captureHeaders: config.proxyDebugCaptureHeaders,
|
||||
captureBodies: config.proxyDebugCaptureBodies,
|
||||
captureStreamChunks: config.proxyDebugCaptureStreamChunks,
|
||||
targetSessionId: (config.proxyDebugTargetSessionId || '').trim(),
|
||||
targetClientKind: (config.proxyDebugTargetClientKind || '').trim(),
|
||||
targetModel: (config.proxyDebugTargetModel || '').trim(),
|
||||
retentionHours: Math.max(1, Math.trunc(config.proxyDebugRetentionHours || 24)),
|
||||
maxBodyBytes: Math.max(1024, Math.trunc(config.proxyDebugMaxBodyBytes || 262_144)),
|
||||
};
|
||||
}
|
||||
|
||||
function serializeHeaders(value: HeadersLike, maxBytes: number): string | null {
|
||||
return stringifyDebugValue(normalizeHeadersValue(value), maxBytes);
|
||||
}
|
||||
|
||||
export function normalizeProxyDebugResponseHeaders(value: HeadersLike): Record<string, unknown> | null {
|
||||
return normalizeHeadersValue(value);
|
||||
}
|
||||
|
||||
export function getProxyDebugCaptureOptions(): ProxyDebugCaptureOptions {
|
||||
return getCaptureOptions();
|
||||
}
|
||||
|
||||
export function shouldTraceProxyDebugRequest(input: {
|
||||
clientKind?: string | null;
|
||||
sessionId?: string | null;
|
||||
requestedModel?: string | null;
|
||||
}, options = getCaptureOptions()): boolean {
|
||||
if (!options.enabled) return false;
|
||||
|
||||
const targetSessionId = options.targetSessionId;
|
||||
if (targetSessionId && (input.sessionId || '').trim() !== targetSessionId) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const targetClientKind = asNormalizedText(options.targetClientKind);
|
||||
if (targetClientKind && asNormalizedText(input.clientKind) !== targetClientKind) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const targetModel = asNormalizedText(options.targetModel);
|
||||
if (targetModel && asNormalizedText(input.requestedModel) !== targetModel) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
export async function deleteExpiredProxyDebugTraces(retentionHours = config.proxyDebugRetentionHours): Promise<number> {
|
||||
const normalizedRetentionHours = Math.max(1, Math.trunc(Number(retentionHours || 24)));
|
||||
const cutoff = formatUtcSqlDateTime(new Date(Date.now() - (normalizedRetentionHours * 60 * 60 * 1000)));
|
||||
const result = await db
|
||||
.delete(schema.proxyDebugTraces)
|
||||
.where(lt(schema.proxyDebugTraces.createdAt, cutoff))
|
||||
.run();
|
||||
return Number(result.changes || 0);
|
||||
}
|
||||
|
||||
async function pruneProxyDebugTracesIfNeeded(nowMs = Date.now(), retentionHours = config.proxyDebugRetentionHours): Promise<void> {
|
||||
if (nowMs - lastPruneAtMs < PRUNE_INTERVAL_MS) return;
|
||||
lastPruneAtMs = nowMs;
|
||||
try {
|
||||
await deleteExpiredProxyDebugTraces(retentionHours);
|
||||
} catch (error) {
|
||||
console.warn('[proxy-debug] failed to prune expired traces', error);
|
||||
}
|
||||
}
|
||||
|
||||
export async function createProxyDebugTrace(input: {
|
||||
downstreamPath: string;
|
||||
clientKind?: string | null;
|
||||
sessionId?: string | null;
|
||||
traceHint?: string | null;
|
||||
requestedModel?: string | null;
|
||||
downstreamApiKeyId?: number | null;
|
||||
requestHeaders?: HeadersLike;
|
||||
requestBody?: unknown;
|
||||
maxBodyBytes?: number;
|
||||
}) {
|
||||
const now = formatUtcSqlDateTime(new Date());
|
||||
const maxBodyBytes = Math.max(1024, Math.trunc(input.maxBodyBytes || config.proxyDebugMaxBodyBytes || 262_144));
|
||||
await pruneProxyDebugTracesIfNeeded(Date.now(), config.proxyDebugRetentionHours);
|
||||
|
||||
const inserted = await db.insert(schema.proxyDebugTraces).values({
|
||||
downstreamPath: input.downstreamPath,
|
||||
clientKind: input.clientKind ?? null,
|
||||
sessionId: input.sessionId ?? null,
|
||||
traceHint: input.traceHint ?? null,
|
||||
requestedModel: input.requestedModel ?? null,
|
||||
downstreamApiKeyId: input.downstreamApiKeyId ?? null,
|
||||
requestHeadersJson: serializeHeaders(input.requestHeaders, maxBodyBytes),
|
||||
requestBodyJson: stringifyDebugValue(input.requestBody, maxBodyBytes),
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
}).run();
|
||||
|
||||
return {
|
||||
id: Number(inserted.lastInsertRowid || 0),
|
||||
createdAt: now,
|
||||
};
|
||||
}
|
||||
|
||||
export async function startProxyDebugTraceSession(input: {
|
||||
downstreamPath: string;
|
||||
clientKind?: string | null;
|
||||
sessionId?: string | null;
|
||||
traceHint?: string | null;
|
||||
requestedModel?: string | null;
|
||||
downstreamApiKeyId?: number | null;
|
||||
requestHeaders?: HeadersLike;
|
||||
requestBody?: unknown;
|
||||
}): Promise<ProxyDebugTraceSession | null> {
|
||||
const options = getCaptureOptions();
|
||||
if (!shouldTraceProxyDebugRequest(input, options)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const trace = await createProxyDebugTrace({
|
||||
downstreamPath: input.downstreamPath,
|
||||
clientKind: input.clientKind ?? null,
|
||||
sessionId: input.sessionId ?? null,
|
||||
traceHint: input.traceHint ?? null,
|
||||
requestedModel: input.requestedModel ?? null,
|
||||
downstreamApiKeyId: input.downstreamApiKeyId ?? null,
|
||||
requestHeaders: options.captureHeaders ? input.requestHeaders : null,
|
||||
requestBody: options.captureBodies ? input.requestBody : null,
|
||||
maxBodyBytes: options.maxBodyBytes,
|
||||
});
|
||||
|
||||
return {
|
||||
traceId: trace.id,
|
||||
options,
|
||||
};
|
||||
}
|
||||
|
||||
export async function updateProxyDebugTraceSelection(traceId: number, input: {
|
||||
stickySessionKey?: string | null;
|
||||
stickyHitChannelId?: number | null;
|
||||
selectedChannelId?: number | null;
|
||||
selectedRouteId?: number | null;
|
||||
selectedAccountId?: number | null;
|
||||
selectedSiteId?: number | null;
|
||||
selectedSitePlatform?: string | null;
|
||||
}) {
|
||||
const now = formatUtcSqlDateTime(new Date());
|
||||
await db.update(schema.proxyDebugTraces).set({
|
||||
stickySessionKey: input.stickySessionKey ?? null,
|
||||
stickyHitChannelId: input.stickyHitChannelId ?? null,
|
||||
selectedChannelId: input.selectedChannelId ?? null,
|
||||
selectedRouteId: input.selectedRouteId ?? null,
|
||||
selectedAccountId: input.selectedAccountId ?? null,
|
||||
selectedSiteId: input.selectedSiteId ?? null,
|
||||
selectedSitePlatform: input.selectedSitePlatform ?? null,
|
||||
updatedAt: now,
|
||||
}).where(eq(schema.proxyDebugTraces.id, traceId)).run();
|
||||
}
|
||||
|
||||
export async function updateProxyDebugTraceCandidates(traceId: number, input: {
|
||||
endpointCandidates?: unknown;
|
||||
endpointRuntimeState?: unknown;
|
||||
decisionSummary?: unknown;
|
||||
}) {
|
||||
const now = formatUtcSqlDateTime(new Date());
|
||||
await db.update(schema.proxyDebugTraces).set({
|
||||
endpointCandidatesJson: stringifyDebugValue(input.endpointCandidates, config.proxyDebugMaxBodyBytes),
|
||||
endpointRuntimeStateJson: stringifyDebugValue(input.endpointRuntimeState, config.proxyDebugMaxBodyBytes),
|
||||
decisionSummaryJson: stringifyDebugValue(input.decisionSummary, config.proxyDebugMaxBodyBytes),
|
||||
updatedAt: now,
|
||||
}).where(eq(schema.proxyDebugTraces.id, traceId)).run();
|
||||
}
|
||||
|
||||
export async function insertProxyDebugAttempt(input: {
|
||||
traceId: number;
|
||||
attemptIndex: number;
|
||||
endpoint: string;
|
||||
requestPath: string;
|
||||
targetUrl: string;
|
||||
runtimeExecutor?: string | null;
|
||||
requestHeaders?: HeadersLike;
|
||||
requestBody?: unknown;
|
||||
responseStatus?: number | null;
|
||||
responseHeaders?: HeadersLike;
|
||||
responseBody?: unknown;
|
||||
rawErrorText?: string | null;
|
||||
recoverApplied?: boolean;
|
||||
downgradeDecision?: boolean;
|
||||
downgradeReason?: string | null;
|
||||
memoryWrite?: unknown;
|
||||
maxBodyBytes?: number;
|
||||
}) {
|
||||
const now = formatUtcSqlDateTime(new Date());
|
||||
const maxBodyBytes = Math.max(1024, Math.trunc(input.maxBodyBytes || config.proxyDebugMaxBodyBytes || 262_144));
|
||||
const inserted = await db.insert(schema.proxyDebugAttempts).values({
|
||||
traceId: input.traceId,
|
||||
attemptIndex: input.attemptIndex,
|
||||
endpoint: input.endpoint,
|
||||
requestPath: input.requestPath,
|
||||
targetUrl: input.targetUrl,
|
||||
runtimeExecutor: input.runtimeExecutor ?? null,
|
||||
requestHeadersJson: serializeHeaders(input.requestHeaders, maxBodyBytes),
|
||||
requestBodyJson: stringifyDebugValue(input.requestBody, maxBodyBytes),
|
||||
responseStatus: input.responseStatus ?? null,
|
||||
responseHeadersJson: serializeHeaders(input.responseHeaders, maxBodyBytes),
|
||||
responseBodyJson: stringifyDebugValue(input.responseBody, maxBodyBytes),
|
||||
rawErrorText: input.rawErrorText ?? null,
|
||||
recoverApplied: input.recoverApplied === true,
|
||||
downgradeDecision: input.downgradeDecision === true,
|
||||
downgradeReason: input.downgradeReason ?? null,
|
||||
memoryWriteJson: stringifyDebugValue(input.memoryWrite, maxBodyBytes),
|
||||
createdAt: now,
|
||||
}).run();
|
||||
|
||||
return {
|
||||
id: Number(inserted.lastInsertRowid || 0),
|
||||
createdAt: now,
|
||||
};
|
||||
}
|
||||
|
||||
export async function updateProxyDebugAttempt(traceId: number, attemptIndex: number, input: {
|
||||
downgradeDecision?: boolean;
|
||||
downgradeReason?: string | null;
|
||||
rawErrorText?: string | null;
|
||||
}) {
|
||||
await db.update(schema.proxyDebugAttempts).set({
|
||||
...(input.downgradeDecision !== undefined ? { downgradeDecision: input.downgradeDecision } : {}),
|
||||
...(input.downgradeReason !== undefined ? { downgradeReason: input.downgradeReason } : {}),
|
||||
...(input.rawErrorText !== undefined ? { rawErrorText: input.rawErrorText } : {}),
|
||||
}).where(and(
|
||||
eq(schema.proxyDebugAttempts.traceId, traceId),
|
||||
eq(schema.proxyDebugAttempts.attemptIndex, attemptIndex),
|
||||
)).run();
|
||||
}
|
||||
|
||||
export async function finalizeProxyDebugTrace(traceId: number, input: {
|
||||
finalStatus?: string | null;
|
||||
finalHttpStatus?: number | null;
|
||||
finalUpstreamPath?: string | null;
|
||||
finalResponseHeaders?: HeadersLike;
|
||||
finalResponseBody?: unknown;
|
||||
maxBodyBytes?: number;
|
||||
}) {
|
||||
const now = formatUtcSqlDateTime(new Date());
|
||||
const maxBodyBytes = Math.max(1024, Math.trunc(input.maxBodyBytes || config.proxyDebugMaxBodyBytes || 262_144));
|
||||
await db.update(schema.proxyDebugTraces).set({
|
||||
finalStatus: input.finalStatus ?? null,
|
||||
finalHttpStatus: input.finalHttpStatus ?? null,
|
||||
finalUpstreamPath: input.finalUpstreamPath ?? null,
|
||||
finalResponseHeadersJson: serializeHeaders(input.finalResponseHeaders, maxBodyBytes),
|
||||
finalResponseBodyJson: stringifyDebugValue(input.finalResponseBody, maxBodyBytes),
|
||||
updatedAt: now,
|
||||
}).where(eq(schema.proxyDebugTraces.id, traceId)).run();
|
||||
}
|
||||
|
||||
export async function listProxyDebugTraces(input: { limit?: number }) {
|
||||
const limit = normalizeLimit(input.limit);
|
||||
return await db.select({
|
||||
id: schema.proxyDebugTraces.id,
|
||||
createdAt: schema.proxyDebugTraces.createdAt,
|
||||
downstreamPath: schema.proxyDebugTraces.downstreamPath,
|
||||
clientKind: schema.proxyDebugTraces.clientKind,
|
||||
sessionId: schema.proxyDebugTraces.sessionId,
|
||||
requestedModel: schema.proxyDebugTraces.requestedModel,
|
||||
selectedChannelId: schema.proxyDebugTraces.selectedChannelId,
|
||||
finalStatus: schema.proxyDebugTraces.finalStatus,
|
||||
finalHttpStatus: schema.proxyDebugTraces.finalHttpStatus,
|
||||
finalUpstreamPath: schema.proxyDebugTraces.finalUpstreamPath,
|
||||
}).from(schema.proxyDebugTraces)
|
||||
.orderBy(desc(schema.proxyDebugTraces.createdAt), desc(schema.proxyDebugTraces.id))
|
||||
.limit(limit)
|
||||
.all();
|
||||
}
|
||||
|
||||
export async function getProxyDebugTraceDetail(traceId: number) {
|
||||
const trace = await db.select().from(schema.proxyDebugTraces)
|
||||
.where(eq(schema.proxyDebugTraces.id, traceId))
|
||||
.get();
|
||||
if (!trace) return null;
|
||||
|
||||
const attempts = await db.select().from(schema.proxyDebugAttempts)
|
||||
.where(eq(schema.proxyDebugAttempts.traceId, traceId))
|
||||
.orderBy(asc(schema.proxyDebugAttempts.attemptIndex), asc(schema.proxyDebugAttempts.id))
|
||||
.all();
|
||||
|
||||
return {
|
||||
trace,
|
||||
attempts,
|
||||
};
|
||||
}
|
||||
|
||||
export async function findLatestProxyDebugTrace(input: {
|
||||
sessionId?: string | null;
|
||||
clientKind?: string | null;
|
||||
requestedModel?: string | null;
|
||||
}) {
|
||||
const conditions = [
|
||||
input.sessionId ? eq(schema.proxyDebugTraces.sessionId, input.sessionId) : null,
|
||||
input.clientKind ? eq(schema.proxyDebugTraces.clientKind, input.clientKind) : null,
|
||||
input.requestedModel ? eq(schema.proxyDebugTraces.requestedModel, input.requestedModel) : null,
|
||||
].filter((condition): condition is NonNullable<typeof condition> => condition !== null);
|
||||
|
||||
let query = db.select().from(schema.proxyDebugTraces);
|
||||
if (conditions.length === 1) {
|
||||
query = query.where(conditions[0]) as typeof query;
|
||||
} else if (conditions.length > 1) {
|
||||
query = query.where(and(...conditions)) as typeof query;
|
||||
}
|
||||
return await query
|
||||
.orderBy(desc(schema.proxyDebugTraces.createdAt), desc(schema.proxyDebugTraces.id))
|
||||
.get();
|
||||
}
|
||||
@@ -0,0 +1,430 @@
|
||||
import { createHash } from 'node:crypto';
|
||||
import type { ConversationFileInputSummary } from '../proxy-core/capabilities/conversationFileCapabilities.js';
|
||||
import type { DownstreamFormat } from '../transformers/shared/normalized.js';
|
||||
import {
|
||||
isEndpointDispatchDeniedError,
|
||||
isUnsupportedMediaTypeError,
|
||||
} from '../transformers/shared/endpointCompatibility.js';
|
||||
|
||||
export type UpstreamEndpointRuntimeEndpoint = 'chat' | 'messages' | 'responses';
|
||||
export type UpstreamEndpointRuntimePreference = DownstreamFormat | 'responses';
|
||||
export type UpstreamEndpointRuntimeMemoryWrite =
|
||||
| {
|
||||
action: 'success';
|
||||
endpoint: UpstreamEndpointRuntimeEndpoint;
|
||||
preferredEndpoint: UpstreamEndpointRuntimeEndpoint;
|
||||
stateKey: string;
|
||||
timestampMs: number;
|
||||
}
|
||||
| {
|
||||
action: 'failure';
|
||||
endpoint: UpstreamEndpointRuntimeEndpoint;
|
||||
blockedEndpoint: UpstreamEndpointRuntimeEndpoint;
|
||||
preferredEndpoint: UpstreamEndpointRuntimeEndpoint | null;
|
||||
stateKey: string;
|
||||
timestampMs: number;
|
||||
};
|
||||
|
||||
export type EndpointCapabilityProfile = {
|
||||
modelKey: string;
|
||||
preferMessagesForClaudeModel: boolean;
|
||||
hasImageInput: boolean;
|
||||
hasAudioInput: boolean;
|
||||
hasNonImageFileInput: boolean;
|
||||
hasRemoteDocumentUrl: boolean;
|
||||
wantsNativeResponsesReasoning: boolean;
|
||||
};
|
||||
|
||||
type EndpointRuntimeState = {
|
||||
preferredEndpoint: UpstreamEndpointRuntimeEndpoint | null;
|
||||
preferredUpdatedAtMs: number;
|
||||
lastTouchedAtMs: number;
|
||||
blockedUntilMsByEndpoint: Partial<Record<UpstreamEndpointRuntimeEndpoint, number>>;
|
||||
};
|
||||
|
||||
const ENDPOINT_RUNTIME_PREFERRED_TTL_MS = 24 * 60 * 60 * 1000;
|
||||
const ENDPOINT_RUNTIME_BLOCK_TTL_MS = 6 * 60 * 60 * 1000;
|
||||
const MAX_ENDPOINT_RUNTIME_STATES = 512;
|
||||
export const MAX_ENDPOINT_RUNTIME_MODEL_KEY_LENGTH = 64;
|
||||
export const MODEL_KEY_HASH_SUFFIX_LENGTH = 8;
|
||||
|
||||
const endpointRuntimeStates = new Map<string, EndpointRuntimeState>();
|
||||
|
||||
function asTrimmedString(value: unknown): string {
|
||||
return typeof value === 'string' ? value.trim() : '';
|
||||
}
|
||||
|
||||
function isClaudeFamilyModel(modelName: string): boolean {
|
||||
const normalized = asTrimmedString(modelName).toLowerCase();
|
||||
if (!normalized) return false;
|
||||
return normalized === 'claude' || normalized.startsWith('claude-') || normalized.includes('claude');
|
||||
}
|
||||
|
||||
export function boundEndpointRuntimeModelKey(value: string): string {
|
||||
if (value.length <= MAX_ENDPOINT_RUNTIME_MODEL_KEY_LENGTH) {
|
||||
return value;
|
||||
}
|
||||
|
||||
const prefix = value.slice(0, MAX_ENDPOINT_RUNTIME_MODEL_KEY_LENGTH);
|
||||
const hash = createHash('sha256')
|
||||
.update(value)
|
||||
.digest('hex')
|
||||
.slice(0, MODEL_KEY_HASH_SUFFIX_LENGTH);
|
||||
return `${prefix}-${hash}`;
|
||||
}
|
||||
|
||||
function normalizeEndpointRuntimeModelKey(...values: Array<unknown>): string {
|
||||
for (const value of values) {
|
||||
const normalized = asTrimmedString(value).toLowerCase();
|
||||
if (normalized) return boundEndpointRuntimeModelKey(normalized);
|
||||
}
|
||||
return boundEndpointRuntimeModelKey('unknown-model');
|
||||
}
|
||||
|
||||
export function buildEndpointCapabilityProfile(input?: {
|
||||
modelName?: string;
|
||||
requestedModelHint?: string;
|
||||
requestCapabilities?: {
|
||||
hasNonImageFileInput?: boolean;
|
||||
conversationFileSummary?: ConversationFileInputSummary;
|
||||
wantsNativeResponsesReasoning?: boolean;
|
||||
};
|
||||
}): EndpointCapabilityProfile {
|
||||
const conversationFileSummary = input?.requestCapabilities?.conversationFileSummary;
|
||||
return {
|
||||
modelKey: normalizeEndpointRuntimeModelKey(input?.modelName, input?.requestedModelHint),
|
||||
preferMessagesForClaudeModel: (
|
||||
isClaudeFamilyModel(asTrimmedString(input?.modelName))
|
||||
|| isClaudeFamilyModel(asTrimmedString(input?.requestedModelHint))
|
||||
),
|
||||
hasImageInput: conversationFileSummary?.hasImage === true,
|
||||
hasAudioInput: conversationFileSummary?.hasAudio === true,
|
||||
hasNonImageFileInput: (
|
||||
conversationFileSummary?.hasDocument === true
|
||||
|| input?.requestCapabilities?.hasNonImageFileInput === true
|
||||
),
|
||||
hasRemoteDocumentUrl: (
|
||||
conversationFileSummary?.hasRemoteDocumentUrl === true
|
||||
),
|
||||
wantsNativeResponsesReasoning: input?.requestCapabilities?.wantsNativeResponsesReasoning === true,
|
||||
};
|
||||
}
|
||||
|
||||
function shouldUseEndpointRuntimeMemory(capabilityProfile: EndpointCapabilityProfile): boolean {
|
||||
return (
|
||||
!capabilityProfile.hasImageInput
|
||||
&& !capabilityProfile.hasAudioInput
|
||||
&& !capabilityProfile.hasNonImageFileInput
|
||||
);
|
||||
}
|
||||
|
||||
function buildEndpointRuntimeStateKey(input: {
|
||||
siteId: number;
|
||||
downstreamFormat: UpstreamEndpointRuntimePreference;
|
||||
capabilityProfile: EndpointCapabilityProfile;
|
||||
}): string {
|
||||
const capabilityProfile = input.capabilityProfile;
|
||||
return [
|
||||
String(input.siteId),
|
||||
input.downstreamFormat,
|
||||
capabilityProfile.modelKey,
|
||||
capabilityProfile.hasNonImageFileInput ? 'files' : 'nofiles',
|
||||
capabilityProfile.hasRemoteDocumentUrl ? 'remoteurl' : 'noremoteurl',
|
||||
capabilityProfile.wantsNativeResponsesReasoning ? 'reasoning' : 'noreasoning',
|
||||
].join(':');
|
||||
}
|
||||
|
||||
function sweepEndpointRuntimeStates(nowMs = Date.now()): void {
|
||||
for (const [key, state] of endpointRuntimeStates.entries()) {
|
||||
const hasActiveBlock = Object.values(state.blockedUntilMsByEndpoint).some((untilMs) => (
|
||||
typeof untilMs === 'number' && untilMs > nowMs
|
||||
));
|
||||
const preferredFresh = (
|
||||
!!state.preferredEndpoint
|
||||
&& (state.preferredUpdatedAtMs + ENDPOINT_RUNTIME_PREFERRED_TTL_MS) > nowMs
|
||||
);
|
||||
const recentlyTouched = (state.lastTouchedAtMs + ENDPOINT_RUNTIME_PREFERRED_TTL_MS) > nowMs;
|
||||
if (!hasActiveBlock && !preferredFresh && !recentlyTouched) {
|
||||
endpointRuntimeStates.delete(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function enforceEndpointRuntimeStateLimit(): void {
|
||||
if (endpointRuntimeStates.size <= MAX_ENDPOINT_RUNTIME_STATES) return;
|
||||
|
||||
const entries = [...endpointRuntimeStates.entries()]
|
||||
.sort((left, right) => left[1].lastTouchedAtMs - right[1].lastTouchedAtMs);
|
||||
const overflowCount = endpointRuntimeStates.size - MAX_ENDPOINT_RUNTIME_STATES;
|
||||
for (const [key] of entries.slice(0, overflowCount)) {
|
||||
endpointRuntimeStates.delete(key);
|
||||
}
|
||||
}
|
||||
|
||||
function getOrCreateEndpointRuntimeState(key: string, nowMs = Date.now()): EndpointRuntimeState {
|
||||
sweepEndpointRuntimeStates(nowMs);
|
||||
const existing = endpointRuntimeStates.get(key);
|
||||
if (existing) {
|
||||
existing.lastTouchedAtMs = nowMs;
|
||||
return existing;
|
||||
}
|
||||
|
||||
const initial: EndpointRuntimeState = {
|
||||
preferredEndpoint: null,
|
||||
preferredUpdatedAtMs: nowMs,
|
||||
lastTouchedAtMs: nowMs,
|
||||
blockedUntilMsByEndpoint: {},
|
||||
};
|
||||
endpointRuntimeStates.set(key, initial);
|
||||
enforceEndpointRuntimeStateLimit();
|
||||
return initial;
|
||||
}
|
||||
|
||||
function maybeDeleteEndpointRuntimeState(key: string, nowMs = Date.now()): void {
|
||||
const state = endpointRuntimeStates.get(key);
|
||||
if (!state) return;
|
||||
|
||||
const hasActiveBlock = Object.values(state.blockedUntilMsByEndpoint).some((untilMs) => (
|
||||
typeof untilMs === 'number' && untilMs > nowMs
|
||||
));
|
||||
const preferredFresh = (
|
||||
!!state.preferredEndpoint
|
||||
&& (state.preferredUpdatedAtMs + ENDPOINT_RUNTIME_PREFERRED_TTL_MS) > nowMs
|
||||
);
|
||||
if (!hasActiveBlock && !preferredFresh) {
|
||||
endpointRuntimeStates.delete(key);
|
||||
}
|
||||
}
|
||||
|
||||
function inferSuggestedEndpointFromError(errorText?: string | null): UpstreamEndpointRuntimeEndpoint | null {
|
||||
const text = (errorText || '').toLowerCase();
|
||||
if (!text) return null;
|
||||
if (text.includes('/v1/responses')) return 'responses';
|
||||
if (text.includes('/v1/messages')) return 'messages';
|
||||
if (text.includes('/v1/chat/completions')) return 'chat';
|
||||
return null;
|
||||
}
|
||||
|
||||
function shouldBlockEndpointByError(status: number, errorText?: string | null): boolean {
|
||||
if (isEndpointDispatchDeniedError(status, errorText)) return true;
|
||||
if (status === 404 || status === 405 || status === 415 || status === 501) return true;
|
||||
if (isUnsupportedMediaTypeError(status, errorText)) return true;
|
||||
|
||||
const text = (errorText || '').toLowerCase();
|
||||
return (
|
||||
text.includes('convert_request_failed')
|
||||
|| text.includes('endpoint_not_found')
|
||||
|| text.includes('unknown_endpoint')
|
||||
|| text.includes('unsupported_endpoint')
|
||||
|| text.includes('unsupported_path')
|
||||
|| text.includes('not_found_error')
|
||||
|| text.includes('unsupported legacy protocol')
|
||||
|| text.includes('please use /v1/')
|
||||
|| text.includes('does not allow /v1/')
|
||||
|| text.includes('unknown endpoint')
|
||||
|| text.includes('unsupported endpoint')
|
||||
|| text.includes('unsupported path')
|
||||
|| text.includes('unrecognized request url')
|
||||
|| text.includes('no route matched')
|
||||
|| text.includes('does not exist')
|
||||
);
|
||||
}
|
||||
|
||||
function shouldRememberSuccessfulEndpoint(input: {
|
||||
endpoint: UpstreamEndpointRuntimeEndpoint;
|
||||
downstreamFormat: UpstreamEndpointRuntimePreference;
|
||||
}): boolean {
|
||||
if (input.downstreamFormat !== 'responses') return true;
|
||||
return input.endpoint === 'responses';
|
||||
}
|
||||
|
||||
export function getUpstreamEndpointRuntimeStateSnapshot(input: {
|
||||
siteId: number;
|
||||
downstreamFormat: UpstreamEndpointRuntimePreference;
|
||||
modelName?: string;
|
||||
requestedModelHint?: string;
|
||||
requestCapabilities?: {
|
||||
hasNonImageFileInput?: boolean;
|
||||
conversationFileSummary?: ConversationFileInputSummary;
|
||||
wantsNativeResponsesReasoning?: boolean;
|
||||
};
|
||||
}) {
|
||||
const capabilityProfile = buildEndpointCapabilityProfile({
|
||||
modelName: input.modelName,
|
||||
requestedModelHint: input.requestedModelHint,
|
||||
requestCapabilities: input.requestCapabilities,
|
||||
});
|
||||
const enabled = shouldUseEndpointRuntimeMemory(capabilityProfile);
|
||||
const stateKey = buildEndpointRuntimeStateKey({
|
||||
siteId: input.siteId,
|
||||
downstreamFormat: input.downstreamFormat,
|
||||
capabilityProfile,
|
||||
});
|
||||
const nowMs = Date.now();
|
||||
const state = endpointRuntimeStates.get(stateKey);
|
||||
const preferredEndpoint = (
|
||||
enabled
|
||||
&& state?.preferredEndpoint
|
||||
&& (state.preferredUpdatedAtMs + ENDPOINT_RUNTIME_PREFERRED_TTL_MS) > nowMs
|
||||
&& !(
|
||||
typeof state.blockedUntilMsByEndpoint[state.preferredEndpoint] === 'number'
|
||||
&& (state.blockedUntilMsByEndpoint[state.preferredEndpoint] as number) > nowMs
|
||||
)
|
||||
) ? state.preferredEndpoint : null;
|
||||
|
||||
return {
|
||||
enabled,
|
||||
stateKey,
|
||||
preferredEndpoint,
|
||||
blockedEndpoints: enabled
|
||||
? (['chat', 'messages', 'responses'] as UpstreamEndpointRuntimeEndpoint[]).filter((endpoint) => {
|
||||
const untilMs = state?.blockedUntilMsByEndpoint[endpoint];
|
||||
return typeof untilMs === 'number' && untilMs > nowMs;
|
||||
})
|
||||
: [],
|
||||
};
|
||||
}
|
||||
|
||||
export function applyUpstreamEndpointRuntimePreference(
|
||||
candidates: UpstreamEndpointRuntimeEndpoint[],
|
||||
input: {
|
||||
siteId: number;
|
||||
downstreamFormat: UpstreamEndpointRuntimePreference;
|
||||
capabilityProfile: EndpointCapabilityProfile;
|
||||
},
|
||||
nowMs = Date.now(),
|
||||
): UpstreamEndpointRuntimeEndpoint[] {
|
||||
if (!shouldUseEndpointRuntimeMemory(input.capabilityProfile)) {
|
||||
return candidates;
|
||||
}
|
||||
|
||||
const key = buildEndpointRuntimeStateKey({
|
||||
siteId: input.siteId,
|
||||
downstreamFormat: input.downstreamFormat,
|
||||
capabilityProfile: input.capabilityProfile,
|
||||
});
|
||||
const state = endpointRuntimeStates.get(key);
|
||||
if (!state || candidates.length <= 1) return candidates;
|
||||
state.lastTouchedAtMs = nowMs;
|
||||
|
||||
const blocked = new Set<UpstreamEndpointRuntimeEndpoint>();
|
||||
for (const endpoint of candidates) {
|
||||
const untilMs = state.blockedUntilMsByEndpoint[endpoint];
|
||||
if (typeof untilMs === 'number' && untilMs > nowMs) {
|
||||
blocked.add(endpoint);
|
||||
}
|
||||
}
|
||||
|
||||
let next = candidates.filter((endpoint) => !blocked.has(endpoint));
|
||||
if (next.length === 0) {
|
||||
next = [...candidates];
|
||||
}
|
||||
|
||||
const preferredFresh = (
|
||||
!!state.preferredEndpoint
|
||||
&& (state.preferredUpdatedAtMs + ENDPOINT_RUNTIME_PREFERRED_TTL_MS) > nowMs
|
||||
);
|
||||
if (preferredFresh && state.preferredEndpoint && next.includes(state.preferredEndpoint)) {
|
||||
next = [
|
||||
state.preferredEndpoint,
|
||||
...next.filter((endpoint) => endpoint !== state.preferredEndpoint),
|
||||
];
|
||||
}
|
||||
|
||||
maybeDeleteEndpointRuntimeState(key, nowMs);
|
||||
return next;
|
||||
}
|
||||
|
||||
export function resetUpstreamEndpointRuntimeState(): void {
|
||||
endpointRuntimeStates.clear();
|
||||
}
|
||||
|
||||
export function recordUpstreamEndpointSuccess(input: {
|
||||
siteId: number;
|
||||
endpoint: UpstreamEndpointRuntimeEndpoint;
|
||||
downstreamFormat: UpstreamEndpointRuntimePreference;
|
||||
modelName?: string;
|
||||
requestedModelHint?: string;
|
||||
requestCapabilities?: {
|
||||
hasNonImageFileInput?: boolean;
|
||||
conversationFileSummary?: ConversationFileInputSummary;
|
||||
wantsNativeResponsesReasoning?: boolean;
|
||||
};
|
||||
}): UpstreamEndpointRuntimeMemoryWrite | null {
|
||||
const capabilityProfile = buildEndpointCapabilityProfile({
|
||||
modelName: input.modelName,
|
||||
requestedModelHint: input.requestedModelHint,
|
||||
requestCapabilities: input.requestCapabilities,
|
||||
});
|
||||
if (!shouldUseEndpointRuntimeMemory(capabilityProfile)) return null;
|
||||
if (!shouldRememberSuccessfulEndpoint(input)) return null;
|
||||
|
||||
const nowMs = Date.now();
|
||||
const key = buildEndpointRuntimeStateKey({
|
||||
siteId: input.siteId,
|
||||
downstreamFormat: input.downstreamFormat,
|
||||
capabilityProfile,
|
||||
});
|
||||
const state = getOrCreateEndpointRuntimeState(key, nowMs);
|
||||
state.preferredEndpoint = input.endpoint;
|
||||
state.preferredUpdatedAtMs = nowMs;
|
||||
delete state.blockedUntilMsByEndpoint[input.endpoint];
|
||||
return {
|
||||
action: 'success',
|
||||
endpoint: input.endpoint,
|
||||
preferredEndpoint: input.endpoint,
|
||||
stateKey: key,
|
||||
timestampMs: nowMs,
|
||||
};
|
||||
}
|
||||
|
||||
export function recordUpstreamEndpointFailure(input: {
|
||||
siteId: number;
|
||||
endpoint: UpstreamEndpointRuntimeEndpoint;
|
||||
downstreamFormat: UpstreamEndpointRuntimePreference;
|
||||
status: number;
|
||||
errorText?: string | null;
|
||||
modelName?: string;
|
||||
requestedModelHint?: string;
|
||||
requestCapabilities?: {
|
||||
hasNonImageFileInput?: boolean;
|
||||
conversationFileSummary?: ConversationFileInputSummary;
|
||||
wantsNativeResponsesReasoning?: boolean;
|
||||
};
|
||||
}): UpstreamEndpointRuntimeMemoryWrite | null {
|
||||
const capabilityProfile = buildEndpointCapabilityProfile({
|
||||
modelName: input.modelName,
|
||||
requestedModelHint: input.requestedModelHint,
|
||||
requestCapabilities: input.requestCapabilities,
|
||||
});
|
||||
if (!shouldUseEndpointRuntimeMemory(capabilityProfile)) return null;
|
||||
if (!shouldBlockEndpointByError(input.status, input.errorText)) return null;
|
||||
|
||||
const nowMs = Date.now();
|
||||
const key = buildEndpointRuntimeStateKey({
|
||||
siteId: input.siteId,
|
||||
downstreamFormat: input.downstreamFormat,
|
||||
capabilityProfile,
|
||||
});
|
||||
const state = getOrCreateEndpointRuntimeState(key, nowMs);
|
||||
state.blockedUntilMsByEndpoint[input.endpoint] = nowMs + ENDPOINT_RUNTIME_BLOCK_TTL_MS;
|
||||
|
||||
const suggestedEndpoint = inferSuggestedEndpointFromError(input.errorText);
|
||||
if (suggestedEndpoint && suggestedEndpoint !== input.endpoint) {
|
||||
state.preferredEndpoint = suggestedEndpoint;
|
||||
state.preferredUpdatedAtMs = nowMs;
|
||||
delete state.blockedUntilMsByEndpoint[suggestedEndpoint];
|
||||
}
|
||||
return {
|
||||
action: 'failure',
|
||||
endpoint: input.endpoint,
|
||||
blockedEndpoint: input.endpoint,
|
||||
preferredEndpoint: (
|
||||
suggestedEndpoint && suggestedEndpoint !== input.endpoint
|
||||
? suggestedEndpoint
|
||||
: null
|
||||
),
|
||||
stateKey: key,
|
||||
timestampMs: nowMs,
|
||||
};
|
||||
}
|
||||
@@ -351,6 +351,72 @@ export type ProxyLogsResponse = {
|
||||
summary: ProxyLogsSummary;
|
||||
};
|
||||
|
||||
export type ProxyDebugTraceListItem = {
|
||||
id: number;
|
||||
createdAt: string;
|
||||
downstreamPath: string;
|
||||
clientKind?: string | null;
|
||||
sessionId?: string | null;
|
||||
requestedModel?: string | null;
|
||||
selectedChannelId?: number | null;
|
||||
finalStatus?: string | null;
|
||||
finalHttpStatus?: number | null;
|
||||
finalUpstreamPath?: string | null;
|
||||
};
|
||||
|
||||
export type ProxyDebugTraceDetail = {
|
||||
trace: {
|
||||
id: number;
|
||||
createdAt?: string | null;
|
||||
updatedAt?: string | null;
|
||||
downstreamPath?: string | null;
|
||||
clientKind?: string | null;
|
||||
sessionId?: string | null;
|
||||
traceHint?: string | null;
|
||||
requestedModel?: string | null;
|
||||
stickySessionKey?: string | null;
|
||||
stickyHitChannelId?: number | null;
|
||||
selectedChannelId?: number | null;
|
||||
selectedRouteId?: number | null;
|
||||
selectedAccountId?: number | null;
|
||||
selectedSiteId?: number | null;
|
||||
selectedSitePlatform?: string | null;
|
||||
endpointCandidatesJson?: string | null;
|
||||
endpointRuntimeStateJson?: string | null;
|
||||
decisionSummaryJson?: string | null;
|
||||
requestHeadersJson?: string | null;
|
||||
requestBodyJson?: string | null;
|
||||
finalStatus?: string | null;
|
||||
finalHttpStatus?: number | null;
|
||||
finalUpstreamPath?: string | null;
|
||||
finalResponseHeadersJson?: string | null;
|
||||
finalResponseBodyJson?: string | null;
|
||||
};
|
||||
attempts: Array<{
|
||||
id: number;
|
||||
attemptIndex: number;
|
||||
endpoint: string;
|
||||
requestPath: string;
|
||||
targetUrl: string;
|
||||
runtimeExecutor?: string | null;
|
||||
requestHeadersJson?: string | null;
|
||||
requestBodyJson?: string | null;
|
||||
responseStatus?: number | null;
|
||||
responseHeadersJson?: string | null;
|
||||
responseBodyJson?: string | null;
|
||||
rawErrorText?: string | null;
|
||||
recoverApplied?: boolean | null;
|
||||
downgradeDecision?: boolean | null;
|
||||
downgradeReason?: string | null;
|
||||
memoryWriteJson?: string | null;
|
||||
createdAt?: string | null;
|
||||
}>;
|
||||
};
|
||||
|
||||
export type ProxyDebugTracesResponse = {
|
||||
items: ProxyDebugTraceListItem[];
|
||||
};
|
||||
|
||||
export type OAuthProviderInfo = {
|
||||
provider: string;
|
||||
label: string;
|
||||
@@ -546,6 +612,8 @@ export const api = {
|
||||
getDashboard: () => request('/api/stats/dashboard'),
|
||||
getProxyLogs: (params?: ProxyLogsQuery) => request(`/api/stats/proxy-logs${buildQueryString(params)}`) as Promise<ProxyLogsResponse>,
|
||||
getProxyLogDetail: (id: number) => request(`/api/stats/proxy-logs/${id}`) as Promise<ProxyLogDetail>,
|
||||
getProxyDebugTraces: (params?: { limit?: number }) => request(`/api/stats/proxy-debug/traces${buildQueryString(params)}`) as Promise<ProxyDebugTracesResponse>,
|
||||
getProxyDebugTraceDetail: (id: number) => request(`/api/stats/proxy-debug/traces/${id}`) as Promise<ProxyDebugTraceDetail>,
|
||||
checkModels: (accountId: number) => request(`/api/models/check/${accountId}`, { method: 'POST' }),
|
||||
getSiteDistribution: () => request('/api/stats/site-distribution'),
|
||||
getSiteTrend: (days = 7) => request(`/api/stats/site-trend?days=${days}`),
|
||||
|
||||
@@ -9,7 +9,11 @@ const { apiMock } = vi.hoisted(() => ({
|
||||
apiMock: {
|
||||
getProxyLogs: vi.fn(),
|
||||
getProxyLogDetail: vi.fn(),
|
||||
getProxyDebugTraces: vi.fn(),
|
||||
getProxyDebugTraceDetail: vi.fn(),
|
||||
getRuntimeSettings: vi.fn(),
|
||||
getSites: vi.fn(),
|
||||
updateRuntimeSettings: vi.fn(),
|
||||
},
|
||||
}));
|
||||
|
||||
@@ -97,6 +101,17 @@ describe('ProxyLogs server-driven page', () => {
|
||||
{ id: 9, name: 'main-site', status: 'active' },
|
||||
{ id: 12, name: 'backup-site', status: 'active' },
|
||||
]);
|
||||
apiMock.getRuntimeSettings.mockResolvedValue({
|
||||
proxyDebugTraceEnabled: false,
|
||||
proxyDebugCaptureHeaders: true,
|
||||
proxyDebugCaptureBodies: false,
|
||||
proxyDebugCaptureStreamChunks: false,
|
||||
proxyDebugTargetSessionId: '',
|
||||
proxyDebugTargetClientKind: '',
|
||||
proxyDebugTargetModel: '',
|
||||
proxyDebugRetentionHours: 24,
|
||||
proxyDebugMaxBodyBytes: 262144,
|
||||
});
|
||||
apiMock.getProxyLogs.mockResolvedValue(buildListResponse());
|
||||
apiMock.getProxyLogDetail.mockResolvedValue({
|
||||
id: 101,
|
||||
@@ -151,6 +166,40 @@ describe('ProxyLogs server-driven page', () => {
|
||||
},
|
||||
},
|
||||
});
|
||||
apiMock.getProxyDebugTraces.mockResolvedValue({
|
||||
items: [
|
||||
{
|
||||
id: 701,
|
||||
createdAt: '2026-03-28 18:00:00',
|
||||
requestedModel: 'gpt-4o',
|
||||
downstreamPath: '/v1/responses',
|
||||
finalStatus: 'failed',
|
||||
finalUpstreamPath: '/responses',
|
||||
clientKind: 'codex',
|
||||
sessionId: 'sess-debug-1',
|
||||
},
|
||||
],
|
||||
});
|
||||
apiMock.getProxyDebugTraceDetail.mockResolvedValue({
|
||||
trace: {
|
||||
id: 701,
|
||||
requestedModel: 'gpt-4o',
|
||||
sessionId: 'sess-debug-1',
|
||||
},
|
||||
attempts: [],
|
||||
});
|
||||
apiMock.updateRuntimeSettings.mockResolvedValue({
|
||||
success: true,
|
||||
proxyDebugTraceEnabled: true,
|
||||
proxyDebugCaptureHeaders: true,
|
||||
proxyDebugCaptureBodies: true,
|
||||
proxyDebugCaptureStreamChunks: false,
|
||||
proxyDebugTargetSessionId: 'sess-debug-1',
|
||||
proxyDebugTargetClientKind: 'codex',
|
||||
proxyDebugTargetModel: 'gpt-4o',
|
||||
proxyDebugRetentionHours: 12,
|
||||
proxyDebugMaxBodyBytes: 131072,
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
@@ -193,6 +242,85 @@ describe('ProxyLogs server-driven page', () => {
|
||||
}
|
||||
});
|
||||
|
||||
it('allows toggling proxy debug settings directly from the proxy logs page', async () => {
|
||||
let root!: WebTestRenderer;
|
||||
|
||||
try {
|
||||
await act(async () => {
|
||||
root = create(
|
||||
<MemoryRouter initialEntries={['/logs']}>
|
||||
<ToastProvider>
|
||||
<ProxyLogs />
|
||||
</ToastProvider>
|
||||
</MemoryRouter>,
|
||||
);
|
||||
});
|
||||
await flushMicrotasks();
|
||||
|
||||
const debugButton = root.root.find((node) => (
|
||||
node.type === 'button'
|
||||
&& typeof node.props.onClick === 'function'
|
||||
&& collectText(node).includes('调试')
|
||||
));
|
||||
|
||||
await act(async () => {
|
||||
debugButton.props.onClick();
|
||||
});
|
||||
await flushMicrotasks();
|
||||
|
||||
expect(apiMock.getRuntimeSettings).toHaveBeenCalled();
|
||||
expect(apiMock.getProxyDebugTraces).toHaveBeenCalled();
|
||||
expect(collectText(root.root)).toContain('最近调试追踪');
|
||||
expect(collectText(root.root)).toContain('sess-debug-1');
|
||||
|
||||
const traceEnabledToggle = root.root.find((node) => (
|
||||
node.type === 'input'
|
||||
&& node.props.type === 'checkbox'
|
||||
&& node.props['data-debug-setting'] === 'trace-enabled'
|
||||
));
|
||||
const captureBodiesToggle = root.root.find((node) => (
|
||||
node.type === 'input'
|
||||
&& node.props.type === 'checkbox'
|
||||
&& node.props['data-debug-setting'] === 'capture-bodies'
|
||||
));
|
||||
const sessionInput = root.root.find((node) => (
|
||||
node.type === 'input'
|
||||
&& node.props['data-debug-setting'] === 'target-session-id'
|
||||
));
|
||||
const retentionInput = root.root.find((node) => (
|
||||
node.type === 'input'
|
||||
&& node.props['data-debug-setting'] === 'retention-hours'
|
||||
));
|
||||
|
||||
await act(async () => {
|
||||
traceEnabledToggle.props.onChange({ target: { checked: true } });
|
||||
captureBodiesToggle.props.onChange({ target: { checked: true } });
|
||||
sessionInput.props.onChange({ target: { value: 'sess-debug-1' } });
|
||||
retentionInput.props.onChange({ target: { value: '12' } });
|
||||
});
|
||||
|
||||
const saveButton = root.root.find((node) => (
|
||||
node.type === 'button'
|
||||
&& typeof node.props.onClick === 'function'
|
||||
&& collectText(node).trim() === '保存调试设置'
|
||||
));
|
||||
|
||||
await act(async () => {
|
||||
saveButton.props.onClick();
|
||||
});
|
||||
await flushMicrotasks();
|
||||
|
||||
expect(apiMock.updateRuntimeSettings).toHaveBeenCalledWith(expect.objectContaining({
|
||||
proxyDebugTraceEnabled: true,
|
||||
proxyDebugCaptureBodies: true,
|
||||
proxyDebugTargetSessionId: 'sess-debug-1',
|
||||
proxyDebugRetentionHours: 12,
|
||||
}));
|
||||
} finally {
|
||||
root?.unmount();
|
||||
}
|
||||
});
|
||||
|
||||
it('keeps the model badge sized to the model name in desktop rows', async () => {
|
||||
let root!: WebTestRenderer;
|
||||
|
||||
|
||||
@@ -2,6 +2,8 @@ import React, { useDeferredValue, useEffect, useMemo, useRef, useState, useCallb
|
||||
import { useLocation, useNavigate } from 'react-router-dom';
|
||||
import {
|
||||
api,
|
||||
type ProxyDebugTraceDetail,
|
||||
type ProxyDebugTraceListItem,
|
||||
type ProxyLogBillingDetails,
|
||||
type ProxyLogClientOption,
|
||||
type ProxyLogDetail,
|
||||
@@ -40,6 +42,24 @@ type ProxyLogSiteFilterOption = {
|
||||
status: string | null;
|
||||
};
|
||||
|
||||
type ProxyDebugSettingsState = {
|
||||
proxyDebugTraceEnabled: boolean;
|
||||
proxyDebugCaptureHeaders: boolean;
|
||||
proxyDebugCaptureBodies: boolean;
|
||||
proxyDebugCaptureStreamChunks: boolean;
|
||||
proxyDebugTargetSessionId: string;
|
||||
proxyDebugTargetClientKind: string;
|
||||
proxyDebugTargetModel: string;
|
||||
proxyDebugRetentionHours: number;
|
||||
proxyDebugMaxBodyBytes: number;
|
||||
};
|
||||
|
||||
type ProxyDebugTraceDetailState = {
|
||||
loading: boolean;
|
||||
data?: ProxyDebugTraceDetail;
|
||||
error?: string;
|
||||
};
|
||||
|
||||
const PAGE_SIZES = [20, 50, 100];
|
||||
const DEFAULT_PAGE_SIZE = 50;
|
||||
const PROXY_LOG_CLIENT_FAMILY_LABELS: Record<string, string> = {
|
||||
@@ -55,6 +75,17 @@ const EMPTY_SUMMARY: ProxyLogsSummary = {
|
||||
totalCost: 0,
|
||||
totalTokensAll: 0,
|
||||
};
|
||||
const DEFAULT_PROXY_DEBUG_SETTINGS: ProxyDebugSettingsState = {
|
||||
proxyDebugTraceEnabled: false,
|
||||
proxyDebugCaptureHeaders: true,
|
||||
proxyDebugCaptureBodies: false,
|
||||
proxyDebugCaptureStreamChunks: false,
|
||||
proxyDebugTargetSessionId: '',
|
||||
proxyDebugTargetClientKind: '',
|
||||
proxyDebugTargetModel: '',
|
||||
proxyDebugRetentionHours: 24,
|
||||
proxyDebugMaxBodyBytes: 262144,
|
||||
};
|
||||
|
||||
function formatLatency(ms: number) {
|
||||
if (ms >= 1000) {
|
||||
@@ -314,6 +345,13 @@ export default function ProxyLogs() {
|
||||
const [sites, setSites] = useState<Array<{ id: number; name: string; status?: string | null }>>([]);
|
||||
const [clientOptions, setClientOptions] = useState<ProxyLogClientOption[]>([]);
|
||||
const [autoRefresh, setAutoRefresh] = useState(false);
|
||||
const [showDebugPanel, setShowDebugPanel] = useState(false);
|
||||
const [debugPanelLoading, setDebugPanelLoading] = useState(false);
|
||||
const [debugPanelSaving, setDebugPanelSaving] = useState(false);
|
||||
const [debugSettings, setDebugSettings] = useState<ProxyDebugSettingsState>(DEFAULT_PROXY_DEBUG_SETTINGS);
|
||||
const [debugTraces, setDebugTraces] = useState<ProxyDebugTraceListItem[]>([]);
|
||||
const [selectedDebugTraceId, setSelectedDebugTraceId] = useState<number | null>(null);
|
||||
const [debugDetailById, setDebugDetailById] = useState<Record<number, ProxyDebugTraceDetailState>>({});
|
||||
const isMobile = useIsMobile(768);
|
||||
const toast = useToast();
|
||||
const loadSeq = useRef(0);
|
||||
@@ -527,6 +565,106 @@ export default function ProxyLogs() {
|
||||
}
|
||||
}, [detailById, toast]);
|
||||
|
||||
const loadDebugPanel = useCallback(async (silent = false) => {
|
||||
if (!silent) setDebugPanelLoading(true);
|
||||
try {
|
||||
const [runtimeSettings, traceResponse] = await Promise.all([
|
||||
api.getRuntimeSettings(),
|
||||
api.getProxyDebugTraces({ limit: 20 }),
|
||||
]);
|
||||
setDebugSettings({
|
||||
proxyDebugTraceEnabled: !!runtimeSettings?.proxyDebugTraceEnabled,
|
||||
proxyDebugCaptureHeaders: runtimeSettings?.proxyDebugCaptureHeaders !== false,
|
||||
proxyDebugCaptureBodies: !!runtimeSettings?.proxyDebugCaptureBodies,
|
||||
proxyDebugCaptureStreamChunks: !!runtimeSettings?.proxyDebugCaptureStreamChunks,
|
||||
proxyDebugTargetSessionId: String(runtimeSettings?.proxyDebugTargetSessionId || ''),
|
||||
proxyDebugTargetClientKind: String(runtimeSettings?.proxyDebugTargetClientKind || ''),
|
||||
proxyDebugTargetModel: String(runtimeSettings?.proxyDebugTargetModel || ''),
|
||||
proxyDebugRetentionHours: Number(runtimeSettings?.proxyDebugRetentionHours || 24),
|
||||
proxyDebugMaxBodyBytes: Number(runtimeSettings?.proxyDebugMaxBodyBytes || 262144),
|
||||
});
|
||||
const items = Array.isArray(traceResponse?.items) ? traceResponse.items : [];
|
||||
setDebugTraces(items);
|
||||
setSelectedDebugTraceId((current) => {
|
||||
if (current && items.some((item) => item.id === current)) return current;
|
||||
return items[0]?.id ?? null;
|
||||
});
|
||||
} catch (error: any) {
|
||||
toast.error(error?.message || '加载代理调试面板失败');
|
||||
} finally {
|
||||
if (!silent) setDebugPanelLoading(false);
|
||||
}
|
||||
}, [toast]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!showDebugPanel) return;
|
||||
void loadDebugPanel();
|
||||
}, [loadDebugPanel, showDebugPanel]);
|
||||
|
||||
const loadDebugTraceDetail = useCallback(async (id: number) => {
|
||||
const existing = debugDetailById[id];
|
||||
if (existing?.loading || existing?.data) return;
|
||||
|
||||
setDebugDetailById((current) => ({
|
||||
...current,
|
||||
[id]: { loading: true },
|
||||
}));
|
||||
|
||||
try {
|
||||
const data = await api.getProxyDebugTraceDetail(id);
|
||||
setDebugDetailById((current) => ({
|
||||
...current,
|
||||
[id]: { loading: false, data },
|
||||
}));
|
||||
} catch (error: any) {
|
||||
const message = error?.message || '加载调试追踪详情失败';
|
||||
setDebugDetailById((current) => ({
|
||||
...current,
|
||||
[id]: { loading: false, error: message },
|
||||
}));
|
||||
toast.error(message);
|
||||
}
|
||||
}, [debugDetailById, toast]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!showDebugPanel || !selectedDebugTraceId) return;
|
||||
void loadDebugTraceDetail(selectedDebugTraceId);
|
||||
}, [loadDebugTraceDetail, selectedDebugTraceId, showDebugPanel]);
|
||||
|
||||
const handleSaveDebugSettings = useCallback(async () => {
|
||||
setDebugPanelSaving(true);
|
||||
try {
|
||||
const updated = await api.updateRuntimeSettings({
|
||||
proxyDebugTraceEnabled: debugSettings.proxyDebugTraceEnabled,
|
||||
proxyDebugCaptureHeaders: debugSettings.proxyDebugCaptureHeaders,
|
||||
proxyDebugCaptureBodies: debugSettings.proxyDebugCaptureBodies,
|
||||
proxyDebugCaptureStreamChunks: debugSettings.proxyDebugCaptureStreamChunks,
|
||||
proxyDebugTargetSessionId: debugSettings.proxyDebugTargetSessionId.trim(),
|
||||
proxyDebugTargetClientKind: debugSettings.proxyDebugTargetClientKind.trim(),
|
||||
proxyDebugTargetModel: debugSettings.proxyDebugTargetModel.trim(),
|
||||
proxyDebugRetentionHours: Math.max(1, Math.trunc(Number(debugSettings.proxyDebugRetentionHours || 24))),
|
||||
proxyDebugMaxBodyBytes: Math.max(1024, Math.trunc(Number(debugSettings.proxyDebugMaxBodyBytes || 262144))),
|
||||
});
|
||||
setDebugSettings({
|
||||
proxyDebugTraceEnabled: !!updated?.proxyDebugTraceEnabled,
|
||||
proxyDebugCaptureHeaders: updated?.proxyDebugCaptureHeaders !== false,
|
||||
proxyDebugCaptureBodies: !!updated?.proxyDebugCaptureBodies,
|
||||
proxyDebugCaptureStreamChunks: !!updated?.proxyDebugCaptureStreamChunks,
|
||||
proxyDebugTargetSessionId: String(updated?.proxyDebugTargetSessionId || ''),
|
||||
proxyDebugTargetClientKind: String(updated?.proxyDebugTargetClientKind || ''),
|
||||
proxyDebugTargetModel: String(updated?.proxyDebugTargetModel || ''),
|
||||
proxyDebugRetentionHours: Number(updated?.proxyDebugRetentionHours || 24),
|
||||
proxyDebugMaxBodyBytes: Number(updated?.proxyDebugMaxBodyBytes || 262144),
|
||||
});
|
||||
toast.success('代理调试设置已保存');
|
||||
await loadDebugPanel(true);
|
||||
} catch (error: any) {
|
||||
toast.error(error?.message || '保存代理调试设置失败');
|
||||
} finally {
|
||||
setDebugPanelSaving(false);
|
||||
}
|
||||
}, [debugSettings, loadDebugPanel, toast]);
|
||||
|
||||
const handleToggleExpand = useCallback((id: number) => {
|
||||
const shouldExpand = expanded !== id;
|
||||
setExpanded(shouldExpand ? id : null);
|
||||
@@ -534,6 +672,7 @@ export default function ProxyLogs() {
|
||||
void loadDetail(id);
|
||||
}
|
||||
}, [expanded, loadDetail]);
|
||||
const selectedDebugTraceDetail = selectedDebugTraceId ? debugDetailById[selectedDebugTraceId] : undefined;
|
||||
|
||||
const filterControls = (
|
||||
<>
|
||||
@@ -650,6 +789,14 @@ export default function ProxyLogs() {
|
||||
</span>
|
||||
</div>
|
||||
<div style={{ display: 'flex', alignItems: 'center', gap: 6 }}>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setShowDebugPanel((current) => !current)}
|
||||
className={`btn btn-ghost${showDebugPanel ? ' btn-ghost-active' : ''}`}
|
||||
style={{ border: '1px solid var(--color-border)', padding: '6px 14px' }}
|
||||
>
|
||||
{showDebugPanel ? '收起调试' : '调试追踪'}
|
||||
</button>
|
||||
<button
|
||||
onClick={() => setAutoRefresh((v) => !v)}
|
||||
className={`btn btn-ghost${autoRefresh ? ' btn-ghost-active' : ''}`}
|
||||
@@ -684,6 +831,251 @@ export default function ProxyLogs() {
|
||||
)}
|
||||
/>
|
||||
|
||||
{showDebugPanel ? (
|
||||
<div className="card" style={{ marginBottom: 12, display: 'grid', gap: 16 }}>
|
||||
<div style={{ display: 'flex', justifyContent: 'space-between', alignItems: 'center', gap: 12, flexWrap: 'wrap' }}>
|
||||
<div>
|
||||
<div style={{ fontSize: 16, fontWeight: 600 }}>代理调试追踪</div>
|
||||
<div style={{ fontSize: 12, color: 'var(--color-text-muted)' }}>
|
||||
在使用日志页直接开启后，后续请求会记录路由选择、上游 endpoint 尝试和原始请求/响应。
|
||||
</div>
|
||||
</div>
|
||||
<button
|
||||
type="button"
|
||||
className="btn btn-ghost"
|
||||
onClick={() => void loadDebugPanel()}
|
||||
disabled={debugPanelLoading}
|
||||
>
|
||||
{debugPanelLoading ? '刷新中...' : '刷新调试面板'}
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<div style={{ display: 'grid', gap: 12, gridTemplateColumns: isMobile ? '1fr' : 'repeat(2, minmax(0, 1fr))' }}>
|
||||
<label style={{ display: 'flex', alignItems: 'center', gap: 8 }}>
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={debugSettings.proxyDebugTraceEnabled}
|
||||
data-debug-setting="trace-enabled"
|
||||
onChange={(e) => setDebugSettings((current) => ({ ...current, proxyDebugTraceEnabled: !!e.target.checked }))}
|
||||
/>
|
||||
<span>开启调试追踪</span>
|
||||
</label>
|
||||
<label style={{ display: 'flex', alignItems: 'center', gap: 8 }}>
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={debugSettings.proxyDebugCaptureHeaders}
|
||||
data-debug-setting="capture-headers"
|
||||
onChange={(e) => setDebugSettings((current) => ({ ...current, proxyDebugCaptureHeaders: !!e.target.checked }))}
|
||||
/>
|
||||
<span>采集原始请求/响应头</span>
|
||||
</label>
|
||||
<label style={{ display: 'flex', alignItems: 'center', gap: 8 }}>
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={debugSettings.proxyDebugCaptureBodies}
|
||||
data-debug-setting="capture-bodies"
|
||||
onChange={(e) => setDebugSettings((current) => ({ ...current, proxyDebugCaptureBodies: !!e.target.checked }))}
|
||||
/>
|
||||
<span>采集请求体和响应体</span>
|
||||
</label>
|
||||
<label style={{ display: 'flex', alignItems: 'center', gap: 8 }}>
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={debugSettings.proxyDebugCaptureStreamChunks}
|
||||
data-debug-setting="capture-stream-chunks"
|
||||
onChange={(e) => setDebugSettings((current) => ({ ...current, proxyDebugCaptureStreamChunks: !!e.target.checked }))}
|
||||
/>
|
||||
<span>采集流式原始分片</span>
|
||||
</label>
|
||||
<label style={{ display: 'grid', gap: 6 }}>
|
||||
<span style={{ fontSize: 12, color: 'var(--color-text-muted)' }}>目标 Session ID</span>
|
||||
<input
|
||||
type="text"
|
||||
value={debugSettings.proxyDebugTargetSessionId}
|
||||
data-debug-setting="target-session-id"
|
||||
onChange={(e) => setDebugSettings((current) => ({ ...current, proxyDebugTargetSessionId: e.target.value }))}
|
||||
placeholder="留空表示不过滤"
|
||||
/>
|
||||
</label>
|
||||
<label style={{ display: 'grid', gap: 6 }}>
|
||||
<span style={{ fontSize: 12, color: 'var(--color-text-muted)' }}>目标客户端</span>
|
||||
<input
|
||||
type="text"
|
||||
value={debugSettings.proxyDebugTargetClientKind}
|
||||
data-debug-setting="target-client-kind"
|
||||
onChange={(e) => setDebugSettings((current) => ({ ...current, proxyDebugTargetClientKind: e.target.value }))}
|
||||
placeholder="如 codex / claude_code"
|
||||
/>
|
||||
</label>
|
||||
<label style={{ display: 'grid', gap: 6 }}>
|
||||
<span style={{ fontSize: 12, color: 'var(--color-text-muted)' }}>目标模型</span>
|
||||
<input
|
||||
type="text"
|
||||
value={debugSettings.proxyDebugTargetModel}
|
||||
data-debug-setting="target-model"
|
||||
onChange={(e) => setDebugSettings((current) => ({ ...current, proxyDebugTargetModel: e.target.value }))}
|
||||
placeholder="如 gpt-4o"
|
||||
/>
|
||||
</label>
|
||||
<label style={{ display: 'grid', gap: 6 }}>
|
||||
<span style={{ fontSize: 12, color: 'var(--color-text-muted)' }}>保留时长（小时）</span>
|
||||
<input
|
||||
type="number"
|
||||
min={1}
|
||||
value={debugSettings.proxyDebugRetentionHours}
|
||||
data-debug-setting="retention-hours"
|
||||
onChange={(e) => setDebugSettings((current) => ({ ...current, proxyDebugRetentionHours: Number(e.target.value || 1) }))}
|
||||
/>
|
||||
</label>
|
||||
<label style={{ display: 'grid', gap: 6 }}>
|
||||
<span style={{ fontSize: 12, color: 'var(--color-text-muted)' }}>抓取体积上限（字节）</span>
|
||||
<input
|
||||
type="number"
|
||||
min={1024}
|
||||
value={debugSettings.proxyDebugMaxBodyBytes}
|
||||
data-debug-setting="max-body-bytes"
|
||||
onChange={(e) => setDebugSettings((current) => ({ ...current, proxyDebugMaxBodyBytes: Number(e.target.value || 1024) }))}
|
||||
/>
|
||||
</label>
|
||||
</div>
|
||||
|
||||
<div style={{ display: 'flex', gap: 8, flexWrap: 'wrap' }}>
|
||||
<button
|
||||
type="button"
|
||||
className="btn btn-primary"
|
||||
onClick={() => void handleSaveDebugSettings()}
|
||||
disabled={debugPanelSaving}
|
||||
>
|
||||
{debugPanelSaving ? '保存中...' : '保存调试设置'}
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
className="btn btn-ghost"
|
||||
onClick={() => setDebugSettings(DEFAULT_PROXY_DEBUG_SETTINGS)}
|
||||
>
|
||||
重置为默认值
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<div style={{ display: 'grid', gap: 12, gridTemplateColumns: isMobile ? '1fr' : 'minmax(280px, 360px) minmax(0, 1fr)' }}>
|
||||
<div style={{ display: 'grid', gap: 8 }}>
|
||||
<div style={{ fontWeight: 600 }}>最近调试追踪</div>
|
||||
{debugTraces.length === 0 ? (
|
||||
<div style={{ color: 'var(--color-text-muted)', fontSize: 13 }}>
|
||||
暂无调试追踪，开启后等待新请求进入即可。
|
||||
</div>
|
||||
) : (
|
||||
debugTraces.map((trace) => (
|
||||
<button
|
||||
key={trace.id}
|
||||
type="button"
|
||||
className="btn btn-ghost"
|
||||
style={{
|
||||
justifyContent: 'flex-start',
|
||||
textAlign: 'left',
|
||||
border: '1px solid var(--color-border)',
|
||||
background: selectedDebugTraceId === trace.id ? 'color-mix(in srgb, var(--color-primary) 10%, transparent)' : 'transparent',
|
||||
}}
|
||||
onClick={() => {
|
||||
setSelectedDebugTraceId(trace.id);
|
||||
void loadDebugTraceDetail(trace.id);
|
||||
}}
|
||||
>
|
||||
<div style={{ display: 'grid', gap: 4 }}>
|
||||
<div style={{ fontWeight: 600 }}>{trace.sessionId || `trace-${trace.id}`}</div>
|
||||
<div style={{ fontSize: 12, color: 'var(--color-text-muted)' }}>
|
||||
{trace.requestedModel || '-'} · {trace.downstreamPath}
|
||||
</div>
|
||||
<div style={{ fontSize: 12, color: 'var(--color-text-muted)' }}>
|
||||
{trace.finalStatus || 'pending'} · {trace.finalUpstreamPath || '-'}
|
||||
</div>
|
||||
</div>
|
||||
</button>
|
||||
))
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div style={{ display: 'grid', gap: 10 }}>
|
||||
<div style={{ fontWeight: 600 }}>追踪详情</div>
|
||||
{!selectedDebugTraceId ? (
|
||||
<div style={{ color: 'var(--color-text-muted)', fontSize: 13 }}>
|
||||
选择左侧一条追踪后查看完整的路由选择、请求头和响应体。
|
||||
</div>
|
||||
) : selectedDebugTraceDetail?.loading ? (
|
||||
<div style={{ color: 'var(--color-text-muted)', fontSize: 13 }}>加载追踪详情中...</div>
|
||||
) : selectedDebugTraceDetail?.error ? (
|
||||
<div style={{ color: 'var(--color-danger)', fontSize: 13 }}>{selectedDebugTraceDetail.error}</div>
|
||||
) : selectedDebugTraceDetail?.data ? (
|
||||
<div style={{ display: 'grid', gap: 10 }}>
|
||||
<div style={{ display: 'grid', gap: 4, fontSize: 13 }}>
|
||||
<div>下游路径：{selectedDebugTraceDetail.data.trace.downstreamPath || '-'}</div>
|
||||
<div>Session：{selectedDebugTraceDetail.data.trace.sessionId || '-'}</div>
|
||||
<div>模型：{selectedDebugTraceDetail.data.trace.requestedModel || '-'}</div>
|
||||
<div>候选 endpoint：{selectedDebugTraceDetail.data.trace.endpointCandidatesJson || '-'}</div>
|
||||
<div>最终上游路径：{selectedDebugTraceDetail.data.trace.finalUpstreamPath || '-'}</div>
|
||||
</div>
|
||||
<details open>
|
||||
<summary>原始下游请求头</summary>
|
||||
<pre style={{ whiteSpace: 'pre-wrap', margin: 0 }}>{selectedDebugTraceDetail.data.trace.requestHeadersJson || '-'}</pre>
|
||||
</details>
|
||||
<details>
|
||||
<summary>原始下游请求体</summary>
|
||||
<pre style={{ whiteSpace: 'pre-wrap', margin: 0 }}>{selectedDebugTraceDetail.data.trace.requestBodyJson || '-'}</pre>
|
||||
</details>
|
||||
<details>
|
||||
<summary>最终响应</summary>
|
||||
<pre style={{ whiteSpace: 'pre-wrap', margin: 0 }}>{selectedDebugTraceDetail.data.trace.finalResponseBodyJson || '-'}</pre>
|
||||
</details>
|
||||
<div style={{ display: 'grid', gap: 8 }}>
|
||||
<div style={{ fontWeight: 600 }}>Attempt 记录</div>
|
||||
{selectedDebugTraceDetail.data.attempts.length === 0 ? (
|
||||
<div style={{ color: 'var(--color-text-muted)', fontSize: 13 }}>暂无 attempt 记录</div>
|
||||
) : (
|
||||
selectedDebugTraceDetail.data.attempts.map((attempt) => (
|
||||
<details key={attempt.id}>
|
||||
<summary>
|
||||
#{attempt.attemptIndex + 1} · {attempt.endpoint} · {attempt.responseStatus ?? '-'} · {attempt.requestPath}
|
||||
</summary>
|
||||
<pre style={{ whiteSpace: 'pre-wrap', margin: 0 }}>
|
||||
{`targetUrl: ${attempt.targetUrl}
|
||||
runtimeExecutor: ${attempt.runtimeExecutor || '-'}
|
||||
recoverApplied: ${attempt.recoverApplied ? 'true' : 'false'}
|
||||
downgradeDecision: ${attempt.downgradeDecision ? 'true' : 'false'}
|
||||
downgradeReason: ${attempt.downgradeReason || '-'}
|
||||
|
||||
requestHeaders:
|
||||
${attempt.requestHeadersJson || '-'}
|
||||
|
||||
requestBody:
|
||||
${attempt.requestBodyJson || '-'}
|
||||
|
||||
responseHeaders:
|
||||
${attempt.responseHeadersJson || '-'}
|
||||
|
||||
responseBody:
|
||||
${attempt.responseBodyJson || '-'}
|
||||
|
||||
rawErrorText:
|
||||
${attempt.rawErrorText || '-'}
|
||||
|
||||
memoryWrite:
|
||||
${attempt.memoryWriteJson || '-'}`}
|
||||
</pre>
|
||||
</details>
|
||||
))
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
) : (
|
||||
<div style={{ color: 'var(--color-text-muted)', fontSize: 13 }}>
|
||||
暂无追踪详情。
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
) : null}
|
||||
|
||||
{hasInvalidTimeRange && (
|
||||
<div className="alert alert-error" style={{ marginBottom: 12 }}>
|
||||
结束时间必须晚于开始时间
|
||||
|
||||
Reference in New Issue
Block a user