Nvidia/BossAI
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add proxy first-byte timeout and log badges (#383)

Browse Source 
* add proxy first-byte timeout and log badges

* fix first-byte timeout review feedback
This commit is contained in:
Cita
2026-04-02 10:00:11 +08:00
committed by GitHub
parent bcd758edac
commit 180d17adfc
43 changed files with 4049 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files
drizzle/0019_proxy_logs_stream_timing.sql
+2
View File
@@ -0,0 +1,2 @@
ALTER TABLE `proxy_logs` ADD `is_stream` integer;--> statement-breakpoint
ALTER TABLE `proxy_logs` ADD `first_byte_latency_ms` integer;
drizzle/meta/0019_snapshot.json
+2774
View File
File diff suppressed because it is too large Load Diff
drizzle/meta/_journal.json
+7
View File
@@ -127,6 +127,13 @@
"when": 1774959227309,
"tag": "0018_site_api_endpoints",
"breakpoints": true
},
{
"idx": 18,
"version": "6",
"when": 1775058998448,
"tag": "0019_proxy_logs_stream_timing",
"breakpoints": true
}
]
}
src/server/config.ts
+1
View File
@@ -118,6 +118,7 @@ export function buildConfig(env: NodeJS.ProcessEnv) {
dbSsl: parseBoolean(env.DB_SSL, false),
requestBodyLimit: DEFAULT_REQUEST_BODY_LIMIT,
routingFallbackUnitCost: Math.max(1e-6, parseNumber(env.ROUTING_FALLBACK_UNIT_COST, 1)),
proxyFirstByteTimeoutSec: Math.max(0, Math.trunc(parseNumber(env.PROXY_FIRST_BYTE_TIMEOUT_SEC, 0))),
tokenRouterFailureCooldownMaxSec: normalizeTokenRouterFailureCooldownMaxSec(
parseNumber(env.TOKEN_ROUTER_FAILURE_COOLDOWN_MAX_SEC, TOKEN_ROUTER_FAILURE_COOLDOWN_MAX_SEC_CEILING),
) ?? TOKEN_ROUTER_FAILURE_COOLDOWN_MAX_SEC_CEILING,
src/server/db/generated/mysql.bootstrap.sql
+1 -1
View File
@@ -8,7 +8,7 @@ CREATE TABLE IF NOT EXISTS `model_availability` (`id` INT AUTO_INCREMENT NOT NUL
CREATE TABLE IF NOT EXISTS `proxy_debug_traces` (`id` INT AUTO_INCREMENT NOT NULL PRIMARY KEY, `downstream_path` TEXT NOT NULL, `client_kind` TEXT, `session_id` TEXT, `trace_hint` TEXT, `requested_model` TEXT, `downstream_api_key_id` INT, `request_headers_json` JSON, `request_body_json` JSON, `sticky_session_key` TEXT, `sticky_hit_channel_id` INT, `selected_channel_id` INT, `selected_route_id` INT, `selected_account_id` INT, `selected_site_id` INT, `selected_site_platform` TEXT, `endpoint_candidates_json` JSON, `endpoint_runtime_state_json` JSON, `decision_summary_json` JSON, `final_status` TEXT, `final_http_status` INT, `final_upstream_path` TEXT, `final_response_headers_json` JSON, `final_response_body_json` JSON, `created_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), '%Y-%m-%d %H:%i:%s')), `updated_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), '%Y-%m-%d %H:%i:%s')));
CREATE TABLE IF NOT EXISTS `proxy_debug_attempts` (`id` INT AUTO_INCREMENT NOT NULL PRIMARY KEY, `trace_id` INT NOT NULL, `attempt_index` INT NOT NULL, `endpoint` TEXT NOT NULL, `request_path` TEXT NOT NULL, `target_url` TEXT NOT NULL, `runtime_executor` TEXT, `request_headers_json` JSON, `request_body_json` JSON, `response_status` INT, `response_headers_json` JSON, `response_body_json` JSON, `raw_error_text` TEXT, `recover_applied` BOOLEAN DEFAULT false, `downgrade_decision` BOOLEAN DEFAULT false, `downgrade_reason` TEXT, `memory_write_json` JSON, `created_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), '%Y-%m-%d %H:%i:%s')), FOREIGN KEY (`trace_id`) REFERENCES `proxy_debug_traces`(`id`) ON DELETE CASCADE);
CREATE TABLE IF NOT EXISTS `proxy_files` (`id` INT AUTO_INCREMENT NOT NULL PRIMARY KEY, `public_id` TEXT NOT NULL, `owner_type` TEXT NOT NULL, `owner_id` TEXT NOT NULL, `filename` TEXT NOT NULL, `mime_type` TEXT NOT NULL, `purpose` TEXT, `byte_size` INT NOT NULL, `sha256` TEXT NOT NULL, `content_base64` TEXT NOT NULL, `created_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), '%Y-%m-%d %H:%i:%s')), `updated_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), '%Y-%m-%d %H:%i:%s')), `deleted_at` VARCHAR(191));
CREATE TABLE IF NOT EXISTS `proxy_logs` (`id` INT AUTO_INCREMENT NOT NULL PRIMARY KEY, `route_id` INT, `channel_id` INT, `account_id` INT, `model_requested` TEXT, `model_actual` TEXT, `status` TEXT, `http_status` INT, `latency_ms` INT, `prompt_tokens` INT, `completion_tokens` INT, `total_tokens` INT, `estimated_cost` DOUBLE, `error_message` TEXT, `retry_count` INT DEFAULT 0, `created_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), '%Y-%m-%d %H:%i:%s')), `billing_details` JSON, `downstream_api_key_id` INT, `client_family` TEXT, `client_app_id` TEXT, `client_app_name` TEXT, `client_confidence` TEXT);
CREATE TABLE IF NOT EXISTS `proxy_logs` (`id` INT AUTO_INCREMENT NOT NULL PRIMARY KEY, `route_id` INT, `channel_id` INT, `account_id` INT, `model_requested` TEXT, `model_actual` TEXT, `status` TEXT, `http_status` INT, `latency_ms` INT, `prompt_tokens` INT, `completion_tokens` INT, `total_tokens` INT, `estimated_cost` DOUBLE, `error_message` TEXT, `retry_count` INT DEFAULT 0, `created_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), '%Y-%m-%d %H:%i:%s')), `billing_details` JSON, `downstream_api_key_id` INT, `client_family` TEXT, `client_app_id` TEXT, `client_app_name` TEXT, `client_confidence` TEXT, `is_stream` BOOLEAN, `first_byte_latency_ms` INT);
CREATE TABLE IF NOT EXISTS `proxy_video_tasks` (`id` INT AUTO_INCREMENT NOT NULL PRIMARY KEY, `public_id` TEXT NOT NULL, `upstream_video_id` TEXT NOT NULL, `site_url` TEXT NOT NULL, `token_value` TEXT NOT NULL, `requested_model` TEXT, `actual_model` TEXT, `channel_id` INT, `account_id` INT, `status_snapshot` JSON, `upstream_response_meta` JSON, `last_upstream_status` INT, `last_polled_at` VARCHAR(191), `created_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), '%Y-%m-%d %H:%i:%s')), `updated_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), '%Y-%m-%d %H:%i:%s')));
CREATE TABLE IF NOT EXISTS `token_routes` (`id` INT AUTO_INCREMENT NOT NULL PRIMARY KEY, `model_pattern` TEXT NOT NULL, `model_mapping` JSON, `enabled` BOOLEAN DEFAULT true, `created_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), '%Y-%m-%d %H:%i:%s')), `updated_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), '%Y-%m-%d %H:%i:%s')), `display_name` TEXT, `display_icon` TEXT, `decision_snapshot` JSON, `decision_refreshed_at` VARCHAR(191), `routing_strategy` VARCHAR(191) DEFAULT 'weighted', `route_mode` VARCHAR(191) DEFAULT 'pattern');
CREATE TABLE IF NOT EXISTS `route_channels` (`id` INT AUTO_INCREMENT NOT NULL PRIMARY KEY, `route_id` INT NOT NULL, `account_id` INT NOT NULL, `token_id` INT, `source_model` TEXT, `priority` INT DEFAULT 0, `weight` INT DEFAULT 10, `enabled` BOOLEAN DEFAULT true, `manual_override` BOOLEAN DEFAULT false, `success_count` INT DEFAULT 0, `fail_count` INT DEFAULT 0, `total_latency_ms` INT DEFAULT 0, `total_cost` DOUBLE DEFAULT 0, `last_used_at` VARCHAR(191), `last_selected_at` VARCHAR(191), `last_fail_at` VARCHAR(191), `consecutive_fail_count` INT NOT NULL DEFAULT 0, `cooldown_level` INT NOT NULL DEFAULT 0, `cooldown_until` VARCHAR(191), FOREIGN KEY (`account_id`) REFERENCES `accounts`(`id`) ON DELETE CASCADE, FOREIGN KEY (`route_id`) REFERENCES `token_routes`(`id`) ON DELETE CASCADE, FOREIGN KEY (`token_id`) REFERENCES `account_tokens`(`id`) ON DELETE SET NULL);
src/server/db/generated/mysql.upgrade.sql
+2 -4
View File
@@ -1,4 +1,2 @@
CREATE TABLE IF NOT EXISTS `site_api_endpoints` (`id` INT AUTO_INCREMENT NOT NULL PRIMARY KEY, `site_id` INT NOT NULL, `url` TEXT NOT NULL, `enabled` BOOLEAN DEFAULT true, `sort_order` INT DEFAULT 0, `cooldown_until` VARCHAR(191), `last_selected_at` VARCHAR(191), `last_failed_at` VARCHAR(191), `last_failure_reason` TEXT, `created_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), '%Y-%m-%d %H:%i:%s')), `updated_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), '%Y-%m-%d %H:%i:%s')), FOREIGN KEY (`site_id`) REFERENCES `sites`(`id`) ON DELETE CASCADE);
CREATE UNIQUE INDEX `site_api_endpoints_site_url_unique` ON `site_api_endpoints` (`site_id`, `url`(191));
CREATE INDEX `site_api_endpoints_site_cooldown_idx` ON `site_api_endpoints` (`site_id`, `cooldown_until`);
CREATE INDEX `site_api_endpoints_site_enabled_sort_idx` ON `site_api_endpoints` (`site_id`, `enabled`, `sort_order`);
ALTER TABLE `proxy_logs` ADD COLUMN `is_stream` BOOLEAN;
ALTER TABLE `proxy_logs` ADD COLUMN `first_byte_latency_ms` INT;
src/server/db/generated/postgres.bootstrap.sql
+1 -1
View File
@@ -8,7 +8,7 @@ CREATE TABLE IF NOT EXISTS "model_availability" ("id" INTEGER GENERATED BY DEFAU
CREATE TABLE IF NOT EXISTS "proxy_debug_traces" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL PRIMARY KEY, "downstream_path" TEXT NOT NULL, "client_kind" TEXT, "session_id" TEXT, "trace_hint" TEXT, "requested_model" TEXT, "downstream_api_key_id" INTEGER, "request_headers_json" JSONB, "request_body_json" JSONB, "sticky_session_key" TEXT, "sticky_hit_channel_id" INTEGER, "selected_channel_id" INTEGER, "selected_route_id" INTEGER, "selected_account_id" INTEGER, "selected_site_id" INTEGER, "selected_site_platform" TEXT, "endpoint_candidates_json" JSONB, "endpoint_runtime_state_json" JSONB, "decision_summary_json" JSONB, "final_status" TEXT, "final_http_status" INTEGER, "final_upstream_path" TEXT, "final_response_headers_json" JSONB, "final_response_body_json" JSONB, "created_at" TEXT DEFAULT to_char(timezone('UTC', CURRENT_TIMESTAMP), 'YYYY-MM-DD HH24:MI:SS'), "updated_at" TEXT DEFAULT to_char(timezone('UTC', CURRENT_TIMESTAMP), 'YYYY-MM-DD HH24:MI:SS'));
CREATE TABLE IF NOT EXISTS "proxy_debug_attempts" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL PRIMARY KEY, "trace_id" INTEGER NOT NULL, "attempt_index" INTEGER NOT NULL, "endpoint" TEXT NOT NULL, "request_path" TEXT NOT NULL, "target_url" TEXT NOT NULL, "runtime_executor" TEXT, "request_headers_json" JSONB, "request_body_json" JSONB, "response_status" INTEGER, "response_headers_json" JSONB, "response_body_json" JSONB, "raw_error_text" TEXT, "recover_applied" BOOLEAN DEFAULT false, "downgrade_decision" BOOLEAN DEFAULT false, "downgrade_reason" TEXT, "memory_write_json" JSONB, "created_at" TEXT DEFAULT to_char(timezone('UTC', CURRENT_TIMESTAMP), 'YYYY-MM-DD HH24:MI:SS'), FOREIGN KEY ("trace_id") REFERENCES "proxy_debug_traces"("id") ON DELETE CASCADE);
CREATE TABLE IF NOT EXISTS "proxy_files" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL PRIMARY KEY, "public_id" TEXT NOT NULL, "owner_type" TEXT NOT NULL, "owner_id" TEXT NOT NULL, "filename" TEXT NOT NULL, "mime_type" TEXT NOT NULL, "purpose" TEXT, "byte_size" INTEGER NOT NULL, "sha256" TEXT NOT NULL, "content_base64" TEXT NOT NULL, "created_at" TEXT DEFAULT to_char(timezone('UTC', CURRENT_TIMESTAMP), 'YYYY-MM-DD HH24:MI:SS'), "updated_at" TEXT DEFAULT to_char(timezone('UTC', CURRENT_TIMESTAMP), 'YYYY-MM-DD HH24:MI:SS'), "deleted_at" TEXT);
CREATE TABLE IF NOT EXISTS "proxy_logs" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL PRIMARY KEY, "route_id" INTEGER, "channel_id" INTEGER, "account_id" INTEGER, "model_requested" TEXT, "model_actual" TEXT, "status" TEXT, "http_status" INTEGER, "latency_ms" INTEGER, "prompt_tokens" INTEGER, "completion_tokens" INTEGER, "total_tokens" INTEGER, "estimated_cost" DOUBLE PRECISION, "error_message" TEXT, "retry_count" INTEGER DEFAULT 0, "created_at" TEXT DEFAULT to_char(timezone('UTC', CURRENT_TIMESTAMP), 'YYYY-MM-DD HH24:MI:SS'), "billing_details" JSONB, "downstream_api_key_id" INTEGER, "client_family" TEXT, "client_app_id" TEXT, "client_app_name" TEXT, "client_confidence" TEXT);
CREATE TABLE IF NOT EXISTS "proxy_logs" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL PRIMARY KEY, "route_id" INTEGER, "channel_id" INTEGER, "account_id" INTEGER, "model_requested" TEXT, "model_actual" TEXT, "status" TEXT, "http_status" INTEGER, "latency_ms" INTEGER, "prompt_tokens" INTEGER, "completion_tokens" INTEGER, "total_tokens" INTEGER, "estimated_cost" DOUBLE PRECISION, "error_message" TEXT, "retry_count" INTEGER DEFAULT 0, "created_at" TEXT DEFAULT to_char(timezone('UTC', CURRENT_TIMESTAMP), 'YYYY-MM-DD HH24:MI:SS'), "billing_details" JSONB, "downstream_api_key_id" INTEGER, "client_family" TEXT, "client_app_id" TEXT, "client_app_name" TEXT, "client_confidence" TEXT, "is_stream" BOOLEAN, "first_byte_latency_ms" INTEGER);
CREATE TABLE IF NOT EXISTS "proxy_video_tasks" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL PRIMARY KEY, "public_id" TEXT NOT NULL, "upstream_video_id" TEXT NOT NULL, "site_url" TEXT NOT NULL, "token_value" TEXT NOT NULL, "requested_model" TEXT, "actual_model" TEXT, "channel_id" INTEGER, "account_id" INTEGER, "status_snapshot" JSONB, "upstream_response_meta" JSONB, "last_upstream_status" INTEGER, "last_polled_at" TEXT, "created_at" TEXT DEFAULT to_char(timezone('UTC', CURRENT_TIMESTAMP), 'YYYY-MM-DD HH24:MI:SS'), "updated_at" TEXT DEFAULT to_char(timezone('UTC', CURRENT_TIMESTAMP), 'YYYY-MM-DD HH24:MI:SS'));
CREATE TABLE IF NOT EXISTS "token_routes" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL PRIMARY KEY, "model_pattern" TEXT NOT NULL, "model_mapping" JSONB, "enabled" BOOLEAN DEFAULT true, "created_at" TEXT DEFAULT to_char(timezone('UTC', CURRENT_TIMESTAMP), 'YYYY-MM-DD HH24:MI:SS'), "updated_at" TEXT DEFAULT to_char(timezone('UTC', CURRENT_TIMESTAMP), 'YYYY-MM-DD HH24:MI:SS'), "display_name" TEXT, "display_icon" TEXT, "decision_snapshot" JSONB, "decision_refreshed_at" TEXT, "routing_strategy" TEXT DEFAULT 'weighted', "route_mode" TEXT DEFAULT 'pattern');
CREATE TABLE IF NOT EXISTS "route_channels" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL PRIMARY KEY, "route_id" INTEGER NOT NULL, "account_id" INTEGER NOT NULL, "token_id" INTEGER, "source_model" TEXT, "priority" INTEGER DEFAULT 0, "weight" INTEGER DEFAULT 10, "enabled" BOOLEAN DEFAULT true, "manual_override" BOOLEAN DEFAULT false, "success_count" INTEGER DEFAULT 0, "fail_count" INTEGER DEFAULT 0, "total_latency_ms" INTEGER DEFAULT 0, "total_cost" DOUBLE PRECISION DEFAULT 0, "last_used_at" TEXT, "last_selected_at" TEXT, "last_fail_at" TEXT, "consecutive_fail_count" INTEGER NOT NULL DEFAULT 0, "cooldown_level" INTEGER NOT NULL DEFAULT 0, "cooldown_until" TEXT, FOREIGN KEY ("account_id") REFERENCES "accounts"("id") ON DELETE CASCADE, FOREIGN KEY ("route_id") REFERENCES "token_routes"("id") ON DELETE CASCADE, FOREIGN KEY ("token_id") REFERENCES "account_tokens"("id") ON DELETE SET NULL);
src/server/db/generated/postgres.upgrade.sql
+2 -4
View File
@@ -1,4 +1,2 @@
CREATE TABLE IF NOT EXISTS "site_api_endpoints" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL PRIMARY KEY, "site_id" INTEGER NOT NULL, "url" TEXT NOT NULL, "enabled" BOOLEAN DEFAULT true, "sort_order" INTEGER DEFAULT 0, "cooldown_until" TEXT, "last_selected_at" TEXT, "last_failed_at" TEXT, "last_failure_reason" TEXT, "created_at" TEXT DEFAULT to_char(timezone('UTC', CURRENT_TIMESTAMP), 'YYYY-MM-DD HH24:MI:SS'), "updated_at" TEXT DEFAULT to_char(timezone('UTC', CURRENT_TIMESTAMP), 'YYYY-MM-DD HH24:MI:SS'), FOREIGN KEY ("site_id") REFERENCES "sites"("id") ON DELETE CASCADE);
CREATE UNIQUE INDEX "site_api_endpoints_site_url_unique" ON "site_api_endpoints" ("site_id", "url");
CREATE INDEX "site_api_endpoints_site_cooldown_idx" ON "site_api_endpoints" ("site_id", "cooldown_until");
CREATE INDEX "site_api_endpoints_site_enabled_sort_idx" ON "site_api_endpoints" ("site_id", "enabled", "sort_order");
ALTER TABLE "proxy_logs" ADD COLUMN "is_stream" BOOLEAN;
ALTER TABLE "proxy_logs" ADD COLUMN "first_byte_latency_ms" INTEGER;
src/server/db/generated/schemaContract.json
+12
View File
@@ -949,6 +949,18 @@
"notNull": false,
"defaultValue": null,
"primaryKey": false
},
"is_stream": {
"logicalType": "boolean",
"notNull": false,
"defaultValue": null,
"primaryKey": false
},
"first_byte_latency_ms": {
"logicalType": "integer",
"notNull": false,
"defaultValue": null,
"primaryKey": false
}
}
},
src/server/db/index.ts
+112
View File
@@ -51,6 +51,7 @@ let pgPool: pg.Pool | null = null;
let proxyLogBillingDetailsColumnAvailable: boolean | null = null;
let proxyLogDownstreamApiKeyIdColumnAvailable: boolean | null = null;
let proxyLogClientColumnsAvailable: boolean | null = null;
let proxyLogStreamTimingColumnsAvailable: boolean | null = null;
function buildMysqlPoolOptions(
connectionString = config.dbUrl,
@@ -663,6 +664,21 @@ function ensureProxyLogClientSchema() {
proxyLogClientColumnsAvailable = true;
}
function ensureProxyLogStreamTimingSchema() {
if (!tableExists('proxy_logs')) {
return;
}
if (!tableColumnExists('proxy_logs', 'is_stream')) {
execSqliteLegacyCompat('ALTER TABLE proxy_logs ADD COLUMN is_stream integer;');
}
if (!tableColumnExists('proxy_logs', 'first_byte_latency_ms')) {
execSqliteLegacyCompat('ALTER TABLE proxy_logs ADD COLUMN first_byte_latency_ms integer;');
}
proxyLogStreamTimingColumnsAvailable = true;
}
function normalizeSchemaErrorMessage(error: unknown): string {
if (typeof error === 'object' && error && 'message' in error) {
return String((error as { message?: unknown }).message || '');
@@ -988,10 +1004,106 @@ export async function ensureProxyLogClientColumns(): Promise<boolean> {
}
}
export async function hasProxyLogStreamTimingColumns(): Promise<boolean> {
if (proxyLogStreamTimingColumnsAvailable !== null) {
return proxyLogStreamTimingColumnsAvailable;
}
const requiredColumns = ['is_stream', 'first_byte_latency_ms'];
if (runtimeDbDialect === 'sqlite') {
proxyLogStreamTimingColumnsAvailable = tableExists('proxy_logs')
&& requiredColumns.every((columnName) => tableColumnExists('proxy_logs', columnName));
return proxyLogStreamTimingColumnsAvailable;
}
if (runtimeDbDialect === 'mysql') {
if (!mysqlPool) return false;
const [rows] = await mysqlPool.query(
'SELECT column_name FROM information_schema.columns WHERE table_schema = DATABASE() AND table_name = ? AND column_name IN (?, ?)',
['proxy_logs', ...requiredColumns],
) as [Array<{ column_name?: string }>, unknown];
const available = new Set(
Array.isArray(rows)
? rows.map((row) => String(row?.column_name || '').trim().toLowerCase()).filter(Boolean)
: [],
);
proxyLogStreamTimingColumnsAvailable = requiredColumns.every((columnName) => available.has(columnName));
return proxyLogStreamTimingColumnsAvailable;
}
if (!pgPool) return false;
const result = await pgPool.query(
'SELECT column_name FROM information_schema.columns WHERE table_schema = current_schema() AND table_name = $1 AND column_name = ANY($2::text[])',
['proxy_logs', requiredColumns],
);
const available = new Set(
result.rows.map((row) => String((row as { column_name?: string }).column_name || '').trim().toLowerCase()).filter(Boolean),
);
proxyLogStreamTimingColumnsAvailable = requiredColumns.every((columnName) => available.has(columnName));
return proxyLogStreamTimingColumnsAvailable;
}
export async function ensureProxyLogStreamTimingColumns(): Promise<boolean> {
const requiredColumns = [
{ name: 'is_stream', sqliteType: 'integer', mysqlType: 'BOOLEAN NULL', postgresType: 'BOOLEAN' },
{ name: 'first_byte_latency_ms', sqliteType: 'integer', mysqlType: 'INT NULL', postgresType: 'INTEGER' },
];
if (runtimeDbDialect === 'sqlite') {
ensureProxyLogStreamTimingSchema();
proxyLogStreamTimingColumnsAvailable = tableExists('proxy_logs')
&& requiredColumns.every((column) => tableColumnExists('proxy_logs', column.name));
return proxyLogStreamTimingColumnsAvailable;
}
if (await hasProxyLogStreamTimingColumns()) {
return true;
}
try {
if (runtimeDbDialect === 'mysql') {
if (!mysqlPool) return false;
for (const column of requiredColumns) {
const [rows] = await mysqlPool.query('SHOW COLUMNS FROM `proxy_logs` LIKE ?', [column.name]);
if (Array.isArray(rows) && rows.length > 0) continue;
await executeLegacyCompat(
(statement) => mysqlPool!.query(statement).then(() => undefined),
`ALTER TABLE \`proxy_logs\` ADD COLUMN \`${column.name}\` ${column.mysqlType}`,
);
}
} else {
if (!pgPool) return false;
for (const column of requiredColumns) {
const result = await pgPool.query(
'SELECT 1 FROM information_schema.columns WHERE table_schema = current_schema() AND table_name = $1 AND column_name = $2 LIMIT 1',
['proxy_logs', column.name],
);
if (Number(result.rowCount || 0) > 0) continue;
await executeLegacyCompat(
(statement) => pgPool!.query(statement).then(() => undefined),
`ALTER TABLE "proxy_logs" ADD COLUMN "${column.name}" ${column.postgresType}`,
);
}
}
proxyLogStreamTimingColumnsAvailable = true;
return true;
} catch (error) {
if (isDuplicateColumnError(error)) {
proxyLogStreamTimingColumnsAvailable = await hasProxyLogStreamTimingColumns();
return proxyLogStreamTimingColumnsAvailable;
}
proxyLogStreamTimingColumnsAvailable = false;
console.warn('[db] failed to ensure proxy_logs stream timing columns', error);
return false;
}
}
function resetSchemaCapabilityCache() {
proxyLogBillingDetailsColumnAvailable = null;
proxyLogDownstreamApiKeyIdColumnAvailable = null;
proxyLogClientColumnsAvailable = null;
proxyLogStreamTimingColumnsAvailable = null;
}
async function sqliteProxyQuery(sqlText: string, params: unknown[], method: SqlMethod) {
src/server/db/legacySchemaCompat.test.ts
+2
View File
@@ -4,6 +4,8 @@ import { classifyLegacyCompatMutation } from './legacySchemaCompat.js';
describe('legacy schema compat boundary', () => {
it('allows only explicitly registered legacy upgrade shims', () => {
expect(classifyLegacyCompatMutation('ALTER TABLE proxy_logs ADD COLUMN billing_details text;')).toBe('legacy');
expect(classifyLegacyCompatMutation('ALTER TABLE proxy_logs ADD COLUMN is_stream integer;')).toBe('legacy');
expect(classifyLegacyCompatMutation('ALTER TABLE proxy_logs ADD COLUMN first_byte_latency_ms integer;')).toBe('legacy');
expect(classifyLegacyCompatMutation('ALTER TABLE proxy_logs ADD COLUMN client_app_id text;')).toBe('legacy');
expect(classifyLegacyCompatMutation('CREATE INDEX proxy_logs_client_app_id_created_at_idx ON proxy_logs(client_app_id, created_at);')).toBe('legacy');
expect(classifyLegacyCompatMutation('UPDATE "sites" SET "use_system_proxy" = FALSE WHERE "use_system_proxy" IS NULL')).toBe('legacy');
src/server/db/legacySchemaCompat.ts
+2
View File
@@ -54,6 +54,8 @@ const BOOTSTRAP_OWNED_LEGACY_COLUMNS = [
'downstream_api_keys.group_name',
'downstream_api_keys.tags',
'proxy_logs.billing_details',
'proxy_logs.is_stream',
'proxy_logs.first_byte_latency_ms',
'proxy_logs.client_family',
'proxy_logs.client_app_id',
'proxy_logs.client_app_name',
src/server/db/migrate.ts
+3
View File
@@ -70,6 +70,9 @@ const VERIFIED_SCHEMA_MARKERS: SchemaMarker[] = [
{ table: 'downstream_api_keys', column: 'tags' },
// 0012: value_status column on account_tokens
{ table: 'account_tokens', column: 'value_status' },
// 0019: proxy log stream/timing columns
{ table: 'proxy_logs', column: 'is_stream' },
{ table: 'proxy_logs', column: 'first_byte_latency_ms' },
];
src/server/db/schema.ts
+2
View File
@@ -205,6 +205,8 @@ export const proxyLogs = sqliteTable('proxy_logs', {
modelActual: text('model_actual'),
status: text('status'), // 'success' | 'failed' | 'retried'
httpStatus: integer('http_status'),
isStream: integer('is_stream', { mode: 'boolean' }),
firstByteLatencyMs: integer('first_byte_latency_ms'),
latencyMs: integer('latency_ms'),
promptTokens: integer('prompt_tokens'),
completionTokens: integer('completion_tokens'),
src/server/db/schemaParity.test.ts
+6
View File
@@ -85,16 +85,22 @@ describe('database schema parity', () => {
const postgresBootstrap = readFileSync(resolve(generatedDir, 'postgres.bootstrap.sql'), 'utf8');
expect(contract.tables.proxy_logs?.columns.downstream_api_key_id?.logicalType).toBe('integer');
expect(contract.tables.proxy_logs?.columns.is_stream?.logicalType).toBe('boolean');
expect(contract.tables.proxy_logs?.columns.first_byte_latency_ms?.logicalType).toBe('integer');
expect(contract.tables.proxy_logs?.columns.client_app_id?.logicalType).toBe('text');
expect(contract.tables.proxy_logs?.columns.client_family?.logicalType).toBe('text');
expect(contract.indexes.some((index) => index.name === 'proxy_logs_downstream_api_key_created_at_idx')).toBe(true);
expect(contract.indexes.some((index) => index.name === 'proxy_logs_client_app_id_created_at_idx')).toBe(true);
expect(contract.indexes.some((index) => index.name === 'proxy_logs_client_family_created_at_idx')).toBe(true);
expect(mysqlBootstrap).toContain('`downstream_api_key_id`');
expect(mysqlBootstrap).toContain('`is_stream`');
expect(mysqlBootstrap).toContain('`first_byte_latency_ms`');
expect(mysqlBootstrap).toContain('`proxy_logs_downstream_api_key_created_at_idx`');
expect(mysqlBootstrap).toContain('`client_app_id`');
expect(mysqlBootstrap).toContain('`proxy_logs_client_app_id_created_at_idx`');
expect(postgresBootstrap).toContain('"downstream_api_key_id"');
expect(postgresBootstrap).toContain('"is_stream"');
expect(postgresBootstrap).toContain('"first_byte_latency_ms"');
expect(postgresBootstrap).toContain('"proxy_logs_downstream_api_key_created_at_idx"');
expect(postgresBootstrap).toContain('"client_app_id"');
expect(postgresBootstrap).toContain('"proxy_logs_client_app_id_created_at_idx"');
src/server/index.ts
+7
View File
@@ -60,6 +60,7 @@ import {
ensureProxyLogClientColumns,
ensureProxyLogDownstreamApiKeyIdColumn,
ensureProxyLogBillingDetailsColumn,
ensureProxyLogStreamTimingColumns,
ensureRouteGroupingCompatibilityColumns,
ensureSiteCompatibilityColumns,
runtimeDbDialect,
@@ -304,6 +305,11 @@ function applyRuntimeSettings(settingsMap: Map<string, string>) {
config.routingFallbackUnitCost = Math.max(1e-6, routingFallbackUnitCost);
}
const proxyFirstByteTimeoutSec = parseSettingFromMap<number>(settingsMap, 'proxy_first_byte_timeout_sec');
if (typeof proxyFirstByteTimeoutSec === 'number' && Number.isFinite(proxyFirstByteTimeoutSec) && proxyFirstByteTimeoutSec >= 0) {
config.proxyFirstByteTimeoutSec = Math.max(0, Math.trunc(proxyFirstByteTimeoutSec));
}
const tokenRouterFailureCooldownMaxSec = parseSettingFromMap<number>(settingsMap, 'token_router_failure_cooldown_max_sec');
const normalizedFailureCooldownMaxSec = normalizeTokenRouterFailureCooldownMaxSec(tokenRouterFailureCooldownMaxSec);
if (normalizedFailureCooldownMaxSec != null) {
@@ -417,6 +423,7 @@ try {
await ensureSiteCompatibilityColumns();
await ensureRouteGroupingCompatibilityColumns();
await ensureProxyFileCompatibilityColumns();
await ensureProxyLogStreamTimingColumns();
await ensureProxyLogClientColumns();
await ensureProxyLogDownstreamApiKeyIdColumn();
const finalRows = await db.select().from(schema.settings).all();
src/server/proxy-core/executors/types.ts
+8 -1
View File
@@ -35,6 +35,7 @@ export type RuntimeDispatchInput = {
siteUrl: string;
request: ProxyRuntimeRequest;
targetUrl?: string;
signal?: AbortSignal;
buildInit: (requestUrl: string, request: ProxyRuntimeRequest) => Promise<UndiciRequestInit> | UndiciRequestInit;
};
@@ -72,7 +73,13 @@ export async function performFetch(
requestUrl = input.targetUrl || buildUpstreamUrl(input.siteUrl, request.path),
): Promise<RuntimeResponse> {
const init = await input.buildInit(requestUrl, request);
return fetch(requestUrl, init);
const combinedSignal = input.signal && init.signal
? AbortSignal.any([input.signal, init.signal as AbortSignal])
: (input.signal ?? init.signal);
return fetch(requestUrl, {
...init,
signal: combinedSignal,
});
}
function hasZstdContentEncoding(contentEncoding: string | null): boolean {
src/server/proxy-core/firstByteTimeout.test.ts
+62
View File
@@ -0,0 +1,62 @@
import { describe, expect, it } from 'vitest';
import {
fetchWithObservedFirstByte,
getObservedResponseMeta,
isObservedFirstByteTimeoutResponse,
} from './firstByteTimeout.js';
function buildDelayedResponse(bodyText: string, delayMs: number, status = 200): Response {
const encoder = new TextEncoder();
const body = new ReadableStream<Uint8Array>({
start(controller) {
setTimeout(() => {
controller.enqueue(encoder.encode(bodyText));
controller.close();
}, delayMs);
},
});
return new Response(body, {
status,
headers: { 'content-type': 'text/plain; charset=utf-8' },
});
}
describe('fetchWithObservedFirstByte', () => {
it('replays the first chunk and records first-byte latency when upstream responds in time', async () => {
const response = await fetchWithObservedFirstByte(
async () => buildDelayedResponse('hello world', 5),
{
firstByteTimeoutMs: 100,
startedAtMs: Date.now(),
},
);
expect(response.status).toBe(200);
expect(await response.text()).toBe('hello world');
const meta = getObservedResponseMeta(response);
expect(meta?.timedOutBeforeFirstByte).toBe(false);
expect(meta?.firstByteLatencyMs).not.toBeNull();
expect((meta?.firstByteLatencyMs || 0)).toBeGreaterThanOrEqual(0);
expect((meta?.firstByteLatencyMs || 0)).toBeLessThan(100);
});
it('returns a synthetic timeout response when upstream sends no first byte before the deadline', async () => {
const response = await fetchWithObservedFirstByte(
async () => buildDelayedResponse('too late', 80),
{
firstByteTimeoutMs: 10,
startedAtMs: Date.now(),
},
);
expect(response.status).toBe(408);
expect(await response.text()).toContain('first byte timeout');
expect(isObservedFirstByteTimeoutResponse(response)).toBe(true);
const meta = getObservedResponseMeta(response);
expect(meta?.timedOutBeforeFirstByte).toBe(true);
expect(meta?.firstByteLatencyMs).toBeNull();
});
});
src/server/proxy-core/firstByteTimeout.ts
+179
View File
@@ -0,0 +1,179 @@
import { Headers, Response } from 'undici';
type ObservedResponseMeta = {
firstByteLatencyMs: number | null;
timedOutBeforeFirstByte: boolean;
};
const observedResponseMeta = new WeakMap<Response, ObservedResponseMeta>();
function setObservedResponseMeta<T extends Response>(response: T, meta: ObservedResponseMeta): T {
observedResponseMeta.set(response, meta);
return response;
}
function clearTimer(timer: ReturnType<typeof setTimeout> | null) {
if (!timer) return;
clearTimeout(timer);
}
function buildFirstByteTimeoutMessage(timeoutMs: number): string {
const seconds = Math.max(1, Math.round(timeoutMs / 1000));
return `first byte timeout (${seconds}s)`;
}
function buildObservedTimeoutResponse(timeoutMs: number): Response {
return setObservedResponseMeta(new Response(buildFirstByteTimeoutMessage(timeoutMs), {
status: 408,
headers: { 'content-type': 'text/plain; charset=utf-8' },
}), {
firstByteLatencyMs: null,
timedOutBeforeFirstByte: true,
});
}
async function cancelReaderQuietly(reader: ReadableStreamDefaultReader<Uint8Array> | null) {
if (!reader) return;
try {
await reader.cancel();
} catch {
// Ignore cancellation errors from already-closed streams.
}
try {
reader.releaseLock();
} catch {
// Ignore release errors from already-released readers.
}
}
function buildReplayResponse<T extends Response>(
response: T,
reader: ReadableStreamDefaultReader<Uint8Array>,
firstChunk: ReadableStreamReadResult<Uint8Array>,
): T {
let firstChunkDelivered = false;
let readerReleased = false;
const releaseReader = () => {
if (readerReleased) return;
readerReleased = true;
try {
reader.releaseLock();
} catch {
// Ignore release failures from cancelled/closed readers.
}
};
const stream = new ReadableStream<Uint8Array>({
async pull(controller) {
if (!firstChunkDelivered) {
firstChunkDelivered = true;
if (firstChunk.done) {
controller.close();
releaseReader();
return;
}
controller.enqueue(firstChunk.value);
return;
}
try {
const next = await reader.read();
if (next.done) {
controller.close();
releaseReader();
return;
}
controller.enqueue(next.value);
} catch (error) {
controller.error(error);
releaseReader();
}
},
async cancel(reason) {
try {
await reader.cancel(reason);
} catch {
// Ignore cancellation failures from already-closed streams.
}
releaseReader();
},
});
return new Response(stream, {
status: response.status,
statusText: response.statusText,
headers: new Headers(response.headers),
}) as T;
}
export async function fetchWithObservedFirstByte<T extends Response>(
dispatch: (signal?: AbortSignal) => Promise<T>,
options: {
firstByteTimeoutMs?: number;
startedAtMs?: number;
} = {},
): Promise<T> {
const timeoutMs = Math.max(0, Math.trunc(options.firstByteTimeoutMs ?? 0));
const startedAtMs = options.startedAtMs ?? Date.now();
const controller = timeoutMs > 0 ? new AbortController() : null;
let timer: ReturnType<typeof setTimeout> | null = null;
let timedOutBeforeFirstByte = false;
const timeoutSentinel = Symbol('first-byte-timeout');
const timeoutPromise = timeoutMs > 0
? new Promise<typeof timeoutSentinel>((resolve) => {
timer = setTimeout(() => {
timedOutBeforeFirstByte = true;
controller?.abort(new Error(buildFirstByteTimeoutMessage(timeoutMs)));
resolve(timeoutSentinel);
}, timeoutMs);
})
: null;
try {
const dispatched = timeoutPromise
? await Promise.race([dispatch(controller?.signal), timeoutPromise])
: await dispatch(controller?.signal);
if (dispatched === timeoutSentinel) {
return buildObservedTimeoutResponse(timeoutMs) as T;
}
const response = dispatched as T;
if (!response.body) {
clearTimer(timer);
return setObservedResponseMeta(response, {
firstByteLatencyMs: Math.max(0, Date.now() - startedAtMs),
timedOutBeforeFirstByte: false,
});
}
const reader = response.body.getReader() as ReadableStreamDefaultReader<Uint8Array>;
const firstChunk = timeoutPromise
? await Promise.race([reader.read(), timeoutPromise])
: await reader.read();
if (firstChunk === timeoutSentinel) {
await cancelReaderQuietly(reader);
return buildObservedTimeoutResponse(timeoutMs) as T;
}
clearTimer(timer);
return setObservedResponseMeta(buildReplayResponse(response, reader, firstChunk), {
firstByteLatencyMs: Math.max(0, Date.now() - startedAtMs),
timedOutBeforeFirstByte: false,
});
} catch (error) {
clearTimer(timer);
if (timedOutBeforeFirstByte && timeoutMs > 0) {
return buildObservedTimeoutResponse(timeoutMs) as T;
}
throw error;
} finally {
clearTimer(timer);
}
}
export function getObservedResponseMeta(response: Response | null | undefined): ObservedResponseMeta | null {
if (!response) return null;
return observedResponseMeta.get(response) ?? null;
}
export function isObservedFirstByteTimeoutResponse(response: Response | null | undefined): boolean {
return getObservedResponseMeta(response)?.timedOutBeforeFirstByte === true;
}
src/server/proxy-core/orchestration/endpointFlow.ts
+37 -8
View File
@@ -1,5 +1,6 @@
import { fetch } from 'undici';
import { readRuntimeResponseText } from '../executors/types.js';
import { fetchWithObservedFirstByte, isObservedFirstByteTimeoutResponse } from '../firstByteTimeout.js';
import { withSiteProxyRequestInit } from '../../services/siteProxy.js';
import {
buildUpstreamUrl,
@@ -69,7 +70,9 @@ export type ExecuteEndpointFlowInput = {
dispatchRequest?: (
request: BuiltEndpointRequest,
targetUrl: string,
signal?: AbortSignal,
) => Promise<Awaited<ReturnType<typeof fetch>>>;
firstByteTimeoutMs?: number;
tryRecover?: (ctx: EndpointAttemptContext) => Promise<EndpointRecoverResult>;
shouldDowngrade?: (ctx: EndpointAttemptContext) => boolean;
shouldAbortRemainingEndpoints?: (ctx: EndpointAttemptContext & { errText: string }) => boolean;
@@ -117,13 +120,23 @@ export async function executeEndpointFlow(input: ExecuteEndpointFlowInput): Prom
? buildUpstreamUrl(input.proxyUrl, request.path)
: defaultTarget;
let response = input.dispatchRequest
? await input.dispatchRequest(request, targetUrl)
: await fetch(targetUrl, await withSiteProxyRequestInit(targetUrl, {
method: 'POST',
headers: request.headers,
body: JSON.stringify(request.body),
}));
const attemptStartedAtMs = Date.now();
let response = await fetchWithObservedFirstByte(
async (signal) => (
input.dispatchRequest
? await input.dispatchRequest(request, targetUrl, signal)
: await fetch(targetUrl, await withSiteProxyRequestInit(targetUrl, {
method: 'POST',
headers: request.headers,
body: JSON.stringify(request.body),
signal,
}))
),
{
firstByteTimeoutMs: input.firstByteTimeoutMs,
startedAtMs: attemptStartedAtMs,
},
);
if (response.ok) {
await runEndpointFlowHook(input.onAttemptSuccess, {
@@ -151,6 +164,23 @@ export async function executeEndpointFlow(input: ExecuteEndpointFlowInput): Prom
rawErrText,
recoverApplied: false,
};
const isLastEndpoint = endpointIndex >= endpointCount - 1;
if (isObservedFirstByteTimeoutResponse(response) && !isLastEndpoint) {
const errText = rawErrText.trim() || 'first byte timeout';
const timeoutContext = {
...baseContext,
errText,
};
await runEndpointFlowHook(input.onAttemptFailure, timeoutContext, 'onAttemptFailure');
finalStatus = response.status || 408;
finalErrText = errText;
finalRawErrText = rawErrText;
if (input.disableCrossProtocolFallback) {
break;
}
continue;
}
if (input.tryRecover) {
const recovered = await input.tryRecover(baseContext);
@@ -192,7 +222,6 @@ export async function executeEndpointFlow(input: ExecuteEndpointFlowInput): Prom
errText,
}, 'onAttemptFailure');
const isLastEndpoint = endpointIndex >= endpointCount - 1;
if (input.disableCrossProtocolFallback && !isLastEndpoint) {
finalStatus = response.status;
finalErrText = errText;
src/server/proxy-core/surfaces/chatSurface.ts
+11
View File
@@ -45,6 +45,7 @@ import {
unwrapGeminiCliPayload,
} from '../../routes/proxy/geminiCliCompat.js';
import { summarizeConversationFileInputsInOpenAiBody } from '../capabilities/conversationFileCapabilities.js';
import { getObservedResponseMeta } from '../firstByteTimeout.js';
import { getRuntimeResponseReader, readRuntimeResponseText } from '../executors/types.js';
import { detectDownstreamClientContext } from '../../routes/proxy/downstreamClientContext.js';
import { getProxyMaxChannelRetries } from '../../services/proxyChannelRetry.js';
@@ -394,6 +395,7 @@ export async function handleChatSurfaceRequest(
return executeEndpointFlow({
siteUrl: siteApiBaseUrl,
disableCrossProtocolFallback: config.disableCrossProtocolFallback,
firstByteTimeoutMs: Math.max(0, Math.trunc((config.proxyFirstByteTimeoutSec || 0) * 1000)),
endpointCandidates,
buildRequest: (endpoint) => buildEndpointRequest(endpoint),
dispatchRequest,
@@ -525,6 +527,7 @@ export async function handleChatSurfaceRequest(
const upstream = endpointResult.upstream;
const successfulUpstreamPath = endpointResult.upstreamPath;
const firstByteLatencyMs = getObservedResponseMeta(upstream)?.firstByteLatencyMs ?? null;
if (isStream) {
const upstreamContentType = (upstream.headers.get('content-type') || '').toLowerCase();
@@ -557,6 +560,8 @@ export async function handleChatSurfaceRequest(
parsedUsage,
upstreamUsagePresent,
requestStartedAtMs: startTime,
isStream: true,
firstByteLatencyMs,
latencyMs,
retryCount,
upstreamPath: successfulUpstreamPath,
@@ -918,6 +923,8 @@ export async function handleChatSurfaceRequest(
parsedUsage,
upstreamUsagePresent,
requestStartedAtMs: startTime,
isStream: false,
firstByteLatencyMs,
latencyMs: latency,
retryCount,
upstreamPath: successfulUpstreamPath,
@@ -961,6 +968,7 @@ export async function handleChatSurfaceRequest(
status: endpointFailureStatus || 502,
errText: err.message || 'unknown error',
rawErrText: err.rawErrText || err.message || 'unknown error',
isStream,
latencyMs: Date.now() - startTime,
retryCount,
});
@@ -985,6 +993,7 @@ export async function handleChatSurfaceRequest(
requestedModel,
modelName,
errorMessage: err?.message || 'network failure',
isStream,
latencyMs: Date.now() - startTime,
retryCount,
});
@@ -1387,6 +1396,7 @@ export async function handleClaudeCountTokensSurfaceRequest(
status: endpointFailureStatus || 502,
errText: error.message || 'unknown error',
rawErrText: error.rawErrText || error.message || 'unknown error',
isStream: false,
latencyMs: Date.now() - startTime,
retryCount,
});
@@ -1407,6 +1417,7 @@ export async function handleClaudeCountTokensSurfaceRequest(
requestedModel,
modelName,
errorMessage: error?.message || 'network failure',
isStream: false,
latencyMs: Date.now() - startTime,
retryCount,
});
src/server/proxy-core/surfaces/geminiSurface.ts
+61 -4
View File
@@ -40,6 +40,7 @@ import { detectDownstreamClientContext, type DownstreamClientContext } from '../
import { insertProxyLog } from '../../services/proxyLogStore.js';
import { summarizeConversationFileInputsInOpenAiBody } from '../capabilities/conversationFileCapabilities.js';
import { getRuntimeResponseReader, readRuntimeResponseText } from '../executors/types.js';
import { fetchWithObservedFirstByte, getObservedResponseMeta } from '../firstByteTimeout.js';
import { getProxyMaxChannelRetries } from '../../services/proxyChannelRetry.js';
import { shouldAbortSameSiteEndpointFallback } from '../../services/proxyRetryPolicy.js';
import {
@@ -253,6 +254,8 @@ async function logProxy(
promptTokens = 0,
completionTokens = 0,
totalTokens = 0,
isStream = false,
firstByteLatencyMs: number | null = null,
) {
try {
const createdAt = formatUtcSqlDateTime(new Date());
@@ -274,6 +277,8 @@ async function logProxy(
modelActual: selected.actualModel || modelRequested,
status,
httpStatus,
isStream,
firstByteLatencyMs,
latencyMs,
promptTokens,
completionTokens,
@@ -611,6 +616,7 @@ export async function geminiProxyRoute(app: FastifyInstance) {
const isInternalGemini = isInternalGeminiPlatform(selected.site.platform);
const isDirectGeminiFamily = isDirectGeminiFamilyPlatform(selected.site.platform);
const startTime = Date.now();
const firstByteTimeoutMs = Math.max(0, Math.trunc((config.proxyFirstByteTimeoutSec || 0) * 1000));
let upstreamPath = '';
try {
@@ -698,11 +704,12 @@ export async function geminiProxyRoute(app: FastifyInstance) {
requestHeaders,
targetUrl,
runtimeExecutor,
dispatch: () => (
dispatch: (signal?: AbortSignal) => (
isInternalGemini
? dispatchRuntimeRequest({
siteUrl: selected.site.url,
targetUrl,
signal,
request: {
endpoint: 'chat',
path: upstreamPath,
@@ -728,13 +735,22 @@ export async function geminiProxyRoute(app: FastifyInstance) {
method: 'POST',
headers: requestHeaders,
body: JSON.stringify(requestBody),
signal,
})
),
};
};
let directDispatchState = buildDirectDispatchState();
let upstream = await directDispatchState.dispatch();
const dispatchWithObservedFirstByte = async () => fetchWithObservedFirstByte(
(signal) => directDispatchState.dispatch(signal),
{
firstByteTimeoutMs,
startedAtMs: Date.now(),
},
);
let upstream = await dispatchWithObservedFirstByte();
let firstByteLatencyMs = getObservedResponseMeta(upstream)?.firstByteLatencyMs ?? null;
let contentType = upstream.headers.get('content-type') || 'application/json';
let recoverApplied = false;
if (upstream.status === 401 && oauth) {
@@ -748,7 +764,8 @@ export async function geminiProxyRoute(app: FastifyInstance) {
};
oauth = getOauthInfoFromAccount(selected.account);
directDispatchState = buildDirectDispatchState();
upstream = await directDispatchState.dispatch();
upstream = await dispatchWithObservedFirstByte();
firstByteLatencyMs = getObservedResponseMeta(upstream)?.firstByteLatencyMs ?? null;
contentType = upstream.headers.get('content-type') || 'application/json';
recoverApplied = true;
} catch {
@@ -791,6 +808,11 @@ export async function geminiProxyRoute(app: FastifyInstance) {
downstreamPath,
upstreamPath,
clientContext,
0,
0,
0,
isStreamAction,
firstByteLatencyMs,
);
if (canRetryChannelSelection(retryCount, forcedChannelId)) {
retryCount += 1;
@@ -829,6 +851,11 @@ export async function geminiProxyRoute(app: FastifyInstance) {
downstreamPath,
upstreamPath,
clientContext,
0,
0,
0,
isStreamAction,
firstByteLatencyMs,
);
await safeInsertSurfaceProxyDebugAttempt(debugTrace, {
attemptIndex: retryCount,
@@ -913,6 +940,8 @@ export async function geminiProxyRoute(app: FastifyInstance) {
parsedUsage.promptTokens,
parsedUsage.completionTokens,
parsedUsage.totalTokens,
isStreamAction,
firstByteLatencyMs,
);
await safeInsertSurfaceProxyDebugAttempt(debugTrace, {
attemptIndex: retryCount,
@@ -965,6 +994,8 @@ export async function geminiProxyRoute(app: FastifyInstance) {
parsedUsage.promptTokens,
parsedUsage.completionTokens,
parsedUsage.totalTokens,
isStreamAction,
firstByteLatencyMs,
);
await safeInsertSurfaceProxyDebugAttempt(debugTrace, {
attemptIndex: retryCount,
@@ -1029,6 +1060,8 @@ export async function geminiProxyRoute(app: FastifyInstance) {
parsedUsage.promptTokens,
parsedUsage.completionTokens,
parsedUsage.totalTokens,
isStreamAction,
firstByteLatencyMs,
);
await safeInsertSurfaceProxyDebugAttempt(debugTrace, {
attemptIndex: retryCount,
@@ -1074,6 +1107,11 @@ export async function geminiProxyRoute(app: FastifyInstance) {
downstreamPath,
upstreamPath,
clientContext,
0,
0,
0,
isStreamAction,
firstByteLatencyMs,
);
await safeInsertSurfaceProxyDebugAttempt(debugTrace, {
attemptIndex: retryCount,
@@ -1192,10 +1230,15 @@ export async function geminiProxyRoute(app: FastifyInstance) {
};
};
const channelProxyUrl = resolveChannelProxyUrl(selected.site, selected.account.extraConfig);
const dispatchRequest = (compatibilityRequest: BuiltEndpointRequest, targetUrl?: string) => (
const dispatchRequest = (
compatibilityRequest: BuiltEndpointRequest,
targetUrl?: string,
signal?: AbortSignal,
) => (
dispatchRuntimeRequest({
siteUrl: selected.site.url,
targetUrl,
signal,
request: compatibilityRequest,
buildInit: async (_requestUrl, requestForFetch) => withSiteRecordProxyRequestInit(selected.site, {
method: 'POST',
@@ -1221,6 +1264,7 @@ export async function geminiProxyRoute(app: FastifyInstance) {
const endpointResult = await executeEndpointFlow({
siteUrl: selected.site.url,
disableCrossProtocolFallback: config.disableCrossProtocolFallback,
firstByteTimeoutMs,
endpointCandidates,
buildRequest: (endpoint) => buildEndpointRequest(endpoint),
dispatchRequest,
@@ -1311,6 +1355,11 @@ export async function geminiProxyRoute(app: FastifyInstance) {
downstreamPath,
null,
clientContext,
0,
0,
0,
isStreamAction,
null,
);
if (canRetryChannelSelection(retryCount, forcedChannelId)) {
retryCount += 1;
@@ -1324,6 +1373,7 @@ export async function geminiProxyRoute(app: FastifyInstance) {
upstreamPath = endpointResult.upstreamPath;
const upstream = endpointResult.upstream;
const firstByteLatencyMs = getObservedResponseMeta(upstream)?.firstByteLatencyMs ?? null;
const rawText = await readRuntimeResponseText(upstream);
let upstreamData: unknown = rawText;
try {
@@ -1355,6 +1405,8 @@ export async function geminiProxyRoute(app: FastifyInstance) {
parsedUsage.promptTokens,
parsedUsage.completionTokens,
parsedUsage.totalTokens,
isStreamAction,
firstByteLatencyMs,
);
const downstreamPayload = isGeminiCliDownstream
? { response: geminiResponse }
@@ -1397,6 +1449,11 @@ export async function geminiProxyRoute(app: FastifyInstance) {
downstreamPath,
upstreamPath || null,
clientContext,
0,
0,
0,
isStreamAction,
null,
);
if (canRetryChannelSelection(retryCount, forcedChannelId)) {
retryCount += 1;
src/server/proxy-core/surfaces/openAiResponsesSurface.ts
+9
View File
@@ -44,6 +44,7 @@ import {
unwrapGeminiCliPayload,
} from '../../routes/proxy/geminiCliCompat.js';
import { isCodexResponsesSurface } from '../cliProfiles/codexProfile.js';
import { getObservedResponseMeta } from '../firstByteTimeout.js';
import { getRuntimeResponseReader, readRuntimeResponseText } from '../executors/types.js';
import { runCodexHttpSessionTask } from '../runtime/codexHttpSessionQueue.js';
import {
@@ -580,6 +581,7 @@ export async function handleOpenAiResponsesSurfaceRequest(
return executeEndpointFlow({
siteUrl: siteApiBaseUrl,
disableCrossProtocolFallback: config.disableCrossProtocolFallback,
firstByteTimeoutMs: Math.max(0, Math.trunc((config.proxyFirstByteTimeoutSec || 0) * 1000)),
endpointCandidates,
buildRequest: (endpoint) => buildEndpointRequest(endpoint),
dispatchRequest,
@@ -712,6 +714,7 @@ export async function handleOpenAiResponsesSurfaceRequest(
const upstream = endpointResult.upstream;
const successfulUpstreamPath = endpointResult.upstreamPath;
const firstByteLatencyMs = getObservedResponseMeta(upstream)?.firstByteLatencyMs ?? null;
const finalizeStreamSuccess = async (
parsedUsage: UsageSummary,
latency: number,
@@ -726,6 +729,8 @@ export async function handleOpenAiResponsesSurfaceRequest(
parsedUsage,
upstreamUsagePresent,
requestStartedAtMs: startTime,
isStream: true,
firstByteLatencyMs,
latencyMs: latency,
retryCount,
upstreamPath: successfulUpstreamPath,
@@ -1174,6 +1179,8 @@ export async function handleOpenAiResponsesSurfaceRequest(
parsedUsage,
upstreamUsagePresent,
requestStartedAtMs: startTime,
isStream: false,
firstByteLatencyMs,
latencyMs: latency,
retryCount,
upstreamPath: successfulUpstreamPath,
@@ -1219,6 +1226,7 @@ export async function handleOpenAiResponsesSurfaceRequest(
status: endpointFailureStatus || 502,
errText: err?.message || 'unknown error',
rawErrText: err?.rawErrText || err?.message || 'unknown error',
isStream,
latencyMs: Date.now() - startTime,
retryCount,
});
@@ -1243,6 +1251,7 @@ export async function handleOpenAiResponsesSurfaceRequest(
requestedModel,
modelName,
errorMessage: err?.message || 'network failure',
isStream,
latencyMs: Date.now() - startTime,
retryCount,
});
src/server/proxy-core/surfaces/sharedSurface.ts
+33 -1
View File
@@ -191,6 +191,8 @@ export async function writeSurfaceProxyLog(input: {
modelRequested: string;
status: string;
httpStatus: number;
isStream?: boolean | null;
firstByteLatencyMs?: number | null;
latencyMs: number;
errorMessage: string | null;
retryCount: number;
@@ -227,6 +229,8 @@ export async function writeSurfaceProxyLog(input: {
modelActual: input.selected.actualModel ?? null,
status: input.status,
httpStatus: input.httpStatus,
isStream: input.isStream ?? null,
firstByteLatencyMs: input.firstByteLatencyMs ?? null,
latencyMs: input.latencyMs,
promptTokens: input.promptTokens ?? null,
completionTokens: input.completionTokens ?? null,
@@ -255,10 +259,12 @@ export function createSurfaceDispatchRequest(input: {
return (
request: BuiltEndpointRequest,
targetUrl?: string,
) => (
signal?: AbortSignal,
) => (
dispatchRuntimeRequest({
siteUrl: input.siteUrl ?? input.site.url,
targetUrl,
signal,
request,
buildInit: (_requestUrl, requestForFetch) => withSiteRecordProxyRequestInit(input.site, {
method: 'POST',
@@ -326,6 +332,8 @@ export async function recordSurfaceSuccess(input: {
parsedUsage: SurfaceUsageSummary;
upstreamUsagePresent?: boolean;
requestStartedAtMs: number;
isStream?: boolean | null;
firstByteLatencyMs?: number | null;
latencyMs: number;
retryCount: number;
upstreamPath?: string | null;
@@ -334,6 +342,8 @@ export async function recordSurfaceSuccess(input: {
modelRequested: string;
status: string;
httpStatus: number;
isStream?: boolean | null;
firstByteLatencyMs?: number | null;
latencyMs: number;
errorMessage: string | null;
retryCount: number;
@@ -427,6 +437,8 @@ export async function recordSurfaceSuccess(input: {
modelRequested: input.requestedModel,
status: 'success',
httpStatus: 200,
isStream: input.isStream ?? null,
firstByteLatencyMs: input.firstByteLatencyMs ?? null,
latencyMs: input.latencyMs,
errorMessage: null,
retryCount: input.retryCount,
@@ -458,6 +470,8 @@ export function createSurfaceFailureToolkit(input: {
modelRequested: string;
status: string;
httpStatus: number;
isStream?: boolean | null;
firstByteLatencyMs?: number | null;
latencyMs: number;
errorMessage: string | null;
retryCount: number;
@@ -474,6 +488,8 @@ export function createSurfaceFailureToolkit(input: {
modelRequested: args.modelRequested,
status: args.status,
httpStatus: args.httpStatus,
isStream: args.isStream ?? null,
firstByteLatencyMs: args.firstByteLatencyMs ?? null,
latencyMs: args.latencyMs,
errorMessage: args.errorMessage,
retryCount: args.retryCount,
@@ -510,6 +526,8 @@ export function createSurfaceFailureToolkit(input: {
status: number;
errText: string;
rawErrText?: string | null;
isStream?: boolean | null;
firstByteLatencyMs?: number | null;
latencyMs: number;
retryCount: number;
}): Promise<SurfaceFailureOutcome> {
@@ -524,6 +542,8 @@ export function createSurfaceFailureToolkit(input: {
modelRequested: args.requestedModel,
status: 'failed',
httpStatus: args.status,
isStream: args.isStream ?? null,
firstByteLatencyMs: args.firstByteLatencyMs ?? null,
latencyMs: args.latencyMs,
errorMessage: args.errText,
retryCount: args.retryCount,
@@ -570,6 +590,8 @@ export function createSurfaceFailureToolkit(input: {
requestedModel: string;
modelName: string;
failure: { status: number; reason: string };
isStream?: boolean | null;
firstByteLatencyMs?: number | null;
latencyMs: number;
retryCount: number;
promptTokens?: number | null;
@@ -587,6 +609,8 @@ export function createSurfaceFailureToolkit(input: {
modelRequested: args.requestedModel,
status: 'failed',
httpStatus: args.failure.status,
isStream: args.isStream ?? null,
firstByteLatencyMs: args.firstByteLatencyMs ?? null,
latencyMs: args.latencyMs,
errorMessage: args.failure.reason,
retryCount: args.retryCount,
@@ -623,6 +647,8 @@ export function createSurfaceFailureToolkit(input: {
requestedModel: string;
modelName: string;
errorMessage: string;
isStream?: boolean | null;
firstByteLatencyMs?: number | null;
latencyMs: number;
retryCount: number;
}): Promise<SurfaceFailureOutcome> {
@@ -635,6 +661,8 @@ export function createSurfaceFailureToolkit(input: {
modelRequested: args.requestedModel,
status: 'failed',
httpStatus: 0,
isStream: args.isStream ?? null,
firstByteLatencyMs: args.firstByteLatencyMs ?? null,
latencyMs: args.latencyMs,
errorMessage: args.errorMessage,
retryCount: args.retryCount,
@@ -665,6 +693,8 @@ export function createSurfaceFailureToolkit(input: {
requestedModel: string;
modelName: string;
errorMessage: string | null;
isStream?: boolean | null;
firstByteLatencyMs?: number | null;
latencyMs: number;
retryCount: number;
promptTokens?: number | null;
@@ -692,6 +722,8 @@ export function createSurfaceFailureToolkit(input: {
modelRequested: args.requestedModel,
status: 'failed',
httpStatus: args.httpStatus ?? 200,
isStream: args.isStream ?? null,
firstByteLatencyMs: args.firstByteLatencyMs ?? null,
latencyMs: args.latencyMs,
errorMessage,
retryCount: args.retryCount,
src/server/routes/api/settings.events.test.ts
+27
View File
@@ -65,6 +65,7 @@ describe('settings and auth events', () => {
(config as any).proxyDebugRetentionHours = 24;
(config as any).proxyDebugMaxBodyBytes = 262144;
config.routingFallbackUnitCost = 1;
(config as any).proxyFirstByteTimeoutSec = 0;
(config as any).tokenRouterFailureCooldownMaxSec = 30 * 24 * 60 * 60;
(config as any).disableCrossProtocolFallback = false;
(config as any).telegramEnabled = false;
@@ -508,6 +509,32 @@ describe('settings and auth events', () => {
expect(saved?.value).toBe(JSON.stringify(thirtyDaysSec));
});
it('persists and returns first-byte timeout from runtime settings', async () => {
const updateResponse = await app.inject({
method: 'PUT',
url: '/api/settings/runtime',
payload: {
proxyFirstByteTimeoutSec: 7,
},
});
expect(updateResponse.statusCode).toBe(200);
const updated = updateResponse.json() as { proxyFirstByteTimeoutSec?: number };
expect(updated.proxyFirstByteTimeoutSec).toBe(7);
expect((config as any).proxyFirstByteTimeoutSec).toBe(7);
const saved = await db.select().from(schema.settings).where(eq(schema.settings.key, 'proxy_first_byte_timeout_sec')).get();
expect(saved?.value).toBe(JSON.stringify(7));
const getResponse = await app.inject({
method: 'GET',
url: '/api/settings/runtime',
});
expect(getResponse.statusCode).toBe(200);
const runtime = getResponse.json() as { proxyFirstByteTimeoutSec?: number };
expect(runtime.proxyFirstByteTimeoutSec).toBe(7);
});
it('persists and returns disable cross protocol fallback from runtime settings', async () => {
const updateResponse = await app.inject({
method: 'PUT',
src/server/routes/api/settings.ts
+21
View File
@@ -95,6 +95,7 @@ interface RuntimeSettingsBody {
notifyCooldownSec?: number;
adminIpAllowlist?: string[] | string;
routingFallbackUnitCost?: number;
proxyFirstByteTimeoutSec?: number;
tokenRouterFailureCooldownMaxSec?: number;
routingWeights?: Partial<RoutingWeights>;
proxyErrorKeywords?: string[] | string;
@@ -681,6 +682,12 @@ function applyImportedSettingToRuntime(key: string, value: unknown) {
config.routingFallbackUnitCost = Math.max(1e-6, n);
return;
}
case 'proxy_first_byte_timeout_sec': {
const n = Number(value);
if (!Number.isFinite(n) || n < 0) return;
config.proxyFirstByteTimeoutSec = Math.max(0, Math.trunc(n));
return;
}
case 'token_router_failure_cooldown_max_sec': {
const normalized = normalizeTokenRouterFailureCooldownMaxSec(value);
if (normalized == null) return;
@@ -717,6 +724,7 @@ function getRuntimeSettingsResponse(currentAdminIp = '') {
proxyDebugRetentionHours: config.proxyDebugRetentionHours,
proxyDebugMaxBodyBytes: config.proxyDebugMaxBodyBytes,
routingFallbackUnitCost: config.routingFallbackUnitCost,
proxyFirstByteTimeoutSec: config.proxyFirstByteTimeoutSec,
tokenRouterFailureCooldownMaxSec: config.tokenRouterFailureCooldownMaxSec,
routingWeights: config.routingWeights,
webhookUrl: config.webhookUrl,
@@ -1639,6 +1647,19 @@ export async function settingsRoutes(app: FastifyInstance) {
upsertSetting('routing_fallback_unit_cost', normalized);
}
if (body.proxyFirstByteTimeoutSec !== undefined) {
const nextProxyFirstByteTimeoutSec = Number(body.proxyFirstByteTimeoutSec);
if (!Number.isFinite(nextProxyFirstByteTimeoutSec) || nextProxyFirstByteTimeoutSec < 0) {
return reply.code(400).send({ success: false, message: '首字超时必须是大于等于 0 的数字（秒）' });
}
const normalized = Math.max(0, Math.trunc(nextProxyFirstByteTimeoutSec));
if (normalized !== config.proxyFirstByteTimeoutSec) {
changedLabels.push(`首字超时（${config.proxyFirstByteTimeoutSec}s -> ${normalized}s）`);
}
config.proxyFirstByteTimeoutSec = normalized;
upsertSetting('proxy_first_byte_timeout_sec', normalized);
}
if (body.tokenRouterFailureCooldownMaxSec !== undefined) {
const normalized = normalizeTokenRouterFailureCooldownMaxSec(body.tokenRouterFailureCooldownMaxSec);
if (normalized == null) {
src/server/routes/api/stats.proxy-logs.test.ts
+16
View File
@@ -75,6 +75,8 @@ describe('stats proxy logs routes', () => {
modelRequested: 'gpt-4o',
modelActual: 'gpt-4o',
status: 'success',
isStream: 1,
firstByteLatencyMs: 45,
clientFamily: 'generic',
clientAppId: 'cherry_studio',
clientAppName: 'Cherry Studio',
@@ -92,6 +94,8 @@ describe('stats proxy logs routes', () => {
modelRequested: 'gpt-4o-mini',
modelActual: 'gpt-4o-mini',
status: 'failed',
isStream: 0,
firstByteLatencyMs: 12,
clientFamily: 'codex',
promptTokens: 8,
completionTokens: 2,
@@ -105,6 +109,8 @@ describe('stats proxy logs routes', () => {
modelRequested: 'gpt-4.1',
modelActual: 'gpt-4.1',
status: 'retried',
isStream: 1,
firstByteLatencyMs: 30,
promptTokens: 20,
completionTokens: 4,
totalTokens: 24,
@@ -117,6 +123,8 @@ describe('stats proxy logs routes', () => {
modelRequested: 'claude-3-7-sonnet',
modelActual: 'claude-3-7-sonnet',
status: 'success',
isStream: 1,
firstByteLatencyMs: 88,
promptTokens: 40,
completionTokens: 10,
totalTokens: 50,
@@ -163,6 +171,8 @@ describe('stats proxy logs routes', () => {
expect(body.items[0]?.clientAppId).toBe(null);
expect(body.items[0]?.clientAppName).toBe(null);
expect(body.items[0]?.clientConfidence).toBe(null);
expect(body.items[0]?.isStream).toBe(false);
expect(body.items[0]?.firstByteLatencyMs).toBe(12);
expect(body.items[0]).not.toHaveProperty('billingDetails');
expect(body.clientOptions).toEqual([
{ value: 'family:codex', label: '协议 · Codex' },
@@ -204,6 +214,8 @@ describe('stats proxy logs routes', () => {
modelRequested: 'gpt-5',
modelActual: 'gpt-5',
status: 'success',
isStream: 1,
firstByteLatencyMs: 64,
clientFamily: 'codex',
clientAppId: 'cherry_studio',
clientAppName: 'Cherry Studio',
@@ -238,6 +250,8 @@ describe('stats proxy logs routes', () => {
clientAppId: string | null;
clientAppName: string | null;
clientConfidence: string | null;
isStream: boolean | null;
firstByteLatencyMs: number | null;
billingDetails: Record<string, unknown> | null;
};
@@ -251,6 +265,8 @@ describe('stats proxy logs routes', () => {
expect(body.clientAppId).toBe('cherry_studio');
expect(body.clientAppName).toBe('Cherry Studio');
expect(body.clientConfidence).toBe('exact');
expect(body.isStream).toBe(true);
expect(body.firstByteLatencyMs).toBe(64);
expect(body.billingDetails).toMatchObject({
breakdown: { totalCost: 0.12 },
usage: { promptTokens: 100, completionTokens: 20 },
src/server/routes/api/stats.ts
+4
View File
@@ -497,6 +497,10 @@ function mapProxyLogRow(
const legacyMeta = parseProxyLogMessageMeta(typeof row.proxy_logs.errorMessage === 'string' ? row.proxy_logs.errorMessage : '');
return {
...row.proxy_logs,
isStream: row.proxy_logs.isStream == null ? null : Boolean(row.proxy_logs.isStream),
firstByteLatencyMs: typeof row.proxy_logs.firstByteLatencyMs === 'number'
? row.proxy_logs.firstByteLatencyMs
: null,
...(options?.includeBillingDetails
? { billingDetails: parseProxyLogBillingDetails(row.proxy_logs.billingDetails) }
: {}),
src/server/routes/proxy/completions.ts
+39 -9
View File
@@ -1,5 +1,6 @@
import { FastifyInstance, FastifyReply, FastifyRequest } from 'fastify';
import { fetch } from 'undici';
import { config } from '../../config.js';
import { tokenRouter } from '../../services/tokenRouter.js';
import { reportProxyAllFailed, reportTokenExpired } from '../../services/alertService.js';
import { isTokenExpiredError } from '../../services/alertRules.js';
@@ -17,6 +18,7 @@ import { getProxyAuthContext } from '../../middleware/auth.js';
import { buildUpstreamUrl } from './upstreamUrl.js';
import { detectDownstreamClientContext, type DownstreamClientContext } from './downstreamClientContext.js';
import { insertProxyLog } from '../../services/proxyLogStore.js';
import { fetchWithObservedFirstByte, getObservedResponseMeta } from '../../proxy-core/firstByteTimeout.js';
import { getProxyMaxChannelRetries } from '../../services/proxyChannelRetry.js';
import { runWithSiteApiEndpointPool, SiteApiEndpointRequestError } from '../../services/siteApiEndpointService.js';
import {
@@ -48,6 +50,7 @@ export async function completionsProxyRoute(app: FastifyInstance) {
});
const isStream = body.stream === true;
const firstByteTimeoutMs = Math.max(0, Math.trunc((config.proxyFirstByteTimeoutSec || 0) * 1000));
const excludeChannelIds: number[] = [];
let retryCount = 0;
@@ -77,23 +80,37 @@ export async function completionsProxyRoute(app: FastifyInstance) {
const forwardBody = { ...body, model: upstreamModel };
const startTime = Date.now();
try {
const upstream = await runWithSiteApiEndpointPool(selected.site, async (target) => {
const { upstream, firstByteLatencyMs } = await runWithSiteApiEndpointPool(selected.site, async (target) => {
const attemptStartedAtMs = Date.now();
const targetUrl = buildUpstreamUrl(target.baseUrl, '/v1/completions');
const response = await fetch(targetUrl, withSiteRecordProxyRequestInit(selected.site, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Authorization': `Bearer ${selected.tokenValue}`,
const response = await fetchWithObservedFirstByte(
async (signal) => fetch(targetUrl, withSiteRecordProxyRequestInit(selected.site, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Authorization': `Bearer ${selected.tokenValue}`,
},
body: JSON.stringify(forwardBody),
signal,
}, getProxyUrlFromExtraConfig(selected.account.extraConfig))),
{
firstByteTimeoutMs,
startedAtMs: attemptStartedAtMs,
},
body: JSON.stringify(forwardBody),
}, getProxyUrlFromExtraConfig(selected.account.extraConfig)));
);
const observedFirstByteLatencyMs = getObservedResponseMeta(response)?.firstByteLatencyMs ?? null;
if (!response.ok) {
const errText = await response.text().catch(() => 'unknown error');
throw new SiteApiEndpointRequestError(errText || 'unknown error', {
status: response.status,
rawErrText: errText || null,
firstByteLatencyMs: observedFirstByteLatencyMs,
});
}
return response;
return {
upstream: response,
firstByteLatencyMs: observedFirstByteLatencyMs,
};
});
if (isStream) {
@@ -191,6 +208,8 @@ export async function completionsProxyRoute(app: FastifyInstance) {
billingDetails,
clientContext,
downstreamPath,
isStream,
firstByteLatencyMs,
);
return;
}
@@ -228,6 +247,8 @@ export async function completionsProxyRoute(app: FastifyInstance) {
null,
clientContext,
downstreamPath,
isStream,
firstByteLatencyMs,
);
if (shouldRetryProxyRequest(failure.status, errText) && canRetryChannelSelection(retryCount, forcedChannelId)) {
@@ -288,11 +309,14 @@ export async function completionsProxyRoute(app: FastifyInstance) {
billingDetails,
clientContext,
downstreamPath,
isStream,
firstByteLatencyMs,
);
return reply.send(data);
} catch (err: any) {
const status = err instanceof SiteApiEndpointRequestError ? (err.status || 0) : 0;
const errorText = err?.message || 'network failure';
const firstByteLatencyMs = err instanceof SiteApiEndpointRequestError ? err.firstByteLatencyMs : null;
await recordTokenRouterEventBestEffort('record channel failure', () => tokenRouter.recordFailure(selected.channel.id, {
status,
errorText,
@@ -314,6 +338,8 @@ export async function completionsProxyRoute(app: FastifyInstance) {
null,
clientContext,
downstreamPath,
isStream,
firstByteLatencyMs,
);
if (status > 0 && isTokenExpiredError({ status, message: errorText })) {
await reportTokenExpired({
@@ -355,6 +381,8 @@ async function logProxy(
billingDetails: unknown = null,
clientContext: DownstreamClientContext | null = null,
downstreamPath = '/v1/completions',
isStream: boolean,
firstByteLatencyMs: number | null,
) {
try {
const createdAt = formatUtcSqlDateTime(new Date());
@@ -376,6 +404,8 @@ async function logProxy(
modelActual: selected.actualModel || modelRequested,
status,
httpStatus,
isStream,
firstByteLatencyMs,
latencyMs,
promptTokens,
completionTokens,
src/server/routes/proxy/embeddings.ts
+31 -8
View File
@@ -1,5 +1,6 @@
import { FastifyInstance, FastifyReply, FastifyRequest } from 'fastify';
import { fetch } from 'undici';
import { config } from '../../config.js';
import { tokenRouter } from '../../services/tokenRouter.js';
import { reportProxyAllFailed, reportTokenExpired } from '../../services/alertService.js';
import { isTokenExpiredError } from '../../services/alertRules.js';
@@ -16,6 +17,7 @@ import { getProxyAuthContext } from '../../middleware/auth.js';
import { buildUpstreamUrl } from './upstreamUrl.js';
import { detectDownstreamClientContext, type DownstreamClientContext } from './downstreamClientContext.js';
import { insertProxyLog } from '../../services/proxyLogStore.js';
import { fetchWithObservedFirstByte, getObservedResponseMeta } from '../../proxy-core/firstByteTimeout.js';
import { getProxyMaxChannelRetries } from '../../services/proxyChannelRetry.js';
import { runWithSiteApiEndpointPool, SiteApiEndpointRequestError } from '../../services/siteApiEndpointService.js';
import {
@@ -45,6 +47,7 @@ export async function embeddingsProxyRoute(app: FastifyInstance) {
headers: request.headers as Record<string, unknown>,
body,
});
const firstByteTimeoutMs = Math.max(0, Math.trunc((config.proxyFirstByteTimeoutSec || 0) * 1000));
const excludeChannelIds: number[] = [];
let retryCount = 0;
@@ -71,16 +74,25 @@ export async function embeddingsProxyRoute(app: FastifyInstance) {
const forwardBody = { ...body, model: upstreamModel };
const startTime = Date.now();
try {
const { upstream, text } = await runWithSiteApiEndpointPool(selected.site, async (target) => {
const { upstream, text, firstByteLatencyMs } = await runWithSiteApiEndpointPool(selected.site, async (target) => {
const attemptStartedAtMs = Date.now();
const targetUrl = buildUpstreamUrl(target.baseUrl, '/v1/embeddings');
const response = await fetch(targetUrl, withSiteRecordProxyRequestInit(selected.site, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Authorization': `Bearer ${selected.tokenValue}`,
const response = await fetchWithObservedFirstByte(
async (signal) => fetch(targetUrl, withSiteRecordProxyRequestInit(selected.site, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Authorization': `Bearer ${selected.tokenValue}`,
},
body: JSON.stringify(forwardBody),
signal,
}, getProxyUrlFromExtraConfig(selected.account.extraConfig))),
{
firstByteTimeoutMs,
startedAtMs: attemptStartedAtMs,
},
body: JSON.stringify(forwardBody),
}, getProxyUrlFromExtraConfig(selected.account.extraConfig)));
);
const observedFirstByteLatencyMs = getObservedResponseMeta(response)?.firstByteLatencyMs ?? null;
const status = response.status;
let responseText = '';
try {
@@ -89,6 +101,7 @@ export async function embeddingsProxyRoute(app: FastifyInstance) {
if (!response.ok) {
throw new SiteApiEndpointRequestError('unknown error', {
status,
firstByteLatencyMs: observedFirstByteLatencyMs,
cause: error,
});
}
@@ -98,11 +111,13 @@ export async function embeddingsProxyRoute(app: FastifyInstance) {
throw new SiteApiEndpointRequestError(responseText || 'unknown error', {
status,
rawErrText: responseText || null,
firstByteLatencyMs: observedFirstByteLatencyMs,
});
}
return {
upstream: response,
text: responseText,
firstByteLatencyMs: observedFirstByteLatencyMs,
};
});
@@ -140,11 +155,13 @@ export async function embeddingsProxyRoute(app: FastifyInstance) {
logProxy(
selected, requestedModel, 'success', upstream.status, latency, null, retryCount, downstreamApiKeyId,
resolvedUsage.promptTokens, resolvedUsage.completionTokens, resolvedUsage.totalTokens, estimatedCost, billingDetails, clientContext, downstreamPath,
false, firstByteLatencyMs,
);
return reply.code(upstream.status).send(data);
} catch (err: any) {
const status = err instanceof SiteApiEndpointRequestError ? (err.status || 0) : 0;
const errorText = err?.message || 'network failure';
const firstByteLatencyMs = err instanceof SiteApiEndpointRequestError ? err.firstByteLatencyMs : null;
await recordTokenRouterEventBestEffort('record channel failure', () => tokenRouter.recordFailure(selected.channel.id, {
status,
errorText,
@@ -166,6 +183,8 @@ export async function embeddingsProxyRoute(app: FastifyInstance) {
null,
clientContext,
downstreamPath,
false,
firstByteLatencyMs,
);
if (status > 0 && isTokenExpiredError({ status, message: errorText })) {
await reportTokenExpired({
@@ -210,6 +229,8 @@ async function logProxy(
billingDetails: unknown = null,
clientContext: DownstreamClientContext | null = null,
downstreamPath = '/v1/embeddings',
isStream = false,
firstByteLatencyMs: number | null = null,
) {
try {
const createdAt = formatUtcSqlDateTime(new Date());
@@ -231,6 +252,8 @@ async function logProxy(
modelActual: selected.actualModel || modelRequested,
status,
httpStatus,
isStream,
firstByteLatencyMs,
latencyMs,
promptTokens,
completionTokens,
src/server/routes/proxy/endpointFlow.first-byte-timeout.test.ts
+114
View File
@@ -0,0 +1,114 @@
import { describe, expect, it } from 'vitest';
import type { BuiltEndpointRequest } from './endpointFlow.js';
function requestFor(path: string): BuiltEndpointRequest {
return {
endpoint: 'responses',
path,
headers: { 'content-type': 'application/json' },
body: { model: 'gpt-5.2', input: 'hello' },
};
}
function buildDelayedResponse(
bodyText: string,
delayMs: number,
status = 200,
_signal?: AbortSignal,
): Response {
const encoder = new TextEncoder();
const body = new ReadableStream<Uint8Array>({
start(controller) {
setTimeout(() => {
controller.enqueue(encoder.encode(bodyText));
controller.close();
}, delayMs);
},
});
return new Response(body, {
status,
headers: { 'content-type': 'application/json' },
});
}
describe('executeEndpointFlow first-byte timeout', () => {
it('falls through to the next endpoint candidate when the current endpoint times out before any output', async () => {
const { executeEndpointFlow } = await import('./endpointFlow.js');
let timedOutSignal: AbortSignal | undefined;
const dispatchRequest = async (
request: BuiltEndpointRequest,
_targetUrl?: string,
signal?: AbortSignal,
) => (
request.path === '/v1/responses'
? (
timedOutSignal = signal,
buildDelayedResponse(JSON.stringify({ ok: false }), 60, 200, signal)
)
: new Response(JSON.stringify({ ok: true }), {
status: 200,
headers: { 'content-type': 'application/json' },
})
) as unknown as Awaited<ReturnType<typeof import('undici').fetch>>;
const failures: string[] = [];
const result = await executeEndpointFlow({
siteUrl: 'https://example.com',
endpointCandidates: ['responses', 'chat'],
buildRequest: (endpoint: 'responses' | 'chat') => endpoint === 'responses'
? requestFor('/v1/responses')
: { ...requestFor('/v1/chat/completions'), endpoint },
dispatchRequest,
firstByteTimeoutMs: 10,
onAttemptFailure: (ctx: { errText: string }) => {
failures.push(ctx.errText);
},
});
expect(result.ok).toBe(true);
if (result.ok) {
expect(result.upstreamPath).toBe('/v1/chat/completions');
}
expect(timedOutSignal).toBeDefined();
expect(timedOutSignal?.aborted).toBe(true);
expect(failures).toHaveLength(1);
expect(failures[0]).toContain('first byte timeout');
});
it('treats first-byte timeout as terminal when cross-protocol fallback is disabled', async () => {
const { executeEndpointFlow } = await import('./endpointFlow.js');
const attemptedPaths: string[] = [];
const result = await executeEndpointFlow({
siteUrl: 'https://example.com',
endpointCandidates: ['responses', 'chat'],
disableCrossProtocolFallback: true,
buildRequest: (endpoint: 'responses' | 'chat') => endpoint === 'responses'
? requestFor('/v1/responses')
: { ...requestFor('/v1/chat/completions'), endpoint },
dispatchRequest: async (
request: BuiltEndpointRequest,
_targetUrl?: string,
signal?: AbortSignal,
) => {
attemptedPaths.push(request.path);
return (
request.path === '/v1/responses'
? buildDelayedResponse(JSON.stringify({ ok: false }), 60, 200, signal)
: new Response(JSON.stringify({ ok: true }), {
status: 200,
headers: { 'content-type': 'application/json' },
})
) as unknown as Awaited<ReturnType<typeof import('undici').fetch>>;
},
firstByteTimeoutMs: 10,
});
expect(result.ok).toBe(false);
if (!result.ok) {
expect(result.status).toBe(408);
expect(result.errText).toContain('first byte timeout');
}
expect(attemptedPaths).toEqual(['/v1/responses']);
});
});
src/server/routes/proxy/endpointFlow.test.ts
+7 -3
View File
@@ -2,9 +2,13 @@ import { beforeEach, describe, expect, it, vi } from 'vitest';
import { fetch } from 'undici';
import type { BuiltEndpointRequest } from './endpointFlow.js';
vi.mock('undici', () => ({
fetch: vi.fn(),
}));
vi.mock('undici', async () => {
const actual = await vi.importActual<typeof import('undici')>('undici');
return {
...actual,
fetch: vi.fn(),
};
});
vi.mock('../../services/siteProxy.js', () => ({
withSiteProxyRequestInit: async (_targetUrl: string, init: RequestInit) => init,
src/server/routes/proxy/images.ts
+84 -12
View File
@@ -1,5 +1,6 @@
import { FastifyInstance, FastifyReply, FastifyRequest } from 'fastify';
import { fetch } from 'undici';
import { config } from '../../config.js';
import { tokenRouter } from '../../services/tokenRouter.js';
import { reportProxyAllFailed, reportTokenExpired } from '../../services/alertService.js';
import { isTokenExpiredError } from '../../services/alertRules.js';
@@ -15,6 +16,7 @@ import { getProxyAuthContext } from '../../middleware/auth.js';
import { buildUpstreamUrl } from './upstreamUrl.js';
import { detectDownstreamClientContext, type DownstreamClientContext } from './downstreamClientContext.js';
import { insertProxyLog } from '../../services/proxyLogStore.js';
import { fetchWithObservedFirstByte, getObservedResponseMeta } from '../../proxy-core/firstByteTimeout.js';
import { getProxyMaxChannelRetries } from '../../services/proxyChannelRetry.js';
import { runWithSiteApiEndpointPool, SiteApiEndpointRequestError } from '../../services/siteApiEndpointService.js';
import {
@@ -43,6 +45,7 @@ export async function imagesProxyRoute(app: FastifyInstance) {
headers: request.headers as Record<string, unknown>,
body,
});
const firstByteTimeoutMs = Math.max(0, Math.trunc((config.proxyFirstByteTimeoutSec || 0) * 1000));
const excludeChannelIds: number[] = [];
let retryCount = 0;
@@ -72,25 +75,37 @@ export async function imagesProxyRoute(app: FastifyInstance) {
const startTime = Date.now();
try {
const { upstream, text } = await runWithSiteApiEndpointPool(selected.site, async (target) => {
const { upstream, text, firstByteLatencyMs } = await runWithSiteApiEndpointPool(selected.site, async (target) => {
const attemptStartedAtMs = Date.now();
const targetUrl = buildUpstreamUrl(target.baseUrl, '/v1/images/generations');
const response = await fetch(targetUrl, withSiteRecordProxyRequestInit(selected.site, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Authorization': `Bearer ${selected.tokenValue}`,
const response = await fetchWithObservedFirstByte(
async (signal) => fetch(targetUrl, withSiteRecordProxyRequestInit(selected.site, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Authorization': `Bearer ${selected.tokenValue}`,
},
body: JSON.stringify(forwardBody),
signal,
}, getProxyUrlFromExtraConfig(selected.account.extraConfig))),
{
firstByteTimeoutMs,
startedAtMs: attemptStartedAtMs,
},
body: JSON.stringify(forwardBody),
}, getProxyUrlFromExtraConfig(selected.account.extraConfig)));
);
const observedFirstByteLatencyMs = getObservedResponseMeta(response)?.firstByteLatencyMs ?? null;
const responseText = await response.text();
if (!response.ok) {
throw new SiteApiEndpointRequestError(responseText || 'unknown error', {
status: response.status,
rawErrText: responseText || null,
firstByteLatencyMs: observedFirstByteLatencyMs,
});
}
return {
upstream: response,
text: responseText,
firstByteLatencyMs: observedFirstByteLatencyMs,
};
});
@@ -113,6 +128,8 @@ export async function imagesProxyRoute(app: FastifyInstance) {
0,
downstreamPath,
clientContext,
false,
firstByteLatencyMs,
);
if (canRetryChannelSelection(retryCount, forcedChannelId)) {
retryCount++;
@@ -145,11 +162,26 @@ export async function imagesProxyRoute(app: FastifyInstance) {
await recordTokenRouterEventBestEffort('record downstream cost usage', () => (
recordDownstreamCostUsage(request, estimatedCost)
));
logProxy(selected, requestedModel, 'success', upstream.status, latency, null, retryCount, downstreamApiKeyId, estimatedCost, downstreamPath, clientContext);
logProxy(
selected,
requestedModel,
'success',
upstream.status,
latency,
null,
retryCount,
downstreamApiKeyId,
estimatedCost,
downstreamPath,
clientContext,
false,
firstByteLatencyMs,
);
return reply.code(upstream.status).send(data.value);
} catch (err: any) {
const status = err instanceof SiteApiEndpointRequestError ? (err.status || 0) : 0;
const errorText = err?.message || 'network failure';
const firstByteLatencyMs = err instanceof SiteApiEndpointRequestError ? err.firstByteLatencyMs : null;
await recordTokenRouterEventBestEffort('record channel failure', () => tokenRouter.recordFailure(selected.channel.id, {
status,
errorText,
@@ -167,6 +199,8 @@ export async function imagesProxyRoute(app: FastifyInstance) {
0,
downstreamPath,
clientContext,
false,
firstByteLatencyMs,
);
if (status > 0 && isTokenExpiredError({ status, message: errorText })) {
await reportTokenExpired({
@@ -216,6 +250,7 @@ export async function imagesProxyRoute(app: FastifyInstance) {
headers: request.headers as Record<string, unknown>,
body: jsonBody || Object.fromEntries(multipartForm?.entries?.() || []),
});
const firstByteTimeoutMs = Math.max(0, Math.trunc((config.proxyFirstByteTimeoutSec || 0) * 1000));
const excludeChannelIds: number[] = [];
let retryCount = 0;
@@ -244,7 +279,8 @@ export async function imagesProxyRoute(app: FastifyInstance) {
const startTime = Date.now();
try {
const { upstream, text } = await runWithSiteApiEndpointPool(selected.site, async (target) => {
const { upstream, text, firstByteLatencyMs } = await runWithSiteApiEndpointPool(selected.site, async (target) => {
const attemptStartedAtMs = Date.now();
const targetUrl = buildUpstreamUrl(target.baseUrl, '/v1/images/edits');
const requestInit = multipartForm
? withSiteRecordProxyRequestInit(selected.site, {
@@ -267,16 +303,29 @@ export async function imagesProxyRoute(app: FastifyInstance) {
model: upstreamModel,
}),
}, getProxyUrlFromExtraConfig(selected.account.extraConfig));
const response = await fetch(targetUrl, requestInit);
const response = await fetchWithObservedFirstByte(
async (signal) => fetch(targetUrl, {
...requestInit,
signal,
}),
{
firstByteTimeoutMs,
startedAtMs: attemptStartedAtMs,
},
);
const observedFirstByteLatencyMs = getObservedResponseMeta(response)?.firstByteLatencyMs ?? null;
const responseText = await response.text();
if (!response.ok) {
throw new SiteApiEndpointRequestError(responseText || 'unknown error', {
status: response.status,
rawErrText: responseText || null,
firstByteLatencyMs: observedFirstByteLatencyMs,
});
}
return {
upstream: response,
text: responseText,
firstByteLatencyMs: observedFirstByteLatencyMs,
};
});
@@ -299,6 +348,8 @@ export async function imagesProxyRoute(app: FastifyInstance) {
0,
downstreamPath,
clientContext,
false,
firstByteLatencyMs,
);
if (canRetryChannelSelection(retryCount, forcedChannelId)) {
retryCount++;
@@ -331,11 +382,26 @@ export async function imagesProxyRoute(app: FastifyInstance) {
await recordTokenRouterEventBestEffort('record downstream cost usage', () => (
recordDownstreamCostUsage(request, estimatedCost)
));
logProxy(selected, requestedModel, 'success', upstream.status, latency, null, retryCount, downstreamApiKeyId, estimatedCost, downstreamPath, clientContext);
logProxy(
selected,
requestedModel,
'success',
upstream.status,
latency,
null,
retryCount,
downstreamApiKeyId,
estimatedCost,
downstreamPath,
clientContext,
false,
firstByteLatencyMs,
);
return reply.code(upstream.status).send(data.value);
} catch (err: any) {
const status = err instanceof SiteApiEndpointRequestError ? (err.status || 0) : 0;
const errorText = err?.message || 'network failure';
const firstByteLatencyMs = err instanceof SiteApiEndpointRequestError ? err.firstByteLatencyMs : null;
await recordTokenRouterEventBestEffort('record channel failure', () => tokenRouter.recordFailure(selected.channel.id, {
status,
errorText,
@@ -353,6 +419,8 @@ export async function imagesProxyRoute(app: FastifyInstance) {
0,
downstreamPath,
clientContext,
false,
firstByteLatencyMs,
);
if (status > 0 && isTokenExpiredError({ status, message: errorText })) {
await reportTokenExpired({
@@ -402,6 +470,8 @@ async function logProxy(
estimatedCost = 0,
downstreamPath = '/v1/images/generations',
clientContext: DownstreamClientContext | null = null,
isStream = false,
firstByteLatencyMs: number | null = null,
) {
try {
const createdAt = formatUtcSqlDateTime(new Date());
@@ -423,6 +493,8 @@ async function logProxy(
modelActual: selected.actualModel || modelRequested,
status,
httpStatus,
isStream,
firstByteLatencyMs,
latencyMs,
promptTokens: 0,
completionTokens: 0,
src/server/routes/proxy/search.ts
+44 -9
View File
@@ -1,5 +1,6 @@
import { FastifyInstance, FastifyReply, FastifyRequest } from 'fastify';
import { fetch } from 'undici';
import { config } from '../../config.js';
import { tokenRouter } from '../../services/tokenRouter.js';
import { reportProxyAllFailed, reportTokenExpired } from '../../services/alertService.js';
import { isTokenExpiredError } from '../../services/alertRules.js';
@@ -13,6 +14,7 @@ import { getProxyAuthContext } from '../../middleware/auth.js';
import { buildUpstreamUrl } from './upstreamUrl.js';
import { detectDownstreamClientContext, type DownstreamClientContext } from './downstreamClientContext.js';
import { insertProxyLog } from '../../services/proxyLogStore.js';
import { fetchWithObservedFirstByte, getObservedResponseMeta } from '../../proxy-core/firstByteTimeout.js';
import { getProxyMaxChannelRetries } from '../../services/proxyChannelRetry.js';
import { runWithSiteApiEndpointPool, SiteApiEndpointRequestError } from '../../services/siteApiEndpointService.js';
import {
@@ -71,6 +73,7 @@ export async function searchProxyRoute(app: FastifyInstance) {
headers: request.headers as Record<string, unknown>,
body,
});
const firstByteTimeoutMs = Math.max(0, Math.trunc((config.proxyFirstByteTimeoutSec || 0) * 1000));
const excludeChannelIds: number[] = [];
let retryCount = 0;
@@ -104,25 +107,37 @@ export async function searchProxyRoute(app: FastifyInstance) {
const startTime = Date.now();
try {
const { upstream, text } = await runWithSiteApiEndpointPool(selected.site, async (target) => {
const { upstream, text, firstByteLatencyMs } = await runWithSiteApiEndpointPool(selected.site, async (target) => {
const attemptStartedAtMs = Date.now();
const targetUrl = buildUpstreamUrl(target.baseUrl, '/v1/search');
const response = await fetch(targetUrl, withSiteRecordProxyRequestInit(selected.site, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
Authorization: `Bearer ${selected.tokenValue}`,
const response = await fetchWithObservedFirstByte(
async (signal) => fetch(targetUrl, withSiteRecordProxyRequestInit(selected.site, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
Authorization: `Bearer ${selected.tokenValue}`,
},
body: JSON.stringify(forwardBody),
signal,
}, getProxyUrlFromExtraConfig(selected.account.extraConfig))),
{
firstByteTimeoutMs,
startedAtMs: attemptStartedAtMs,
},
body: JSON.stringify(forwardBody),
}, getProxyUrlFromExtraConfig(selected.account.extraConfig)));
);
const observedFirstByteLatencyMs = getObservedResponseMeta(response)?.firstByteLatencyMs ?? null;
const responseText = await response.text();
if (!response.ok) {
throw new SiteApiEndpointRequestError(responseText || 'unknown error', {
status: response.status,
rawErrText: responseText || null,
firstByteLatencyMs: observedFirstByteLatencyMs,
});
}
return {
upstream: response,
text: responseText,
firstByteLatencyMs: observedFirstByteLatencyMs,
};
});
@@ -134,11 +149,25 @@ export async function searchProxyRoute(app: FastifyInstance) {
tokenRouter.recordSuccess(selected.channel.id, latency, 0, upstreamModel)
));
recordDownstreamCostUsage(request, 0);
logProxy(selected, requestedModel, 'success', upstream.status, latency, null, retryCount, downstreamApiKeyId, clientContext, downstreamPath);
logProxy(
selected,
requestedModel,
'success',
upstream.status,
latency,
null,
retryCount,
downstreamApiKeyId,
clientContext,
downstreamPath,
false,
firstByteLatencyMs,
);
return reply.code(upstream.status).send(data);
} catch (error: any) {
const status = error instanceof SiteApiEndpointRequestError ? (error.status || 0) : 0;
const errorText = error?.message || 'network error';
const firstByteLatencyMs = error instanceof SiteApiEndpointRequestError ? error.firstByteLatencyMs : null;
await recordTokenRouterEventBestEffort('record channel failure', () => tokenRouter.recordFailure(selected.channel.id, {
status,
errorText,
@@ -155,6 +184,8 @@ export async function searchProxyRoute(app: FastifyInstance) {
downstreamApiKeyId,
clientContext,
downstreamPath,
false,
firstByteLatencyMs,
);
if (status > 0 && isTokenExpiredError({ status, message: errorText })) {
await reportTokenExpired({
@@ -194,6 +225,8 @@ async function logProxy(
downstreamApiKeyId: number | null = null,
clientContext: DownstreamClientContext | null = null,
downstreamPath = '/v1/search',
isStream = false,
firstByteLatencyMs: number | null = null,
) {
try {
const createdAt = formatUtcSqlDateTime(new Date());
@@ -206,6 +239,8 @@ async function logProxy(
modelActual: selected.actualModel || modelRequested,
status,
httpStatus,
isStream,
firstByteLatencyMs,
latencyMs,
promptTokens: 0,
completionTokens: 0,
src/server/services/proxyLogStore.test.ts
+49
View File
@@ -4,6 +4,7 @@ const {
hasProxyLogBillingDetailsColumnMock,
hasProxyLogClientColumnsMock,
hasProxyLogDownstreamApiKeyIdColumnMock,
hasProxyLogStreamTimingColumnsMock,
dbInsertMock,
dbInsertValuesMock,
dbInsertRunMock,
@@ -12,6 +13,7 @@ const {
hasProxyLogBillingDetailsColumnMock: vi.fn(),
hasProxyLogClientColumnsMock: vi.fn(),
hasProxyLogDownstreamApiKeyIdColumnMock: vi.fn(),
hasProxyLogStreamTimingColumnsMock: vi.fn(),
dbInsertMock: vi.fn(),
dbInsertValuesMock: vi.fn(),
dbInsertRunMock: vi.fn(),
@@ -24,6 +26,8 @@ const {
modelActual: 'model_actual',
status: 'status',
httpStatus: 'http_status',
isStream: 'is_stream',
firstByteLatencyMs: 'first_byte_latency_ms',
latencyMs: 'latency_ms',
promptTokens: 'prompt_tokens',
completionTokens: 'completion_tokens',
@@ -50,6 +54,7 @@ vi.mock('../db/index.js', () => ({
hasProxyLogBillingDetailsColumn: (...args: unknown[]) => hasProxyLogBillingDetailsColumnMock(...args),
hasProxyLogClientColumns: (...args: unknown[]) => hasProxyLogClientColumnsMock(...args),
hasProxyLogDownstreamApiKeyIdColumn: (...args: unknown[]) => hasProxyLogDownstreamApiKeyIdColumnMock(...args),
hasProxyLogStreamTimingColumns: (...args: unknown[]) => hasProxyLogStreamTimingColumnsMock(...args),
}));
import { insertProxyLog, parseProxyLogBillingDetails, withProxyLogSelectFields } from './proxyLogStore.js';
@@ -59,12 +64,14 @@ describe('proxyLogStore', () => {
hasProxyLogBillingDetailsColumnMock.mockReset();
hasProxyLogClientColumnsMock.mockReset();
hasProxyLogDownstreamApiKeyIdColumnMock.mockReset();
hasProxyLogStreamTimingColumnsMock.mockReset();
dbInsertMock.mockReset();
dbInsertValuesMock.mockReset();
dbInsertRunMock.mockReset();
hasProxyLogBillingDetailsColumnMock.mockResolvedValue(false);
hasProxyLogClientColumnsMock.mockResolvedValue(false);
hasProxyLogDownstreamApiKeyIdColumnMock.mockResolvedValue(false);
hasProxyLogStreamTimingColumnsMock.mockResolvedValue(false);
dbInsertMock.mockReturnValue({
values: (...args: unknown[]) => dbInsertValuesMock(...args),
@@ -89,6 +96,23 @@ describe('proxyLogStore', () => {
expect(runner.mock.calls[1][0].fields.billingDetails).toBeUndefined();
});
it('retries proxy log selects without stream timing fields when the columns are missing', async () => {
hasProxyLogStreamTimingColumnsMock.mockResolvedValue(true);
const runner = vi.fn()
.mockRejectedValueOnce(new Error('column proxy_logs.first_byte_latency_ms does not exist'))
.mockResolvedValueOnce([{ id: 1 }]);
await expect(withProxyLogSelectFields(runner)).resolves.toEqual([{ id: 1 }]);
expect(runner).toHaveBeenCalledTimes(2);
expect(runner.mock.calls[0][0].includeStreamTimingFields).toBe(true);
expect(runner.mock.calls[0][0].fields.isStream).toBe('is_stream');
expect(runner.mock.calls[0][0].fields.firstByteLatencyMs).toBe('first_byte_latency_ms');
expect(runner.mock.calls[1][0].includeStreamTimingFields).toBe(false);
expect(runner.mock.calls[1][0].fields.isStream).toBeUndefined();
expect(runner.mock.calls[1][0].fields.firstByteLatencyMs).toBeUndefined();
});
it('accepts parsed billing details objects for helper-level callers', () => {
expect(parseProxyLogBillingDetails({
source: 'pricing',
@@ -220,4 +244,29 @@ describe('proxyLogStore', () => {
expect(dbInsertValuesMock.mock.calls[1][0].clientAppName).toBeUndefined();
expect(dbInsertValuesMock.mock.calls[1][0].clientConfidence).toBeUndefined();
});
it('retries proxy log inserts without stream timing fields when those columns are missing', async () => {
hasProxyLogStreamTimingColumnsMock.mockResolvedValue(true);
dbInsertRunMock
.mockRejectedValueOnce(new Error('column proxy_logs.is_stream does not exist'))
.mockResolvedValueOnce(undefined);
await insertProxyLog({
modelRequested: 'gpt-5',
isStream: true,
firstByteLatencyMs: 42,
});
expect(dbInsertValuesMock).toHaveBeenCalledTimes(2);
expect(dbInsertValuesMock.mock.calls[0][0]).toMatchObject({
modelRequested: 'gpt-5',
isStream: true,
firstByteLatencyMs: 42,
});
expect(dbInsertValuesMock.mock.calls[1][0]).toMatchObject({
modelRequested: 'gpt-5',
});
expect(dbInsertValuesMock.mock.calls[1][0].isStream).toBeUndefined();
expect(dbInsertValuesMock.mock.calls[1][0].firstByteLatencyMs).toBeUndefined();
});
});
src/server/services/proxyLogStore.ts
+68 -1
View File
@@ -4,6 +4,7 @@ import {
hasProxyLogBillingDetailsColumn,
hasProxyLogClientColumns,
hasProxyLogDownstreamApiKeyIdColumn,
hasProxyLogStreamTimingColumns,
} from '../db/index.js';
export type ProxyLogInsertInput = {
@@ -15,6 +16,8 @@ export type ProxyLogInsertInput = {
modelActual?: string | null;
status?: string | null;
httpStatus?: number | null;
isStream?: boolean | null;
firstByteLatencyMs?: number | null;
latencyMs?: number | null;
promptTokens?: number | null;
completionTokens?: number | null;
@@ -61,12 +64,21 @@ function buildProxyLogClientSelectFields() {
};
}
function buildProxyLogStreamTimingSelectFields() {
return {
isStream: schema.proxyLogs.isStream,
firstByteLatencyMs: schema.proxyLogs.firstByteLatencyMs,
};
}
function buildProxyLogSelectFields(options?: {
includeBillingDetails?: boolean;
includeClientFields?: boolean;
includeStreamTimingFields?: boolean;
}) {
return {
...buildProxyLogCoreSelectFields(),
...(options?.includeStreamTimingFields ? buildProxyLogStreamTimingSelectFields() : {}),
...(options?.includeClientFields ? buildProxyLogClientSelectFields() : {}),
...(options?.includeBillingDetails ? { billingDetails: schema.proxyLogs.billingDetails } : {}),
};
@@ -81,31 +93,37 @@ export type ProxyLogSelectFields = ReturnType<typeof buildProxyLogSelectFields>;
export type ResolvedProxyLogSelectFields = {
includeBillingDetails: boolean;
includeClientFields: boolean;
includeStreamTimingFields: boolean;
fields: ProxyLogSelectFields;
};
export async function resolveProxyLogSelectFields(options?: {
includeBillingDetails?: boolean;
includeClientFields?: boolean;
includeStreamTimingFields?: boolean;
}) {
const includeBillingDetails = options?.includeBillingDetails === true
&& await hasProxyLogBillingDetailsColumn();
const includeClientFields = options?.includeClientFields !== false
&& await hasProxyLogClientColumns();
const includeStreamTimingFields = options?.includeStreamTimingFields !== false
&& await hasProxyLogStreamTimingColumns();
return {
includeBillingDetails,
includeClientFields,
includeStreamTimingFields,
fields: buildProxyLogSelectFields({
includeBillingDetails,
includeClientFields,
includeStreamTimingFields,
}),
};
}
export async function withProxyLogSelectFields<T>(
runner: (selection: ResolvedProxyLogSelectFields) => Promise<T>,
options?: { includeBillingDetails?: boolean; includeClientFields?: boolean },
options?: { includeBillingDetails?: boolean; includeClientFields?: boolean; includeStreamTimingFields?: boolean },
): Promise<T> {
let selection = await resolveProxyLogSelectFields(options);
@@ -117,9 +135,11 @@ export async function withProxyLogSelectFields<T>(
selection = {
includeBillingDetails: false,
includeClientFields: selection.includeClientFields,
includeStreamTimingFields: selection.includeStreamTimingFields,
fields: buildProxyLogSelectFields({
includeBillingDetails: false,
includeClientFields: selection.includeClientFields,
includeStreamTimingFields: selection.includeStreamTimingFields,
}),
};
continue;
@@ -129,9 +149,25 @@ export async function withProxyLogSelectFields<T>(
selection = {
includeBillingDetails: selection.includeBillingDetails,
includeClientFields: false,
includeStreamTimingFields: selection.includeStreamTimingFields,
fields: buildProxyLogSelectFields({
includeBillingDetails: selection.includeBillingDetails,
includeClientFields: false,
includeStreamTimingFields: selection.includeStreamTimingFields,
}),
};
continue;
}
if (selection.includeStreamTimingFields && isMissingProxyLogStreamTimingColumnsError(error)) {
selection = {
includeBillingDetails: selection.includeBillingDetails,
includeClientFields: selection.includeClientFields,
includeStreamTimingFields: false,
fields: buildProxyLogSelectFields({
includeBillingDetails: selection.includeBillingDetails,
includeClientFields: selection.includeClientFields,
includeStreamTimingFields: false,
}),
};
continue;
@@ -204,6 +240,22 @@ export function isMissingProxyLogClientColumnsError(error: unknown): boolean {
);
}
export function isMissingProxyLogStreamTimingColumnsError(error: unknown): boolean {
const lowered = normalizeProxyLogStoreErrorMessage(error);
const hasStreamTimingColumnReference = [
'is_stream',
'first_byte_latency_ms',
].some((columnName) => lowered.includes(columnName));
return hasStreamTimingColumnReference
&& (
lowered.includes('does not exist')
|| lowered.includes('unknown column')
|| lowered.includes('no such column')
|| lowered.includes('has no column named')
);
}
export async function insertProxyLog(input: ProxyLogInsertInput): Promise<void> {
const baseValues = {
routeId: input.routeId ?? null,
@@ -237,14 +289,24 @@ export async function insertProxyLog(input: ProxyLogInsertInput): Promise<void>
].some((value) => value != null && String(value).trim().length > 0);
const includeClientFields = requestedClientFields
&& await hasProxyLogClientColumns();
const requestedStreamTimingFields = input.isStream != null || input.firstByteLatencyMs != null;
const includeStreamTimingFields = requestedStreamTimingFields
&& await hasProxyLogStreamTimingColumns();
let allowBillingDetails = includeBillingDetails;
let allowDownstreamApiKeyId = includeDownstreamApiKeyId;
let allowClientFields = includeClientFields;
let allowStreamTimingFields = includeStreamTimingFields;
while (true) {
const values = {
...baseValues,
...(allowStreamTimingFields
? {
isStream: input.isStream ?? null,
firstByteLatencyMs: input.firstByteLatencyMs ?? null,
}
: {}),
...(allowBillingDetails ? { billingDetails: serializedBillingDetails } : {}),
...(allowDownstreamApiKeyId ? { downstreamApiKeyId: input.downstreamApiKeyId } : {}),
...(allowClientFields
@@ -276,6 +338,11 @@ export async function insertProxyLog(input: ProxyLogInsertInput): Promise<void>
continue;
}
if (allowStreamTimingFields && isMissingProxyLogStreamTimingColumnsError(error)) {
allowStreamTimingFields = false;
continue;
}
throw error;
}
}
src/server/services/proxyRetryPolicy.ts
+1 -1
View File
@@ -16,7 +16,7 @@ const MODEL_UNSUPPORTED_PATTERNS: RegExp[] = [
];
export const RETRYABLE_TIMEOUT_PATTERNS: RegExp[] = [
/(request timed out|connection timed out|read timeout|\btimed out\b)/i,
/(request timed out|connection timed out|read timeout|first byte timeout|\btimed out\b)/i,
];
const RETRYABLE_CHANNEL_LOCAL_PATTERNS: RegExp[] = [
src/server/services/siteApiEndpointService.ts
+10 -1
View File
@@ -49,14 +49,23 @@ export interface RecordedSiteApiEndpointFailure extends SiteApiEndpointFailureDi
export class SiteApiEndpointRequestError extends Error {
readonly status: number | null;
readonly rawErrText: string | null;
readonly firstByteLatencyMs: number | null;
constructor(message: string, options?: { status?: number | null; rawErrText?: string | null; cause?: unknown }) {
constructor(message: string, options?: {
status?: number | null;
rawErrText?: string | null;
firstByteLatencyMs?: number | null;
cause?: unknown;
}) {
super(message, options?.cause !== undefined ? { cause: options.cause } : undefined);
this.name = 'SiteApiEndpointRequestError';
this.status = typeof options?.status === 'number' ? options.status : null;
this.rawErrText = typeof options?.rawErrText === 'string' && options.rawErrText.trim()
? options.rawErrText
: null;
this.firstByteLatencyMs = typeof options?.firstByteLatencyMs === 'number' && Number.isFinite(options.firstByteLatencyMs)
? options.firstByteLatencyMs
: null;
}
}
src/web/api.ts
+3
View File
@@ -387,6 +387,7 @@ export type RuntimeSettingsPayload = {
notifyCooldownSec?: number;
adminIpAllowlist?: string[] | string;
routingFallbackUnitCost?: number;
proxyFirstByteTimeoutSec?: number;
tokenRouterFailureCooldownMaxSec?: number;
routingWeights?: RuntimeRoutingWeightsPayload;
proxyErrorKeywords?: string[] | string;
@@ -437,6 +438,8 @@ export type ProxyLogListItem = {
modelActual: string;
status: string;
latencyMs: number;
isStream?: boolean | null;
firstByteLatencyMs?: number | null;
totalTokens: number | null;
retryCount: number;
accountId?: number | null;
src/web/pages/ProxyLogs.server-driven.test.tsx
+8
View File
@@ -58,6 +58,8 @@ function buildListResponse(overrides?: Partial<{
modelActual: 'gpt-4o',
status: 'success',
latencyMs: 120,
firstByteLatencyMs: 35,
isStream: true,
promptTokens: 10,
completionTokens: 5,
totalTokens: 15,
@@ -146,6 +148,8 @@ describe('ProxyLogs server-driven page', () => {
modelActual: 'gpt-4o',
status: 'success',
latencyMs: 120,
firstByteLatencyMs: 35,
isStream: true,
promptTokens: 10,
completionTokens: 5,
totalTokens: 15,
@@ -264,6 +268,8 @@ describe('ProxyLogs server-driven page', () => {
expect(text).toContain('Codex');
expect(text).toContain('推测');
expect(text).toContain('下游 Key: 移动端灰度');
expect(text).toContain('流式');
expect(text).toContain('首字');
} finally {
await act(async () => {
root?.unmount();
@@ -801,6 +807,8 @@ describe('ProxyLogs server-driven page', () => {
modelActual: 'gpt-4o',
status: 'success',
latencyMs: 120,
firstByteLatencyMs: 22,
isStream: false,
promptTokens: 10,
completionTokens: 5,
totalTokens: 15,
src/web/pages/ProxyLogs.tsx
+74
View File
@@ -265,6 +265,28 @@ function latencyBgColor(ms: number) {
return 'color-mix(in srgb, var(--color-success) 12%, transparent)';
}
function firstByteColor(ms: number) {
if (ms >= 3000) return 'var(--color-danger)';
if (ms >= 1000) return 'var(--color-warning)';
return 'var(--color-primary)';
}
function firstByteBgColor(ms: number) {
if (ms >= 3000) return 'color-mix(in srgb, var(--color-danger) 12%, transparent)';
if (ms >= 1000) return 'color-mix(in srgb, var(--color-warning) 12%, transparent)';
return 'color-mix(in srgb, var(--color-primary) 12%, transparent)';
}
function formatStreamModeLabel(isStream: boolean | null | undefined) {
if (isStream == null) return null;
return isStream ? '流式' : '非流';
}
function formatFirstByteLabel(ms: number | null | undefined) {
if (!Number.isFinite(ms) || typeof ms !== 'number' || ms < 0) return null;
return `首字 ${formatLatency(ms)}`;
}
function formatCompactNumber(value: number, digits = 6) {
if (!Number.isFinite(value)) return '0';
const formatted = value.toFixed(digits).replace(/\.?0+$/, '');
@@ -1877,6 +1899,8 @@ export default function ProxyLogs() {
const downstreamKeySummary = renderDownstreamKeySummary(detailLog);
const isExpanded = expanded === log.id;
const clientDisplay = resolveProxyLogClientDisplay(detailLog);
const streamModeLabel = formatStreamModeLabel(detailLog.isStream);
const firstByteLabel = formatFirstByteLabel(detailLog.firstByteLatencyMs);
return (
<MobileCard
@@ -1911,6 +1935,24 @@ export default function ProxyLogs() {
{clientDisplay.secondary}
</span>
) : null}
{streamModeLabel ? (
<span className="badge badge-muted" style={{ fontSize: 10 }}>
{streamModeLabel}
</span>
) : null}
{firstByteLabel ? (
<span
className="badge"
style={{
fontSize: 10,
color: firstByteColor(detailLog.firstByteLatencyMs ?? 0),
background: firstByteBgColor(detailLog.firstByteLatencyMs ?? 0),
borderColor: 'transparent',
}}
>
{firstByteLabel}
</span>
) : null}
</div>
<div className="mobile-summary-grid">
<div className="mobile-summary-metric">
@@ -1934,6 +1976,8 @@ export default function ProxyLogs() {
<div className="mobile-card-extra">
<MobileField label="时间" value={formatDateTimeLocal(log.createdAt)} />
<MobileField label="站点" value={<SiteBadgeLink siteId={siteIdByName.get(String(log.siteName || '').trim())} siteName={log.siteName} badgeStyle={{ fontSize: 11 }} />} />
{streamModeLabel ? <MobileField label="模式" value={streamModeLabel} /> : null}
{firstByteLabel ? <MobileField label="首字" value={firstByteLabel.replace(/^首字\s*/, '')} /> : null}
<MobileField label="重试" value={log.retryCount > 0 ? log.retryCount : 0} />
<MobileField label="用量来源" value={formatProxyLogUsageSource(detailLog.usageSource ?? pathMeta.usageSource) || '--'} />
{detailState?.loading && <div style={{ color: 'var(--color-text-muted)' }}>加载详情中...</div>}
@@ -1983,6 +2027,8 @@ export default function ProxyLogs() {
const billingDetailSummary = detail ? formatBillingDetailSummary(detailLog) : null;
const billingProcessLines = detail ? buildBillingProcessLines(detailLog) : [];
const downstreamKeySummary = renderDownstreamKeySummary(detailLog);
const streamModeLabel = formatStreamModeLabel(detailLog.isStream);
const firstByteLabel = formatFirstByteLabel(detailLog.firstByteLatencyMs);
return (
<React.Fragment key={log.id}>
@@ -2015,6 +2061,28 @@ export default function ProxyLogs() {
{downstreamKeySummary}
</div>
) : null}
{(streamModeLabel || firstByteLabel) ? (
<div style={{ display: 'flex', gap: 6, flexWrap: 'wrap' }}>
{streamModeLabel ? (
<span className="badge badge-muted" style={{ fontSize: 10 }}>
{streamModeLabel}
</span>
) : null}
{firstByteLabel ? (
<span
className="badge"
style={{
fontSize: 10,
color: firstByteColor(detailLog.firstByteLatencyMs ?? 0),
background: firstByteBgColor(detailLog.firstByteLatencyMs ?? 0),
borderColor: 'transparent',
}}
>
{firstByteLabel}
</span>
) : null}
</div>
) : null}
</div>
</td>
<td style={{ fontSize: 12, color: 'var(--color-text-secondary)' }}>
@@ -2084,6 +2152,12 @@ export default function ProxyLogs() {
<>{' -> '}实际模型: <strong style={{ color: 'var(--color-text-primary)' }}>{detailLog.modelActual}</strong></>
)}
，状态: <strong style={{ color: detailLog.status === 'success' ? 'var(--color-success)' : 'var(--color-danger)' }}>{detailLog.status === 'success' ? '成功' : '失败'}</strong>
{streamModeLabel && (
<>，模式: <strong style={{ color: 'var(--color-text-primary)' }}>{streamModeLabel}</strong></>
)}
{firstByteLabel && (
<>，首字: <strong style={{ color: firstByteColor(detailLog.firstByteLatencyMs ?? 0) }}>{formatLatency(detailLog.firstByteLatencyMs ?? 0)}</strong></>
)}
，用时: <strong style={{ color: latencyColor(detailLog.latencyMs) }}>{formatLatency(detailLog.latencyMs)}</strong>
{detail && (
<>
src/web/pages/Settings.tsx
+34
View File
@@ -66,6 +66,7 @@ type RuntimeSettings = {
proxySessionChannelConcurrencyLimit: number;
proxySessionChannelQueueWaitMs: number;
routingFallbackUnitCost: number;
proxyFirstByteTimeoutSec: number;
routeFailureCooldownMaxValue: number;
routeFailureCooldownMaxUnit: RouteCooldownUnit;
routingWeights: RoutingWeights;
@@ -244,6 +245,7 @@ export default function Settings() {
proxySessionChannelConcurrencyLimit: 2,
proxySessionChannelQueueWaitMs: 1500,
routingFallbackUnitCost: 1,
proxyFirstByteTimeoutSec: 0,
routeFailureCooldownMaxValue: 30,
routeFailureCooldownMaxUnit: 'day',
routingWeights: defaultWeights,
@@ -496,6 +498,9 @@ export default function Settings() {
routingFallbackUnitCost: Number(runtimeInfo.routingFallbackUnitCost) > 0
? Number(runtimeInfo.routingFallbackUnitCost)
: 1,
proxyFirstByteTimeoutSec: Number(runtimeInfo.proxyFirstByteTimeoutSec) >= 0
? Math.trunc(Number(runtimeInfo.proxyFirstByteTimeoutSec))
: 0,
routeFailureCooldownMaxValue: routeCooldownInput.value,
routeFailureCooldownMaxUnit: routeCooldownInput.unit,
routingWeights: {
@@ -942,6 +947,9 @@ export default function Settings() {
await api.updateRuntimeSettings({
routingWeights: runtime.routingWeights,
routingFallbackUnitCost: runtime.routingFallbackUnitCost,
proxyFirstByteTimeoutSec: Number.isFinite(runtime.proxyFirstByteTimeoutSec)
? Math.max(0, Math.trunc(runtime.proxyFirstByteTimeoutSec))
: 0,
tokenRouterFailureCooldownMaxSec: toRouteCooldownSeconds(
runtime.routeFailureCooldownMaxValue,
runtime.routeFailureCooldownMaxUnit,
@@ -1739,6 +1747,32 @@ export default function Settings() {
</span>
</label>
<div style={{ marginBottom: 12 }}>
<div style={{ fontSize: 12, color: 'var(--color-text-muted)', marginBottom: 6 }}>
首字超时（无首包 / 首 token）
</div>
<input
type="number"
min={0}
step={1}
aria-label="首字超时秒数"
value={runtime.proxyFirstByteTimeoutSec}
onChange={(e) => {
const nextValue = Number(e.target.value);
setRuntime((prev) => ({
...prev,
proxyFirstByteTimeoutSec: Number.isFinite(nextValue) && nextValue >= 0
? Math.trunc(nextValue)
: prev.proxyFirstByteTimeoutSec,
}));
}}
style={inputStyle}
/>
<div style={{ fontSize: 12, color: 'var(--color-text-muted)', lineHeight: 1.7, marginTop: 6 }}>
`0` 表示关闭。只有在指定时间内完全没有任何首包 / 首 token 返回时才切换，已经开始输出的请求不会被这项超时打断。
</div>
</div>
<div className={`anim-collapse ${showAdvancedRouting ? 'is-open' : ''}`.trim()}>
<div className="anim-collapse-inner" style={{ paddingTop: 2 }}>
<div style={{ display: 'grid', gridTemplateColumns: isMobile ? '1fr' : '1fr 1fr', gap: 12 }}>
src/web/pages/settings.route-cooldown.test.tsx
+56
View File
@@ -57,6 +57,7 @@ describe('Settings route cooldown cap', () => {
logCleanupProgramLogsEnabled: true,
logCleanupRetentionDays: 14,
routingFallbackUnitCost: 1,
proxyFirstByteTimeoutSec: 0,
routingWeights: {},
tokenRouterFailureCooldownMaxSec: 30 * 24 * 60 * 60,
adminIpAllowlist: [],
@@ -127,6 +128,7 @@ describe('Settings route cooldown cap', () => {
usageWeight: 0.3,
},
routingFallbackUnitCost: 1,
proxyFirstByteTimeoutSec: 0,
tokenRouterFailureCooldownMaxSec: 10,
disableCrossProtocolFallback: false,
});
@@ -146,6 +148,7 @@ describe('Settings route cooldown cap', () => {
logCleanupProgramLogsEnabled: true,
logCleanupRetentionDays: 14,
routingFallbackUnitCost: 1,
proxyFirstByteTimeoutSec: 0,
routingWeights: {},
tokenRouterFailureCooldownMaxSec: 10,
adminIpAllowlist: [],
@@ -181,4 +184,57 @@ describe('Settings route cooldown cap', () => {
root?.unmount();
}
});
it('saves the first-byte timeout seconds alongside other routing runtime settings', async () => {
let root!: ReactTestRenderer;
try {
await act(async () => {
root = create(
<MemoryRouter>
<ToastProvider>
<Settings />
</ToastProvider>
</MemoryRouter>,
);
});
await flushMicrotasks();
const firstByteInput = root.root.find((node) => (
node.type === 'input'
&& node.props.type === 'number'
&& node.props['aria-label'] === '首字超时秒数'
));
await act(async () => {
firstByteInput.props.onChange({ target: { value: '7' } });
});
const saveButton = root.root.find((node) => (
node.type === 'button'
&& typeof node.props.onClick === 'function'
&& collectText(node).trim() === '保存路由策略'
));
await act(async () => {
saveButton.props.onClick();
});
await flushMicrotasks();
expect(apiMock.updateRuntimeSettings).toHaveBeenCalledWith({
routingWeights: {
baseWeightFactor: 0.5,
valueScoreFactor: 0.5,
costWeight: 0.4,
balanceWeight: 0.3,
usageWeight: 0.3,
},
routingFallbackUnitCost: 1,
proxyFirstByteTimeoutSec: 7,
tokenRouterFailureCooldownMaxSec: 30 * 24 * 60 * 60,
disableCrossProtocolFallback: false,
});
} finally {
root?.unmount();
}
});
});