Nvidia/BossAI
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[codex] Split site API endpoint pool from panel host (#373)

Browse Source 
* feat: add site api endpoint schema scaffolding

* feat: wire site api endpoints into backup migration

* feat: add site api endpoint selection service

* feat: add site api endpoint contracts

* feat: route discovery through site api endpoints

* feat: route proxy traffic through site api endpoints

* feat: add site api endpoint pool editor

* fix: keep proxy video tasks on site endpoint pool

* fix: restore core test compatibility for site endpoint pool

* fix: address endpoint pool review findings
This commit is contained in:
Cita
2026-04-01 20:58:37 +08:00
committed by GitHub
parent ce77bde95a
commit 76c85d0a13
49 changed files with 7226 additions and 986 deletions
Download Patch File Download Diff File Expand all files Collapse all files
drizzle/0018_site_api_endpoints.sql
+18
View File
@@ -0,0 +1,18 @@
CREATE TABLE `site_api_endpoints` (
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
`site_id` integer NOT NULL,
`url` text NOT NULL,
`enabled` integer DEFAULT true,
`sort_order` integer DEFAULT 0,
`cooldown_until` text,
`last_selected_at` text,
`last_failed_at` text,
`last_failure_reason` text,
`created_at` text DEFAULT (datetime('now')),
`updated_at` text DEFAULT (datetime('now')),
FOREIGN KEY (`site_id`) REFERENCES `sites`(`id`) ON UPDATE no action ON DELETE cascade
);
--> statement-breakpoint
CREATE UNIQUE INDEX `site_api_endpoints_site_url_unique` ON `site_api_endpoints` (`site_id`,`url`);--> statement-breakpoint
CREATE INDEX `site_api_endpoints_site_enabled_sort_idx` ON `site_api_endpoints` (`site_id`,`enabled`,`sort_order`);--> statement-breakpoint
CREATE INDEX `site_api_endpoints_site_cooldown_idx` ON `site_api_endpoints` (`site_id`,`cooldown_until`);
drizzle/meta/0018_snapshot.json
+2760
View File
File diff suppressed because it is too large Load Diff
drizzle/meta/_journal.json
+7
View File
@@ -120,6 +120,13 @@
"when": 1774692431588,
"tag": "0017_proxy_debug_traces",
"breakpoints": true
},
{
"idx": 17,
"version": "6",
"when": 1774959227309,
"tag": "0018_site_api_endpoints",
"breakpoints": true
}
]
}
src/server/db/generated/mysql.bootstrap.sql
+4
View File
@@ -15,6 +15,7 @@ CREATE TABLE IF NOT EXISTS `route_channels` (`id` INT AUTO_INCREMENT NOT NULL PR
CREATE TABLE IF NOT EXISTS `route_group_sources` (`id` INT AUTO_INCREMENT NOT NULL PRIMARY KEY, `group_route_id` INT NOT NULL, `source_route_id` INT NOT NULL, FOREIGN KEY (`group_route_id`) REFERENCES `token_routes`(`id`) ON DELETE CASCADE, FOREIGN KEY (`source_route_id`) REFERENCES `token_routes`(`id`) ON DELETE CASCADE);
CREATE TABLE IF NOT EXISTS `settings` (`key` VARCHAR(191) NOT NULL PRIMARY KEY, `value` TEXT);
CREATE TABLE IF NOT EXISTS `site_announcements` (`id` INT AUTO_INCREMENT NOT NULL PRIMARY KEY, `site_id` INT NOT NULL, `platform` TEXT NOT NULL, `source_key` TEXT NOT NULL, `title` TEXT NOT NULL, `content` TEXT NOT NULL, `level` VARCHAR(191) NOT NULL DEFAULT 'info', `source_url` TEXT, `starts_at` VARCHAR(191), `ends_at` VARCHAR(191), `upstream_created_at` VARCHAR(191), `upstream_updated_at` VARCHAR(191), `first_seen_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), '%Y-%m-%d %H:%i:%s')), `last_seen_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), '%Y-%m-%d %H:%i:%s')), `read_at` VARCHAR(191), `dismissed_at` VARCHAR(191), `raw_payload` TEXT, FOREIGN KEY (`site_id`) REFERENCES `sites`(`id`) ON DELETE CASCADE);
CREATE TABLE IF NOT EXISTS `site_api_endpoints` (`id` INT AUTO_INCREMENT NOT NULL PRIMARY KEY, `site_id` INT NOT NULL, `url` TEXT NOT NULL, `enabled` BOOLEAN DEFAULT true, `sort_order` INT DEFAULT 0, `cooldown_until` VARCHAR(191), `last_selected_at` VARCHAR(191), `last_failed_at` VARCHAR(191), `last_failure_reason` TEXT, `created_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), '%Y-%m-%d %H:%i:%s')), `updated_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), '%Y-%m-%d %H:%i:%s')), FOREIGN KEY (`site_id`) REFERENCES `sites`(`id`) ON DELETE CASCADE);
CREATE TABLE IF NOT EXISTS `site_disabled_models` (`id` INT AUTO_INCREMENT NOT NULL PRIMARY KEY, `site_id` INT NOT NULL, `model_name` TEXT NOT NULL, `created_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), '%Y-%m-%d %H:%i:%s')), FOREIGN KEY (`site_id`) REFERENCES `sites`(`id`) ON DELETE CASCADE);
CREATE TABLE IF NOT EXISTS `token_model_availability` (`id` INT AUTO_INCREMENT NOT NULL PRIMARY KEY, `token_id` INT NOT NULL, `model_name` TEXT NOT NULL, `available` BOOLEAN, `latency_ms` INT, `checked_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), '%Y-%m-%d %H:%i:%s')), FOREIGN KEY (`token_id`) REFERENCES `account_tokens`(`id`) ON DELETE CASCADE);
CREATE UNIQUE INDEX `downstream_api_keys_key_unique` ON `downstream_api_keys` (`key`(191));
@@ -24,6 +25,7 @@ CREATE UNIQUE INDEX `proxy_files_public_id_unique` ON `proxy_files` (`public_id`
CREATE UNIQUE INDEX `proxy_video_tasks_public_id_unique` ON `proxy_video_tasks` (`public_id`(191));
CREATE UNIQUE INDEX `route_group_sources_group_source_unique` ON `route_group_sources` (`group_route_id`, `source_route_id`);
CREATE UNIQUE INDEX `site_announcements_site_source_key_unique` ON `site_announcements` (`site_id`, `source_key`(191));
CREATE UNIQUE INDEX `site_api_endpoints_site_url_unique` ON `site_api_endpoints` (`site_id`, `url`(191));
CREATE UNIQUE INDEX `site_disabled_models_site_model_unique` ON `site_disabled_models` (`site_id`, `model_name`(191));
CREATE UNIQUE INDEX `sites_platform_url_unique` ON `sites` (`platform`(191), `url`(191));
CREATE UNIQUE INDEX `token_model_availability_token_model_unique` ON `token_model_availability` (`token_id`, `model_name`(191));
@@ -69,6 +71,8 @@ CREATE INDEX `route_channels_token_id_idx` ON `route_channels` (`token_id`);
CREATE INDEX `route_group_sources_source_route_id_idx` ON `route_group_sources` (`source_route_id`);
CREATE INDEX `site_announcements_read_at_idx` ON `site_announcements` (`read_at`);
CREATE INDEX `site_announcements_site_id_first_seen_at_idx` ON `site_announcements` (`site_id`, `first_seen_at`);
CREATE INDEX `site_api_endpoints_site_cooldown_idx` ON `site_api_endpoints` (`site_id`, `cooldown_until`);
CREATE INDEX `site_api_endpoints_site_enabled_sort_idx` ON `site_api_endpoints` (`site_id`, `enabled`, `sort_order`);
CREATE INDEX `site_disabled_models_site_id_idx` ON `site_disabled_models` (`site_id`);
CREATE INDEX `sites_status_idx` ON `sites` (`status`(191));
CREATE INDEX `token_model_availability_available_idx` ON `token_model_availability` (`available`);
src/server/db/generated/mysql.upgrade.sql
+4 -8
View File
@@ -1,8 +1,4 @@
CREATE TABLE IF NOT EXISTS `proxy_debug_traces` (`id` INT AUTO_INCREMENT NOT NULL PRIMARY KEY, `downstream_path` TEXT NOT NULL, `client_kind` TEXT, `session_id` TEXT, `trace_hint` TEXT, `requested_model` TEXT, `downstream_api_key_id` INT, `request_headers_json` JSON, `request_body_json` JSON, `sticky_session_key` TEXT, `sticky_hit_channel_id` INT, `selected_channel_id` INT, `selected_route_id` INT, `selected_account_id` INT, `selected_site_id` INT, `selected_site_platform` TEXT, `endpoint_candidates_json` JSON, `endpoint_runtime_state_json` JSON, `decision_summary_json` JSON, `final_status` TEXT, `final_http_status` INT, `final_upstream_path` TEXT, `final_response_headers_json` JSON, `final_response_body_json` JSON, `created_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), '%Y-%m-%d %H:%i:%s')), `updated_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), '%Y-%m-%d %H:%i:%s')));
CREATE TABLE IF NOT EXISTS `proxy_debug_attempts` (`id` INT AUTO_INCREMENT NOT NULL PRIMARY KEY, `trace_id` INT NOT NULL, `attempt_index` INT NOT NULL, `endpoint` TEXT NOT NULL, `request_path` TEXT NOT NULL, `target_url` TEXT NOT NULL, `runtime_executor` TEXT, `request_headers_json` JSON, `request_body_json` JSON, `response_status` INT, `response_headers_json` JSON, `response_body_json` JSON, `raw_error_text` TEXT, `recover_applied` BOOLEAN DEFAULT false, `downgrade_decision` BOOLEAN DEFAULT false, `downgrade_reason` TEXT, `memory_write_json` JSON, `created_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), '%Y-%m-%d %H:%i:%s')), FOREIGN KEY (`trace_id`) REFERENCES `proxy_debug_traces`(`id`) ON DELETE CASCADE);
CREATE UNIQUE INDEX `proxy_debug_attempts_trace_attempt_unique` ON `proxy_debug_attempts` (`trace_id`, `attempt_index`);
CREATE INDEX `proxy_debug_attempts_trace_created_at_idx` ON `proxy_debug_attempts` (`trace_id`, `created_at`);
CREATE INDEX `proxy_debug_traces_created_at_idx` ON `proxy_debug_traces` (`created_at`);
CREATE INDEX `proxy_debug_traces_final_status_created_at_idx` ON `proxy_debug_traces` (`final_status`(191), `created_at`);
CREATE INDEX `proxy_debug_traces_model_created_at_idx` ON `proxy_debug_traces` (`requested_model`(191), `created_at`);
CREATE INDEX `proxy_debug_traces_session_created_at_idx` ON `proxy_debug_traces` (`session_id`(191), `created_at`);
CREATE TABLE IF NOT EXISTS `site_api_endpoints` (`id` INT AUTO_INCREMENT NOT NULL PRIMARY KEY, `site_id` INT NOT NULL, `url` TEXT NOT NULL, `enabled` BOOLEAN DEFAULT true, `sort_order` INT DEFAULT 0, `cooldown_until` VARCHAR(191), `last_selected_at` VARCHAR(191), `last_failed_at` VARCHAR(191), `last_failure_reason` TEXT, `created_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), '%Y-%m-%d %H:%i:%s')), `updated_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), '%Y-%m-%d %H:%i:%s')), FOREIGN KEY (`site_id`) REFERENCES `sites`(`id`) ON DELETE CASCADE);
CREATE UNIQUE INDEX `site_api_endpoints_site_url_unique` ON `site_api_endpoints` (`site_id`, `url`(191));
CREATE INDEX `site_api_endpoints_site_cooldown_idx` ON `site_api_endpoints` (`site_id`, `cooldown_until`);
CREATE INDEX `site_api_endpoints_site_enabled_sort_idx` ON `site_api_endpoints` (`site_id`, `enabled`, `sort_order`);
src/server/db/generated/postgres.bootstrap.sql
+4
View File
@@ -15,6 +15,7 @@ CREATE TABLE IF NOT EXISTS "route_channels" ("id" INTEGER GENERATED BY DEFAULT A
CREATE TABLE IF NOT EXISTS "route_group_sources" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL PRIMARY KEY, "group_route_id" INTEGER NOT NULL, "source_route_id" INTEGER NOT NULL, FOREIGN KEY ("group_route_id") REFERENCES "token_routes"("id") ON DELETE CASCADE, FOREIGN KEY ("source_route_id") REFERENCES "token_routes"("id") ON DELETE CASCADE);
CREATE TABLE IF NOT EXISTS "settings" ("key" TEXT NOT NULL PRIMARY KEY, "value" TEXT);
CREATE TABLE IF NOT EXISTS "site_announcements" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL PRIMARY KEY, "site_id" INTEGER NOT NULL, "platform" TEXT NOT NULL, "source_key" TEXT NOT NULL, "title" TEXT NOT NULL, "content" TEXT NOT NULL, "level" TEXT NOT NULL DEFAULT 'info', "source_url" TEXT, "starts_at" TEXT, "ends_at" TEXT, "upstream_created_at" TEXT, "upstream_updated_at" TEXT, "first_seen_at" TEXT DEFAULT to_char(timezone('UTC', CURRENT_TIMESTAMP), 'YYYY-MM-DD HH24:MI:SS'), "last_seen_at" TEXT DEFAULT to_char(timezone('UTC', CURRENT_TIMESTAMP), 'YYYY-MM-DD HH24:MI:SS'), "read_at" TEXT, "dismissed_at" TEXT, "raw_payload" TEXT, FOREIGN KEY ("site_id") REFERENCES "sites"("id") ON DELETE CASCADE);
CREATE TABLE IF NOT EXISTS "site_api_endpoints" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL PRIMARY KEY, "site_id" INTEGER NOT NULL, "url" TEXT NOT NULL, "enabled" BOOLEAN DEFAULT true, "sort_order" INTEGER DEFAULT 0, "cooldown_until" TEXT, "last_selected_at" TEXT, "last_failed_at" TEXT, "last_failure_reason" TEXT, "created_at" TEXT DEFAULT to_char(timezone('UTC', CURRENT_TIMESTAMP), 'YYYY-MM-DD HH24:MI:SS'), "updated_at" TEXT DEFAULT to_char(timezone('UTC', CURRENT_TIMESTAMP), 'YYYY-MM-DD HH24:MI:SS'), FOREIGN KEY ("site_id") REFERENCES "sites"("id") ON DELETE CASCADE);
CREATE TABLE IF NOT EXISTS "site_disabled_models" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL PRIMARY KEY, "site_id" INTEGER NOT NULL, "model_name" TEXT NOT NULL, "created_at" TEXT DEFAULT to_char(timezone('UTC', CURRENT_TIMESTAMP), 'YYYY-MM-DD HH24:MI:SS'), FOREIGN KEY ("site_id") REFERENCES "sites"("id") ON DELETE CASCADE);
CREATE TABLE IF NOT EXISTS "token_model_availability" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL PRIMARY KEY, "token_id" INTEGER NOT NULL, "model_name" TEXT NOT NULL, "available" BOOLEAN, "latency_ms" INTEGER, "checked_at" TEXT DEFAULT to_char(timezone('UTC', CURRENT_TIMESTAMP), 'YYYY-MM-DD HH24:MI:SS'), FOREIGN KEY ("token_id") REFERENCES "account_tokens"("id") ON DELETE CASCADE);
CREATE UNIQUE INDEX "downstream_api_keys_key_unique" ON "downstream_api_keys" ("key");
@@ -24,6 +25,7 @@ CREATE UNIQUE INDEX "proxy_files_public_id_unique" ON "proxy_files" ("public_id"
CREATE UNIQUE INDEX "proxy_video_tasks_public_id_unique" ON "proxy_video_tasks" ("public_id");
CREATE UNIQUE INDEX "route_group_sources_group_source_unique" ON "route_group_sources" ("group_route_id", "source_route_id");
CREATE UNIQUE INDEX "site_announcements_site_source_key_unique" ON "site_announcements" ("site_id", "source_key");
CREATE UNIQUE INDEX "site_api_endpoints_site_url_unique" ON "site_api_endpoints" ("site_id", "url");
CREATE UNIQUE INDEX "site_disabled_models_site_model_unique" ON "site_disabled_models" ("site_id", "model_name");
CREATE UNIQUE INDEX "sites_platform_url_unique" ON "sites" ("platform", "url");
CREATE UNIQUE INDEX "token_model_availability_token_model_unique" ON "token_model_availability" ("token_id", "model_name");
@@ -69,6 +71,8 @@ CREATE INDEX "route_channels_token_id_idx" ON "route_channels" ("token_id");
CREATE INDEX "route_group_sources_source_route_id_idx" ON "route_group_sources" ("source_route_id");
CREATE INDEX "site_announcements_read_at_idx" ON "site_announcements" ("read_at");
CREATE INDEX "site_announcements_site_id_first_seen_at_idx" ON "site_announcements" ("site_id", "first_seen_at");
CREATE INDEX "site_api_endpoints_site_cooldown_idx" ON "site_api_endpoints" ("site_id", "cooldown_until");
CREATE INDEX "site_api_endpoints_site_enabled_sort_idx" ON "site_api_endpoints" ("site_id", "enabled", "sort_order");
CREATE INDEX "site_disabled_models_site_id_idx" ON "site_disabled_models" ("site_id");
CREATE INDEX "sites_status_idx" ON "sites" ("status");
CREATE INDEX "token_model_availability_available_idx" ON "token_model_availability" ("available");
src/server/db/generated/postgres.upgrade.sql
+4 -8
View File
@@ -1,8 +1,4 @@
CREATE TABLE IF NOT EXISTS "proxy_debug_traces" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL PRIMARY KEY, "downstream_path" TEXT NOT NULL, "client_kind" TEXT, "session_id" TEXT, "trace_hint" TEXT, "requested_model" TEXT, "downstream_api_key_id" INTEGER, "request_headers_json" JSONB, "request_body_json" JSONB, "sticky_session_key" TEXT, "sticky_hit_channel_id" INTEGER, "selected_channel_id" INTEGER, "selected_route_id" INTEGER, "selected_account_id" INTEGER, "selected_site_id" INTEGER, "selected_site_platform" TEXT, "endpoint_candidates_json" JSONB, "endpoint_runtime_state_json" JSONB, "decision_summary_json" JSONB, "final_status" TEXT, "final_http_status" INTEGER, "final_upstream_path" TEXT, "final_response_headers_json" JSONB, "final_response_body_json" JSONB, "created_at" TEXT DEFAULT to_char(timezone('UTC', CURRENT_TIMESTAMP), 'YYYY-MM-DD HH24:MI:SS'), "updated_at" TEXT DEFAULT to_char(timezone('UTC', CURRENT_TIMESTAMP), 'YYYY-MM-DD HH24:MI:SS'));
CREATE TABLE IF NOT EXISTS "proxy_debug_attempts" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL PRIMARY KEY, "trace_id" INTEGER NOT NULL, "attempt_index" INTEGER NOT NULL, "endpoint" TEXT NOT NULL, "request_path" TEXT NOT NULL, "target_url" TEXT NOT NULL, "runtime_executor" TEXT, "request_headers_json" JSONB, "request_body_json" JSONB, "response_status" INTEGER, "response_headers_json" JSONB, "response_body_json" JSONB, "raw_error_text" TEXT, "recover_applied" BOOLEAN DEFAULT false, "downgrade_decision" BOOLEAN DEFAULT false, "downgrade_reason" TEXT, "memory_write_json" JSONB, "created_at" TEXT DEFAULT to_char(timezone('UTC', CURRENT_TIMESTAMP), 'YYYY-MM-DD HH24:MI:SS'), FOREIGN KEY ("trace_id") REFERENCES "proxy_debug_traces"("id") ON DELETE CASCADE);
CREATE UNIQUE INDEX "proxy_debug_attempts_trace_attempt_unique" ON "proxy_debug_attempts" ("trace_id", "attempt_index");
CREATE INDEX "proxy_debug_attempts_trace_created_at_idx" ON "proxy_debug_attempts" ("trace_id", "created_at");
CREATE INDEX "proxy_debug_traces_created_at_idx" ON "proxy_debug_traces" ("created_at");
CREATE INDEX "proxy_debug_traces_final_status_created_at_idx" ON "proxy_debug_traces" ("final_status", "created_at");
CREATE INDEX "proxy_debug_traces_model_created_at_idx" ON "proxy_debug_traces" ("requested_model", "created_at");
CREATE INDEX "proxy_debug_traces_session_created_at_idx" ON "proxy_debug_traces" ("session_id", "created_at");
CREATE TABLE IF NOT EXISTS "site_api_endpoints" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL PRIMARY KEY, "site_id" INTEGER NOT NULL, "url" TEXT NOT NULL, "enabled" BOOLEAN DEFAULT true, "sort_order" INTEGER DEFAULT 0, "cooldown_until" TEXT, "last_selected_at" TEXT, "last_failed_at" TEXT, "last_failure_reason" TEXT, "created_at" TEXT DEFAULT to_char(timezone('UTC', CURRENT_TIMESTAMP), 'YYYY-MM-DD HH24:MI:SS'), "updated_at" TEXT DEFAULT to_char(timezone('UTC', CURRENT_TIMESTAMP), 'YYYY-MM-DD HH24:MI:SS'), FOREIGN KEY ("site_id") REFERENCES "sites"("id") ON DELETE CASCADE);
CREATE UNIQUE INDEX "site_api_endpoints_site_url_unique" ON "site_api_endpoints" ("site_id", "url");
CREATE INDEX "site_api_endpoints_site_cooldown_idx" ON "site_api_endpoints" ("site_id", "cooldown_until");
CREATE INDEX "site_api_endpoints_site_enabled_sort_idx" ON "site_api_endpoints" ("site_id", "enabled", "sort_order");
src/server/db/generated/schemaContract.json
+117
View File
@@ -1308,6 +1308,76 @@
}
}
},
"site_api_endpoints": {
"columns": {
"id": {
"logicalType": "integer",
"notNull": true,
"defaultValue": null,
"primaryKey": true
},
"site_id": {
"logicalType": "integer",
"notNull": true,
"defaultValue": null,
"primaryKey": false
},
"url": {
"logicalType": "text",
"notNull": true,
"defaultValue": null,
"primaryKey": false
},
"enabled": {
"logicalType": "boolean",
"notNull": false,
"defaultValue": "true",
"primaryKey": false
},
"sort_order": {
"logicalType": "integer",
"notNull": false,
"defaultValue": "0",
"primaryKey": false
},
"cooldown_until": {
"logicalType": "datetime",
"notNull": false,
"defaultValue": null,
"primaryKey": false
},
"last_selected_at": {
"logicalType": "datetime",
"notNull": false,
"defaultValue": null,
"primaryKey": false
},
"last_failed_at": {
"logicalType": "datetime",
"notNull": false,
"defaultValue": null,
"primaryKey": false
},
"last_failure_reason": {
"logicalType": "text",
"notNull": false,
"defaultValue": null,
"primaryKey": false
},
"created_at": {
"logicalType": "datetime",
"notNull": false,
"defaultValue": "datetime('now')",
"primaryKey": false
},
"updated_at": {
"logicalType": "datetime",
"notNull": false,
"defaultValue": "datetime('now')",
"primaryKey": false
}
}
},
"site_disabled_models": {
"columns": {
"id": {
@@ -1967,6 +2037,34 @@
],
"unique": true
},
{
"name": "site_api_endpoints_site_cooldown_idx",
"table": "site_api_endpoints",
"columns": [
"site_id",
"cooldown_until"
],
"unique": false
},
{
"name": "site_api_endpoints_site_enabled_sort_idx",
"table": "site_api_endpoints",
"columns": [
"site_id",
"enabled",
"sort_order"
],
"unique": false
},
{
"name": "site_api_endpoints_site_url_unique",
"table": "site_api_endpoints",
"columns": [
"site_id",
"url"
],
"unique": true
},
{
"name": "site_disabled_models_site_id_idx",
"table": "site_disabled_models",
@@ -2106,6 +2204,14 @@
"source_key"
]
},
{
"name": "site_api_endpoints_site_url_unique",
"table": "site_api_endpoints",
"columns": [
"site_id",
"url"
]
},
{
"name": "site_disabled_models_site_model_unique",
"table": "site_disabled_models",
@@ -2253,6 +2359,17 @@
],
"onDelete": "CASCADE"
},
{
"table": "site_api_endpoints",
"columns": [
"site_id"
],
"referencedTable": "sites",
"referencedColumns": [
"id"
],
"onDelete": "CASCADE"
},
{
"table": "site_disabled_models",
"columns": [
src/server/db/schema.ts
+18
View File
@@ -22,6 +22,24 @@ export const sites = sqliteTable('sites', {
platformUrlUnique: uniqueIndex('sites_platform_url_unique').on(table.platform, table.url),
}));
export const siteApiEndpoints = sqliteTable('site_api_endpoints', {
id: integer('id').primaryKey({ autoIncrement: true }),
siteId: integer('site_id').notNull().references(() => sites.id, { onDelete: 'cascade' }),
url: text('url').notNull(),
enabled: integer('enabled', { mode: 'boolean' }).default(true),
sortOrder: integer('sort_order').default(0),
cooldownUntil: text('cooldown_until'),
lastSelectedAt: text('last_selected_at'),
lastFailedAt: text('last_failed_at'),
lastFailureReason: text('last_failure_reason'),
createdAt: text('created_at').default(sql`(datetime('now'))`),
updatedAt: text('updated_at').default(sql`(datetime('now'))`),
}, (table) => ({
siteUrlUnique: uniqueIndex('site_api_endpoints_site_url_unique').on(table.siteId, table.url),
siteEnabledSortIdx: index('site_api_endpoints_site_enabled_sort_idx').on(table.siteId, table.enabled, table.sortOrder),
siteCooldownIdx: index('site_api_endpoints_site_cooldown_idx').on(table.siteId, table.cooldownUntil),
}));
export const siteDisabledModels = sqliteTable('site_disabled_models', {
id: integer('id').primaryKey({ autoIncrement: true }),
siteId: integer('site_id').notNull().references(() => sites.id, { onDelete: 'cascade' }),
src/server/db/siteSchemaCompatibility.test.ts
+27
View File
@@ -33,6 +33,25 @@ function createInspector(
describe('ensureSiteSchemaCompatibility', () => {
it.each([
{
dialect: 'sqlite' as const,
expectedSql: [
'ALTER TABLE sites ADD COLUMN proxy_url text;',
'ALTER TABLE sites ADD COLUMN use_system_proxy integer DEFAULT 0;',
'UPDATE sites SET use_system_proxy = 0 WHERE use_system_proxy IS NULL;',
'ALTER TABLE sites ADD COLUMN custom_headers text;',
'ALTER TABLE sites ADD COLUMN external_checkin_url text;',
'ALTER TABLE sites ADD COLUMN global_weight real DEFAULT 1;',
'UPDATE sites SET global_weight = 1 WHERE global_weight IS NULL OR global_weight <= 0;',
'CREATE TABLE IF NOT EXISTS site_api_endpoints (id integer PRIMARY KEY AUTOINCREMENT NOT NULL, site_id integer NOT NULL REFERENCES sites(id) ON DELETE cascade, url text NOT NULL, enabled integer DEFAULT 1, sort_order integer DEFAULT 0, cooldown_until text, last_selected_at text, last_failed_at text, last_failure_reason text, created_at text DEFAULT (datetime(\'now\')), updated_at text DEFAULT (datetime(\'now\')));',
'CREATE UNIQUE INDEX IF NOT EXISTS site_api_endpoints_site_url_unique ON site_api_endpoints (site_id, url);',
'CREATE INDEX IF NOT EXISTS site_api_endpoints_site_enabled_sort_idx ON site_api_endpoints (site_id, enabled, sort_order);',
'CREATE INDEX IF NOT EXISTS site_api_endpoints_site_cooldown_idx ON site_api_endpoints (site_id, cooldown_until);',
'CREATE TABLE IF NOT EXISTS site_disabled_models (id integer PRIMARY KEY AUTOINCREMENT NOT NULL, site_id integer NOT NULL REFERENCES sites(id) ON DELETE cascade, model_name text NOT NULL, created_at text DEFAULT (datetime(\'now\')));',
'CREATE UNIQUE INDEX IF NOT EXISTS site_disabled_models_site_model_unique ON site_disabled_models (site_id, model_name);',
'CREATE INDEX IF NOT EXISTS site_disabled_models_site_id_idx ON site_disabled_models (site_id);',
],
},
{
dialect: 'postgres' as const,
expectedSql: [
@@ -43,6 +62,10 @@ describe('ensureSiteSchemaCompatibility', () => {
'ALTER TABLE "sites" ADD COLUMN "external_checkin_url" TEXT',
'ALTER TABLE "sites" ADD COLUMN "global_weight" DOUBLE PRECISION DEFAULT 1',
'UPDATE "sites" SET "global_weight" = 1 WHERE "global_weight" IS NULL OR "global_weight" <= 0',
'CREATE TABLE IF NOT EXISTS "site_api_endpoints" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, "site_id" INTEGER NOT NULL REFERENCES "sites"("id") ON DELETE CASCADE, "url" TEXT NOT NULL, "enabled" BOOLEAN DEFAULT TRUE, "sort_order" INTEGER DEFAULT 0, "cooldown_until" TEXT, "last_selected_at" TEXT, "last_failed_at" TEXT, "last_failure_reason" TEXT, "created_at" TEXT DEFAULT to_char(timezone(\'UTC\', CURRENT_TIMESTAMP), \'YYYY-MM-DD HH24:MI:SS\'), "updated_at" TEXT DEFAULT to_char(timezone(\'UTC\', CURRENT_TIMESTAMP), \'YYYY-MM-DD HH24:MI:SS\'))',
'CREATE UNIQUE INDEX IF NOT EXISTS "site_api_endpoints_site_url_unique" ON "site_api_endpoints" ("site_id", "url")',
'CREATE INDEX IF NOT EXISTS "site_api_endpoints_site_enabled_sort_idx" ON "site_api_endpoints" ("site_id", "enabled", "sort_order")',
'CREATE INDEX IF NOT EXISTS "site_api_endpoints_site_cooldown_idx" ON "site_api_endpoints" ("site_id", "cooldown_until")',
'CREATE TABLE IF NOT EXISTS "site_disabled_models" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, "site_id" INTEGER NOT NULL REFERENCES "sites"("id") ON DELETE CASCADE, "model_name" TEXT NOT NULL, "created_at" TEXT)',
'CREATE UNIQUE INDEX IF NOT EXISTS "site_disabled_models_site_model_unique" ON "site_disabled_models" ("site_id", "model_name")',
'CREATE INDEX IF NOT EXISTS "site_disabled_models_site_id_idx" ON "site_disabled_models" ("site_id")',
@@ -58,6 +81,10 @@ describe('ensureSiteSchemaCompatibility', () => {
'ALTER TABLE `sites` ADD COLUMN `external_checkin_url` TEXT NULL',
'ALTER TABLE `sites` ADD COLUMN `global_weight` DOUBLE DEFAULT 1',
'UPDATE `sites` SET `global_weight` = 1 WHERE `global_weight` IS NULL OR `global_weight` <= 0',
'CREATE TABLE IF NOT EXISTS `site_api_endpoints` (`id` INT AUTO_INCREMENT PRIMARY KEY, `site_id` INT NOT NULL, `url` TEXT NOT NULL, `enabled` BOOLEAN DEFAULT TRUE, `sort_order` INT DEFAULT 0, `cooldown_until` VARCHAR(191) NULL, `last_selected_at` VARCHAR(191) NULL, `last_failed_at` VARCHAR(191) NULL, `last_failure_reason` TEXT NULL, `created_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), \'%Y-%m-%d %H:%i:%s\')), `updated_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), \'%Y-%m-%d %H:%i:%s\')), CONSTRAINT `site_api_endpoints_site_fk` FOREIGN KEY (`site_id`) REFERENCES `sites`(`id`) ON DELETE CASCADE)',
'CREATE UNIQUE INDEX `site_api_endpoints_site_url_unique` ON `site_api_endpoints` (`site_id`, `url`(191))',
'CREATE INDEX `site_api_endpoints_site_enabled_sort_idx` ON `site_api_endpoints` (`site_id`, `enabled`, `sort_order`)',
'CREATE INDEX `site_api_endpoints_site_cooldown_idx` ON `site_api_endpoints` (`site_id`, `cooldown_until`(191))',
'CREATE TABLE IF NOT EXISTS `site_disabled_models` (`id` INT AUTO_INCREMENT PRIMARY KEY, `site_id` INT NOT NULL, `model_name` VARCHAR(191) NOT NULL, `created_at` TEXT NULL, CONSTRAINT `site_disabled_models_site_fk` FOREIGN KEY (`site_id`) REFERENCES `sites`(`id`) ON DELETE CASCADE)',
'CREATE UNIQUE INDEX `site_disabled_models_site_model_unique` ON `site_disabled_models` (`site_id`, `model_name`(191))',
'CREATE INDEX `site_disabled_models_site_id_idx` ON `site_disabled_models` (`site_id`)',
src/server/db/siteSchemaCompatibility.ts
+25
View File
@@ -73,6 +73,31 @@ export const SITE_COLUMN_COMPATIBILITY_SPECS: SiteColumnCompatibilitySpec[] = [
];
export const SITE_TABLE_COMPATIBILITY_SPECS: SiteTableCompatibilitySpec[] = [
{
table: 'site_api_endpoints',
createSql: {
sqlite: 'CREATE TABLE IF NOT EXISTS site_api_endpoints (id integer PRIMARY KEY AUTOINCREMENT NOT NULL, site_id integer NOT NULL REFERENCES sites(id) ON DELETE cascade, url text NOT NULL, enabled integer DEFAULT 1, sort_order integer DEFAULT 0, cooldown_until text, last_selected_at text, last_failed_at text, last_failure_reason text, created_at text DEFAULT (datetime(\'now\')), updated_at text DEFAULT (datetime(\'now\')));',
mysql: 'CREATE TABLE IF NOT EXISTS `site_api_endpoints` (`id` INT AUTO_INCREMENT PRIMARY KEY, `site_id` INT NOT NULL, `url` TEXT NOT NULL, `enabled` BOOLEAN DEFAULT TRUE, `sort_order` INT DEFAULT 0, `cooldown_until` VARCHAR(191) NULL, `last_selected_at` VARCHAR(191) NULL, `last_failed_at` VARCHAR(191) NULL, `last_failure_reason` TEXT NULL, `created_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), \'%Y-%m-%d %H:%i:%s\')), `updated_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), \'%Y-%m-%d %H:%i:%s\')), CONSTRAINT `site_api_endpoints_site_fk` FOREIGN KEY (`site_id`) REFERENCES `sites`(`id`) ON DELETE CASCADE)',
postgres: 'CREATE TABLE IF NOT EXISTS "site_api_endpoints" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, "site_id" INTEGER NOT NULL REFERENCES "sites"("id") ON DELETE CASCADE, "url" TEXT NOT NULL, "enabled" BOOLEAN DEFAULT TRUE, "sort_order" INTEGER DEFAULT 0, "cooldown_until" TEXT, "last_selected_at" TEXT, "last_failed_at" TEXT, "last_failure_reason" TEXT, "created_at" TEXT DEFAULT to_char(timezone(\'UTC\', CURRENT_TIMESTAMP), \'YYYY-MM-DD HH24:MI:SS\'), "updated_at" TEXT DEFAULT to_char(timezone(\'UTC\', CURRENT_TIMESTAMP), \'YYYY-MM-DD HH24:MI:SS\'))',
},
postCreateSql: {
sqlite: [
'CREATE UNIQUE INDEX IF NOT EXISTS site_api_endpoints_site_url_unique ON site_api_endpoints (site_id, url);',
'CREATE INDEX IF NOT EXISTS site_api_endpoints_site_enabled_sort_idx ON site_api_endpoints (site_id, enabled, sort_order);',
'CREATE INDEX IF NOT EXISTS site_api_endpoints_site_cooldown_idx ON site_api_endpoints (site_id, cooldown_until);',
],
mysql: [
'CREATE UNIQUE INDEX `site_api_endpoints_site_url_unique` ON `site_api_endpoints` (`site_id`, `url`(191))',
'CREATE INDEX `site_api_endpoints_site_enabled_sort_idx` ON `site_api_endpoints` (`site_id`, `enabled`, `sort_order`)',
'CREATE INDEX `site_api_endpoints_site_cooldown_idx` ON `site_api_endpoints` (`site_id`, `cooldown_until`(191))',
],
postgres: [
'CREATE UNIQUE INDEX IF NOT EXISTS "site_api_endpoints_site_url_unique" ON "site_api_endpoints" ("site_id", "url")',
'CREATE INDEX IF NOT EXISTS "site_api_endpoints_site_enabled_sort_idx" ON "site_api_endpoints" ("site_id", "enabled", "sort_order")',
'CREATE INDEX IF NOT EXISTS "site_api_endpoints_site_cooldown_idx" ON "site_api_endpoints" ("site_id", "cooldown_until")',
],
},
},
{
table: 'site_disabled_models',
createSql: {
src/server/proxy-core/surfaces/chatSurface.ts
+298 -253
View File
@@ -60,6 +60,7 @@ import {
selectSurfaceChannelForAttempt,
trySurfaceOauthRefreshRecovery,
} from './sharedSurface.js';
import { runWithSiteApiEndpointPool, SiteApiEndpointRequestError } from '../../services/siteApiEndpointService.js';
import {
buildSurfaceProxyDebugResponseHeaders,
captureSurfaceProxyDebugSuccessResponseBody,
@@ -316,67 +317,145 @@ export async function handleChatSurfaceRequest(
downstreamHeaders: request.headers as Record<string, unknown>,
})
);
const buildEndpointRequest = (
endpoint: 'chat' | 'messages' | 'responses',
options: { forceNormalizeClaudeBody?: boolean } = {},
) => {
const upstreamStream = isStream || (isCodexSite && endpoint === 'responses');
const endpointRequest = buildUpstreamEndpointRequest({
endpoint,
modelName,
stream: upstreamStream,
tokenValue: selected.tokenValue,
oauthProvider: oauth?.provider,
oauthProjectId: oauth?.projectId,
sitePlatform: selected.site.platform,
siteUrl: selected.site.url,
openaiBody: resolvedOpenAiBody,
downstreamFormat,
claudeOriginalBody,
forceNormalizeClaudeBody: options.forceNormalizeClaudeBody,
downstreamHeaders: request.headers as Record<string, unknown>,
providerHeaders: buildProviderHeaders(),
codexSessionCacheKey,
});
return {
endpoint,
path: endpointRequest.path,
headers: endpointRequest.headers,
body: endpointRequest.body as Record<string, unknown>,
runtime: endpointRequest.runtime,
};
};
const dispatchRequest = createSurfaceDispatchRequest({
site: selected.site,
accountExtraConfig: selected.account.extraConfig,
});
const endpointStrategy = downstreamTransformer.compatibility.createEndpointStrategy({
downstreamFormat,
endpointCandidates,
modelName,
requestedModelHint: requestedModel,
sitePlatform: selected.site.platform,
isStream: isStream || isCodexSite,
buildRequest: ({ endpoint, forceNormalizeClaudeBody }) => buildEndpointRequest(
endpoint,
{ forceNormalizeClaudeBody },
),
dispatchRequest,
});
const tryRecover = async (ctx: Parameters<NonNullable<typeof endpointStrategy.tryRecover>>[0]) => {
if ((ctx.response.status === 401 || ctx.response.status === 403) && oauth) {
const recovered = await trySurfaceOauthRefreshRecovery({
ctx,
selected,
siteUrl: selected.site.url,
buildRequest: (endpoint) => buildEndpointRequest(endpoint),
dispatchRequest,
const executeEndpointResultForSiteApiBaseUrl = async (siteApiBaseUrl: string) => {
const buildEndpointRequest = (
endpoint: 'chat' | 'messages' | 'responses',
options: { forceNormalizeClaudeBody?: boolean } = {},
) => {
const upstreamStream = isStream || (isCodexSite && endpoint === 'responses');
const endpointRequest = buildUpstreamEndpointRequest({
endpoint,
modelName,
stream: upstreamStream,
tokenValue: selected.tokenValue,
oauthProvider: oauth?.provider,
oauthProjectId: oauth?.projectId,
sitePlatform: selected.site.platform,
siteUrl: siteApiBaseUrl,
openaiBody: resolvedOpenAiBody,
downstreamFormat,
claudeOriginalBody,
forceNormalizeClaudeBody: options.forceNormalizeClaudeBody,
downstreamHeaders: request.headers as Record<string, unknown>,
providerHeaders: buildProviderHeaders(),
codexSessionCacheKey,
});
if (recovered?.upstream?.ok) {
return recovered;
return {
endpoint,
path: endpointRequest.path,
headers: endpointRequest.headers,
body: endpointRequest.body as Record<string, unknown>,
runtime: endpointRequest.runtime,
};
};
const dispatchRequest = createSurfaceDispatchRequest({
site: selected.site,
siteUrl: siteApiBaseUrl,
accountExtraConfig: selected.account.extraConfig,
});
const endpointStrategy = downstreamTransformer.compatibility.createEndpointStrategy({
downstreamFormat,
endpointCandidates,
modelName,
requestedModelHint: requestedModel,
sitePlatform: selected.site.platform,
isStream: isStream || isCodexSite,
buildRequest: ({ endpoint, forceNormalizeClaudeBody }) => buildEndpointRequest(
endpoint,
{ forceNormalizeClaudeBody },
),
dispatchRequest,
});
const tryRecover = async (ctx: Parameters<NonNullable<typeof endpointStrategy.tryRecover>>[0]) => {
if ((ctx.response.status === 401 || ctx.response.status === 403) && oauth) {
const recovered = await trySurfaceOauthRefreshRecovery({
ctx,
selected,
siteUrl: siteApiBaseUrl,
buildRequest: (endpoint) => buildEndpointRequest(endpoint),
dispatchRequest,
});
if (recovered?.upstream?.ok) {
return recovered;
}
}
}
return endpointStrategy.tryRecover(ctx);
return endpointStrategy.tryRecover(ctx);
};
const debugAttemptBase = reserveSurfaceProxyDebugAttemptBase(debugTrace, endpointCandidates.length);
return executeEndpointFlow({
siteUrl: siteApiBaseUrl,
disableCrossProtocolFallback: config.disableCrossProtocolFallback,
endpointCandidates,
buildRequest: (endpoint) => buildEndpointRequest(endpoint),
dispatchRequest,
tryRecover,
onAttemptFailure: async (ctx) => {
const memoryWrite = recordUpstreamEndpointFailure({
...endpointRuntimeContext,
endpoint: ctx.request.endpoint,
status: ctx.response.status,
errorText: ctx.rawErrText,
});
await safeInsertSurfaceProxyDebugAttempt(debugTrace, {
attemptIndex: debugAttemptBase + ctx.endpointIndex,
endpoint: ctx.request.endpoint,
requestPath: ctx.request.path,
targetUrl: ctx.targetUrl,
runtimeExecutor: ctx.request.runtime?.executor || 'default',
requestHeaders: ctx.request.headers,
requestBody: ctx.request.body,
responseStatus: ctx.response.status,
responseHeaders: buildSurfaceProxyDebugResponseHeaders(ctx.response),
responseBody: parseSurfaceProxyDebugTextPayload(ctx.rawErrText),
rawErrorText: ctx.rawErrText,
recoverApplied: ctx.recoverApplied === true,
downgradeDecision: false,
downgradeReason: null,
memoryWrite,
});
},
onAttemptSuccess: async (ctx) => {
const memoryWrite = recordUpstreamEndpointSuccess({
...endpointRuntimeContext,
endpoint: ctx.request.endpoint,
});
const responseBody = await captureSurfaceProxyDebugSuccessResponseBody(debugTrace, ctx);
await safeInsertSurfaceProxyDebugAttempt(debugTrace, {
attemptIndex: debugAttemptBase + ctx.endpointIndex,
endpoint: ctx.request.endpoint,
requestPath: ctx.request.path,
targetUrl: ctx.targetUrl,
runtimeExecutor: ctx.request.runtime?.executor || 'default',
requestHeaders: ctx.request.headers,
requestBody: ctx.request.body,
responseStatus: ctx.response.status,
responseHeaders: buildSurfaceProxyDebugResponseHeaders(ctx.response),
responseBody,
rawErrorText: null,
recoverApplied: ctx.recoverApplied === true,
downgradeDecision: false,
downgradeReason: null,
memoryWrite,
});
},
shouldDowngrade: endpointStrategy.shouldDowngrade,
onDowngrade: async (ctx) => {
await safeUpdateSurfaceProxyDebugAttempt(debugTrace, debugAttemptBase + ctx.endpointIndex, {
downgradeDecision: true,
downgradeReason: ctx.errText,
rawErrorText: ctx.rawErrText,
});
return failureToolkit.log({
selected,
modelRequested: requestedModel,
status: 'failed',
httpStatus: ctx.response.status,
latencyMs: Date.now() - startTime,
errorMessage: ctx.errText,
retryCount,
});
},
});
};
let startTime = Date.now();
const leaseResult = await acquireSurfaceChannelLease({
@@ -418,113 +497,18 @@ export async function handleChatSurfaceRequest(
const channelLease = leaseResult.lease;
try {
const debugAttemptBase = reserveSurfaceProxyDebugAttemptBase(debugTrace, endpointCandidates.length);
const endpointResult = await executeEndpointFlow({
siteUrl: selected.site.url,
disableCrossProtocolFallback: config.disableCrossProtocolFallback,
endpointCandidates,
buildRequest: (endpoint) => buildEndpointRequest(endpoint),
dispatchRequest,
tryRecover,
onAttemptFailure: async (ctx) => {
const memoryWrite = recordUpstreamEndpointFailure({
...endpointRuntimeContext,
endpoint: ctx.request.endpoint,
status: ctx.response.status,
errorText: ctx.rawErrText,
});
await safeInsertSurfaceProxyDebugAttempt(debugTrace, {
attemptIndex: debugAttemptBase + ctx.endpointIndex,
endpoint: ctx.request.endpoint,
requestPath: ctx.request.path,
targetUrl: ctx.targetUrl,
runtimeExecutor: ctx.request.runtime?.executor || 'default',
requestHeaders: ctx.request.headers,
requestBody: ctx.request.body,
responseStatus: ctx.response.status,
responseHeaders: buildSurfaceProxyDebugResponseHeaders(ctx.response),
responseBody: parseSurfaceProxyDebugTextPayload(ctx.rawErrText),
rawErrorText: ctx.rawErrText,
recoverApplied: ctx.recoverApplied === true,
downgradeDecision: false,
downgradeReason: null,
memoryWrite,
});
},
onAttemptSuccess: async (ctx) => {
const memoryWrite = recordUpstreamEndpointSuccess({
...endpointRuntimeContext,
endpoint: ctx.request.endpoint,
});
const responseBody = await captureSurfaceProxyDebugSuccessResponseBody(debugTrace, ctx);
await safeInsertSurfaceProxyDebugAttempt(debugTrace, {
attemptIndex: debugAttemptBase + ctx.endpointIndex,
endpoint: ctx.request.endpoint,
requestPath: ctx.request.path,
targetUrl: ctx.targetUrl,
runtimeExecutor: ctx.request.runtime?.executor || 'default',
requestHeaders: ctx.request.headers,
requestBody: ctx.request.body,
responseStatus: ctx.response.status,
responseHeaders: buildSurfaceProxyDebugResponseHeaders(ctx.response),
responseBody,
rawErrorText: null,
recoverApplied: ctx.recoverApplied === true,
downgradeDecision: false,
downgradeReason: null,
memoryWrite,
});
},
shouldDowngrade: endpointStrategy.shouldDowngrade,
onDowngrade: async (ctx) => {
await safeUpdateSurfaceProxyDebugAttempt(debugTrace, debugAttemptBase + ctx.endpointIndex, {
downgradeDecision: true,
downgradeReason: ctx.errText,
rawErrorText: ctx.rawErrText,
});
return failureToolkit.log({
selected,
modelRequested: requestedModel,
status: 'failed',
httpStatus: ctx.response.status,
latencyMs: Date.now() - startTime,
errorMessage: ctx.errText,
retryCount,
});
},
});
if (!endpointResult.ok) {
clearSurfaceStickyChannel({
stickySessionKey,
selected,
});
const failureOutcome = await failureToolkit.handleUpstreamFailure({
selected,
requestedModel,
modelName,
status: endpointResult.status || 502,
errText: endpointResult.errText || 'unknown error',
rawErrText: endpointResult.rawErrText,
latencyMs: Date.now() - startTime,
retryCount,
});
const terminalFailureOutcome = failureOutcome.action === 'retry'
? (canRetryChannelSelection(retryCount, forcedChannelId)
? null
: finalizeRetryAsUpstreamFailure(endpointResult.status || 502, endpointResult.errText || 'unknown error'))
: failureOutcome;
if (!terminalFailureOutcome) {
retryCount += 1;
continue;
const endpointResult = await runWithSiteApiEndpointPool(selected.site, async (target) => {
const result = await executeEndpointResultForSiteApiBaseUrl(target.baseUrl);
if (!result.ok) {
const upstreamFailure = new SiteApiEndpointRequestError(result.errText || 'unknown error', {
status: result.status || 502,
rawErrText: result.rawErrText || result.errText || 'unknown error',
}) as SiteApiEndpointRequestError & { siteApiEndpointUpstreamFailure?: boolean };
upstreamFailure.siteApiEndpointUpstreamFailure = true;
throw upstreamFailure;
}
await finalizeDebugFailure(
terminalFailureOutcome.status,
terminalFailureOutcome.payload,
null,
);
return reply.code(terminalFailureOutcome.status).send(terminalFailureOutcome.payload);
}
return result;
});
const upstream = endpointResult.upstream;
const successfulUpstreamPath = endpointResult.upstreamPath;
@@ -944,14 +928,48 @@ export async function handleChatSurfaceRequest(
});
return reply.send(downstreamResponse);
} catch (err: any) {
clearSurfaceStickyChannel({
stickySessionKey,
selected,
});
const failureOutcome = await failureToolkit.handleExecutionError({
} catch (err: any) {
clearSurfaceStickyChannel({
stickySessionKey,
selected,
});
const endpointFailureStatus = typeof err?.status === 'number' ? err.status : null;
const isSiteApiEndpointFailure = (
err instanceof SiteApiEndpointRequestError
|| err?.name === 'SiteApiEndpointRequestError'
|| err?.siteApiEndpointUpstreamFailure === true
|| (endpointFailureStatus !== null && endpointFailureStatus >= 500)
);
if (isSiteApiEndpointFailure) {
const failureOutcome = await failureToolkit.handleUpstreamFailure({
selected,
requestedModel,
modelName,
status: endpointFailureStatus || 502,
errText: err.message || 'unknown error',
rawErrText: err.rawErrText || err.message || 'unknown error',
latencyMs: Date.now() - startTime,
retryCount,
});
const terminalFailureOutcome = failureOutcome.action === 'retry'
? (canRetryChannelSelection(retryCount, forcedChannelId)
? null
: finalizeRetryAsUpstreamFailure(endpointFailureStatus || 502, err.message || 'unknown error'))
: failureOutcome;
if (!terminalFailureOutcome) {
retryCount += 1;
continue;
}
await finalizeDebugFailure(
terminalFailureOutcome.status,
terminalFailureOutcome.payload,
null,
);
return reply.code(terminalFailureOutcome.status).send(terminalFailureOutcome.payload);
}
const failureOutcome = await failureToolkit.handleExecutionError({
selected,
requestedModel,
modelName,
errorMessage: err?.message || 'network failure',
latencyMs: Date.now() - startTime,
@@ -1229,92 +1247,89 @@ export async function handleClaudeCountTokensSurfaceRequest(
};
try {
let upstreamRequest = buildRequest();
const dispatchRequest = createSurfaceDispatchRequest({
site: selected.site,
accountExtraConfig: selected.account.extraConfig,
});
let upstream = await dispatchRequest(upstreamRequest);
let recoverApplied = false;
const countTokensResult = await runWithSiteApiEndpointPool(selected.site, async (target) => {
let upstreamRequest = buildRequest();
const dispatchRequest = createSurfaceDispatchRequest({
site: selected.site,
siteUrl: target.baseUrl,
accountExtraConfig: selected.account.extraConfig,
});
let upstream = await dispatchRequest(upstreamRequest);
let recoverApplied = false;
if ((upstream.status === 401 || upstream.status === 403) && oauth) {
const recoverContext = {
request: upstreamRequest,
response: upstream,
rawErrText: '',
if ((upstream.status === 401 || upstream.status === 403) && oauth) {
const recoverContext = {
request: upstreamRequest,
response: upstream,
rawErrText: '',
};
const recovered = await trySurfaceOauthRefreshRecovery({
ctx: recoverContext,
selected,
siteUrl: target.baseUrl,
buildRequest: () => buildRequest(),
dispatchRequest,
captureFailureBody: false,
});
if (recovered?.upstream?.ok) {
upstreamRequest = buildRequest();
upstream = recovered.upstream;
recoverApplied = true;
} else {
upstreamRequest = recoverContext.request;
upstream = recoverContext.response;
}
}
const latency = Date.now() - startTime;
const contentType = upstream.headers.get('content-type') || 'application/json';
const text = await readRuntimeResponseText(upstream);
let payload: unknown = text;
try {
payload = JSON.parse(text);
} catch {
payload = text;
}
await safeInsertSurfaceProxyDebugAttempt(debugTrace, {
attemptIndex: retryCount,
endpoint: upstreamRequest.endpoint,
requestPath: upstreamRequest.path,
targetUrl: `${target.baseUrl}${upstreamRequest.path}`,
runtimeExecutor: upstreamRequest.runtime?.executor || 'default',
requestHeaders: upstreamRequest.headers,
requestBody: upstreamRequest.body,
responseStatus: upstream.status,
responseHeaders: buildSurfaceProxyDebugResponseHeaders(upstream),
responseBody: payload,
rawErrorText: upstream.ok ? null : text,
recoverApplied,
downgradeDecision: false,
downgradeReason: null,
memoryWrite: null,
});
if (!upstream.ok) {
const errText = typeof payload === 'string' ? payload : JSON.stringify(payload);
throw new SiteApiEndpointRequestError(errText || 'unknown error', {
status: upstream.status,
rawErrText: typeof payload === 'string' ? payload : text,
});
}
return {
upstream,
upstreamRequest,
contentType,
payload,
latency,
};
const recovered = await trySurfaceOauthRefreshRecovery({
ctx: recoverContext,
selected,
siteUrl: selected.site.url,
buildRequest: () => buildRequest(),
dispatchRequest,
captureFailureBody: false,
});
if (recovered?.upstream?.ok) {
upstreamRequest = buildRequest();
upstream = recovered.upstream;
recoverApplied = true;
} else {
upstreamRequest = recoverContext.request;
upstream = recoverContext.response;
}
}
const latency = Date.now() - startTime;
const contentType = upstream.headers.get('content-type') || 'application/json';
const text = await readRuntimeResponseText(upstream);
let payload: unknown = text;
try {
payload = JSON.parse(text);
} catch {
payload = text;
}
await safeInsertSurfaceProxyDebugAttempt(debugTrace, {
attemptIndex: retryCount,
endpoint: upstreamRequest.endpoint,
requestPath: upstreamRequest.path,
targetUrl: `${selected.site.url}${upstreamRequest.path}`,
runtimeExecutor: upstreamRequest.runtime?.executor || 'default',
requestHeaders: upstreamRequest.headers,
requestBody: upstreamRequest.body,
responseStatus: upstream.status,
responseHeaders: buildSurfaceProxyDebugResponseHeaders(upstream),
responseBody: payload,
rawErrorText: upstream.ok ? null : text,
recoverApplied,
downgradeDecision: false,
downgradeReason: null,
memoryWrite: null,
});
if (!upstream.ok) {
clearSurfaceStickyChannel({
stickySessionKey,
selected,
});
const failureOutcome = await failureToolkit.handleUpstreamFailure({
selected,
requestedModel,
modelName,
status: upstream.status,
errText: typeof payload === 'string' ? payload : JSON.stringify(payload),
rawErrText: typeof payload === 'string' ? payload : text,
latencyMs: latency,
retryCount,
});
const terminalFailureOutcome = failureOutcome.action === 'retry'
? (canRetryChannelSelection(retryCount, forcedChannelId)
? null
: finalizeRetryAsUpstreamFailure(upstream.status, typeof payload === 'string' ? payload : JSON.stringify(payload)))
: failureOutcome;
if (!terminalFailureOutcome) {
retryCount += 1;
continue;
}
await finalizeDebugFailure(terminalFailureOutcome.status, terminalFailureOutcome.payload, upstreamRequest.path);
return reply.code(terminalFailureOutcome.status).send(terminalFailureOutcome.payload);
}
const {
upstream,
upstreamRequest,
contentType,
payload,
latency,
} = countTokensResult;
tokenRouter.recordSuccess(selected.channel.id, latency, 0, modelName);
recordDownstreamCostUsage(request, 0);
@@ -1344,6 +1359,36 @@ export async function handleClaudeCountTokensSurfaceRequest(
stickySessionKey,
selected,
});
const endpointFailureStatus = typeof error?.status === 'number' ? error.status : null;
const isSiteApiEndpointFailure = (
error instanceof SiteApiEndpointRequestError
|| error?.name === 'SiteApiEndpointRequestError'
|| error?.siteApiEndpointUpstreamFailure === true
|| (endpointFailureStatus !== null && endpointFailureStatus >= 500)
);
if (isSiteApiEndpointFailure) {
const failureOutcome = await failureToolkit.handleUpstreamFailure({
selected,
requestedModel,
modelName,
status: endpointFailureStatus || 502,
errText: error.message || 'unknown error',
rawErrText: error.rawErrText || error.message || 'unknown error',
latencyMs: Date.now() - startTime,
retryCount,
});
const terminalFailureOutcome = failureOutcome.action === 'retry'
? (canRetryChannelSelection(retryCount, forcedChannelId)
? null
: finalizeRetryAsUpstreamFailure(endpointFailureStatus || 502, error.message || 'unknown error'))
: failureOutcome;
if (!terminalFailureOutcome) {
retryCount += 1;
continue;
}
await finalizeDebugFailure(terminalFailureOutcome.status, terminalFailureOutcome.payload, null);
return reply.code(terminalFailureOutcome.status).send(terminalFailureOutcome.payload);
}
const failureOutcome = await failureToolkit.handleExecutionError({
selected,
requestedModel,
src/server/proxy-core/surfaces/openAiResponsesSurface.ts
+213 -194
View File
@@ -70,6 +70,7 @@ import {
safeUpdateSurfaceProxyDebugSelection,
startSurfaceProxyDebugTrace,
} from '../../services/proxyDebugTraceRuntime.js';
import { runWithSiteApiEndpointPool, SiteApiEndpointRequestError } from '../../services/siteApiEndpointService.js';
import {
buildForcedChannelUnavailableMessage,
canRetryChannelSelection,
@@ -423,122 +424,84 @@ export async function handleOpenAiResponsesSurfaceRequest(
downstreamHeaders: request.headers as Record<string, unknown>,
})
);
const buildEndpointRequest = (endpoint: 'chat' | 'messages' | 'responses') => {
const upstreamStream = isStream || (isCodexSite && endpoint === 'responses');
const endpointRequest = buildUpstreamEndpointRequest({
endpoint,
modelName,
stream: upstreamStream,
tokenValue: selected.tokenValue,
oauthProvider: oauth?.provider,
oauthProjectId: oauth?.projectId,
sitePlatform: selected.site.platform,
siteUrl: selected.site.url,
openaiBody: openAiBody,
downstreamFormat: 'responses',
responsesOriginalBody: normalizedResponsesBody,
downstreamHeaders: request.headers as Record<string, unknown>,
providerHeaders: buildProviderHeaders(),
});
const upstreamPath = (
isCompactRequest && endpoint === 'responses'
? `${endpointRequest.path}/compact`
: endpointRequest.path
);
return {
endpoint,
path: upstreamPath,
headers: endpointRequest.headers,
body: endpointRequest.body as Record<string, unknown>,
runtime: endpointRequest.runtime,
};
};
const baseDispatchRequest = createSurfaceDispatchRequest({
site: selected.site,
accountExtraConfig: selected.account.extraConfig,
});
const dispatchRequest = (
endpointRequest: BuiltEndpointRequest,
targetUrl?: string,
) => {
if (!isCodexSite || endpointRequest.path !== '/responses') {
return baseDispatchRequest(endpointRequest, targetUrl);
}
const sessionId = getCodexSessionHeaderValue(endpointRequest.headers);
return runCodexHttpSessionTask(
sessionId,
() => baseDispatchRequest(endpointRequest, targetUrl),
);
};
const endpointStrategy = openAiResponsesTransformer.compatibility.createEndpointStrategy({
isStream: isStream || isCodexSite,
requiresNativeResponsesFileUrl,
dispatchRequest,
});
const tryRecover = async (ctx: Parameters<NonNullable<typeof endpointStrategy.tryRecover>>[0]) => {
if (oauth && shouldRefreshOauthResponsesRequest({
oauthProvider: oauth.provider,
status: ctx.response.status,
response: ctx.response,
rawErrText: ctx.rawErrText || '',
})) {
const recovered = await trySurfaceOauthRefreshRecovery({
ctx,
selected,
siteUrl: selected.site.url,
buildRequest: (endpoint) => buildEndpointRequest(endpoint),
dispatchRequest,
const executeEndpointResultForSiteApiBaseUrl = async (siteApiBaseUrl: string) => {
const buildEndpointRequest = (endpoint: 'chat' | 'messages' | 'responses') => {
const upstreamStream = isStream || (isCodexSite && endpoint === 'responses');
const endpointRequest = buildUpstreamEndpointRequest({
endpoint,
modelName,
stream: upstreamStream,
tokenValue: selected.tokenValue,
oauthProvider: oauth?.provider,
oauthProjectId: oauth?.projectId,
sitePlatform: selected.site.platform,
siteUrl: siteApiBaseUrl,
openaiBody: openAiBody,
downstreamFormat: 'responses',
responsesOriginalBody: normalizedResponsesBody,
downstreamHeaders: request.headers as Record<string, unknown>,
providerHeaders: buildProviderHeaders(),
});
if (recovered?.upstream?.ok) {
return recovered;
const upstreamPath = (
isCompactRequest && endpoint === 'responses'
? `${endpointRequest.path}/compact`
: endpointRequest.path
);
return {
endpoint,
path: upstreamPath,
headers: endpointRequest.headers,
body: endpointRequest.body as Record<string, unknown>,
runtime: endpointRequest.runtime,
};
};
const baseDispatchRequest = createSurfaceDispatchRequest({
site: selected.site,
siteUrl: siteApiBaseUrl,
accountExtraConfig: selected.account.extraConfig,
});
const dispatchRequest = (
endpointRequest: BuiltEndpointRequest,
targetUrl?: string,
) => {
if (!isCodexSite || !endpointRequest.path.startsWith('/responses')) {
return baseDispatchRequest(endpointRequest, targetUrl);
}
}
return endpointStrategy.tryRecover(ctx);
};
const sessionId = getCodexSessionHeaderValue(endpointRequest.headers);
return runCodexHttpSessionTask(
sessionId,
() => baseDispatchRequest(endpointRequest, targetUrl),
);
};
const endpointStrategy = openAiResponsesTransformer.compatibility.createEndpointStrategy({
isStream: isStream || isCodexSite,
requiresNativeResponsesFileUrl,
dispatchRequest,
});
const tryRecover = async (ctx: Parameters<NonNullable<typeof endpointStrategy.tryRecover>>[0]) => {
if (oauth && shouldRefreshOauthResponsesRequest({
oauthProvider: oauth.provider,
status: ctx.response.status,
response: ctx.response,
rawErrText: ctx.rawErrText || '',
})) {
const recovered = await trySurfaceOauthRefreshRecovery({
ctx,
selected,
siteUrl: siteApiBaseUrl,
buildRequest: (endpoint) => buildEndpointRequest(endpoint),
dispatchRequest,
});
if (recovered?.upstream?.ok) {
return recovered;
}
}
return endpointStrategy.tryRecover(ctx);
};
const startTime = Date.now();
const leaseResult = await acquireSurfaceChannelLease({
stickySessionKey,
selected,
});
if (leaseResult.status === 'timeout') {
clearSurfaceStickyChannel({
stickySessionKey,
selected,
});
const busyMessage = buildSurfaceChannelBusyMessage(leaseResult.waitMs);
await failureToolkit.log({
selected,
modelRequested: requestedModel,
status: 'failed',
httpStatus: 503,
latencyMs: leaseResult.waitMs,
errorMessage: busyMessage,
retryCount,
});
if (retryCount < maxRetries && canRetryChannelSelection(retryCount, forcedChannelId)) {
retryCount += 1;
continue;
}
await finalizeDebugFailure(503, {
error: {
message: busyMessage,
type: 'server_error',
},
});
return reply.code(503).send({
error: {
message: busyMessage,
type: 'server_error',
},
});
}
const channelLease = leaseResult.lease;
try {
const debugAttemptBase = reserveSurfaceProxyDebugAttemptBase(debugTrace, endpointCandidates.length);
const endpointResult = await executeEndpointFlow({
siteUrl: selected.site.url,
return executeEndpointFlow({
siteUrl: siteApiBaseUrl,
disableCrossProtocolFallback: config.disableCrossProtocolFallback,
endpointCandidates,
buildRequest: (endpoint) => buildEndpointRequest(endpoint),
@@ -611,77 +574,99 @@ export async function handleOpenAiResponsesSurfaceRequest(
});
},
});
if (!endpointResult.ok) {
clearSurfaceStickyChannel({
stickySessionKey,
selected,
});
const failureOutcome = await failureToolkit.handleUpstreamFailure({
selected,
requestedModel,
modelName,
status: endpointResult.status || 502,
errText: endpointResult.errText || 'unknown error',
rawErrText: endpointResult.rawErrText,
latencyMs: Date.now() - startTime,
retryCount,
});
const terminalFailureOutcome = failureOutcome.action === 'retry'
? (canRetryChannelSelection(retryCount, forcedChannelId)
? null
: finalizeRetryAsUpstreamFailure(endpointResult.status || 502, endpointResult.errText || 'unknown error'))
: failureOutcome;
if (!terminalFailureOutcome) {
retryCount += 1;
continue;
}
await finalizeDebugFailure(
terminalFailureOutcome.status,
terminalFailureOutcome.payload,
null,
);
return reply.code(terminalFailureOutcome.status).send(terminalFailureOutcome.payload);
}
const upstream = endpointResult.upstream;
const successfulUpstreamPath = endpointResult.upstreamPath;
const finalizeStreamSuccess = async (
parsedUsage: UsageSummary,
latency: number,
streamDebugBody: unknown,
upstreamUsagePresent: boolean,
) => {
try {
await recordSurfaceSuccess({
selected,
requestedModel,
modelName,
parsedUsage,
upstreamUsagePresent,
requestStartedAtMs: startTime,
latencyMs: latency,
retryCount,
upstreamPath: successfulUpstreamPath,
logSuccess: failureToolkit.log,
recordDownstreamCost: (estimatedCost) => {
recordDownstreamCostUsage(request, estimatedCost);
},
bestEffortMetrics: {
errorLabel: '[responses] post-stream bookkeeping failed:',
},
});
} catch (error) {
console.error('[responses] post-stream success logging failed:', error);
}
await finalizeDebugSuccess(
200,
successfulUpstreamPath,
buildSurfaceProxyDebugResponseHeaders(upstream),
streamDebugBody,
);
};
const startTime = Date.now();
const leaseResult = await acquireSurfaceChannelLease({
stickySessionKey,
selected,
});
if (leaseResult.status === 'timeout') {
clearSurfaceStickyChannel({
stickySessionKey,
selected,
});
const busyMessage = buildSurfaceChannelBusyMessage(leaseResult.waitMs);
await failureToolkit.log({
selected,
modelRequested: requestedModel,
status: 'failed',
httpStatus: 503,
latencyMs: leaseResult.waitMs,
errorMessage: busyMessage,
retryCount,
});
if (retryCount < maxRetries && canRetryChannelSelection(retryCount, forcedChannelId)) {
retryCount += 1;
continue;
}
await finalizeDebugFailure(503, {
error: {
message: busyMessage,
type: 'server_error',
},
});
return reply.code(503).send({
error: {
message: busyMessage,
type: 'server_error',
},
});
}
const channelLease = leaseResult.lease;
try {
const endpointResult = await runWithSiteApiEndpointPool(selected.site, async (target) => {
const result = await executeEndpointResultForSiteApiBaseUrl(target.baseUrl);
if (!result.ok) {
const upstreamFailure = new SiteApiEndpointRequestError(result.errText || 'unknown error', {
status: result.status || 502,
rawErrText: result.rawErrText || result.errText || 'unknown error',
}) as SiteApiEndpointRequestError & { siteApiEndpointUpstreamFailure?: boolean };
upstreamFailure.siteApiEndpointUpstreamFailure = true;
throw upstreamFailure;
}
return result;
});
const upstream = endpointResult.upstream;
const successfulUpstreamPath = endpointResult.upstreamPath;
const finalizeStreamSuccess = async (
parsedUsage: UsageSummary,
latency: number,
streamDebugBody: unknown,
upstreamUsagePresent: boolean,
) => {
try {
await recordSurfaceSuccess({
selected,
requestedModel,
modelName,
parsedUsage,
upstreamUsagePresent,
requestStartedAtMs: startTime,
latencyMs: latency,
retryCount,
upstreamPath: successfulUpstreamPath,
logSuccess: failureToolkit.log,
recordDownstreamCost: (estimatedCost) => {
recordDownstreamCostUsage(request, estimatedCost);
},
bestEffortMetrics: {
errorLabel: '[responses] post-stream bookkeeping failed:',
},
});
} catch (error) {
console.error('[responses] post-stream success logging failed:', error);
}
await finalizeDebugSuccess(
200,
successfulUpstreamPath,
buildSurfaceProxyDebugResponseHeaders(upstream),
streamDebugBody,
);
};
if (isStream) {
const upstreamContentType = (upstream.headers.get('content-type') || '').toLowerCase();
const startSseResponse = () => {
@@ -1126,22 +1111,56 @@ export async function handleOpenAiResponsesSurfaceRequest(
stickySessionKey,
selected,
});
const endpointFailureStatus = typeof err?.status === 'number' ? err.status : null;
const isSiteApiEndpointFailure = (
err instanceof SiteApiEndpointRequestError
|| err?.name === 'SiteApiEndpointRequestError'
|| err?.siteApiEndpointUpstreamFailure === true
|| (endpointFailureStatus !== null && endpointFailureStatus >= 500)
);
if (isSiteApiEndpointFailure) {
const failureOutcome = await failureToolkit.handleUpstreamFailure({
selected,
requestedModel,
modelName,
status: endpointFailureStatus || 502,
errText: err?.message || 'unknown error',
rawErrText: err?.rawErrText || err?.message || 'unknown error',
latencyMs: Date.now() - startTime,
retryCount,
});
const terminalFailureOutcome = failureOutcome.action === 'retry'
? (canRetryChannelSelection(retryCount, forcedChannelId)
? null
: finalizeRetryAsUpstreamFailure(endpointFailureStatus || 502, err?.message || 'unknown error'))
: failureOutcome;
if (!terminalFailureOutcome) {
retryCount += 1;
continue;
}
await finalizeDebugFailure(
terminalFailureOutcome.status,
terminalFailureOutcome.payload,
null,
);
return reply.code(terminalFailureOutcome.status).send(terminalFailureOutcome.payload);
}
const failureOutcome = await failureToolkit.handleExecutionError({
selected,
requestedModel,
modelName,
errorMessage: err?.message || 'network failure',
latencyMs: Date.now() - startTime,
retryCount,
});
const terminalFailureOutcome = failureOutcome.action === 'retry'
? (canRetryChannelSelection(retryCount, forcedChannelId)
? null
: finalizeRetryAsExecutionFailure(err?.message || 'network failure'))
: failureOutcome;
if (!terminalFailureOutcome) {
retryCount += 1;
continue;
modelName,
errorMessage: err?.message || 'network failure',
latencyMs: Date.now() - startTime,
retryCount,
});
const terminalFailureOutcome = failureOutcome.action === 'retry'
? (canRetryChannelSelection(retryCount, forcedChannelId)
? null
: finalizeRetryAsExecutionFailure(err?.message || 'network failure'))
: failureOutcome;
if (!terminalFailureOutcome) {
retryCount += 1;
continue;
}
await finalizeDebugFailure(
terminalFailureOutcome.status,
src/server/proxy-core/surfaces/sharedSurface.ts
+3 -2
View File
@@ -249,14 +249,15 @@ export async function writeSurfaceProxyLog(input: {
export function createSurfaceDispatchRequest(input: {
site: SiteProxyConfigLike & { url: string };
accountExtraConfig?: string | null;
siteUrl?: string;
}) {
const channelProxyUrl = resolveChannelProxyUrl(input.site, input.accountExtraConfig);
return (
request: BuiltEndpointRequest,
targetUrl?: string,
) => (
) => (
dispatchRuntimeRequest({
siteUrl: input.site.url,
siteUrl: input.siteUrl ?? input.site.url,
targetUrl,
request,
buildInit: (_requestUrl, requestForFetch) => withSiteRecordProxyRequestInit(input.site, {
src/server/routes/api/accounts.apiSite.test.ts
+271
View File
@@ -0,0 +1,271 @@
import Fastify, { type FastifyInstance } from 'fastify';
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
import { mkdtempSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
const verifyTokenMock = vi.fn();
const getModelsMock = vi.fn();
vi.mock('../../services/platforms/index.js', () => ({
getAdapter: () => ({
verifyToken: (...args: unknown[]) => verifyTokenMock(...args),
getModels: (...args: unknown[]) => getModelsMock(...args),
}),
}));
type DbModule = typeof import('../../db/index.js');
describe('accounts api endpoint host selection', { timeout: 15_000 }, () => {
let app: FastifyInstance;
let db: DbModule['db'];
let schema: DbModule['schema'];
let dataDir = '';
beforeAll(async () => {
dataDir = mkdtempSync(join(tmpdir(), 'metapi-accounts-api-site-'));
process.env.DATA_DIR = dataDir;
await import('../../db/migrate.js');
const dbModule = await import('../../db/index.js');
const routesModule = await import('./accounts.js');
db = dbModule.db;
schema = dbModule.schema;
app = Fastify();
await app.register(routesModule.accountsRoutes);
});
beforeEach(async () => {
verifyTokenMock.mockReset();
getModelsMock.mockReset();
await db.delete(schema.proxyLogs).run();
await db.delete(schema.checkinLogs).run();
await db.delete(schema.routeChannels).run();
await db.delete(schema.tokenRoutes).run();
await db.delete(schema.tokenModelAvailability).run();
await db.delete(schema.modelAvailability).run();
await db.delete(schema.accountTokens).run();
await db.delete(schema.accounts).run();
await db.delete(schema.siteApiEndpoints).run();
await db.delete(schema.sites).run();
});
afterAll(async () => {
await app.close();
delete process.env.DATA_DIR;
});
it('uses the configured ai endpoint for API key verification', async () => {
getModelsMock.mockResolvedValueOnce(['gpt-4o-mini']);
const site = await db.insert(schema.sites).values({
name: 'Nihao Panel',
url: 'https://nih.cc',
platform: 'new-api',
status: 'active',
}).returning().get();
await db.insert(schema.siteApiEndpoints).values({
siteId: site.id,
url: 'https://api.nih.cc',
enabled: true,
sortOrder: 0,
}).run();
const response = await app.inject({
method: 'POST',
url: '/api/accounts/verify-token',
payload: {
siteId: site.id,
accessToken: 'sk-nihao',
credentialMode: 'apikey',
},
});
expect(response.statusCode).toBe(200);
expect(response.json()).toMatchObject({
success: true,
tokenType: 'apikey',
modelCount: 1,
});
expect(getModelsMock).toHaveBeenCalledWith('https://api.nih.cc', 'sk-nihao', undefined);
expect(verifyTokenMock).not.toHaveBeenCalled();
});
it('rotates API key verification across configured ai endpoints after a retryable failure', async () => {
getModelsMock
.mockRejectedValueOnce(new Error('HTTP 502: temporary upstream failure'))
.mockResolvedValueOnce(['gpt-4o-mini']);
const site = await db.insert(schema.sites).values({
name: 'Nihao Pool',
url: 'https://nih.cc',
platform: 'new-api',
status: 'active',
}).returning().get();
await db.insert(schema.siteApiEndpoints).values([
{
siteId: site.id,
url: 'https://api-a.nih.cc',
enabled: true,
sortOrder: 0,
},
{
siteId: site.id,
url: 'https://api-b.nih.cc',
enabled: true,
sortOrder: 1,
},
]).run();
const response = await app.inject({
method: 'POST',
url: '/api/accounts/verify-token',
payload: {
siteId: site.id,
accessToken: 'sk-rotate',
credentialMode: 'apikey',
},
});
expect(response.statusCode).toBe(200);
expect(response.json()).toMatchObject({
success: true,
tokenType: 'apikey',
modelCount: 1,
});
expect(getModelsMock).toHaveBeenNthCalledWith(1, 'https://api-a.nih.cc', 'sk-rotate', undefined);
expect(getModelsMock).toHaveBeenNthCalledWith(2, 'https://api-b.nih.cc', 'sk-rotate', undefined);
const endpoints = await db.select().from(schema.siteApiEndpoints).all();
const firstEndpoint = endpoints.find((item) => item.url === 'https://api-a.nih.cc');
const secondEndpoint = endpoints.find((item) => item.url === 'https://api-b.nih.cc');
expect(firstEndpoint?.cooldownUntil).toBeTruthy();
expect(firstEndpoint?.lastFailureReason).toContain('HTTP 502');
expect(secondEndpoint?.lastSelectedAt).toBeTruthy();
});
it('keeps session verification on the panel host even when api endpoints exist', async () => {
verifyTokenMock.mockResolvedValueOnce({
tokenType: 'session',
userInfo: { username: 'nihao-user' },
balance: 12.5,
apiToken: 'sk-derived',
});
const site = await db.insert(schema.sites).values({
name: 'Nihao Panel',
url: 'https://nih.cc',
platform: 'new-api',
status: 'active',
}).returning().get();
await db.insert(schema.siteApiEndpoints).values({
siteId: site.id,
url: 'https://api.nih.cc',
enabled: true,
sortOrder: 0,
}).run();
const response = await app.inject({
method: 'POST',
url: '/api/accounts/verify-token',
payload: {
siteId: site.id,
accessToken: 'session-token',
credentialMode: 'session',
},
});
expect(response.statusCode).toBe(200);
expect(response.json()).toMatchObject({
success: true,
tokenType: 'session',
apiToken: 'sk-derived',
});
expect(verifyTokenMock).toHaveBeenCalledWith('https://nih.cc', 'session-token', undefined);
expect(getModelsMock).not.toHaveBeenCalled();
});
it('uses the configured ai endpoint when adding an API key connection', async () => {
getModelsMock.mockResolvedValueOnce(['gpt-4o-mini']);
const site = await db.insert(schema.sites).values({
name: 'Nihao Panel',
url: 'https://nih.cc',
platform: 'new-api',
status: 'active',
}).returning().get();
await db.insert(schema.siteApiEndpoints).values({
siteId: site.id,
url: 'https://api.nih.cc',
enabled: true,
sortOrder: 0,
}).run();
const response = await app.inject({
method: 'POST',
url: '/api/accounts',
payload: {
siteId: site.id,
accessToken: 'sk-nihao-create',
credentialMode: 'apikey',
},
});
expect(response.statusCode).toBe(200);
expect(response.json()).toMatchObject({
tokenType: 'apikey',
});
expect(getModelsMock).toHaveBeenCalledWith('https://api.nih.cc', 'sk-nihao-create', undefined);
});
it('rotates API key account creation across configured ai endpoints after a retryable failure', async () => {
getModelsMock
.mockRejectedValueOnce(new Error('HTTP 502: temporary upstream failure'))
.mockResolvedValueOnce(['gpt-4o-mini']);
const site = await db.insert(schema.sites).values({
name: 'Nihao Create Pool',
url: 'https://nih.cc',
platform: 'new-api',
status: 'active',
}).returning().get();
await db.insert(schema.siteApiEndpoints).values([
{
siteId: site.id,
url: 'https://api-create-a.nih.cc',
enabled: true,
sortOrder: 0,
},
{
siteId: site.id,
url: 'https://api-create-b.nih.cc',
enabled: true,
sortOrder: 1,
},
]).run();
const response = await app.inject({
method: 'POST',
url: '/api/accounts',
payload: {
siteId: site.id,
accessToken: 'sk-nihao-create-rotate',
credentialMode: 'apikey',
},
});
expect(response.statusCode).toBe(200);
expect(response.json()).toMatchObject({
tokenType: 'apikey',
});
expect(getModelsMock).toHaveBeenNthCalledWith(1, 'https://api-create-a.nih.cc', 'sk-nihao-create-rotate', undefined);
expect(getModelsMock).toHaveBeenNthCalledWith(2, 'https://api-create-b.nih.cc', 'sk-nihao-create-rotate', undefined);
});
});
src/server/routes/api/accounts.ts
+78 -22
View File
@@ -42,6 +42,7 @@ import {
parseAccountUpdatePayload,
parseAccountVerifyTokenPayload,
} from '../../contracts/accountsRoutePayloads.js';
import { requireSiteApiBaseUrl, runWithSiteApiEndpointPool } from '../../services/siteApiEndpointService.js';
type AccountWithSiteRow = {
accounts: typeof schema.accounts.$inferSelect;
@@ -314,6 +315,31 @@ function isVerificationTimeoutError(error: unknown): boolean {
return lowered.includes('timeout') || lowered.includes('timed out') || lowered.includes('abort');
}
function buildAccountVerifyTimeoutMessage(): string {
return `Token verification timed out (${Math.max(1, Math.round(ACCOUNT_VERIFY_TIMEOUT_MS / 1000))}s)`;
}
async function getModelsWithSiteApiEndpointPool(
site: typeof schema.sites.$inferSelect,
adapter: NonNullable<ReturnType<typeof getAdapter>>,
accessToken: string,
platformUserId?: number,
): Promise<string[]> {
const timeoutMessage = buildAccountVerifyTimeoutMessage();
const deadline = Date.now() + ACCOUNT_VERIFY_TIMEOUT_MS;
return runWithSiteApiEndpointPool(site, (target) => {
const remainingMs = deadline - Date.now();
if (remainingMs <= 0) {
throw new Error(timeoutMessage);
}
return withTimeout(
() => adapter.getModels(target.baseUrl, accessToken, platformUserId),
remainingMs,
timeoutMessage,
);
});
}
function resolveUserIdFailureReason(message: string, hasProvidedUserId: boolean): VerifyFailureReason {
const lowered = String(message || '').trim().toLowerCase();
if (!lowered) return null;
@@ -671,7 +697,9 @@ export async function accountsRoutes(app: FastifyInstance) {
: undefined;
const hasProvidedUserId = parsedPlatformUserId !== undefined;
const skipRawShieldDetection = normalizedPlatform === 'new-api' || normalizedPlatform === 'anyrouter';
const diagnoseVerificationFailure = async (): Promise<VerifyFailureReason> => {
const diagnoseVerificationFailure = async (
options: { useApiEndpointPool?: boolean } = {},
): Promise<VerifyFailureReason> => {
const parseFailureReason = (bodyText: string, contentType: string): VerifyFailureReason => {
const text = bodyText || '';
const ct = (contentType || '').toLowerCase();
@@ -720,21 +748,39 @@ export async function accountsRoutes(app: FastifyInstance) {
});
}
for (const headers of headerVariants) {
try {
const testRes = await fetch(
`${site.url}/api/user/self`,
withSiteRecordProxyRequestInit(site, {
headers,
signal: AbortSignal.timeout(ACCOUNT_VERIFY_DIAG_TIMEOUT_MS),
}),
);
const bodyText = await testRes.text();
const contentType = testRes.headers.get('content-type') || '';
const reason = parseFailureReason(bodyText, contentType);
if (reason) return reason;
} catch { }
const tryBaseUrl = async (baseUrl: string): Promise<VerifyFailureReason> => {
let sawNetworkError = false;
let sawResponse = false;
for (const headers of headerVariants) {
try {
const testRes = await fetch(
`${baseUrl.replace(/\/+$/, '')}/api/user/self`,
withSiteRecordProxyRequestInit(site, {
headers,
signal: AbortSignal.timeout(ACCOUNT_VERIFY_DIAG_TIMEOUT_MS),
}),
);
sawResponse = true;
const bodyText = await testRes.text();
const contentType = testRes.headers.get('content-type') || '';
const reason = parseFailureReason(bodyText, contentType);
if (reason) return reason;
} catch {
sawNetworkError = true;
}
}
if (sawNetworkError && !sawResponse) {
throw new Error(`diagnostic request timed out for ${baseUrl}`);
}
return null;
};
if (options.useApiEndpointPool) {
const diagnosticBaseUrl = await requireSiteApiBaseUrl(site);
return await tryBaseUrl(diagnosticBaseUrl);
}
return await tryBaseUrl(site.url);
} catch { }
return null;
@@ -768,7 +814,9 @@ export async function accountsRoutes(app: FastifyInstance) {
};
if (!hasProvidedUserId && (normalizedPlatform === 'new-api' || normalizedPlatform === 'anyrouter')) {
const preflightReason = await diagnoseVerificationFailure();
const preflightReason = await diagnoseVerificationFailure({
useApiEndpointPool: credentialMode === 'apikey',
});
if (preflightReason === 'needs-user-id') {
return buildVerificationFailureResponse(preflightReason);
}
@@ -776,10 +824,11 @@ export async function accountsRoutes(app: FastifyInstance) {
if (credentialMode === 'apikey') {
try {
const models = await withTimeout(
() => adapter.getModels(site.url, accessToken, parsedPlatformUserId),
ACCOUNT_VERIFY_TIMEOUT_MS,
`Token verification timed out (${Math.max(1, Math.round(ACCOUNT_VERIFY_TIMEOUT_MS / 1000))}s)`,
const models = await getModelsWithSiteApiEndpointPool(
site,
adapter,
accessToken,
parsedPlatformUserId,
);
const availableModels = Array.isArray(models) ? models.filter((item) => typeof item === 'string' && item.trim().length > 0) : [];
if (availableModels.length === 0) {
@@ -796,7 +845,9 @@ export async function accountsRoutes(app: FastifyInstance) {
};
} catch (err: any) {
if (isVerificationTimeoutError(err)) {
const failure = buildVerificationFailureResponse(await diagnoseVerificationFailure());
const failure = buildVerificationFailureResponse(await diagnoseVerificationFailure({
useApiEndpointPool: true,
}));
if (failure) return failure;
}
return {
@@ -1114,7 +1165,12 @@ export async function accountsRoutes(app: FastifyInstance) {
if (!apiToken) apiToken = rawAccessToken;
} else {
try {
const models = await adapter.getModels(site.url, rawAccessToken, body.platformUserId);
const models = await getModelsWithSiteApiEndpointPool(
site,
adapter,
rawAccessToken,
body.platformUserId,
);
verifiedModels = Array.isArray(models)
? models.filter((item) => typeof item === 'string' && item.trim().length > 0)
: [];
src/server/routes/api/sites.api-endpoints.test.ts
+226
View File
@@ -0,0 +1,226 @@
import Fastify, { type FastifyInstance } from 'fastify';
import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest';
import { mkdtempSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { asc, eq } from 'drizzle-orm';
type DbModule = typeof import('../../db/index.js');
describe('sites api endpoints', () => {
let app: FastifyInstance;
let db: DbModule['db'];
let schema: DbModule['schema'];
let dataDir = '';
beforeAll(async () => {
dataDir = mkdtempSync(join(tmpdir(), 'metapi-sites-api-endpoints-'));
process.env.DATA_DIR = dataDir;
await import('../../db/migrate.js');
const dbModule = await import('../../db/index.js');
const routesModule = await import('./sites.js');
db = dbModule.db;
schema = dbModule.schema;
app = Fastify();
await app.register(routesModule.sitesRoutes);
});
beforeEach(async () => {
await db.delete(schema.accounts).run();
await db.delete(schema.siteApiEndpoints).run();
await db.delete(schema.sites).run();
});
afterAll(async () => {
await app.close();
delete process.env.DATA_DIR;
});
it('accepts apiEndpoints on create and returns them in stable sort order with server-owned metadata', async () => {
const response = await app.inject({
method: 'POST',
url: '/api/sites',
payload: {
name: 'split-host-site',
url: 'https://panel.example.com',
platform: 'new-api',
apiEndpoints: [
{
url: 'https://api-b.example.com',
enabled: false,
sortOrder: 1,
cooldownUntil: '2099-01-01T00:00:00.000Z',
lastFailureReason: 'should-be-ignored',
},
{
url: 'https://api-a.example.com/',
enabled: true,
sortOrder: 0,
lastSelectedAt: '2099-01-01T00:00:00.000Z',
},
],
},
});
expect(response.statusCode).toBe(200);
const payload = response.json() as {
id: number;
apiEndpoints?: Array<Record<string, unknown>>;
};
expect(payload.apiEndpoints).toEqual([
expect.objectContaining({
url: 'https://api-a.example.com',
enabled: true,
sortOrder: 0,
cooldownUntil: null,
lastSelectedAt: null,
lastFailedAt: null,
lastFailureReason: null,
}),
expect.objectContaining({
url: 'https://api-b.example.com',
enabled: false,
sortOrder: 1,
cooldownUntil: null,
lastSelectedAt: null,
lastFailedAt: null,
lastFailureReason: null,
}),
]);
const stored = await db.select().from(schema.siteApiEndpoints)
.where(eq(schema.siteApiEndpoints.siteId, payload.id))
.orderBy(asc(schema.siteApiEndpoints.sortOrder), asc(schema.siteApiEndpoints.id))
.all();
expect(stored.map((row) => row.url)).toEqual([
'https://api-a.example.com',
'https://api-b.example.com',
]);
});
it('replaces apiEndpoints on update', async () => {
const created = await app.inject({
method: 'POST',
url: '/api/sites',
payload: {
name: 'replace-site',
url: 'https://panel.example.com',
platform: 'new-api',
apiEndpoints: [
{ url: 'https://api-old.example.com', enabled: true, sortOrder: 0 },
],
},
});
expect(created.statusCode).toBe(200);
const site = created.json() as { id: number };
const updated = await app.inject({
method: 'PUT',
url: `/api/sites/${site.id}`,
payload: {
apiEndpoints: [
{ url: 'https://api-b.example.com', enabled: true, sortOrder: 1 },
{ url: 'https://api-a.example.com', enabled: false, sortOrder: 0 },
],
},
});
expect(updated.statusCode).toBe(200);
const payload = updated.json() as { apiEndpoints?: Array<Record<string, unknown>> };
expect(payload.apiEndpoints).toEqual([
expect.objectContaining({
url: 'https://api-a.example.com',
enabled: false,
sortOrder: 0,
}),
expect.objectContaining({
url: 'https://api-b.example.com',
enabled: true,
sortOrder: 1,
}),
]);
const stored = await db.select().from(schema.siteApiEndpoints)
.where(eq(schema.siteApiEndpoints.siteId, site.id))
.orderBy(asc(schema.siteApiEndpoints.sortOrder), asc(schema.siteApiEndpoints.id))
.all();
expect(stored.map((row) => row.url)).toEqual([
'https://api-a.example.com',
'https://api-b.example.com',
]);
});
it('rejects duplicate api endpoint urls under the same site after normalization', async () => {
const response = await app.inject({
method: 'POST',
url: '/api/sites',
payload: {
name: 'duplicate-endpoints-site',
url: 'https://panel.example.com',
platform: 'new-api',
apiEndpoints: [
{ url: 'https://api.example.com/', enabled: true, sortOrder: 0 },
{ url: 'https://api.example.com', enabled: false, sortOrder: 1 },
],
},
});
expect(response.statusCode).toBe(400);
expect((response.json() as { error?: string }).error).toContain('Duplicate apiEndpoints url');
});
it('includes apiEndpoints in GET /api/sites using stable endpoint order', async () => {
const site = await db.insert(schema.sites).values({
name: 'listed-site',
url: 'https://panel.example.com',
platform: 'new-api',
status: 'active',
}).returning().get();
await db.insert(schema.siteApiEndpoints).values([
{
siteId: site.id,
url: 'https://api-b.example.com',
enabled: true,
sortOrder: 1,
cooldownUntil: '2026-03-31T12:05:00.000Z',
lastFailedAt: '2026-03-31T12:00:00.000Z',
lastFailureReason: 'HTTP 502',
},
{
siteId: site.id,
url: 'https://api-a.example.com',
enabled: false,
sortOrder: 0,
},
]).run();
const response = await app.inject({
method: 'GET',
url: '/api/sites',
});
expect(response.statusCode).toBe(200);
const payload = response.json() as Array<{
id: number;
apiEndpoints?: Array<Record<string, unknown>>;
}>;
const listed = payload.find((row) => row.id === site.id);
expect(listed?.apiEndpoints).toEqual([
expect.objectContaining({
url: 'https://api-a.example.com',
enabled: false,
sortOrder: 0,
}),
expect.objectContaining({
url: 'https://api-b.example.com',
enabled: true,
sortOrder: 1,
cooldownUntil: '2026-03-31T12:05:00.000Z',
lastFailureReason: 'HTTP 502',
}),
]);
});
});
src/server/routes/api/sites.ts
+219 -16
View File
@@ -1,7 +1,7 @@
import { FastifyInstance, FastifyReply } from 'fastify';
import { db, schema } from '../../db/index.js';
import { insertAndGetById } from '../../db/insertHelpers.js';
import { and, eq } from 'drizzle-orm';
import { getInsertedRowId } from '../../db/insertHelpers.js';
import { and, asc, eq, inArray } from 'drizzle-orm';
import { detectSite } from '../../services/siteDetector.js';
import { invalidateSiteProxyCache, normalizeSiteUrl, parseSiteProxyUrlInput } from '../../services/siteProxy.js';
import { formatUtcSqlDateTime } from '../../services/localTimeService.js';
@@ -16,6 +16,7 @@ import {
parseSiteUpdatePayload,
} from '../../contracts/siteRoutePayloads.js';
import { getSiteInitializationPreset } from '../../../shared/siteInitializationPresets.js';
import { normalizeSiteApiEndpointBaseUrl } from '../../services/siteApiEndpointService.js';
function normalizeSiteStatus(input: unknown): 'active' | 'disabled' | null {
if (input === undefined || input === null) return null;
@@ -103,6 +104,165 @@ function normalizeSitePlatform(value: string | undefined): string | null {
return normalized || null;
}
type SiteApiEndpointInputRow = {
url: string;
enabled: boolean;
sortOrder: number;
};
function normalizeSiteApiEndpointBoolean(input: unknown): boolean | null {
return normalizePinnedFlag(input);
}
function normalizeSiteApiEndpointsInput(input: unknown): {
valid: boolean;
present: boolean;
apiEndpoints: SiteApiEndpointInputRow[];
error?: string;
} {
if (input === undefined) {
return { valid: true, present: false, apiEndpoints: [] };
}
if (input === null) {
return { valid: true, present: true, apiEndpoints: [] };
}
if (!Array.isArray(input)) {
return {
valid: false,
present: true,
apiEndpoints: [],
error: 'Invalid apiEndpoints. Expected an array.',
};
}
const seenUrls = new Set<string>();
const apiEndpoints: SiteApiEndpointInputRow[] = [];
for (let index = 0; index < input.length; index += 1) {
const row = input[index];
if (!row || typeof row !== 'object') {
return {
valid: false,
present: true,
apiEndpoints: [],
error: 'Invalid apiEndpoints item. Expected an object.',
};
}
const rawUrl = typeof (row as { url?: unknown }).url === 'string'
? (row as { url: string }).url
: '';
const normalizedUrl = normalizeSiteApiEndpointBaseUrl(rawUrl);
if (!normalizedUrl) {
return {
valid: false,
present: true,
apiEndpoints: [],
error: 'Invalid apiEndpoints url. Expected a valid http(s) URL.',
};
}
let parsedUrl: URL;
try {
parsedUrl = new URL(normalizedUrl);
} catch {
return {
valid: false,
present: true,
apiEndpoints: [],
error: 'Invalid apiEndpoints url. Expected a valid http(s) URL.',
};
}
if (parsedUrl.protocol !== 'http:' && parsedUrl.protocol !== 'https:') {
return {
valid: false,
present: true,
apiEndpoints: [],
error: 'Invalid apiEndpoints url. Expected a valid http(s) URL.',
};
}
if (seenUrls.has(normalizedUrl)) {
return {
valid: false,
present: true,
apiEndpoints: [],
error: `Duplicate apiEndpoints url: ${normalizedUrl}`,
};
}
seenUrls.add(normalizedUrl);
const normalizedEnabled = normalizeSiteApiEndpointBoolean((row as { enabled?: unknown }).enabled);
if ((row as { enabled?: unknown }).enabled !== undefined && normalizedEnabled === null) {
return {
valid: false,
present: true,
apiEndpoints: [],
error: 'Invalid apiEndpoints enabled value. Expected boolean.',
};
}
const normalizedSortOrder = normalizeSortOrder((row as { sortOrder?: unknown }).sortOrder);
if ((row as { sortOrder?: unknown }).sortOrder !== undefined && normalizedSortOrder === null) {
return {
valid: false,
present: true,
apiEndpoints: [],
error: 'Invalid apiEndpoints sortOrder value. Expected non-negative integer.',
};
}
apiEndpoints.push({
url: normalizedUrl,
enabled: normalizedEnabled ?? true,
sortOrder: normalizedSortOrder ?? index,
});
}
return { valid: true, present: true, apiEndpoints };
}
async function loadSiteApiEndpointsBySiteIds(siteIds: number[]) {
if (siteIds.length === 0) {
return new Map<number, Array<typeof schema.siteApiEndpoints.$inferSelect>>();
}
const rows = await db.select().from(schema.siteApiEndpoints)
.where(inArray(schema.siteApiEndpoints.siteId, siteIds))
.orderBy(
asc(schema.siteApiEndpoints.siteId),
asc(schema.siteApiEndpoints.sortOrder),
asc(schema.siteApiEndpoints.id),
)
.all();
const bySiteId = new Map<number, Array<typeof schema.siteApiEndpoints.$inferSelect>>();
for (const row of rows) {
const current = bySiteId.get(row.siteId) || [];
current.push({
...row,
url: normalizeSiteApiEndpointBaseUrl(row.url),
});
bySiteId.set(row.siteId, current);
}
return bySiteId;
}
async function attachSiteApiEndpoints<T extends { id: number }>(siteRows: T[]) {
const bySiteId = await loadSiteApiEndpointsBySiteIds(siteRows.map((row) => row.id));
return siteRows.map((row) => ({
...row,
apiEndpoints: bySiteId.get(row.id) || [],
}));
}
async function loadSiteWithApiEndpoints(siteId: number) {
const site = await db.select().from(schema.sites).where(eq(schema.sites.id, siteId)).get();
if (!site) return null;
const [hydrated] = await attachSiteApiEndpoints([site]);
return hydrated || null;
}
function getErrorChain(error: unknown): ErrorLike[] {
const chain: ErrorLike[] = [];
const seen = new Set<unknown>();
@@ -272,6 +432,7 @@ export async function sitesRoutes(app: FastifyInstance) {
// List all sites
app.get('/api/sites', async () => {
const siteRows = await db.select().from(schema.sites).all();
const siteRowsWithApiEndpoints = await attachSiteApiEndpoints(siteRows);
const accountRows = await db.select({
siteId: schema.accounts.siteId,
balance: schema.accounts.balance,
@@ -285,7 +446,7 @@ export async function sitesRoutes(app: FastifyInstance) {
subscriptionBySiteId[row.siteId] = aggregateSiteSubscription(subscriptionBySiteId[row.siteId], row.extraConfig);
}
return siteRows.map((site) => ({
return siteRowsWithApiEndpoints.map((site) => ({
...site,
totalBalance: Math.round((totalBalanceBySiteId[site.id] || 0) * 1_000_000) / 1_000_000,
subscriptionSummary: subscriptionBySiteId[site.id] || null,
@@ -298,7 +459,7 @@ export async function sitesRoutes(app: FastifyInstance) {
if (!parsedBody.success) {
return reply.code(400).send({ error: parsedBody.error });
}
const createBody = parsedBody.data as typeof parsedBody.data & { apiEndpoints?: unknown };
const {
name,
url,
@@ -312,7 +473,8 @@ export async function sitesRoutes(app: FastifyInstance) {
isPinned,
sortOrder,
globalWeight,
} = parsedBody.data;
apiEndpoints,
} = createBody;
const normalizedStatus = normalizeSiteStatus(status);
if (status !== undefined && !normalizedStatus) {
return reply.code(400).send({ error: 'Invalid site status. Expected active or disabled.' });
@@ -351,6 +513,10 @@ export async function sitesRoutes(app: FastifyInstance) {
if (initializationPresetId != null && initializationPresetId !== '' && !explicitInitializationPreset) {
return reply.code(400).send({ error: 'Invalid initializationPresetId.' });
}
const normalizedApiEndpoints = normalizeSiteApiEndpointsInput(apiEndpoints);
if (!normalizedApiEndpoints.valid) {
return reply.code(400).send({ error: normalizedApiEndpoints.error || 'Invalid apiEndpoints.' });
}
const existingSites = await db.select().from(schema.sites).all();
const maxSortOrder = existingSites.reduce((max, site) => Math.max(max, site.sortOrder || 0), -1);
@@ -380,10 +546,8 @@ export async function sitesRoutes(app: FastifyInstance) {
let inserted;
try {
inserted = await insertAndGetById<typeof schema.sites.$inferSelect>({
table: schema.sites,
idColumn: schema.sites.id,
values: {
inserted = await db.transaction(async (tx) => {
const siteInsert = await tx.insert(schema.sites).values({
name,
url: canonicalUrl,
platform: detectedPlatform,
@@ -395,9 +559,19 @@ export async function sitesRoutes(app: FastifyInstance) {
isPinned: normalizedPinned ?? false,
sortOrder: normalizedSortOrder ?? (maxSortOrder + 1),
globalWeight: normalizedGlobalWeight ?? 1,
},
insertErrorMessage: 'Create site failed',
loadErrorMessage: 'Create site failed',
}).run();
const siteId = getInsertedRowId(siteInsert);
if (siteId && normalizedApiEndpoints.present && normalizedApiEndpoints.apiEndpoints.length > 0) {
await tx.insert(schema.siteApiEndpoints).values(
normalizedApiEndpoints.apiEndpoints.map((row) => ({
siteId,
url: row.url,
enabled: row.enabled,
sortOrder: row.sortOrder,
})),
).run();
}
return siteInsert;
});
} catch (error) {
if (isSitesPlatformUrlConflict(error)) {
@@ -405,9 +579,17 @@ export async function sitesRoutes(app: FastifyInstance) {
}
throw error;
}
const siteId = getInsertedRowId(inserted);
if (!siteId) {
return reply.code(500).send({ error: 'Create site failed' });
}
const result = await loadSiteWithApiEndpoints(siteId);
if (!result) {
return reply.code(500).send({ error: 'Create site failed' });
}
invalidateSiteCaches();
return {
...inserted,
...result,
...(responseInitializationPresetId ? { initializationPresetId: responseInitializationPresetId } : {}),
};
});
@@ -430,7 +612,7 @@ export async function sitesRoutes(app: FastifyInstance) {
}
const updates: any = {};
const body = parsedBody.data;
const body = parsedBody.data as typeof parsedBody.data & { apiEndpoints?: unknown };
const normalizedStatus = normalizeSiteStatus(body.status);
if (body.status !== undefined && !normalizedStatus) {
return reply.code(400).send({ error: 'Invalid site status. Expected active or disabled.' });
@@ -463,6 +645,10 @@ export async function sitesRoutes(app: FastifyInstance) {
if (!normalizedCustomHeaders.valid) {
return reply.code(400).send({ error: normalizedCustomHeaders.error || 'Invalid customHeaders.' });
}
const normalizedApiEndpoints = normalizeSiteApiEndpointsInput(body.apiEndpoints);
if (!normalizedApiEndpoints.valid) {
return reply.code(400).send({ error: normalizedApiEndpoints.error || 'Invalid apiEndpoints.' });
}
const canonicalPlatform = normalizeSitePlatform(body.platform);
const nextUrl = body.url !== undefined ? normalizeCanonicalSiteUrl(body.url) : existingSite.url;
@@ -498,7 +684,24 @@ export async function sitesRoutes(app: FastifyInstance) {
if (body.globalWeight !== undefined) updates.globalWeight = normalizedGlobalWeight;
updates.updatedAt = new Date().toISOString();
try {
await db.update(schema.sites).set(updates).where(eq(schema.sites.id, id)).run();
await db.transaction(async (tx) => {
await tx.update(schema.sites).set(updates).where(eq(schema.sites.id, id)).run();
if (normalizedApiEndpoints.present) {
await tx.delete(schema.siteApiEndpoints)
.where(eq(schema.siteApiEndpoints.siteId, id))
.run();
if (normalizedApiEndpoints.apiEndpoints.length > 0) {
await tx.insert(schema.siteApiEndpoints).values(
normalizedApiEndpoints.apiEndpoints.map((row) => ({
siteId: id,
url: row.url,
enabled: row.enabled,
sortOrder: row.sortOrder,
})),
).run();
}
}
});
} catch (error) {
if (isSitesPlatformUrlConflict(error)) {
return sendSiteBindingConflict(reply, nextPlatform, nextUrl);
@@ -512,7 +715,7 @@ export async function sitesRoutes(app: FastifyInstance) {
invalidateSiteCaches();
return await db.select().from(schema.sites).where(eq(schema.sites.id, id)).get();
return await loadSiteWithApiEndpoints(id);
});
// Delete a site
src/server/routes/proxy/chat.codex-oauth.test.ts
+25 -1
View File
@@ -55,6 +55,7 @@ vi.mock('../../services/modelPricingService.js', () => ({
vi.mock('../../services/proxyRetryPolicy.js', () => ({
shouldRetryProxyRequest: () => false,
RETRYABLE_TIMEOUT_PATTERNS: [/(request timed out|connection timed out|read timeout|\btimed out\b)/i],
}));
vi.mock('../../services/proxyUsageFallbackService.js', () => ({
@@ -68,12 +69,33 @@ vi.mock('../../services/oauth/refreshSingleflight.js', () => ({
vi.mock('../../db/index.js', () => ({
db: {
insert: (arg: any) => dbInsertMock(arg),
select: () => ({
from: () => ({
where: () => ({
orderBy: () => ({
all: async () => [],
}),
}),
}),
}),
update: () => ({
set: () => ({
where: () => ({
run: async () => undefined,
}),
}),
}),
},
hasProxyLogBillingDetailsColumn: async () => false,
hasProxyLogClientColumns: async () => false,
hasProxyLogDownstreamApiKeyIdColumn: async () => false,
schema: {
proxyLogs: {},
siteApiEndpoints: {
id: {},
siteId: {},
sortOrder: {},
},
},
}));
@@ -144,7 +166,9 @@ describe('chat proxy codex oauth compatibility', () => {
});
afterAll(async () => {
await app.close();
if (app) {
await app.close();
}
});
it('strips codex-unsupported responses fields and aggregates the SSE result for /v1/messages callers', async () => {
src/server/routes/proxy/chat.count-tokens.test.ts
+25 -1
View File
@@ -59,6 +59,7 @@ vi.mock('../../services/modelPricingService.js', () => ({
vi.mock('../../services/proxyRetryPolicy.js', () => ({
shouldRetryProxyRequest: () => false,
RETRYABLE_TIMEOUT_PATTERNS: [/(request timed out|connection timed out|read timeout|\btimed out\b)/i],
}));
vi.mock('../../services/proxyUsageFallbackService.js', () => ({
@@ -85,12 +86,33 @@ vi.mock('../../services/oauth/quota.js', () => ({
vi.mock('../../db/index.js', () => ({
db: {
insert: (arg: any) => dbInsertMock(arg),
select: () => ({
from: () => ({
where: () => ({
orderBy: () => ({
all: async () => [],
}),
}),
}),
}),
update: () => ({
set: () => ({
where: () => ({
run: async () => undefined,
}),
}),
}),
},
hasProxyLogBillingDetailsColumn: async () => false,
hasProxyLogClientColumns: async () => false,
hasProxyLogDownstreamApiKeyIdColumn: async () => false,
schema: {
proxyLogs: {},
siteApiEndpoints: {
id: {},
siteId: {},
sortOrder: {},
},
},
}));
@@ -147,7 +169,9 @@ describe('claude count_tokens proxy route', () => {
});
afterAll(async () => {
await app.close();
if (app) {
await app.close();
}
});
it('forwards /v1/messages/count_tokens to the claude count_tokens upstream path', async () => {
src/server/routes/proxy/chat.siteApiEndpoint.test.ts
+239
View File
@@ -0,0 +1,239 @@
import Fastify, { type FastifyInstance } from 'fastify';
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
import { mkdtempSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { asc, eq } from 'drizzle-orm';
import { config } from '../../config.js';
import { resetUpstreamEndpointRuntimeState } from '../../services/upstreamEndpointRuntimeMemory.js';
const fetchMock = vi.fn();
const selectChannelMock = vi.fn();
const selectNextChannelMock = vi.fn();
const recordSuccessMock = vi.fn();
const recordFailureMock = vi.fn();
const refreshModelsAndRebuildRoutesMock = vi.fn();
const reportProxyAllFailedMock = vi.fn();
const reportTokenExpiredMock = vi.fn();
const estimateProxyCostMock = vi.fn(async (_arg?: any) => 0);
const buildProxyBillingDetailsMock = vi.fn(async (_arg?: any) => null);
const fetchModelPricingCatalogMock = vi.fn(async (_arg?: any): Promise<any> => null);
const resolveProxyUsageWithSelfLogFallbackMock = vi.fn(async ({ usage }: any) => ({
...usage,
estimatedCostFromQuota: 0,
recoveredFromSelfLog: false,
}));
const insertProxyLogMock = vi.fn();
vi.mock('undici', () => ({
fetch: (...args: unknown[]) => fetchMock(...args),
}));
vi.mock('../../services/tokenRouter.js', () => ({
tokenRouter: {
selectChannel: (...args: unknown[]) => selectChannelMock(...args),
selectNextChannel: (...args: unknown[]) => selectNextChannelMock(...args),
recordSuccess: (...args: unknown[]) => recordSuccessMock(...args),
recordFailure: (...args: unknown[]) => recordFailureMock(...args),
},
}));
vi.mock('../../services/modelService.js', () => ({
refreshModelsAndRebuildRoutes: (...args: unknown[]) => refreshModelsAndRebuildRoutesMock(...args),
}));
vi.mock('../../services/alertService.js', () => ({
reportProxyAllFailed: (...args: unknown[]) => reportProxyAllFailedMock(...args),
reportTokenExpired: (...args: unknown[]) => reportTokenExpiredMock(...args),
}));
vi.mock('../../services/alertRules.js', () => ({
isTokenExpiredError: () => false,
}));
vi.mock('../../services/modelPricingService.js', () => ({
estimateProxyCost: (arg: any) => estimateProxyCostMock(arg),
buildProxyBillingDetails: (arg: any) => buildProxyBillingDetailsMock(arg),
fetchModelPricingCatalog: (arg: any) => fetchModelPricingCatalogMock(arg),
}));
vi.mock('../../services/proxyRetryPolicy.js', () => ({
shouldRetryProxyRequest: () => false,
RETRYABLE_TIMEOUT_PATTERNS: [/(request timed out|connection timed out|read timeout|\btimed out\b)/i],
}));
vi.mock('../../services/proxyUsageFallbackService.js', () => ({
resolveProxyUsageWithSelfLogFallback: (arg: any) => resolveProxyUsageWithSelfLogFallbackMock(arg),
}));
vi.mock('../../services/proxyLogStore.js', () => ({
insertProxyLog: (...args: unknown[]) => insertProxyLogMock(...args),
}));
type DbModule = typeof import('../../db/index.js');
describe('chat proxy site api endpoint rotation', () => {
let app: FastifyInstance;
let db: DbModule['db'];
let schema: DbModule['schema'];
let dataDir = '';
beforeAll(async () => {
dataDir = mkdtempSync(join(tmpdir(), 'metapi-chat-site-api-endpoint-'));
process.env.DATA_DIR = dataDir;
await import('../../db/migrate.js');
const dbModule = await import('../../db/index.js');
const routesModule = await import('./chat.js');
db = dbModule.db;
schema = dbModule.schema;
app = Fastify();
await app.register(routesModule.chatProxyRoute);
});
beforeEach(async () => {
fetchMock.mockReset();
selectChannelMock.mockReset();
selectNextChannelMock.mockReset();
recordSuccessMock.mockReset();
recordFailureMock.mockReset();
refreshModelsAndRebuildRoutesMock.mockReset();
reportProxyAllFailedMock.mockReset();
reportTokenExpiredMock.mockReset();
estimateProxyCostMock.mockClear();
buildProxyBillingDetailsMock.mockClear();
fetchModelPricingCatalogMock.mockReset();
resolveProxyUsageWithSelfLogFallbackMock.mockClear();
insertProxyLogMock.mockReset();
resetUpstreamEndpointRuntimeState();
await db.delete(schema.proxyLogs).run();
await db.delete(schema.routeChannels).run();
await db.delete(schema.tokenRoutes).run();
await db.delete(schema.tokenModelAvailability).run();
await db.delete(schema.modelAvailability).run();
await db.delete(schema.accountTokens).run();
await db.delete(schema.accounts).run();
await db.delete(schema.siteApiEndpoints).run();
await db.delete(schema.sites).run();
fetchModelPricingCatalogMock.mockResolvedValue(null);
(config as any).codexHeaderDefaults = {
userAgent: '',
betaFeatures: '',
};
(config as any).payloadRules = {
default: [],
defaultRaw: [],
override: [],
overrideRaw: [],
filter: [],
};
(config as any).disableCrossProtocolFallback = false;
config.proxyEmptyContentFailEnabled = false;
config.proxyErrorKeywords = [];
});
afterAll(async () => {
if (app) {
await app.close();
}
delete process.env.DATA_DIR;
});
it('rotates to the next configured ai endpoint for retryable /v1/chat/completions failures', async () => {
const site = await db.insert(schema.sites).values({
name: 'nihao-panel',
url: 'https://nih.cc',
platform: 'openai',
status: 'active',
}).returning().get();
const account = await db.insert(schema.accounts).values({
siteId: site.id,
username: 'nihao-user',
accessToken: '',
apiToken: 'sk-nihao',
status: 'active',
checkinEnabled: false,
extraConfig: JSON.stringify({ credentialMode: 'apikey' }),
}).returning().get();
await db.insert(schema.siteApiEndpoints).values([
{
siteId: site.id,
url: 'https://api-a.nih.cc',
enabled: true,
sortOrder: 0,
},
{
siteId: site.id,
url: 'https://api-b.nih.cc',
enabled: true,
sortOrder: 1,
},
]).run();
selectChannelMock.mockReturnValue({
channel: { id: 11, routeId: 22 },
site,
account,
tokenName: 'default',
tokenValue: 'sk-nihao',
actualModel: 'gpt-4o-mini',
});
selectNextChannelMock.mockReturnValue(null);
fetchMock
.mockResolvedValueOnce(new Response('bad gateway', { status: 502 }))
.mockResolvedValueOnce(new Response('bad gateway via responses', { status: 502 }))
.mockResolvedValueOnce(new Response('bad gateway via messages', { status: 502 }))
.mockResolvedValueOnce(new Response(JSON.stringify({
id: 'chatcmpl-ok',
object: 'chat.completion',
created: 1_706_000_000,
model: 'gpt-4o-mini',
choices: [{
index: 0,
message: { role: 'assistant', content: 'ok via api-b' },
finish_reason: 'stop',
}],
usage: { prompt_tokens: 1, completion_tokens: 2, total_tokens: 3 },
}), {
status: 200,
headers: { 'content-type': 'application/json' },
}));
const response = await app.inject({
method: 'POST',
url: '/v1/chat/completions',
payload: {
model: 'gpt-4o-mini',
messages: [{ role: 'user', content: 'hi' }],
},
});
expect(response.statusCode).toBe(200);
expect(response.json()?.choices?.[0]?.message?.content).toBe('ok via api-b');
expect(fetchMock).toHaveBeenCalledTimes(4);
expect(String(fetchMock.mock.calls[0]?.[0] || '')).toBe('https://api-a.nih.cc/v1/chat/completions');
expect(String(fetchMock.mock.calls[1]?.[0] || '')).toBe('https://api-a.nih.cc/v1/responses');
expect(String(fetchMock.mock.calls[2]?.[0] || '')).toBe('https://api-a.nih.cc/v1/messages');
expect(String(fetchMock.mock.calls[3]?.[0] || '')).toBe('https://api-b.nih.cc/v1/chat/completions');
expect(selectNextChannelMock).not.toHaveBeenCalled();
expect(recordFailureMock).not.toHaveBeenCalled();
expect(recordSuccessMock).toHaveBeenCalledTimes(1);
const storedEndpoints = await db.select().from(schema.siteApiEndpoints)
.where(eq(schema.siteApiEndpoints.siteId, site.id))
.orderBy(asc(schema.siteApiEndpoints.sortOrder), asc(schema.siteApiEndpoints.id))
.all();
expect(storedEndpoints[0]).toMatchObject({
url: 'https://api-a.nih.cc',
lastFailureReason: 'HTTP 502: [upstream:/v1/messages] Upstream returned HTTP 502: bad gateway via messages',
});
expect(storedEndpoints[0]?.cooldownUntil).toBeTruthy();
expect(storedEndpoints[1]?.lastSelectedAt).toBeTruthy();
});
});
src/server/routes/proxy/chat.stream.test.ts
+25 -1
View File
@@ -60,6 +60,7 @@ vi.mock('../../services/modelPricingService.js', () => ({
vi.mock('../../services/proxyRetryPolicy.js', () => ({
shouldRetryProxyRequest: () => false,
RETRYABLE_TIMEOUT_PATTERNS: [/(request timed out|connection timed out|read timeout|\btimed out\b)/i],
}));
vi.mock('../../services/proxyUsageFallbackService.js', () => ({
@@ -69,12 +70,33 @@ vi.mock('../../services/proxyUsageFallbackService.js', () => ({
vi.mock('../../db/index.js', () => ({
db: {
insert: (arg: any) => dbInsertMock(arg),
select: () => ({
from: () => ({
where: () => ({
orderBy: () => ({
all: async () => [],
}),
}),
}),
}),
update: () => ({
set: () => ({
where: () => ({
run: async () => undefined,
}),
}),
}),
},
hasProxyLogBillingDetailsColumn: async () => false,
hasProxyLogClientColumns: async () => false,
hasProxyLogDownstreamApiKeyIdColumn: async () => false,
schema: {
proxyLogs: {},
siteApiEndpoints: {
id: {},
siteId: {},
sortOrder: {},
},
},
}));
@@ -133,7 +155,9 @@ describe('chat proxy stream behavior', () => {
});
afterAll(async () => {
await app.close();
if (app) {
await app.close();
}
});
it('converts non-SSE upstream streaming responses into SSE events', async () => {
src/server/routes/proxy/completions.siteApiEndpoint.test.ts
+213
View File
@@ -0,0 +1,213 @@
import Fastify, { type FastifyInstance } from 'fastify';
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
import { mkdtempSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { asc, eq } from 'drizzle-orm';
const fetchMock = vi.fn();
const selectChannelMock = vi.fn();
const selectNextChannelMock = vi.fn();
const recordSuccessMock = vi.fn();
const recordFailureMock = vi.fn();
const refreshModelsAndRebuildRoutesMock = vi.fn();
const reportProxyAllFailedMock = vi.fn();
const reportTokenExpiredMock = vi.fn();
const insertProxyLogMock = vi.fn();
const resolveProxyUsageWithSelfLogFallbackMock = vi.fn();
const resolveProxyLogBillingMock = vi.fn();
vi.mock('undici', () => ({
fetch: (...args: unknown[]) => fetchMock(...args),
}));
vi.mock('../../services/tokenRouter.js', () => ({
tokenRouter: {
selectChannel: (...args: unknown[]) => selectChannelMock(...args),
selectNextChannel: (...args: unknown[]) => selectNextChannelMock(...args),
recordSuccess: (...args: unknown[]) => recordSuccessMock(...args),
recordFailure: (...args: unknown[]) => recordFailureMock(...args),
},
invalidateTokenRouterCache: vi.fn(),
}));
vi.mock('../../services/routeRefreshWorkflow.js', () => ({
refreshModelsAndRebuildRoutes: (...args: unknown[]) => refreshModelsAndRebuildRoutesMock(...args),
}));
vi.mock('../../services/alertService.js', () => ({
reportProxyAllFailed: (...args: unknown[]) => reportProxyAllFailedMock(...args),
reportTokenExpired: (...args: unknown[]) => reportTokenExpiredMock(...args),
}));
vi.mock('../../services/proxyLogStore.js', () => ({
insertProxyLog: (...args: unknown[]) => insertProxyLogMock(...args),
}));
vi.mock('../../services/proxyUsageFallbackService.js', () => ({
resolveProxyUsageWithSelfLogFallback: (...args: unknown[]) => resolveProxyUsageWithSelfLogFallbackMock(...args),
}));
vi.mock('./proxyBilling.js', () => ({
resolveProxyLogBilling: (...args: unknown[]) => resolveProxyLogBillingMock(...args),
}));
type DbModule = typeof import('../../db/index.js');
describe('/v1/completions site api endpoint rotation', () => {
let app: FastifyInstance;
let db: DbModule['db'];
let schema: DbModule['schema'];
let dataDir = '';
beforeAll(async () => {
dataDir = mkdtempSync(join(tmpdir(), 'metapi-completions-site-api-endpoint-'));
process.env.DATA_DIR = dataDir;
await import('../../db/migrate.js');
const dbModule = await import('../../db/index.js');
const routesModule = await import('./completions.js');
db = dbModule.db;
schema = dbModule.schema;
app = Fastify();
await app.register(routesModule.completionsProxyRoute);
});
beforeEach(async () => {
fetchMock.mockReset();
selectChannelMock.mockReset();
selectNextChannelMock.mockReset();
recordSuccessMock.mockReset();
recordFailureMock.mockReset();
refreshModelsAndRebuildRoutesMock.mockReset();
reportProxyAllFailedMock.mockReset();
reportTokenExpiredMock.mockReset();
insertProxyLogMock.mockReset();
resolveProxyUsageWithSelfLogFallbackMock.mockReset();
resolveProxyLogBillingMock.mockReset();
resolveProxyUsageWithSelfLogFallbackMock.mockResolvedValue({
promptTokens: 1,
completionTokens: 2,
totalTokens: 3,
});
resolveProxyLogBillingMock.mockResolvedValue({
estimatedCost: 0,
billingDetails: null,
});
await db.delete(schema.proxyLogs).run();
await db.delete(schema.routeChannels).run();
await db.delete(schema.tokenRoutes).run();
await db.delete(schema.tokenModelAvailability).run();
await db.delete(schema.modelAvailability).run();
await db.delete(schema.accountTokens).run();
await db.delete(schema.accounts).run();
await db.delete(schema.siteApiEndpoints).run();
await db.delete(schema.sites).run();
});
afterAll(async () => {
await app.close();
delete process.env.DATA_DIR;
});
it('cools down a retryable failed endpoint and retries the next endpoint within the same site', async () => {
const site = await db.insert(schema.sites).values({
name: 'nihao-panel',
url: 'https://nih.cc',
platform: 'new-api',
status: 'active',
}).returning().get();
const account = await db.insert(schema.accounts).values({
siteId: site.id,
username: 'nihao-user',
accessToken: '',
apiToken: 'sk-nihao',
status: 'active',
checkinEnabled: false,
extraConfig: JSON.stringify({ credentialMode: 'apikey' }),
}).returning().get();
await db.insert(schema.siteApiEndpoints).values([
{
siteId: site.id,
url: 'https://api-a.nih.cc',
enabled: true,
sortOrder: 0,
},
{
siteId: site.id,
url: 'https://api-b.nih.cc',
enabled: true,
sortOrder: 1,
},
]).run();
selectChannelMock.mockResolvedValue({
channel: { id: 11, routeId: 22 },
site,
account,
tokenName: 'default',
tokenValue: 'sk-nihao',
actualModel: 'gpt-4o-mini',
});
selectNextChannelMock.mockResolvedValue(null);
fetchMock
.mockResolvedValueOnce(new Response('bad gateway', { status: 502 }))
.mockResolvedValueOnce(new Response(JSON.stringify({
id: 'cmpl-ok',
object: 'text_completion',
choices: [{ text: 'ok' }],
usage: {
prompt_tokens: 1,
completion_tokens: 2,
total_tokens: 3,
},
}), {
status: 200,
headers: { 'content-type': 'application/json' },
}));
const response = await app.inject({
method: 'POST',
url: '/v1/completions',
headers: {
authorization: 'Bearer sk-downstream',
},
payload: {
model: 'gpt-4o-mini',
prompt: 'hello',
},
});
expect(response.statusCode).toBe(200);
expect(response.json()).toMatchObject({
id: 'cmpl-ok',
object: 'text_completion',
});
expect(fetchMock).toHaveBeenCalledTimes(2);
expect(String(fetchMock.mock.calls[0]?.[0] || '')).toBe('https://api-a.nih.cc/v1/completions');
expect(String(fetchMock.mock.calls[1]?.[0] || '')).toBe('https://api-b.nih.cc/v1/completions');
expect(selectNextChannelMock).not.toHaveBeenCalled();
expect(recordFailureMock).not.toHaveBeenCalled();
expect(recordSuccessMock).toHaveBeenCalledTimes(1);
const storedEndpoints = await db.select().from(schema.siteApiEndpoints)
.where(eq(schema.siteApiEndpoints.siteId, site.id))
.orderBy(asc(schema.siteApiEndpoints.sortOrder), asc(schema.siteApiEndpoints.id))
.all();
expect(storedEndpoints[0]).toMatchObject({
url: 'https://api-a.nih.cc',
lastFailureReason: 'HTTP 502: bad gateway',
});
expect(storedEndpoints[0]?.cooldownUntil).toBeTruthy();
expect(storedEndpoints[1]).toMatchObject({
url: 'https://api-b.nih.cc',
});
expect(storedEndpoints[1]?.lastSelectedAt).toBeTruthy();
});
});
src/server/routes/proxy/completions.ts
+47 -74
View File
@@ -10,14 +10,15 @@ import { ensureModelAllowedForDownstreamKey, getDownstreamRoutingPolicy, recordD
import { withSiteRecordProxyRequestInit } from '../../services/siteProxy.js';
import { getProxyUrlFromExtraConfig } from '../../services/accountExtraConfig.js';
import { composeProxyLogMessage } from './logPathMeta.js';
import { formatUtcSqlDateTime } from '../../services/localTimeService.js';
import { detectProxyFailure } from './proxyFailureJudge.js';
import { resolveProxyLogBilling } from './proxyBilling.js';
import { formatUtcSqlDateTime } from '../../services/localTimeService.js';
import { detectProxyFailure } from './proxyFailureJudge.js';
import { resolveProxyLogBilling } from './proxyBilling.js';
import { getProxyAuthContext } from '../../middleware/auth.js';
import { buildUpstreamUrl } from './upstreamUrl.js';
import { detectDownstreamClientContext, type DownstreamClientContext } from './downstreamClientContext.js';
import { insertProxyLog } from '../../services/proxyLogStore.js';
import { getProxyMaxChannelRetries } from '../../services/proxyChannelRetry.js';
import { runWithSiteApiEndpointPool, SiteApiEndpointRequestError } from '../../services/siteApiEndpointService.js';
import {
buildForcedChannelUnavailableMessage,
canRetryChannelSelection,
@@ -72,68 +73,30 @@ export async function completionsProxyRoute(app: FastifyInstance) {
excludeChannelIds.push(selected.channel.id);
const targetUrl = buildUpstreamUrl(selected.site.url, '/v1/completions');
const upstreamModel = selected.actualModel || requestedModel;
const forwardBody = { ...body, model: upstreamModel };
const startTime = Date.now();
try {
const upstream = await fetch(targetUrl, withSiteRecordProxyRequestInit(selected.site, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Authorization': `Bearer ${selected.tokenValue}`,
},
body: JSON.stringify(forwardBody),
}, getProxyUrlFromExtraConfig(selected.account.extraConfig)));
if (!upstream.ok) {
const errText = await upstream.text().catch(() => 'unknown error');
await recordTokenRouterEventBestEffort('record channel failure', () => tokenRouter.recordFailure(selected.channel.id, {
status: upstream.status,
errorText: errText,
modelName: upstreamModel,
}));
logProxy(
selected,
requestedModel,
'failed',
upstream.status,
Date.now() - startTime,
errText,
retryCount,
downstreamApiKeyId,
0,
0,
0,
0,
null,
clientContext,
downstreamPath,
);
if (isTokenExpiredError({ status: upstream.status, message: errText })) {
await reportTokenExpired({
accountId: selected.account.id,
username: selected.account.username,
siteName: selected.site.name,
detail: `HTTP ${upstream.status}`,
});
}
if (shouldRetryProxyRequest(upstream.status, errText) && canRetryChannelSelection(retryCount, forcedChannelId)) {
retryCount++;
continue;
try {
const upstream = await runWithSiteApiEndpointPool(selected.site, async (target) => {
const targetUrl = buildUpstreamUrl(target.baseUrl, '/v1/completions');
const response = await fetch(targetUrl, withSiteRecordProxyRequestInit(selected.site, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Authorization': `Bearer ${selected.tokenValue}`,
},
body: JSON.stringify(forwardBody),
}, getProxyUrlFromExtraConfig(selected.account.extraConfig)));
if (!response.ok) {
const errText = await response.text().catch(() => 'unknown error');
throw new SiteApiEndpointRequestError(errText || 'unknown error', {
status: response.status,
});
}
await reportProxyAllFailed({
model: requestedModel,
reason: `upstream returned HTTP ${upstream.status}`,
});
return reply.code(upstream.status).send({ error: { message: errText, type: 'upstream_error' } });
}
if (isStream) {
return response;
});
if (isStream) {
reply.raw.writeHead(200, {
'Content-Type': 'text/event-stream',
'Cache-Control': 'no-cache',
@@ -328,18 +291,20 @@ export async function completionsProxyRoute(app: FastifyInstance) {
);
return reply.send(data);
} catch (err: any) {
const status = err instanceof SiteApiEndpointRequestError ? (err.status || 0) : 0;
const errorText = err?.message || 'network failure';
await recordTokenRouterEventBestEffort('record channel failure', () => tokenRouter.recordFailure(selected.channel.id, {
status: 0,
errorText: err.message,
status,
errorText,
modelName: upstreamModel,
}));
logProxy(
selected,
requestedModel,
'failed',
0,
status,
Date.now() - startTime,
err.message,
errorText,
retryCount,
downstreamApiKeyId,
0,
@@ -350,18 +315,26 @@ export async function completionsProxyRoute(app: FastifyInstance) {
clientContext,
downstreamPath,
);
if (canRetryChannelSelection(retryCount, forcedChannelId)) {
if (status > 0 && isTokenExpiredError({ status, message: errorText })) {
await reportTokenExpired({
accountId: selected.account.id,
username: selected.account.username,
siteName: selected.site.name,
detail: `HTTP ${status}`,
});
}
if ((status > 0 ? shouldRetryProxyRequest(status, errorText) : true) && canRetryChannelSelection(retryCount, forcedChannelId)) {
retryCount++;
continue;
}
await reportProxyAllFailed({
model: requestedModel,
reason: err.message || 'network failure',
});
return reply.code(502).send({
error: { message: `Upstream error: ${err.message}`, type: 'upstream_error' },
});
}
await reportProxyAllFailed({
model: requestedModel,
reason: errorText || 'network failure',
});
return reply.code(status || 502).send({
error: { message: status > 0 ? errorText : `Upstream error: ${errorText}`, type: 'upstream_error' },
});
}
}
});
}
src/server/routes/proxy/downstreamClientContext.routes.test.ts
+25 -1
View File
@@ -58,6 +58,7 @@ vi.mock('../../services/modelPricingService.js', () => ({
vi.mock('../../services/proxyRetryPolicy.js', () => ({
shouldRetryProxyRequest: () => false,
RETRYABLE_TIMEOUT_PATTERNS: [/(request timed out|connection timed out|read timeout|\btimed out\b)/i],
}));
vi.mock('../../services/proxyUsageFallbackService.js', () => ({
@@ -67,9 +68,30 @@ vi.mock('../../services/proxyUsageFallbackService.js', () => ({
vi.mock('../../db/index.js', () => ({
db: {
insert: (arg: any) => dbInsertMock(arg),
select: () => ({
from: () => ({
where: () => ({
orderBy: () => ({
all: async () => [],
}),
}),
}),
}),
update: () => ({
set: () => ({
where: () => ({
run: async () => undefined,
}),
}),
}),
},
schema: {
proxyLogs: {},
siteApiEndpoints: {
id: {},
siteId: {},
sortOrder: {},
},
},
hasProxyLogBillingDetailsColumn: async () => false,
hasProxyLogClientColumns: async () => false,
@@ -116,7 +138,9 @@ describe('downstream client context route logging', () => {
});
afterAll(async () => {
await app.close();
if (app) {
await app.close();
}
});
it('includes Codex client and session metadata in /v1/responses failure logs', async () => {
src/server/routes/proxy/embeddings.ts
+63 -70
View File
@@ -12,11 +12,12 @@ import { getProxyUrlFromExtraConfig } from '../../services/accountExtraConfig.js
import { composeProxyLogMessage } from './logPathMeta.js';
import { formatUtcSqlDateTime } from '../../services/localTimeService.js';
import { resolveProxyLogBilling } from './proxyBilling.js';
import { getProxyAuthContext } from '../../middleware/auth.js';
import { getProxyAuthContext } from '../../middleware/auth.js';
import { buildUpstreamUrl } from './upstreamUrl.js';
import { detectDownstreamClientContext, type DownstreamClientContext } from './downstreamClientContext.js';
import { insertProxyLog } from '../../services/proxyLogStore.js';
import { getProxyMaxChannelRetries } from '../../services/proxyChannelRetry.js';
import { runWithSiteApiEndpointPool, SiteApiEndpointRequestError } from '../../services/siteApiEndpointService.js';
import {
buildForcedChannelUnavailableMessage,
canRetryChannelSelection,
@@ -66,68 +67,45 @@ export async function embeddingsProxyRoute(app: FastifyInstance) {
return reply.code(503).send({ error: { message: noChannelMessage, type: 'server_error' } });
}
excludeChannelIds.push(selected.channel.id);
const targetUrl = buildUpstreamUrl(selected.site.url, '/v1/embeddings');
const upstreamModel = selected.actualModel || requestedModel;
const forwardBody = { ...body, model: upstreamModel };
const startTime = Date.now();
try {
const upstream = await fetch(targetUrl, withSiteRecordProxyRequestInit(selected.site, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Authorization': `Bearer ${selected.tokenValue}`,
},
body: JSON.stringify(forwardBody),
}, getProxyUrlFromExtraConfig(selected.account.extraConfig)));
const text = await upstream.text();
if (!upstream.ok) {
await recordTokenRouterEventBestEffort('record channel failure', () => tokenRouter.recordFailure(selected.channel.id, {
status: upstream.status,
errorText: text,
modelName: upstreamModel,
}));
logProxy(
selected,
requestedModel,
'failed',
upstream.status,
Date.now() - startTime,
text,
retryCount,
downstreamApiKeyId,
0,
0,
0,
0,
null,
clientContext,
downstreamPath,
);
if (isTokenExpiredError({ status: upstream.status, message: text })) {
await reportTokenExpired({
accountId: selected.account.id,
username: selected.account.username,
siteName: selected.site.name,
detail: `HTTP ${upstream.status}`,
});
}
if (shouldRetryProxyRequest(upstream.status, text) && canRetryChannelSelection(retryCount, forcedChannelId)) {
retryCount++;
continue;
try {
const { upstream, text } = await runWithSiteApiEndpointPool(selected.site, async (target) => {
const targetUrl = buildUpstreamUrl(target.baseUrl, '/v1/embeddings');
const response = await fetch(targetUrl, withSiteRecordProxyRequestInit(selected.site, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Authorization': `Bearer ${selected.tokenValue}`,
},
body: JSON.stringify(forwardBody),
}, getProxyUrlFromExtraConfig(selected.account.extraConfig)));
const status = response.status;
let responseText = '';
try {
responseText = await response.text();
} catch (error) {
if (!response.ok) {
throw new SiteApiEndpointRequestError('unknown error', {
status,
cause: error,
});
}
throw error;
}
await reportProxyAllFailed({
model: requestedModel,
reason: `upstream returned HTTP ${upstream.status}`,
});
return reply.code(upstream.status).send({ error: { message: text, type: 'upstream_error' } });
}
if (!response.ok) {
throw new SiteApiEndpointRequestError(responseText || 'unknown error', {
status,
rawErrText: responseText || null,
});
}
return {
upstream: response,
text: responseText,
};
});
let data: any = {};
try { data = JSON.parse(text); } catch { data = {}; }
const latency = Date.now() - startTime;
@@ -165,18 +143,20 @@ export async function embeddingsProxyRoute(app: FastifyInstance) {
);
return reply.code(upstream.status).send(data);
} catch (err: any) {
const status = err instanceof SiteApiEndpointRequestError ? (err.status || 0) : 0;
const errorText = err?.message || 'network failure';
await recordTokenRouterEventBestEffort('record channel failure', () => tokenRouter.recordFailure(selected.channel.id, {
status: 0,
errorText: err.message,
status,
errorText,
modelName: upstreamModel,
}));
logProxy(
selected,
requestedModel,
'failed',
0,
status,
Date.now() - startTime,
err.message,
errorText,
retryCount,
downstreamApiKeyId,
0,
@@ -187,16 +167,29 @@ export async function embeddingsProxyRoute(app: FastifyInstance) {
clientContext,
downstreamPath,
);
if (canRetryChannelSelection(retryCount, forcedChannelId)) {
if (status > 0 && isTokenExpiredError({ status, message: errorText })) {
await reportTokenExpired({
accountId: selected.account.id,
username: selected.account.username,
siteName: selected.site.name,
detail: `HTTP ${status}`,
});
}
if ((status > 0 ? shouldRetryProxyRequest(status, errorText) : true) && canRetryChannelSelection(retryCount, forcedChannelId)) {
retryCount++;
continue;
}
await reportProxyAllFailed({
model: requestedModel,
reason: err.message || 'network failure',
});
return reply.code(502).send({ error: { message: err.message, type: 'upstream_error' } });
}
await reportProxyAllFailed({
model: requestedModel,
reason: errorText || 'network failure',
});
return reply.code(status || 502).send({
error: {
message: status > 0 ? errorText : `Upstream error: ${errorText}`,
type: 'upstream_error',
},
});
}
}
});
}
src/server/routes/proxy/images.edits.test.ts
+25 -1
View File
@@ -52,17 +52,39 @@ vi.mock('../../services/modelPricingService.js', () => ({
vi.mock('../../services/proxyRetryPolicy.js', () => ({
shouldRetryProxyRequest: () => false,
RETRYABLE_TIMEOUT_PATTERNS: [/(request timed out|connection timed out|read timeout|\btimed out\b)/i],
}));
vi.mock('../../db/index.js', () => ({
db: {
insert: (arg: any) => dbInsertMock(arg),
select: () => ({
from: () => ({
where: () => ({
orderBy: () => ({
all: async () => [],
}),
}),
}),
}),
update: () => ({
set: () => ({
where: () => ({
run: async () => undefined,
}),
}),
}),
},
hasProxyLogBillingDetailsColumn: async () => false,
hasProxyLogClientColumns: async () => false,
hasProxyLogDownstreamApiKeyIdColumn: async () => false,
schema: {
proxyLogs: {},
siteApiEndpoints: {
id: {},
siteId: {},
sortOrder: {},
},
},
}));
@@ -115,7 +137,9 @@ describe('/v1/images/edits route', () => {
});
afterAll(async () => {
await app.close();
if (app) {
await app.close();
}
});
it('accepts multipart image edit requests and forwards them to /v1/images/edits', async () => {
src/server/routes/proxy/images.ts
+95 -123
View File
@@ -16,6 +16,7 @@ import { buildUpstreamUrl } from './upstreamUrl.js';
import { detectDownstreamClientContext, type DownstreamClientContext } from './downstreamClientContext.js';
import { insertProxyLog } from '../../services/proxyLogStore.js';
import { getProxyMaxChannelRetries } from '../../services/proxyChannelRetry.js';
import { runWithSiteApiEndpointPool, SiteApiEndpointRequestError } from '../../services/siteApiEndpointService.js';
import {
buildForcedChannelUnavailableMessage,
canRetryChannelSelection,
@@ -66,60 +67,32 @@ export async function imagesProxyRoute(app: FastifyInstance) {
}
excludeChannelIds.push(selected.channel.id);
const targetUrl = buildUpstreamUrl(selected.site.url, '/v1/images/generations');
const upstreamModel = selected.actualModel || requestedModel;
const forwardBody = { ...body, model: upstreamModel };
const startTime = Date.now();
try {
const upstream = await fetch(targetUrl, withSiteRecordProxyRequestInit(selected.site, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Authorization': `Bearer ${selected.tokenValue}`,
},
body: JSON.stringify(forwardBody),
}, getProxyUrlFromExtraConfig(selected.account.extraConfig)));
const text = await upstream.text();
if (!upstream.ok) {
await recordTokenRouterEventBestEffort('record channel failure', () => tokenRouter.recordFailure(selected.channel.id, {
status: upstream.status,
errorText: text,
modelName: upstreamModel,
}));
logProxy(
selected,
requestedModel,
'failed',
upstream.status,
Date.now() - startTime,
text,
retryCount,
downstreamApiKeyId,
0,
downstreamPath,
clientContext,
);
if (isTokenExpiredError({ status: upstream.status, message: text })) {
await reportTokenExpired({
accountId: selected.account.id,
username: selected.account.username,
siteName: selected.site.name,
detail: `HTTP ${upstream.status}`,
const { upstream, text } = await runWithSiteApiEndpointPool(selected.site, async (target) => {
const targetUrl = buildUpstreamUrl(target.baseUrl, '/v1/images/generations');
const response = await fetch(targetUrl, withSiteRecordProxyRequestInit(selected.site, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Authorization': `Bearer ${selected.tokenValue}`,
},
body: JSON.stringify(forwardBody),
}, getProxyUrlFromExtraConfig(selected.account.extraConfig)));
const responseText = await response.text();
if (!response.ok) {
throw new SiteApiEndpointRequestError(responseText || 'unknown error', {
status: response.status,
});
}
if (shouldRetryProxyRequest(upstream.status, text) && canRetryChannelSelection(retryCount, forcedChannelId)) {
retryCount++;
continue;
}
await reportProxyAllFailed({
model: requestedModel,
reason: `upstream returned HTTP ${upstream.status}`,
});
return reply.code(upstream.status).send({ error: { message: text, type: 'upstream_error' } });
}
return {
upstream: response,
text: responseText,
};
});
const data = parseUpstreamImageResponse(text);
if (!data.ok) {
@@ -175,34 +148,47 @@ export async function imagesProxyRoute(app: FastifyInstance) {
logProxy(selected, requestedModel, 'success', upstream.status, latency, null, retryCount, downstreamApiKeyId, estimatedCost, downstreamPath, clientContext);
return reply.code(upstream.status).send(data.value);
} catch (err: any) {
const status = err instanceof SiteApiEndpointRequestError ? (err.status || 0) : 0;
const errorText = err?.message || 'network failure';
await recordTokenRouterEventBestEffort('record channel failure', () => tokenRouter.recordFailure(selected.channel.id, {
status: 0,
errorText: err.message,
status,
errorText,
modelName: upstreamModel,
}));
logProxy(
selected,
requestedModel,
'failed',
0,
status,
Date.now() - startTime,
err.message,
errorText,
retryCount,
downstreamApiKeyId,
0,
downstreamPath,
clientContext,
);
if (canRetryChannelSelection(retryCount, forcedChannelId)) {
if (status > 0 && isTokenExpiredError({ status, message: errorText })) {
await reportTokenExpired({
accountId: selected.account.id,
username: selected.account.username,
siteName: selected.site.name,
detail: `HTTP ${status}`,
});
}
if ((status > 0 ? shouldRetryProxyRequest(status, errorText) : true) && canRetryChannelSelection(retryCount, forcedChannelId)) {
retryCount++;
continue;
}
await reportProxyAllFailed({
model: requestedModel,
reason: err.message || 'network failure',
reason: errorText || 'network failure',
});
return reply.code(502).send({
error: { message: `Upstream error: ${err.message}`, type: 'upstream_error' },
return reply.code(status || 502).send({
error: {
message: status > 0 ? errorText : `Upstream error: ${errorText}`,
type: 'upstream_error',
},
});
}
}
@@ -254,72 +240,45 @@ export async function imagesProxyRoute(app: FastifyInstance) {
}
excludeChannelIds.push(selected.channel.id);
const targetUrl = buildUpstreamUrl(selected.site.url, '/v1/images/edits');
const upstreamModel = selected.actualModel || requestedModel;
const startTime = Date.now();
try {
const requestInit = multipartForm
? withSiteRecordProxyRequestInit(selected.site, {
method: 'POST',
headers: {
Authorization: `Bearer ${selected.tokenValue}`,
},
body: cloneFormDataWithOverrides(multipartForm, {
model: upstreamModel,
}) as any,
}, getProxyUrlFromExtraConfig(selected.account.extraConfig))
: withSiteRecordProxyRequestInit(selected.site, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
Authorization: `Bearer ${selected.tokenValue}`,
},
body: JSON.stringify({
...(jsonBody || {}),
model: upstreamModel,
}),
}, getProxyUrlFromExtraConfig(selected.account.extraConfig));
const upstream = await fetch(targetUrl, requestInit);
const text = await upstream.text();
if (!upstream.ok) {
await recordTokenRouterEventBestEffort('record channel failure', () => tokenRouter.recordFailure(selected.channel.id, {
status: upstream.status,
errorText: text,
modelName: upstreamModel,
}));
logProxy(
selected,
requestedModel,
'failed',
upstream.status,
Date.now() - startTime,
text,
retryCount,
downstreamApiKeyId,
0,
downstreamPath,
clientContext,
);
if (isTokenExpiredError({ status: upstream.status, message: text })) {
await reportTokenExpired({
accountId: selected.account.id,
username: selected.account.username,
siteName: selected.site.name,
detail: `HTTP ${upstream.status}`,
const { upstream, text } = await runWithSiteApiEndpointPool(selected.site, async (target) => {
const targetUrl = buildUpstreamUrl(target.baseUrl, '/v1/images/edits');
const requestInit = multipartForm
? withSiteRecordProxyRequestInit(selected.site, {
method: 'POST',
headers: {
Authorization: `Bearer ${selected.tokenValue}`,
},
body: cloneFormDataWithOverrides(multipartForm, {
model: upstreamModel,
}) as any,
}, getProxyUrlFromExtraConfig(selected.account.extraConfig))
: withSiteRecordProxyRequestInit(selected.site, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
Authorization: `Bearer ${selected.tokenValue}`,
},
body: JSON.stringify({
...(jsonBody || {}),
model: upstreamModel,
}),
}, getProxyUrlFromExtraConfig(selected.account.extraConfig));
const response = await fetch(targetUrl, requestInit);
const responseText = await response.text();
if (!response.ok) {
throw new SiteApiEndpointRequestError(responseText || 'unknown error', {
status: response.status,
});
}
if (shouldRetryProxyRequest(upstream.status, text) && canRetryChannelSelection(retryCount, forcedChannelId)) {
retryCount++;
continue;
}
await reportProxyAllFailed({
model: requestedModel,
reason: `upstream returned HTTP ${upstream.status}`,
});
return reply.code(upstream.status).send({ error: { message: text, type: 'upstream_error' } });
}
return {
upstream: response,
text: responseText,
};
});
const data = parseUpstreamImageResponse(text);
if (!data.ok) {
@@ -375,34 +334,47 @@ export async function imagesProxyRoute(app: FastifyInstance) {
logProxy(selected, requestedModel, 'success', upstream.status, latency, null, retryCount, downstreamApiKeyId, estimatedCost, downstreamPath, clientContext);
return reply.code(upstream.status).send(data.value);
} catch (err: any) {
const status = err instanceof SiteApiEndpointRequestError ? (err.status || 0) : 0;
const errorText = err?.message || 'network failure';
await recordTokenRouterEventBestEffort('record channel failure', () => tokenRouter.recordFailure(selected.channel.id, {
status: 0,
errorText: err.message,
status,
errorText,
modelName: upstreamModel,
}));
logProxy(
selected,
requestedModel,
'failed',
0,
status,
Date.now() - startTime,
err.message,
errorText,
retryCount,
downstreamApiKeyId,
0,
downstreamPath,
clientContext,
);
if (canRetryChannelSelection(retryCount, forcedChannelId)) {
if (status > 0 && isTokenExpiredError({ status, message: errorText })) {
await reportTokenExpired({
accountId: selected.account.id,
username: selected.account.username,
siteName: selected.site.name,
detail: `HTTP ${status}`,
});
}
if ((status > 0 ? shouldRetryProxyRequest(status, errorText) : true) && canRetryChannelSelection(retryCount, forcedChannelId)) {
retryCount++;
continue;
}
await reportProxyAllFailed({
model: requestedModel,
reason: err.message || 'network failure',
reason: errorText || 'network failure',
});
return reply.code(502).send({
error: { message: `Upstream error: ${err.message}`, type: 'upstream_error' },
return reply.code(status || 502).send({
error: {
message: status > 0 ? errorText : `Upstream error: ${errorText}`,
type: 'upstream_error',
},
});
}
}
src/server/routes/proxy/responses.codex-oauth.test.ts
+25 -1
View File
@@ -69,6 +69,7 @@ vi.mock('../../services/modelPricingService.js', () => ({
vi.mock('../../services/proxyRetryPolicy.js', () => ({
shouldRetryProxyRequest: () => false,
RETRYABLE_TIMEOUT_PATTERNS: [/(request timed out|connection timed out|read timeout|\btimed out\b)/i],
}));
vi.mock('../../services/proxyUsageFallbackService.js', () => ({
@@ -86,12 +87,33 @@ vi.mock('../../services/oauth/quota.js', () => ({
vi.mock('../../db/index.js', () => ({
db: {
insert: (arg: any) => dbInsertMock(arg),
select: () => ({
from: () => ({
where: () => ({
orderBy: () => ({
all: async () => [],
}),
}),
}),
}),
update: () => ({
set: () => ({
where: () => ({
run: async () => undefined,
}),
}),
}),
},
hasProxyLogBillingDetailsColumn: async () => false,
hasProxyLogClientColumns: async () => false,
hasProxyLogDownstreamApiKeyIdColumn: async () => false,
schema: {
proxyLogs: {},
siteApiEndpoints: {
id: {},
siteId: {},
sortOrder: {},
},
},
}));
@@ -205,7 +227,9 @@ describe('responses proxy codex oauth refresh', () => {
config.proxyStickySessionEnabled = originalProxyStickySessionEnabled;
config.proxySessionChannelConcurrencyLimit = originalProxySessionChannelConcurrencyLimit;
config.proxySessionChannelQueueWaitMs = originalProxySessionChannelQueueWaitMs;
await app.close();
if (app) {
await app.close();
}
});
it('refreshes codex oauth token and retries the same responses request on 401', async () => {
src/server/routes/proxy/responses.compact-upstream.test.ts
+25 -1
View File
@@ -57,6 +57,7 @@ vi.mock('../../services/modelPricingService.js', () => ({
vi.mock('../../services/proxyRetryPolicy.js', () => ({
shouldRetryProxyRequest: () => false,
RETRYABLE_TIMEOUT_PATTERNS: [/(request timed out|connection timed out|read timeout|\btimed out\b)/i],
}));
vi.mock('../../services/proxyUsageFallbackService.js', () => ({
@@ -66,12 +67,33 @@ vi.mock('../../services/proxyUsageFallbackService.js', () => ({
vi.mock('../../db/index.js', () => ({
db: {
insert: (arg: any) => dbInsertMock(arg),
select: () => ({
from: () => ({
where: () => ({
orderBy: () => ({
all: async () => [],
}),
}),
}),
}),
update: () => ({
set: () => ({
where: () => ({
run: async () => undefined,
}),
}),
}),
},
hasProxyLogBillingDetailsColumn: async () => false,
hasProxyLogClientColumns: async () => false,
hasProxyLogDownstreamApiKeyIdColumn: async () => false,
schema: {
proxyLogs: {},
siteApiEndpoints: {
id: {},
siteId: {},
sortOrder: {},
},
},
}));
@@ -112,7 +134,9 @@ describe('responses proxy compact upstream routing', () => {
});
afterAll(async () => {
await app.close();
if (app) {
await app.close();
}
});
it('forwards compact requests to the upstream /v1/responses/compact path first', async () => {
src/server/routes/proxy/responses.websocket.test.ts
+31 -3
View File
@@ -78,6 +78,7 @@ vi.mock('../../services/modelPricingService.js', () => ({
vi.mock('../../services/proxyRetryPolicy.js', () => ({
shouldRetryProxyRequest: () => false,
RETRYABLE_TIMEOUT_PATTERNS: [/(request timed out|connection timed out|read timeout|\btimed out\b)/i],
}));
vi.mock('../../services/proxyUsageFallbackService.js', () => ({
@@ -91,12 +92,33 @@ vi.mock('../../services/oauth/quota.js', () => ({
vi.mock('../../db/index.js', () => ({
db: {
insert: (arg: any) => dbInsertMock(arg),
select: () => ({
from: () => ({
where: () => ({
orderBy: () => ({
all: async () => [],
}),
}),
}),
}),
update: () => ({
set: () => ({
where: () => ({
run: async () => undefined,
}),
}),
}),
},
hasProxyLogBillingDetailsColumn: async () => false,
hasProxyLogClientColumns: async () => false,
hasProxyLogDownstreamApiKeyIdColumn: async () => false,
schema: {
proxyLogs: {},
siteApiEndpoints: {
id: {},
siteId: {},
sortOrder: {},
},
},
}));
@@ -366,9 +388,15 @@ describe('responses websocket transport', () => {
afterAll(async () => {
(config as any).codexUpstreamWebsocketEnabled = originalCodexUpstreamWebsocketEnabled;
await new Promise<void>((resolve) => rejectedUpgradeServer.close(() => resolve()));
await new Promise<void>((resolve) => upstreamServer.close(() => resolve()));
await app.close();
if (rejectedUpgradeServer) {
await new Promise<void>((resolve) => rejectedUpgradeServer.close(() => resolve()));
}
if (upstreamServer) {
await new Promise<void>((resolve) => upstreamServer.close(() => resolve()));
}
if (app) {
await app.close();
}
});
it('accepts response.create over GET /v1/responses websocket and forwards streamed responses events', async () => {
src/server/routes/proxy/search.test.ts
+25 -1
View File
@@ -48,17 +48,39 @@ vi.mock('../../services/modelPricingService.js', () => ({
vi.mock('../../services/proxyRetryPolicy.js', () => ({
shouldRetryProxyRequest: () => false,
RETRYABLE_TIMEOUT_PATTERNS: [/(request timed out|connection timed out|read timeout|\btimed out\b)/i],
}));
vi.mock('../../db/index.js', () => ({
db: {
insert: (arg: any) => dbInsertMock(arg),
select: () => ({
from: () => ({
where: () => ({
orderBy: () => ({
all: async () => [],
}),
}),
}),
}),
update: () => ({
set: () => ({
where: () => ({
run: async () => undefined,
}),
}),
}),
},
hasProxyLogBillingDetailsColumn: async () => false,
hasProxyLogClientColumns: async () => false,
hasProxyLogDownstreamApiKeyIdColumn: async () => false,
schema: {
proxyLogs: {},
siteApiEndpoints: {
id: {},
siteId: {},
sortOrder: {},
},
},
}));
@@ -95,7 +117,9 @@ describe('/v1/search route', () => {
});
afterAll(async () => {
await app.close();
if (app) {
await app.close();
}
});
it('defaults model to __search and forwards to the upstream /v1/search endpoint', async () => {
src/server/routes/proxy/search.ts
+41 -53
View File
@@ -14,6 +14,7 @@ import { buildUpstreamUrl } from './upstreamUrl.js';
import { detectDownstreamClientContext, type DownstreamClientContext } from './downstreamClientContext.js';
import { insertProxyLog } from '../../services/proxyLogStore.js';
import { getProxyMaxChannelRetries } from '../../services/proxyChannelRetry.js';
import { runWithSiteApiEndpointPool, SiteApiEndpointRequestError } from '../../services/siteApiEndpointService.js';
import {
buildForcedChannelUnavailableMessage,
canRetryChannelSelection,
@@ -94,7 +95,6 @@ export async function searchProxyRoute(app: FastifyInstance) {
}
excludeChannelIds.push(selected.channel.id);
const targetUrl = buildUpstreamUrl(selected.site.url, '/v1/search');
const upstreamModel = selected.actualModel || requestedModel;
const forwardBody = {
...body,
@@ -104,52 +104,27 @@ export async function searchProxyRoute(app: FastifyInstance) {
const startTime = Date.now();
try {
const upstream = await fetch(targetUrl, withSiteRecordProxyRequestInit(selected.site, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
Authorization: `Bearer ${selected.tokenValue}`,
},
body: JSON.stringify(forwardBody),
}, getProxyUrlFromExtraConfig(selected.account.extraConfig)));
const text = await upstream.text();
if (!upstream.ok) {
await recordTokenRouterEventBestEffort('record channel failure', () => tokenRouter.recordFailure(selected.channel.id, {
status: upstream.status,
errorText: text,
modelName: upstreamModel,
}));
logProxy(
selected,
requestedModel,
'failed',
upstream.status,
Date.now() - startTime,
text,
retryCount,
downstreamApiKeyId,
clientContext,
downstreamPath,
);
if (isTokenExpiredError({ status: upstream.status, message: text })) {
await reportTokenExpired({
accountId: selected.account.id,
username: selected.account.username,
siteName: selected.site.name,
detail: `HTTP ${upstream.status}`,
const { upstream, text } = await runWithSiteApiEndpointPool(selected.site, async (target) => {
const targetUrl = buildUpstreamUrl(target.baseUrl, '/v1/search');
const response = await fetch(targetUrl, withSiteRecordProxyRequestInit(selected.site, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
Authorization: `Bearer ${selected.tokenValue}`,
},
body: JSON.stringify(forwardBody),
}, getProxyUrlFromExtraConfig(selected.account.extraConfig)));
const responseText = await response.text();
if (!response.ok) {
throw new SiteApiEndpointRequestError(responseText || 'unknown error', {
status: response.status,
});
}
if (shouldRetryProxyRequest(upstream.status, text) && canRetryChannelSelection(retryCount, forcedChannelId)) {
retryCount += 1;
continue;
}
await reportProxyAllFailed({
model: requestedModel,
reason: `upstream returned HTTP ${upstream.status}`,
});
return reply.code(upstream.status).send({ error: { message: text, type: 'upstream_error' } });
}
return {
upstream: response,
text: responseText,
};
});
let data: any = {};
try { data = JSON.parse(text); } catch { data = { data: [] }; }
@@ -162,33 +137,46 @@ export async function searchProxyRoute(app: FastifyInstance) {
logProxy(selected, requestedModel, 'success', upstream.status, latency, null, retryCount, downstreamApiKeyId, clientContext, downstreamPath);
return reply.code(upstream.status).send(data);
} catch (error: any) {
const status = error instanceof SiteApiEndpointRequestError ? (error.status || 0) : 0;
const errorText = error?.message || 'network error';
await recordTokenRouterEventBestEffort('record channel failure', () => tokenRouter.recordFailure(selected.channel.id, {
status: 0,
errorText: error?.message || 'network error',
status,
errorText,
modelName: upstreamModel,
}));
logProxy(
selected,
requestedModel,
'failed',
0,
status,
Date.now() - startTime,
error?.message || 'network error',
errorText,
retryCount,
downstreamApiKeyId,
clientContext,
downstreamPath,
);
if (canRetryChannelSelection(retryCount, forcedChannelId)) {
if (status > 0 && isTokenExpiredError({ status, message: errorText })) {
await reportTokenExpired({
accountId: selected.account.id,
username: selected.account.username,
siteName: selected.site.name,
detail: `HTTP ${status}`,
});
}
if ((status > 0 ? shouldRetryProxyRequest(status, errorText) : true) && canRetryChannelSelection(retryCount, forcedChannelId)) {
retryCount += 1;
continue;
}
await reportProxyAllFailed({
model: requestedModel,
reason: error?.message || 'network failure',
reason: errorText || 'network failure',
});
return reply.code(502).send({
error: { message: error?.message || 'network failure', type: 'upstream_error' },
return reply.code(status || 502).send({
error: {
message: status > 0 ? errorText : `Upstream error: ${errorText}`,
type: 'upstream_error',
},
});
}
}
src/server/routes/proxy/videos.test.ts
+238 -2
View File
@@ -10,10 +10,13 @@ const refreshModelsAndRebuildRoutesMock = vi.fn();
const reportProxyAllFailedMock = vi.fn();
const reportTokenExpiredMock = vi.fn();
const estimateProxyCostMock = vi.fn(async () => 0);
const shouldRetryProxyRequestMock = vi.fn();
const saveProxyVideoTaskMock = vi.fn();
const getProxyVideoTaskByPublicIdMock = vi.fn();
const deleteProxyVideoTaskByPublicIdMock = vi.fn();
const refreshProxyVideoTaskSnapshotMock = vi.fn();
const resolveProxyVideoTaskSiteMock = vi.fn();
let siteApiEndpointRows: Array<Record<string, unknown>> = [];
vi.mock('undici', async () => {
const actual = await vi.importActual<typeof import('undici')>('undici');
@@ -50,7 +53,8 @@ vi.mock('../../services/modelPricingService.js', () => ({
}));
vi.mock('../../services/proxyRetryPolicy.js', () => ({
shouldRetryProxyRequest: () => false,
shouldRetryProxyRequest: (...args: unknown[]) => shouldRetryProxyRequestMock(...args),
RETRYABLE_TIMEOUT_PATTERNS: [/(request timed out|connection timed out|read timeout|\btimed out\b)/i],
}));
vi.mock('../../services/proxyVideoTaskStore.js', () => ({
@@ -58,6 +62,35 @@ vi.mock('../../services/proxyVideoTaskStore.js', () => ({
getProxyVideoTaskByPublicId: (...args: unknown[]) => getProxyVideoTaskByPublicIdMock(...args),
deleteProxyVideoTaskByPublicId: (...args: unknown[]) => deleteProxyVideoTaskByPublicIdMock(...args),
refreshProxyVideoTaskSnapshot: (...args: unknown[]) => refreshProxyVideoTaskSnapshotMock(...args),
resolveProxyVideoTaskSite: (...args: unknown[]) => resolveProxyVideoTaskSiteMock(...args),
}));
vi.mock('../../db/index.js', () => ({
db: {
select: () => ({
from: () => ({
where: () => ({
orderBy: () => ({
all: async () => siteApiEndpointRows,
}),
}),
}),
}),
update: () => ({
set: () => ({
where: () => ({
run: async () => undefined,
}),
}),
}),
},
schema: {
siteApiEndpoints: {
id: {},
siteId: {},
sortOrder: {},
},
},
}));
describe('/v1/videos routes', () => {
@@ -93,10 +126,14 @@ describe('/v1/videos routes', () => {
reportProxyAllFailedMock.mockReset();
reportTokenExpiredMock.mockReset();
estimateProxyCostMock.mockClear();
shouldRetryProxyRequestMock.mockReset();
saveProxyVideoTaskMock.mockReset();
getProxyVideoTaskByPublicIdMock.mockReset();
deleteProxyVideoTaskByPublicIdMock.mockReset();
refreshProxyVideoTaskSnapshotMock.mockReset();
resolveProxyVideoTaskSiteMock.mockReset();
siteApiEndpointRows = [];
shouldRetryProxyRequestMock.mockReturnValue(false);
selectChannelMock.mockReturnValue({
channel: { id: 11, routeId: 22 },
@@ -110,7 +147,9 @@ describe('/v1/videos routes', () => {
});
afterAll(async () => {
await app.close();
if (app) {
await app.close();
}
});
it('creates an upstream video task and stores a local public id mapping', async () => {
@@ -189,7 +228,55 @@ describe('/v1/videos routes', () => {
});
});
it('stores the selected api endpoint base url for split-host video tasks', async () => {
siteApiEndpointRows = [
{
id: 91,
siteId: 44,
url: 'https://api-videos.example.com',
enabled: true,
sortOrder: 0,
},
];
selectChannelMock.mockReturnValue({
channel: { id: 11, routeId: 22 },
site: { id: 44, name: 'demo-site', url: 'https://panel.example.com', platform: 'openai' },
account: { id: 33, username: 'demo-user' },
tokenName: 'default',
tokenValue: 'sk-demo',
actualModel: 'sora-2',
});
saveProxyVideoTaskMock.mockResolvedValue({
publicId: 'vid_local_split',
upstreamVideoId: 'vid_upstream_split',
});
fetchMock.mockResolvedValue(new Response(JSON.stringify({
id: 'vid_upstream_split',
object: 'video',
status: 'queued',
model: 'sora-2',
}), {
status: 200,
headers: { 'content-type': 'application/json' },
}));
const response = await app.inject({
method: 'POST',
url: '/v1/videos',
payload: {
model: 'sora-2',
prompt: 'split-host create',
},
});
expect(response.statusCode).toBe(200);
expect(saveProxyVideoTaskMock).toHaveBeenCalledWith(expect.objectContaining({
siteUrl: 'https://api-videos.example.com',
}));
});
it('resolves local video ids back to the upstream task on GET', async () => {
resolveProxyVideoTaskSiteMock.mockResolvedValue(null);
getProxyVideoTaskByPublicIdMock.mockResolvedValue({
publicId: 'vid_local_123',
upstreamVideoId: 'vid_upstream_123',
@@ -228,6 +315,116 @@ describe('/v1/videos routes', () => {
});
});
it('re-resolves account-backed video tasks through the site api endpoint pool on GET', async () => {
siteApiEndpointRows = [
{
id: 91,
siteId: 44,
url: 'https://api-videos.example.com',
enabled: true,
sortOrder: 0,
},
];
resolveProxyVideoTaskSiteMock.mockResolvedValue({
id: 44,
name: 'demo-site',
url: 'https://panel.example.com',
platform: 'openai',
});
getProxyVideoTaskByPublicIdMock.mockResolvedValue({
publicId: 'vid_local_456',
upstreamVideoId: 'vid_upstream_456',
siteUrl: 'https://panel.example.com',
tokenValue: 'sk-demo',
accountId: 33,
});
fetchMock.mockResolvedValue(new Response(JSON.stringify({
id: 'vid_upstream_456',
object: 'video',
status: 'queued',
model: 'sora-2',
}), {
status: 200,
headers: { 'content-type': 'application/json' },
}));
const response = await app.inject({
method: 'GET',
url: '/v1/videos/vid_local_456',
});
expect(response.statusCode).toBe(200);
const [targetUrl] = fetchMock.mock.calls[0] as [string, RequestInit];
expect(targetUrl).toBe('https://api-videos.example.com/v1/videos/vid_upstream_456');
});
it('uses the persisted api base url when the backing site can no longer be resolved', async () => {
resolveProxyVideoTaskSiteMock.mockResolvedValue(null);
getProxyVideoTaskByPublicIdMock.mockResolvedValue({
publicId: 'vid_local_fallback',
upstreamVideoId: 'vid_upstream_fallback',
siteUrl: 'https://api-fallback.example.com',
tokenValue: 'sk-demo',
accountId: 33,
});
fetchMock.mockResolvedValue(new Response(JSON.stringify({
id: 'vid_upstream_fallback',
object: 'video',
status: 'running',
}), {
status: 200,
headers: { 'content-type': 'application/json' },
}));
const response = await app.inject({
method: 'GET',
url: '/v1/videos/vid_local_fallback',
});
expect(response.statusCode).toBe(200);
const [targetUrl] = fetchMock.mock.calls[0] as [string, RequestInit];
expect(targetUrl).toBe('https://api-fallback.example.com/v1/videos/vid_upstream_fallback');
});
it('preserves retryable upstream failures after mapped GET retries exhaust', async () => {
shouldRetryProxyRequestMock.mockReturnValue(true);
siteApiEndpointRows = [
{
id: 91,
siteId: 44,
url: 'https://api-videos.example.com',
enabled: true,
sortOrder: 0,
},
];
resolveProxyVideoTaskSiteMock.mockResolvedValue({
id: 44,
name: 'demo-site',
url: 'https://panel.example.com',
platform: 'openai',
});
getProxyVideoTaskByPublicIdMock.mockResolvedValue({
publicId: 'vid_local_retry_get',
upstreamVideoId: 'vid_upstream_retry_get',
siteUrl: 'https://api-videos.example.com',
tokenValue: 'sk-demo',
accountId: 33,
});
fetchMock.mockResolvedValue(new Response('temporary unavailable', {
status: 502,
headers: { 'content-type': 'text/plain' },
}));
const response = await app.inject({
method: 'GET',
url: '/v1/videos/vid_local_retry_get',
});
expect(response.statusCode).toBe(502);
expect(response.headers['content-type']).toContain('text/plain');
expect(response.body).toBe('temporary unavailable');
});
it('deletes the upstream task and local mapping on DELETE', async () => {
getProxyVideoTaskByPublicIdMock.mockResolvedValue({
publicId: 'vid_local_123',
@@ -245,4 +442,43 @@ describe('/v1/videos routes', () => {
expect(response.statusCode).toBe(204);
expect(deleteProxyVideoTaskByPublicIdMock).toHaveBeenCalledWith('vid_local_123');
});
it('preserves retryable upstream failures after mapped DELETE retries exhaust', async () => {
shouldRetryProxyRequestMock.mockReturnValue(true);
siteApiEndpointRows = [
{
id: 91,
siteId: 44,
url: 'https://api-videos.example.com',
enabled: true,
sortOrder: 0,
},
];
resolveProxyVideoTaskSiteMock.mockResolvedValue({
id: 44,
name: 'demo-site',
url: 'https://panel.example.com',
platform: 'openai',
});
getProxyVideoTaskByPublicIdMock.mockResolvedValue({
publicId: 'vid_local_retry_delete',
upstreamVideoId: 'vid_upstream_retry_delete',
siteUrl: 'https://api-videos.example.com',
tokenValue: 'sk-demo',
accountId: 33,
});
fetchMock.mockResolvedValue(new Response('temporary unavailable', {
status: 502,
headers: { 'content-type': 'text/plain' },
}));
const response = await app.inject({
method: 'DELETE',
url: '/v1/videos/vid_local_retry_delete',
});
expect(response.statusCode).toBe(502);
expect(response.headers['content-type']).toContain('text/plain');
expect(response.body).toBe('temporary unavailable');
});
});
src/server/routes/proxy/videos.ts
+133 -69
View File
@@ -14,6 +14,7 @@ import {
deleteProxyVideoTaskByPublicId,
getProxyVideoTaskByPublicId,
refreshProxyVideoTaskSnapshot,
resolveProxyVideoTaskSite,
saveProxyVideoTask,
} from '../../services/proxyVideoTaskStore.js';
import { getProxyMaxChannelRetries } from '../../services/proxyChannelRetry.js';
@@ -23,6 +24,7 @@ import {
getTesterForcedChannelId,
selectProxyChannelForAttempt,
} from '../../proxy-core/channelSelection.js';
import { runWithSiteApiEndpointPool, SiteApiEndpointRequestError } from '../../services/siteApiEndpointService.js';
function rewriteVideoResponsePublicId(payload: unknown, publicId: string): unknown {
if (!payload || typeof payload !== 'object') return payload;
@@ -80,60 +82,48 @@ export async function videosProxyRoute(app: FastifyInstance) {
}
excludeChannelIds.push(selected.channel.id);
const targetUrl = buildUpstreamUrl(selected.site.url, '/v1/videos');
const upstreamModel = selected.actualModel || requestedModel;
const startTime = Date.now();
try {
const accountProxy = getProxyUrlFromExtraConfig(selected.account.extraConfig);
const requestInit = multipartForm
? withSiteRecordProxyRequestInit(selected.site, {
method: 'POST',
headers: {
Authorization: `Bearer ${selected.tokenValue}`,
},
body: cloneFormDataWithOverrides(multipartForm, {
model: upstreamModel,
}) as any,
}, accountProxy)
: withSiteRecordProxyRequestInit(selected.site, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
Authorization: `Bearer ${selected.tokenValue}`,
},
body: JSON.stringify({
...(jsonBody || {}),
model: upstreamModel,
}),
}, accountProxy);
const upstream = await fetch(targetUrl, requestInit);
const text = await upstream.text();
if (!upstream.ok) {
await recordTokenRouterEventBestEffort('record channel failure', () => tokenRouter.recordFailure(selected.channel.id, {
status: upstream.status,
errorText: text,
modelName: upstreamModel,
}));
if (isTokenExpiredError({ status: upstream.status, message: text })) {
await reportTokenExpired({
accountId: selected.account.id,
username: selected.account.username,
siteName: selected.site.name,
detail: `HTTP ${upstream.status}`,
const { upstream, text, baseUrl } = await runWithSiteApiEndpointPool(selected.site, async (target) => {
const targetUrl = buildUpstreamUrl(target.baseUrl, '/v1/videos');
const accountProxy = getProxyUrlFromExtraConfig(selected.account.extraConfig);
const requestInit = multipartForm
? withSiteRecordProxyRequestInit(selected.site, {
method: 'POST',
headers: {
Authorization: `Bearer ${selected.tokenValue}`,
},
body: cloneFormDataWithOverrides(multipartForm, {
model: upstreamModel,
}) as any,
}, accountProxy)
: withSiteRecordProxyRequestInit(selected.site, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
Authorization: `Bearer ${selected.tokenValue}`,
},
body: JSON.stringify({
...(jsonBody || {}),
model: upstreamModel,
}),
}, accountProxy);
const response = await fetch(targetUrl, requestInit);
const responseText = await response.text();
if (!response.ok) {
throw new SiteApiEndpointRequestError(responseText || 'unknown error', {
status: response.status,
rawErrText: responseText || null,
});
}
if (shouldRetryProxyRequest(upstream.status, text) && canRetryChannelSelection(retryCount, forcedChannelId)) {
retryCount += 1;
continue;
}
await reportProxyAllFailed({
model: requestedModel,
reason: `upstream returned HTTP ${upstream.status}`,
});
return reply.code(upstream.status).send({ error: { message: text, type: 'upstream_error' } });
}
return {
baseUrl: target.baseUrl,
upstream: response,
text: responseText,
};
});
let data: any = {};
try { data = JSON.parse(text); } catch { data = {}; }
@@ -146,7 +136,7 @@ export async function videosProxyRoute(app: FastifyInstance) {
const mapping = await saveProxyVideoTask({
upstreamVideoId,
siteUrl: selected.site.url,
siteUrl: baseUrl,
tokenValue: selected.tokenValue,
requestedModel,
actualModel: upstreamModel,
@@ -174,21 +164,34 @@ export async function videosProxyRoute(app: FastifyInstance) {
recordDownstreamCostUsage(request, estimatedCost);
return reply.code(upstream.status).send(rewriteVideoResponsePublicId(data, mapping.publicId));
} catch (error: any) {
const status = error instanceof SiteApiEndpointRequestError ? (error.status || 0) : 0;
const errorText = error?.message || 'network failure';
await recordTokenRouterEventBestEffort('record channel failure', () => tokenRouter.recordFailure(selected.channel.id, {
status: 0,
errorText: error?.message || 'network failure',
status,
errorText,
modelName: upstreamModel,
}));
if (canRetryChannelSelection(retryCount, forcedChannelId)) {
if (status > 0 && isTokenExpiredError({ status, message: errorText })) {
await reportTokenExpired({
accountId: selected.account.id,
username: selected.account.username,
siteName: selected.site.name,
detail: `HTTP ${status}`,
});
}
if ((status > 0 ? shouldRetryProxyRequest(status, errorText) : true) && canRetryChannelSelection(retryCount, forcedChannelId)) {
retryCount += 1;
continue;
}
await reportProxyAllFailed({
model: requestedModel,
reason: error?.message || 'network failure',
reason: errorText || 'network failure',
});
return reply.code(502).send({
error: { message: error?.message || 'network failure', type: 'upstream_error' },
return reply.code(status || 502).send({
error: {
message: status > 0 ? errorText : `Upstream error: ${errorText}`,
type: 'upstream_error',
},
});
}
}
@@ -202,13 +205,15 @@ export async function videosProxyRoute(app: FastifyInstance) {
});
}
const targetUrl = buildUpstreamUrl(mapping.siteUrl, `/v1/videos/${encodeURIComponent(mapping.upstreamVideoId)}`);
const upstream = await fetch(targetUrl, await withSiteProxyRequestInit(targetUrl, {
method: 'GET',
headers: {
Authorization: `Bearer ${mapping.tokenValue}`,
},
}));
let upstream: Awaited<ReturnType<typeof fetch>>;
try {
({ upstream } = await requestMappedVideoTaskUpstream(mapping, 'GET'));
} catch (error) {
if (isSiteApiEndpointFailure(error)) {
return sendVideoTaskEndpointFailure(reply, error);
}
throw error;
}
const text = await upstream.text();
try {
const data = JSON.parse(text);
@@ -233,13 +238,15 @@ export async function videosProxyRoute(app: FastifyInstance) {
});
}
const targetUrl = buildUpstreamUrl(mapping.siteUrl, `/v1/videos/${encodeURIComponent(mapping.upstreamVideoId)}`);
const upstream = await fetch(targetUrl, await withSiteProxyRequestInit(targetUrl, {
method: 'DELETE',
headers: {
Authorization: `Bearer ${mapping.tokenValue}`,
},
}));
let upstream: Awaited<ReturnType<typeof fetch>>;
try {
({ upstream } = await requestMappedVideoTaskUpstream(mapping, 'DELETE'));
} catch (error) {
if (isSiteApiEndpointFailure(error)) {
return sendVideoTaskEndpointFailure(reply, error);
}
throw error;
}
if (upstream.ok) {
await deleteProxyVideoTaskByPublicId(mapping.publicId);
return reply.code(upstream.status).send();
@@ -252,6 +259,63 @@ export async function videosProxyRoute(app: FastifyInstance) {
});
}
async function requestMappedVideoTaskUpstream(
mapping: NonNullable<Awaited<ReturnType<typeof getProxyVideoTaskByPublicId>>>,
method: 'GET' | 'DELETE',
): Promise<{ upstream: Awaited<ReturnType<typeof fetch>> }> {
const buildRequest = async (baseUrl: string) => {
const targetUrl = buildUpstreamUrl(baseUrl, `/v1/videos/${encodeURIComponent(mapping.upstreamVideoId)}`);
const upstream = await fetch(targetUrl, await withSiteProxyRequestInit(targetUrl, {
method,
headers: {
Authorization: `Bearer ${mapping.tokenValue}`,
},
}));
if (!upstream.ok) {
const errorText = await upstream.clone().text().catch(() => '');
if (shouldRetryProxyRequest(upstream.status, errorText || `HTTP ${upstream.status}`)) {
throw new SiteApiEndpointRequestError(errorText || `HTTP ${upstream.status}`, {
status: upstream.status,
rawErrText: errorText || null,
});
}
}
return { upstream };
};
const site = await resolveProxyVideoTaskSite(mapping.accountId);
if (site) {
return runWithSiteApiEndpointPool(site, (target) => buildRequest(target.baseUrl));
}
return buildRequest(mapping.siteUrl);
}
function isSiteApiEndpointFailure(error: unknown): error is SiteApiEndpointRequestError {
return error instanceof SiteApiEndpointRequestError
|| (typeof error === 'object' && error !== null && (error as { name?: unknown }).name === 'SiteApiEndpointRequestError');
}
function sendVideoTaskEndpointFailure(
reply: FastifyReply,
error: { status?: number | null; rawErrText?: string | null; message?: string | null },
) {
const status = typeof error.status === 'number' && error.status > 0 ? error.status : 502;
const rawText = (typeof error.rawErrText === 'string' && error.rawErrText.trim())
? error.rawErrText
: (typeof error.message === 'string' ? error.message.trim() : '');
if (!rawText) {
return reply.code(status).send({
error: { message: 'Upstream request failed', type: 'upstream_error' },
});
}
try {
return reply.code(status).send(JSON.parse(rawText));
} catch {
return reply.code(status).type('text/plain').send(rawText);
}
}
async function recordTokenRouterEventBestEffort(
label: string,
operation: () => Promise<unknown>,
src/server/services/backupService.test.ts
+90 -1
View File
@@ -2,7 +2,7 @@ import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vites
import { mkdtempSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { eq } from 'drizzle-orm';
import { asc, eq } from 'drizzle-orm';
type DbModule = typeof import('../db/index.js');
type BackupServiceModule = typeof import('./backupService.js');
@@ -155,6 +155,19 @@ describe('backupService', () => {
createdAt: now,
}).run();
await db.insert(schema.siteApiEndpoints).values({
siteId: site.id,
url: 'https://api-roundtrip.example.com',
enabled: true,
sortOrder: 0,
cooldownUntil: null,
lastSelectedAt: now,
lastFailedAt: null,
lastFailureReason: null,
createdAt: now,
updatedAt: now,
}).run();
await db.insert(schema.modelAvailability).values([
{
accountId: account.id,
@@ -199,6 +212,15 @@ describe('backupService', () => {
expect(exported.accounts.siteDisabledModels).toEqual([
{ siteId: site.id, modelName: 'gpt-hidden' },
]);
expect(exported.accounts.siteApiEndpoints).toEqual([
expect.objectContaining({
siteId: site.id,
url: 'https://api-roundtrip.example.com',
enabled: true,
sortOrder: 0,
cooldownUntil: null,
}),
]);
expect(exported.accounts.manualModels).toEqual([
{ accountId: account.id, modelName: 'gpt-manual' },
]);
@@ -579,6 +601,33 @@ describe('backupService', () => {
createdAt: exportedAt,
}).run();
await db.insert(schema.siteApiEndpoints).values([
{
siteId: site.id,
url: 'https://api-a.example.com',
enabled: true,
sortOrder: 0,
cooldownUntil: null,
lastSelectedAt: exportedAt,
lastFailedAt: null,
lastFailureReason: null,
createdAt: exportedAt,
updatedAt: exportedAt,
},
{
siteId: site.id,
url: 'https://api-b.example.com',
enabled: false,
sortOrder: 1,
cooldownUntil: exportedAt,
lastSelectedAt: null,
lastFailedAt: exportedAt,
lastFailureReason: 'HTTP 429',
createdAt: exportedAt,
updatedAt: exportedAt,
},
]).run();
await db.insert(schema.modelAvailability).values([
{
accountId: account.id,
@@ -678,6 +727,22 @@ describe('backupService', () => {
updatedAt: localRuntimeAt,
}).where(eq(schema.sites.id, site.id)).run();
await db.delete(schema.siteApiEndpoints)
.where(eq(schema.siteApiEndpoints.siteId, site.id))
.run();
await db.insert(schema.siteApiEndpoints).values({
siteId: site.id,
url: 'https://api-local-only.example.com',
enabled: true,
sortOrder: 9,
cooldownUntil: null,
lastSelectedAt: localRuntimeAt,
lastFailedAt: null,
lastFailureReason: null,
createdAt: localRuntimeAt,
updatedAt: localRuntimeAt,
}).run();
await db.update(schema.tokenRoutes).set({
displayName: 'local-route-name',
updatedAt: localRuntimeAt,
@@ -821,6 +886,11 @@ describe('backupService', () => {
const restoredAccount = await db.select().from(schema.accounts).where(eq(schema.accounts.id, account.id)).get();
const restoredRoute = await db.select().from(schema.tokenRoutes).where(eq(schema.tokenRoutes.id, route.id)).get();
const restoredChannel = await db.select().from(schema.routeChannels).where(eq(schema.routeChannels.id, insertedChannel!.id)).get();
const restoredSiteApiEndpoints = await db.select()
.from(schema.siteApiEndpoints)
.where(eq(schema.siteApiEndpoints.siteId, site.id))
.orderBy(asc(schema.siteApiEndpoints.sortOrder), asc(schema.siteApiEndpoints.id))
.all();
const restoredDisabledModels = await db.select().from(schema.siteDisabledModels).all();
const restoredAvailability = await db.select().from(schema.modelAvailability).all();
const restoredTokenAvailability = await db.select().from(schema.tokenModelAvailability).all();
@@ -848,6 +918,25 @@ describe('backupService', () => {
expect(restoredChannel?.cooldownLevel).toBe(2);
expect(restoredChannel?.cooldownUntil).toBe(localRuntimeAt);
expect(restoredSiteApiEndpoints).toEqual([
expect.objectContaining({
siteId: site.id,
url: 'https://api-a.example.com',
enabled: true,
sortOrder: 0,
cooldownUntil: null,
lastFailureReason: null,
}),
expect.objectContaining({
siteId: site.id,
url: 'https://api-b.example.com',
enabled: false,
sortOrder: 1,
cooldownUntil: exportedAt,
lastFailureReason: 'HTTP 429',
}),
]);
expect(restoredDisabledModels).toEqual([
expect.objectContaining({ siteId: site.id, modelName: 'gpt-backup-disabled' }),
]);
src/server/services/backupService.ts
+31
View File
@@ -38,6 +38,7 @@ export interface BackupWebdavState {
}
type SiteRow = typeof schema.sites.$inferSelect;
type SiteApiEndpointRow = typeof schema.siteApiEndpoints.$inferSelect;
type AccountRow = typeof schema.accounts.$inferSelect;
type AccountTokenRow = typeof schema.accountTokens.$inferSelect;
type TokenRouteRow = typeof schema.tokenRoutes.$inferSelect;
@@ -100,6 +101,7 @@ type BackupDownstreamApiKeyRow = Pick<DownstreamApiKeyRow,
interface AccountsBackupSection {
sites: SiteRow[];
siteApiEndpoints?: SiteApiEndpointRow[];
accounts: BackupAccountRow[];
accountTokens: AccountTokenRow[];
tokenRoutes: TokenRouteRow[];
@@ -1282,6 +1284,7 @@ function isSettingValueAcceptable(key: string, value: unknown): boolean {
async function exportAccountsSection(): Promise<AccountsBackupSection> {
const [
sites,
siteApiEndpoints,
accounts,
accountTokens,
tokenRoutes,
@@ -1292,6 +1295,13 @@ async function exportAccountsSection(): Promise<AccountsBackupSection> {
downstreamApiKeys,
] = await Promise.all([
db.select().from(schema.sites).orderBy(asc(schema.sites.id)).all(),
db.select().from(schema.siteApiEndpoints)
.orderBy(
asc(schema.siteApiEndpoints.siteId),
asc(schema.siteApiEndpoints.sortOrder),
asc(schema.siteApiEndpoints.id),
)
.all(),
db.select().from(schema.accounts).orderBy(asc(schema.accounts.id)).all(),
db.select().from(schema.accountTokens).orderBy(asc(schema.accountTokens.id)).all(),
db.select().from(schema.tokenRoutes).orderBy(asc(schema.tokenRoutes.id)).all(),
@@ -1309,6 +1319,7 @@ async function exportAccountsSection(): Promise<AccountsBackupSection> {
return {
sites,
siteApiEndpoints,
accounts: accounts.map(({ balanceUsed: _balanceUsed, lastCheckinAt: _lastCheckinAt, lastBalanceRefresh: _lastBalanceRefresh, ...row }) => row),
accountTokens,
tokenRoutes,
@@ -1389,6 +1400,9 @@ function coerceAccountsSection(input: unknown): AccountsBackupSection | null {
if (!isRecord(input)) return null;
const sites = Array.isArray(input.sites) ? input.sites as SiteRow[] : null;
const siteApiEndpoints = Array.isArray(input.siteApiEndpoints)
? input.siteApiEndpoints as SiteApiEndpointRow[]
: undefined;
const accounts = Array.isArray(input.accounts) ? input.accounts as BackupAccountRow[] : null;
const accountTokens = Array.isArray(input.accountTokens) ? input.accountTokens as AccountTokenRow[] : null;
const tokenRoutes = Array.isArray(input.tokenRoutes) ? input.tokenRoutes as TokenRouteRow[] : null;
@@ -1410,6 +1424,7 @@ function coerceAccountsSection(input: unknown): AccountsBackupSection | null {
return {
sites,
siteApiEndpoints,
accounts,
accountTokens,
tokenRoutes,
@@ -1534,6 +1549,22 @@ async function importAccountsSection(section: AccountsBackupSection): Promise<vo
}).run();
}
for (const row of section.siteApiEndpoints || []) {
await tx.insert(schema.siteApiEndpoints).values({
id: row.id,
siteId: row.siteId,
url: row.url,
enabled: row.enabled ?? true,
sortOrder: row.sortOrder ?? 0,
cooldownUntil: row.cooldownUntil ?? null,
lastSelectedAt: row.lastSelectedAt ?? null,
lastFailedAt: row.lastFailedAt ?? null,
lastFailureReason: row.lastFailureReason ?? null,
createdAt: row.createdAt,
updatedAt: row.updatedAt,
}).run();
}
for (const row of section.accounts) {
const oauthColumns = resolveImportedOauthColumns(row);
const accountKey = importedIndexes.accountKeyById.get(row.id);
src/server/services/databaseMigrationService.test.ts
+76
View File
@@ -14,6 +14,7 @@ function createDbSchemaMock() {
return {
settings: { __table: 'settings' },
sites: { __table: 'sites' },
siteApiEndpoints: { __table: 'siteApiEndpoints' },
siteAnnouncements: { __table: 'siteAnnouncements' },
siteDisabledModels: { __table: 'siteDisabledModels' },
accounts: { __table: 'accounts' },
@@ -245,6 +246,81 @@ describe('databaseMigrationService', () => {
expect(siteStatement?.values[customHeadersIndex]).toBe('{"x-site-scope":"internal"}');
});
it('includes site api endpoints when building migration statements', () => {
const statements = __databaseMigrationServiceTestUtils.buildStatements({
version: 'test',
timestamp: Date.now(),
accounts: {
sites: [{
id: 1,
name: 'demo',
url: 'https://example.com',
platform: 'new-api',
status: 'active',
}],
siteApiEndpoints: [{
id: 9,
siteId: 1,
url: 'https://api.example.com',
enabled: true,
sortOrder: 2,
cooldownUntil: '2026-03-31T12:05:00.000Z',
lastSelectedAt: '2026-03-31T12:00:00.000Z',
lastFailedAt: '2026-03-31T11:59:00.000Z',
lastFailureReason: 'HTTP 502',
createdAt: '2026-03-30T00:00:00.000Z',
updatedAt: '2026-03-31T12:05:00.000Z',
}],
siteAnnouncements: [],
siteDisabledModels: [],
accounts: [],
accountTokens: [],
checkinLogs: [],
modelAvailability: [],
tokenModelAvailability: [],
tokenRoutes: [],
routeChannels: [],
routeGroupSources: [],
proxyLogs: [],
proxyVideoTasks: [],
proxyFiles: [],
downstreamApiKeys: [],
events: [],
},
preferences: {
settings: [],
},
} as any);
const endpointStatement = statements.find((statement) => statement.table === 'site_api_endpoints');
expect(endpointStatement?.columns).toEqual([
'id',
'site_id',
'url',
'enabled',
'sort_order',
'cooldown_until',
'last_selected_at',
'last_failed_at',
'last_failure_reason',
'created_at',
'updated_at',
]);
expect(endpointStatement?.values).toEqual([
9,
1,
'https://api.example.com',
true,
2,
'2026-03-31T12:05:00.000Z',
'2026-03-31T12:00:00.000Z',
'2026-03-31T11:59:00.000Z',
'HTTP 502',
'2026-03-30T00:00:00.000Z',
'2026-03-31T12:05:00.000Z',
]);
});
it('serializes parsed JSON-column values when building migration statements', () => {
const statements = __databaseMigrationServiceTestUtils.buildStatements({
version: 'test',
src/server/services/databaseMigrationService.ts
+38
View File
@@ -29,6 +29,7 @@ type BackupSnapshot = {
timestamp: number;
accounts: {
sites: Array<Record<string, unknown>>;
siteApiEndpoints: Array<Record<string, unknown>>;
siteAnnouncements: Array<Record<string, unknown>>;
siteDisabledModels: Array<Record<string, unknown>>;
accounts: Array<Record<string, unknown>>;
@@ -58,6 +59,7 @@ export interface DatabaseMigrationSummary {
timestamp: number;
rows: {
sites: number;
siteApiEndpoints: number;
siteAnnouncements: number;
siteDisabledModels: number;
accounts: number;
@@ -251,6 +253,7 @@ async function toBackupSnapshot(): Promise<BackupSnapshot> {
timestamp: Date.now(),
accounts: {
sites: await db.select().from(schema.sites).all() as Array<Record<string, unknown>>,
siteApiEndpoints: await db.select().from(schema.siteApiEndpoints).all() as Array<Record<string, unknown>>,
siteAnnouncements: await db.select().from(schema.siteAnnouncements).all() as Array<Record<string, unknown>>,
siteDisabledModels: await db.select().from(schema.siteDisabledModels).all() as Array<Record<string, unknown>>,
accounts: await db.select().from(schema.accounts).all() as Array<Record<string, unknown>>,
@@ -299,6 +302,7 @@ async function clearTargetData(client: SqlClient): Promise<void> {
'accounts',
'site_announcements',
'site_disabled_models',
'site_api_endpoints',
'token_routes',
'sites',
'downstream_api_keys',
@@ -340,6 +344,38 @@ function buildStatements(
});
}
for (const row of snapshot.accounts.siteApiEndpoints || []) {
statements.push({
table: 'site_api_endpoints',
columns: [
'id',
'site_id',
'url',
'enabled',
'sort_order',
'cooldown_until',
'last_selected_at',
'last_failed_at',
'last_failure_reason',
'created_at',
'updated_at',
],
values: [
asNumber(row.id, 0),
asNumber(row.siteId, 0),
asNullableString(row.url),
asBoolean(row.enabled, true),
asNumber(row.sortOrder, 0),
asNullableString(row.cooldownUntil),
asNullableString(row.lastSelectedAt),
asNullableString(row.lastFailedAt),
asNullableString(row.lastFailureReason),
asNullableString(row.createdAt),
asNullableString(row.updatedAt),
],
});
}
for (const row of snapshot.accounts.siteDisabledModels) {
statements.push({
table: 'site_disabled_models',
@@ -708,6 +744,7 @@ async function syncPostgresSequences(client: SqlClient): Promise<void> {
if (client.dialect !== 'postgres') return;
const tables = [
'sites',
'site_api_endpoints',
'site_announcements',
'site_disabled_models',
'accounts',
@@ -776,6 +813,7 @@ export async function migrateCurrentDatabase(input: DatabaseMigrationInput): Pro
timestamp: snapshot.timestamp,
rows: {
sites: snapshot.accounts.sites.length,
siteApiEndpoints: snapshot.accounts.siteApiEndpoints.length,
siteAnnouncements: snapshot.accounts.siteAnnouncements.length,
siteDisabledModels: snapshot.accounts.siteDisabledModels.length,
accounts: snapshot.accounts.accounts.length,
src/server/services/modelService.discovery.test.ts
+263
View File
@@ -75,6 +75,7 @@ describe('refreshModelsForAccount credential discovery', () => {
await db.delete(schema.modelAvailability).run();
await db.delete(schema.accountTokens).run();
await db.delete(schema.accounts).run();
await db.delete(schema.siteApiEndpoints).run();
await db.delete(schema.settings).run();
await db.delete(schema.sites).run();
const { config } = await import('../config.js');
@@ -134,6 +135,49 @@ describe('refreshModelsForAccount credential discovery', () => {
expect(tokenRows).toHaveLength(0);
});
it('uses the configured ai endpoint for direct model discovery credentials', async () => {
getApiTokenMock.mockResolvedValue(null);
getModelsMock.mockImplementation(async (baseUrl: string, token: string) => (
baseUrl === 'https://api.nih.cc' && token === 'session-token'
? ['gpt-4.1']
: []
));
const site = await db.insert(schema.sites).values({
name: 'nihao-panel',
url: 'https://nih.cc',
platform: 'new-api',
status: 'active',
}).returning().get();
await db.insert(schema.siteApiEndpoints).values({
siteId: site.id,
url: 'https://api.nih.cc',
enabled: true,
sortOrder: 0,
}).run();
const account = await db.insert(schema.accounts).values({
siteId: site.id,
username: 'nihao-user',
accessToken: 'session-token',
apiToken: null,
status: 'active',
extraConfig: JSON.stringify({ credentialMode: 'session' }),
}).returning().get();
const result = await refreshModelsForAccount(account.id);
expect(result).toMatchObject({
accountId: account.id,
refreshed: true,
status: 'success',
modelCount: 1,
modelsPreview: ['gpt-4.1'],
});
expect(getModelsMock).toHaveBeenCalledWith('https://api.nih.cc', 'session-token', undefined);
});
it('deduplicates discovered model names before writing availability rows', async () => {
getApiTokenMock.mockResolvedValue(null);
getModelsMock.mockResolvedValue(['? ', '?', 'GPT-4.1', 'gpt-4.1']);
@@ -694,6 +738,145 @@ describe('refreshModelsForAccount credential discovery', () => {
]);
});
it('uses the configured ai endpoint for codex cloud model discovery', async () => {
getApiTokenMock.mockResolvedValue(null);
getModelsMock.mockRejectedValue(new Error('codex plan discovery should not call adapter.getModels'));
undiciFetchMock.mockResolvedValue({
ok: true,
status: 200,
json: async () => ({
models: [
{ id: 'gpt-5.3-codex' },
],
}),
text: async () => JSON.stringify({ ok: true }),
});
const site = await db.insert(schema.sites).values({
name: 'codex-panel-site',
url: 'https://chatgpt.com/panel-codex',
platform: 'codex',
status: 'active',
}).returning().get();
await db.insert(schema.siteApiEndpoints).values({
siteId: site.id,
url: 'https://chatgpt.com/backend-api/codex',
enabled: true,
sortOrder: 0,
}).run();
const account = await db.insert(schema.accounts).values({
siteId: site.id,
username: 'codex-user@example.com',
accessToken: 'oauth-access-token',
apiToken: null,
status: 'active',
extraConfig: JSON.stringify({
credentialMode: 'session',
oauth: {
provider: 'codex',
accountId: 'chatgpt-account-456',
email: 'codex-user@example.com',
planType: 'plus',
},
}),
}).returning().get();
const result = await refreshModelsForAccount(account.id);
expect(result).toMatchObject({
accountId: account.id,
refreshed: true,
status: 'success',
modelCount: 1,
modelsPreview: ['gpt-5.3-codex'],
});
expect(String(undiciFetchMock.mock.calls[0]?.[0] || '')).toBe('https://chatgpt.com/backend-api/codex/models?client_version=1.0.0');
});
it('rotates codex cloud discovery across configured ai endpoints after a retryable failure', async () => {
getApiTokenMock.mockResolvedValue(null);
getModelsMock.mockRejectedValue(new Error('codex plan discovery should not call adapter.getModels'));
undiciFetchMock
.mockResolvedValueOnce({
ok: false,
status: 502,
json: async () => ({ error: 'bad gateway' }),
text: async () => 'bad gateway',
})
.mockResolvedValueOnce({
ok: true,
status: 200,
json: async () => ({
models: [
{ id: 'gpt-5.3-codex' },
],
}),
text: async () => JSON.stringify({ ok: true }),
});
const site = await db.insert(schema.sites).values({
name: 'codex-pool-site',
url: 'https://chatgpt.com/panel-codex',
platform: 'codex',
status: 'active',
}).returning().get();
await db.insert(schema.siteApiEndpoints).values([
{
siteId: site.id,
url: 'https://chatgpt.com/backend-api/codex-a',
enabled: true,
sortOrder: 0,
},
{
siteId: site.id,
url: 'https://chatgpt.com/backend-api/codex-b',
enabled: true,
sortOrder: 1,
},
]).run();
const account = await db.insert(schema.accounts).values({
siteId: site.id,
username: 'codex-pool-user@example.com',
accessToken: 'oauth-access-token',
apiToken: null,
status: 'active',
extraConfig: JSON.stringify({
credentialMode: 'session',
oauth: {
provider: 'codex',
accountId: 'chatgpt-account-789',
email: 'codex-pool-user@example.com',
planType: 'plus',
},
}),
}).returning().get();
const result = await refreshModelsForAccount(account.id);
expect(result).toMatchObject({
accountId: account.id,
refreshed: true,
status: 'success',
modelCount: 1,
modelsPreview: ['gpt-5.3-codex'],
});
expect(String(undiciFetchMock.mock.calls[0]?.[0] || '')).toBe('https://chatgpt.com/backend-api/codex-a/models?client_version=1.0.0');
expect(String(undiciFetchMock.mock.calls[1]?.[0] || '')).toBe('https://chatgpt.com/backend-api/codex-b/models?client_version=1.0.0');
const endpoints = await db.select().from(schema.siteApiEndpoints)
.where(eq(schema.siteApiEndpoints.siteId, site.id))
.all();
const firstEndpoint = endpoints.find((item) => item.url === 'https://chatgpt.com/backend-api/codex-a');
const secondEndpoint = endpoints.find((item) => item.url === 'https://chatgpt.com/backend-api/codex-b');
expect(firstEndpoint?.cooldownUntil).toBeTruthy();
expect(firstEndpoint?.lastFailureReason).toContain('HTTP 502');
expect(secondEndpoint?.lastSelectedAt).toBeTruthy();
});
it('discovers Claude OAuth models from the upstream /v1/models response', async () => {
getApiTokenMock.mockResolvedValue(null);
getModelsMock.mockRejectedValue(new Error('claude oauth discovery should not call adapter.getModels'));
@@ -1276,6 +1459,86 @@ describe('refreshModelsForAccount credential discovery', () => {
});
});
it('rotates antigravity discovery across configured ai endpoints before using built-in fallback hosts', async () => {
getApiTokenMock.mockResolvedValue(null);
getModelsMock.mockRejectedValue(new Error('antigravity oauth discovery should not call adapter.getModels'));
undiciFetchMock
.mockResolvedValueOnce({
ok: false,
status: 503,
json: async () => ({ error: 'endpoint-a unavailable' }),
text: async () => 'endpoint-a unavailable',
})
.mockResolvedValueOnce({
ok: true,
status: 200,
json: async () => ({
models: {
'gemini-3-pro-preview': { displayName: 'Gemini 3 Pro Preview' },
},
}),
text: async () => JSON.stringify({ ok: true }),
});
const site = await db.insert(schema.sites).values({
name: 'antigravity-endpoint-pool-site',
url: 'https://cloudcode-panel.example.com',
platform: 'antigravity',
status: 'active',
}).returning().get();
await db.insert(schema.siteApiEndpoints).values([
{
siteId: site.id,
url: 'https://api-antigravity-a.example.com',
enabled: true,
sortOrder: 0,
},
{
siteId: site.id,
url: 'https://api-antigravity-b.example.com',
enabled: true,
sortOrder: 1,
},
]).run();
const account = await db.insert(schema.accounts).values({
siteId: site.id,
username: 'antigravity-endpoint-pool@example.com',
accessToken: 'antigravity-access-token',
apiToken: null,
status: 'active',
extraConfig: JSON.stringify({
credentialMode: 'session',
oauth: {
provider: 'antigravity',
email: 'antigravity-endpoint-pool@example.com',
projectId: 'project-demo',
},
}),
}).returning().get();
const result = await refreshModelsForAccount(account.id);
expect(result).toMatchObject({
accountId: account.id,
refreshed: true,
status: 'success',
modelCount: 1,
modelsPreview: ['gemini-3-pro-preview'],
});
expect(undiciFetchMock).toHaveBeenCalledTimes(2);
expect(String(undiciFetchMock.mock.calls[0]?.[0] || '')).toBe('https://api-antigravity-a.example.com/v1internal:fetchAvailableModels');
expect(String(undiciFetchMock.mock.calls[1]?.[0] || '')).toBe('https://api-antigravity-b.example.com/v1internal:fetchAvailableModels');
const endpoints = await db.select().from(schema.siteApiEndpoints).all();
const firstEndpoint = endpoints.find((item) => item.url === 'https://api-antigravity-a.example.com');
const secondEndpoint = endpoints.find((item) => item.url === 'https://api-antigravity-b.example.com');
expect(firstEndpoint?.cooldownUntil).toBeTruthy();
expect(firstEndpoint?.lastFailureReason).toContain('HTTP 503');
expect(secondEndpoint?.lastSelectedAt).toBeTruthy();
});
it('preserves manual models after successful model refresh', async () => {
getApiTokenMock.mockResolvedValue(null);
getModelsMock.mockResolvedValue(['gpt-4.1', 'claude-opus-4-6']);
src/server/services/modelService.ts
+27 -2
View File
@@ -26,6 +26,7 @@ import { withAccountProxyOverride } from './siteProxy.js';
import { isCodexPlatform } from './oauth/codexAccount.js';
import { buildStoredOauthStateFromAccount, getOauthInfoFromAccount } from './oauth/oauthAccount.js';
import { refreshOauthAccessTokenSingleflight } from './oauth/refreshSingleflight.js';
import { requireSiteApiBaseUrl } from './siteApiEndpointService.js';
import {
discoverAntigravityModelsFromCloud,
discoverClaudeModelsFromCloud,
@@ -775,6 +776,30 @@ export async function refreshModelsForAccount(
}
}
let aiBaseUrl: string;
try {
aiBaseUrl = await requireSiteApiBaseUrl(site);
} catch (err) {
const rawMessage = (err as { message?: string })?.message || '模型获取失败';
const errorCode = classifyModelDiscoveryError(rawMessage);
const errorMessage = rawMessage;
await setAccountRuntimeHealth(account.id, {
state: 'unhealthy',
reason: errorMessage,
source: 'model-discovery',
checkedAt: new Date().toISOString(),
});
await restorePreviousAvailability();
return buildFailedRefreshResult({
accountId,
errorCode,
errorMessage,
tokenScanned: 0,
discoveredByCredential: false,
discoveredApiToken: !!discoveredApiToken,
});
}
const accountModels = new Map<string, string>(); // lowercase key → original name (first-wins)
const modelLatency = new Map<string, number | null>();
let scannedTokenCount = 0;
@@ -813,7 +838,7 @@ export async function refreshModelsForAccount(
models = normalizeModels(
await withTimeout(
() => withAccountProxyOverride(accountProxyUrl,
() => adapter.getModels(site.url, credential, platformUserId)),
() => adapter.getModels(aiBaseUrl, credential, platformUserId)),
MODEL_DISCOVERY_TIMEOUT_MS,
`model discovery timeout (${Math.round(MODEL_DISCOVERY_TIMEOUT_MS / 1000)}s)`,
),
@@ -841,7 +866,7 @@ export async function refreshModelsForAccount(
models = normalizeModels(
await withTimeout(
() => withAccountProxyOverride(accountProxyUrl,
() => adapter.getModels(site.url, token.token, platformUserId)),
() => adapter.getModels(aiBaseUrl, token.token, platformUserId)),
MODEL_DISCOVERY_TIMEOUT_MS,
`model discovery timeout (${Math.round(MODEL_DISCOVERY_TIMEOUT_MS / 1000)}s)`,
),
src/server/services/platformDiscoveryRegistry.ts
+68 -54
View File
@@ -1,6 +1,7 @@
import { fetch } from 'undici';
import { schema } from '../db/index.js';
import { withSiteRecordProxyRequestInit } from './siteProxy.js';
import { runWithSiteApiEndpointPool } from './siteApiEndpointService.js';
import { getOauthInfoFromAccount } from './oauth/oauthAccount.js';
import { CLAUDE_DEFAULT_ANTHROPIC_VERSION } from './oauth/claudeProvider.js';
import {
@@ -136,15 +137,18 @@ export async function discoverCodexModelsFromCloud(input: {
headers['Chatgpt-Account-Id'] = oauth.accountId;
}
const response = await fetch(
buildCodexModelsEndpoint(input.site.url),
withSiteRecordProxyRequestInit(input.site, { method: 'GET', headers }),
);
if (!response.ok) {
const text = await response.text().catch(() => '');
throw new Error(`HTTP ${response.status}: ${text || 'codex model discovery failed'}`);
}
return normalizeDiscoveredModels(extractCodexModelIds(await response.json()));
const payload = await runWithSiteApiEndpointPool(input.site, async (target) => {
const response = await fetch(
buildCodexModelsEndpoint(target.baseUrl),
withSiteRecordProxyRequestInit(input.site, { method: 'GET', headers }),
);
if (!response.ok) {
const text = await response.text().catch(() => '');
throw new Error(`HTTP ${response.status}: ${text || 'codex model discovery failed'}`);
}
return response.json();
});
return normalizeDiscoveredModels(extractCodexModelIds(payload));
}
export async function discoverClaudeModelsFromCloud(input: {
@@ -155,22 +159,25 @@ export async function discoverClaudeModelsFromCloud(input: {
if (!accessToken) {
throw new Error('claude oauth access token missing');
}
const response = await fetch(
`${input.site.url.replace(/\/+$/, '')}/v1/models`,
withSiteRecordProxyRequestInit(input.site, {
method: 'GET',
headers: {
Authorization: `Bearer ${accessToken}`,
Accept: 'application/json',
'anthropic-version': CLAUDE_DEFAULT_ANTHROPIC_VERSION,
},
}),
);
if (!response.ok) {
const text = await response.text().catch(() => '');
throw new Error(`HTTP ${response.status}: ${text || 'claude oauth model discovery failed'}`);
}
return normalizeDiscoveredModels(extractClaudeModelIds(await response.json()));
const payload = await runWithSiteApiEndpointPool(input.site, async (target) => {
const response = await fetch(
`${target.baseUrl.replace(/\/+$/, '')}/v1/models`,
withSiteRecordProxyRequestInit(input.site, {
method: 'GET',
headers: {
Authorization: `Bearer ${accessToken}`,
Accept: 'application/json',
'anthropic-version': CLAUDE_DEFAULT_ANTHROPIC_VERSION,
},
}),
);
if (!response.ok) {
const text = await response.text().catch(() => '');
throw new Error(`HTTP ${response.status}: ${text || 'claude oauth model discovery failed'}`);
}
return response.json();
});
return normalizeDiscoveredModels(extractClaudeModelIds(payload));
}
export async function validateGeminiCliOauthConnection(input: {
@@ -220,38 +227,45 @@ export async function discoverAntigravityModelsFromCloud(input: {
const oauth = getOauthInfoFromAccount(input.account);
const projectId = (oauth?.projectId || '').trim();
const requestBody = projectId ? { project: projectId } : {};
let lastError = '';
return runWithSiteApiEndpointPool(input.site, async (target) => {
let lastError = '';
const selectedBaseUrl = normalizeBaseUrl(target.baseUrl || ANTIGRAVITY_UPSTREAM_BASE_URL) || ANTIGRAVITY_UPSTREAM_BASE_URL;
const discoveryBaseUrls = target.endpointId
? [selectedBaseUrl]
: buildAntigravityDiscoveryBaseUrls(selectedBaseUrl);
for (const baseUrl of buildAntigravityDiscoveryBaseUrls(input.site.url || ANTIGRAVITY_UPSTREAM_BASE_URL)) {
try {
const response = await fetch(
`${baseUrl}/v1internal:fetchAvailableModels`,
withSiteRecordProxyRequestInit(input.site, {
method: 'POST',
headers: {
Authorization: `Bearer ${accessToken}`,
Accept: 'application/json',
'Content-Type': 'application/json',
'User-Agent': ANTIGRAVITY_MODELS_USER_AGENT,
},
body: JSON.stringify(requestBody),
}),
);
if (!response.ok) {
lastError = await response.text().catch(() => '') || `HTTP ${response.status}`;
continue;
}
for (const discoveryBaseUrl of discoveryBaseUrls) {
try {
const response = await fetch(
`${discoveryBaseUrl}/v1internal:fetchAvailableModels`,
withSiteRecordProxyRequestInit(input.site, {
method: 'POST',
headers: {
Authorization: `Bearer ${accessToken}`,
Accept: 'application/json',
'Content-Type': 'application/json',
'User-Agent': ANTIGRAVITY_MODELS_USER_AGENT,
},
body: JSON.stringify(requestBody),
}),
);
if (!response.ok) {
const text = await response.text().catch(() => '');
lastError = `HTTP ${response.status}: ${text || '未获取到可用模型'}`;
continue;
}
const payload = await response.json();
const models = normalizeDiscoveredModels(extractAntigravityModelIds(payload));
if (models.length > 0) {
return models;
const payload = await response.json();
const models = normalizeDiscoveredModels(extractAntigravityModelIds(payload));
if (models.length > 0) {
return models;
}
lastError = '未获取到可用模型';
} catch (error) {
lastError = error instanceof Error ? `${discoveryBaseUrl}: ${error.message}` : String(error);
}
lastError = '未获取到可用模型';
} catch (error) {
lastError = error instanceof Error ? `${baseUrl}: ${error.message}` : String(error);
}
}
throw new Error(lastError || '未获取到可用模型');
throw new Error(lastError || '未获取到可用模型');
});
}
src/server/services/proxyVideoTaskStore.ts
+12
View File
@@ -89,6 +89,18 @@ export async function getProxyVideoTaskByPublicId(publicId: string): Promise<Pro
};
}
export async function resolveProxyVideoTaskSite(accountId: number | null): Promise<typeof schema.sites.$inferSelect | null> {
if (!Number.isFinite(accountId) || !accountId || accountId <= 0) {
return null;
}
const row = await db.select()
.from(schema.accounts)
.innerJoin(schema.sites, eq(schema.accounts.siteId, schema.sites.id))
.where(eq(schema.accounts.id, accountId))
.get();
return row?.sites ?? null;
}
export async function deleteProxyVideoTaskByPublicId(publicId: string): Promise<void> {
await db.delete(schema.proxyVideoTasks)
.where(eq(schema.proxyVideoTasks.publicId, publicId))
src/server/services/siteApiEndpointService.test.ts
+340
View File
@@ -0,0 +1,340 @@
import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest';
import { mkdtempSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { asc, eq } from 'drizzle-orm';
type DbModule = typeof import('../db/index.js');
type SiteApiEndpointServiceModule = typeof import('./siteApiEndpointService.js');
describe('siteApiEndpointService', () => {
let db: DbModule['db'];
let schema: DbModule['schema'];
let selectSiteApiEndpointTarget: SiteApiEndpointServiceModule['selectSiteApiEndpointTarget'];
let recordSiteApiEndpointFailure: SiteApiEndpointServiceModule['recordSiteApiEndpointFailure'];
let recordSiteApiEndpointSuccess: SiteApiEndpointServiceModule['recordSiteApiEndpointSuccess'];
let dataDir = '';
beforeAll(async () => {
dataDir = mkdtempSync(join(tmpdir(), 'metapi-site-api-endpoint-service-'));
process.env.DATA_DIR = dataDir;
await import('../db/migrate.js');
const dbModule = await import('../db/index.js');
const serviceModule = await import('./siteApiEndpointService.js');
db = dbModule.db;
schema = dbModule.schema;
selectSiteApiEndpointTarget = serviceModule.selectSiteApiEndpointTarget;
recordSiteApiEndpointFailure = serviceModule.recordSiteApiEndpointFailure;
recordSiteApiEndpointSuccess = serviceModule.recordSiteApiEndpointSuccess;
});
beforeEach(async () => {
await db.delete(schema.siteApiEndpoints).run();
await db.delete(schema.sites).run();
});
afterAll(() => {
delete process.env.DATA_DIR;
});
it('returns a synthetic site-url fallback when the site has no configured api endpoints', async () => {
const site = await db.insert(schema.sites).values({
name: 'panel-only-site',
url: 'https://panel.example.com/',
platform: 'new-api',
status: 'active',
}).returning().get();
const selected = await selectSiteApiEndpointTarget(site, '2026-03-31T12:00:00.000Z');
expect(selected).toMatchObject({
kind: 'site-fallback',
siteId: site.id,
endpointId: null,
baseUrl: 'https://panel.example.com',
configuredEndpointCount: 0,
});
});
it('selects the least recently selected enabled endpoint when sort order is tied', async () => {
const site = await db.insert(schema.sites).values({
name: 'pool-site',
url: 'https://panel.example.com',
platform: 'new-api',
status: 'active',
}).returning().get();
await db.insert(schema.siteApiEndpoints).values([
{
siteId: site.id,
url: 'https://api-b.example.com',
enabled: true,
sortOrder: 1,
lastSelectedAt: '2026-03-31T11:59:00.000Z',
},
{
siteId: site.id,
url: 'https://api-a.example.com/',
enabled: true,
sortOrder: 0,
lastSelectedAt: '2026-03-31T11:00:00.000Z',
},
]).run();
const selected = await selectSiteApiEndpointTarget(site, '2026-03-31T12:00:00.000Z');
expect(selected).toMatchObject({
kind: 'endpoint',
siteId: site.id,
baseUrl: 'https://api-a.example.com',
configuredEndpointCount: 2,
endpoint: expect.objectContaining({
url: 'https://api-a.example.com/',
sortOrder: 0,
}),
});
});
it('prefers lower sortOrder before lastSelectedAt when selecting an enabled endpoint', async () => {
const site = await db.insert(schema.sites).values({
name: 'ordered-site',
url: 'https://panel.example.com',
platform: 'new-api',
status: 'active',
}).returning().get();
await db.insert(schema.siteApiEndpoints).values([
{
siteId: site.id,
url: 'https://api-secondary.example.com',
enabled: true,
sortOrder: 1,
lastSelectedAt: '2026-03-31T11:00:00.000Z',
},
{
siteId: site.id,
url: 'https://api-primary.example.com',
enabled: true,
sortOrder: 0,
lastSelectedAt: '2026-03-31T11:59:00.000Z',
},
]).run();
const selected = await selectSiteApiEndpointTarget(site, '2026-03-31T12:00:00.000Z');
expect(selected).toMatchObject({
kind: 'endpoint',
siteId: site.id,
baseUrl: 'https://api-primary.example.com',
configuredEndpointCount: 2,
endpoint: expect.objectContaining({
url: 'https://api-primary.example.com',
sortOrder: 0,
}),
});
});
it('skips disabled endpoints and endpoints that are still cooling down', async () => {
const site = await db.insert(schema.sites).values({
name: 'filtered-site',
url: 'https://panel.example.com',
platform: 'new-api',
status: 'active',
}).returning().get();
await db.insert(schema.siteApiEndpoints).values([
{
siteId: site.id,
url: 'https://api-disabled.example.com',
enabled: false,
sortOrder: 0,
},
{
siteId: site.id,
url: 'https://api-cooling.example.com',
enabled: true,
sortOrder: 1,
cooldownUntil: '2026-03-31T12:05:00.000Z',
},
{
siteId: site.id,
url: 'https://api-ready.example.com',
enabled: true,
sortOrder: 2,
cooldownUntil: '2026-03-31T11:55:00.000Z',
},
]).run();
const selected = await selectSiteApiEndpointTarget(site, '2026-03-31T12:00:00.000Z');
expect(selected).toMatchObject({
kind: 'endpoint',
baseUrl: 'https://api-ready.example.com',
configuredEndpointCount: 3,
});
});
it('returns null when the site has configured api endpoints but none are currently eligible', async () => {
const site = await db.insert(schema.sites).values({
name: 'exhausted-site',
url: 'https://panel.example.com',
platform: 'new-api',
status: 'active',
}).returning().get();
await db.insert(schema.siteApiEndpoints).values([
{
siteId: site.id,
url: 'https://api-disabled.example.com',
enabled: false,
sortOrder: 0,
},
{
siteId: site.id,
url: 'https://api-cooling.example.com',
enabled: true,
sortOrder: 1,
cooldownUntil: '2026-03-31T12:05:00.000Z',
},
]).run();
const selected = await selectSiteApiEndpointTarget(site, '2026-03-31T12:00:00.000Z');
expect(selected).toBeNull();
});
it('records retryable failures with a 5-minute cooldown', async () => {
const site = await db.insert(schema.sites).values({
name: 'retryable-site',
url: 'https://panel.example.com',
platform: 'new-api',
status: 'active',
}).returning().get();
const endpoint = await db.insert(schema.siteApiEndpoints).values({
siteId: site.id,
url: 'https://api-retryable.example.com',
enabled: true,
sortOrder: 0,
}).returning().get();
const result = await recordSiteApiEndpointFailure(endpoint.id, {
status: 502,
message: 'Bad gateway',
}, '2026-03-31T12:00:00.000Z');
expect(result).toMatchObject({
retryable: true,
rotateToNextEndpoint: true,
cooldownUntil: '2026-03-31T12:05:00.000Z',
failureReason: 'HTTP 502: Bad gateway',
});
const stored = await db.select().from(schema.siteApiEndpoints)
.where(eq(schema.siteApiEndpoints.id, endpoint.id))
.get();
expect(stored).toMatchObject({
cooldownUntil: '2026-03-31T12:05:00.000Z',
lastFailedAt: '2026-03-31T12:00:00.000Z',
lastFailureReason: 'HTTP 502: Bad gateway',
});
});
it('parses retryable HTTP status codes from failure messages when no explicit status is provided', async () => {
const site = await db.insert(schema.sites).values({
name: 'message-status-site',
url: 'https://panel.example.com',
platform: 'new-api',
status: 'active',
}).returning().get();
const endpoint = await db.insert(schema.siteApiEndpoints).values({
siteId: site.id,
url: 'https://api-message-status.example.com',
enabled: true,
sortOrder: 0,
}).returning().get();
const result = await recordSiteApiEndpointFailure(endpoint.id, {
message: 'HTTP 502: upstream temporarily unavailable',
}, '2026-03-31T12:00:00.000Z');
expect(result).toMatchObject({
retryable: true,
rotateToNextEndpoint: true,
cooldownUntil: '2026-03-31T12:05:00.000Z',
failureReason: 'HTTP 502: upstream temporarily unavailable',
});
});
it('records auth and validation failures without triggering cooldown rotation', async () => {
const site = await db.insert(schema.sites).values({
name: 'non-retryable-site',
url: 'https://panel.example.com',
platform: 'new-api',
status: 'active',
}).returning().get();
const endpoint = await db.insert(schema.siteApiEndpoints).values({
siteId: site.id,
url: 'https://api-auth.example.com',
enabled: true,
sortOrder: 0,
cooldownUntil: '2026-03-31T11:00:00.000Z',
}).returning().get();
const result = await recordSiteApiEndpointFailure(endpoint.id, {
status: 401,
message: 'Invalid token',
}, '2026-03-31T12:00:00.000Z');
expect(result).toMatchObject({
retryable: false,
rotateToNextEndpoint: false,
cooldownUntil: null,
failureReason: 'HTTP 401: Invalid token',
});
const stored = await db.select().from(schema.siteApiEndpoints)
.where(eq(schema.siteApiEndpoints.id, endpoint.id))
.get();
expect(stored).toMatchObject({
cooldownUntil: null,
lastFailedAt: '2026-03-31T12:00:00.000Z',
lastFailureReason: 'HTTP 401: Invalid token',
});
});
it('clears cooldown metadata and updates lastSelectedAt after a recorded success', async () => {
const site = await db.insert(schema.sites).values({
name: 'success-site',
url: 'https://panel.example.com',
platform: 'new-api',
status: 'active',
}).returning().get();
const endpoint = await db.insert(schema.siteApiEndpoints).values({
siteId: site.id,
url: 'https://api-success.example.com',
enabled: true,
sortOrder: 0,
cooldownUntil: '2026-03-31T12:05:00.000Z',
lastFailedAt: '2026-03-31T12:00:00.000Z',
lastFailureReason: 'HTTP 502: Bad gateway',
}).returning().get();
await recordSiteApiEndpointSuccess(endpoint.id, '2026-03-31T12:01:00.000Z');
const stored = await db.select().from(schema.siteApiEndpoints)
.where(eq(schema.siteApiEndpoints.id, endpoint.id))
.orderBy(asc(schema.siteApiEndpoints.id))
.get();
expect(stored).toMatchObject({
cooldownUntil: null,
lastSelectedAt: '2026-03-31T12:01:00.000Z',
lastFailureReason: null,
});
});
});
src/server/services/siteApiEndpointService.ts
+287
View File
@@ -0,0 +1,287 @@
import { asc, eq } from 'drizzle-orm';
import { db, schema } from '../db/index.js';
import { RETRYABLE_TIMEOUT_PATTERNS } from './proxyRetryPolicy.js';
const RETRYABLE_STATUS_CODES = new Set([408, 429, 500, 502, 503, 504]);
const NON_RETRYABLE_STATUS_CODES = new Set([400, 401, 403, 404, 422]);
const NETWORK_FAILURE_PATTERNS = [
/network error/i,
/fetch failed/i,
/socket hang up/i,
/econnreset/i,
/econnrefused/i,
/enotfound/i,
/ehostunreach/i,
/ecanceled/i,
...RETRYABLE_TIMEOUT_PATTERNS,
];
export const SITE_API_ENDPOINT_COOLDOWN_MS = 5 * 60 * 1000;
type SiteRow = typeof schema.sites.$inferSelect;
type SiteApiEndpointRow = typeof schema.siteApiEndpoints.$inferSelect;
export interface SiteApiEndpointTarget {
kind: 'site-fallback' | 'endpoint';
siteId: number;
endpointId: number | null;
baseUrl: string;
configuredEndpointCount: number;
endpoint: SiteApiEndpointRow | null;
}
export interface SiteApiEndpointFailureInput {
status?: number | null;
message?: string | null;
error?: unknown;
}
export interface SiteApiEndpointFailureDisposition {
retryable: boolean;
rotateToNextEndpoint: boolean;
failureReason: string;
}
export interface RecordedSiteApiEndpointFailure extends SiteApiEndpointFailureDisposition {
cooldownUntil: string | null;
}
export class SiteApiEndpointRequestError extends Error {
readonly status: number | null;
readonly rawErrText: string | null;
constructor(message: string, options?: { status?: number | null; rawErrText?: string | null; cause?: unknown }) {
super(message, options?.cause !== undefined ? { cause: options.cause } : undefined);
this.name = 'SiteApiEndpointRequestError';
this.status = typeof options?.status === 'number' ? options.status : null;
this.rawErrText = typeof options?.rawErrText === 'string' && options.rawErrText.trim()
? options.rawErrText
: null;
}
}
export function normalizeSiteApiEndpointBaseUrl(raw: string): string {
const trimmed = raw.trim();
if (!trimmed) return '';
try {
const parsed = new URL(trimmed);
parsed.search = '';
parsed.hash = '';
return parsed.toString().replace(/\/+$/, '');
} catch {
return trimmed.replace(/\/+$/, '');
}
}
function toIsoTimestamp(now?: string | Date): string {
if (typeof now === 'string' && now.trim()) return now;
if (now instanceof Date) return now.toISOString();
return new Date().toISOString();
}
function compareNullableTimeAsc(left?: string | null, right?: string | null): number {
if (!left && !right) return 0;
if (!left) return -1;
if (!right) return 1;
return left.localeCompare(right);
}
function isEndpointCoolingDown(endpoint: SiteApiEndpointRow, nowIso: string): boolean {
return !!endpoint.cooldownUntil && endpoint.cooldownUntil > nowIso;
}
function extractFailureMessage(input: SiteApiEndpointFailureInput): string {
const direct = typeof input.message === 'string' ? input.message.trim() : '';
if (direct) return direct;
const errorMessage = input.error instanceof Error ? input.error.message.trim() : '';
return errorMessage;
}
function formatFailureReason(status: number | null, message: string): string {
if (status && message) {
if (message.match(new RegExp(`^HTTP\\s+${status}\\b`, 'i'))) {
return message;
}
return `HTTP ${status}: ${message}`;
}
if (status) return `HTTP ${status}`;
return message || 'endpoint failure';
}
function parseStatusFromFailureMessage(message: string): number | null {
const matched = message.match(/\bHTTP\s+(\d{3})\b/i);
if (!matched) return null;
const status = Number.parseInt(matched[1] || '', 10);
return Number.isFinite(status) ? status : null;
}
export function classifySiteApiEndpointFailure(
input: SiteApiEndpointFailureInput,
): SiteApiEndpointFailureDisposition {
const message = extractFailureMessage(input);
const status = typeof input.status === 'number'
? input.status
: parseStatusFromFailureMessage(message);
const failureReason = formatFailureReason(status, message);
if (status !== null) {
if (RETRYABLE_STATUS_CODES.has(status)) {
return { retryable: true, rotateToNextEndpoint: true, failureReason };
}
if (NON_RETRYABLE_STATUS_CODES.has(status)) {
return { retryable: false, rotateToNextEndpoint: false, failureReason };
}
}
if (NETWORK_FAILURE_PATTERNS.some((pattern) => pattern.test(message))) {
return { retryable: true, rotateToNextEndpoint: true, failureReason };
}
return { retryable: false, rotateToNextEndpoint: false, failureReason };
}
export async function selectSiteApiEndpointTarget(
site: SiteRow,
now?: string | Date,
): Promise<SiteApiEndpointTarget | null> {
const nowIso = toIsoTimestamp(now);
const endpoints = await db.select().from(schema.siteApiEndpoints)
.where(eq(schema.siteApiEndpoints.siteId, site.id))
.orderBy(asc(schema.siteApiEndpoints.sortOrder), asc(schema.siteApiEndpoints.id))
.all();
if (endpoints.length === 0) {
return {
kind: 'site-fallback',
siteId: site.id,
endpointId: null,
baseUrl: normalizeSiteApiEndpointBaseUrl(site.url),
configuredEndpointCount: 0,
endpoint: null,
};
}
const eligible = endpoints
.filter((endpoint) => (endpoint.enabled ?? true) && !isEndpointCoolingDown(endpoint, nowIso))
.sort((left, right) => {
const sortOrder = (left.sortOrder ?? 0) - (right.sortOrder ?? 0);
if (sortOrder !== 0) return sortOrder;
const selectionOrder = compareNullableTimeAsc(left.lastSelectedAt, right.lastSelectedAt);
if (selectionOrder !== 0) return selectionOrder;
return (left.id ?? 0) - (right.id ?? 0);
});
const selected = eligible[0];
if (!selected) return null;
return {
kind: 'endpoint',
siteId: site.id,
endpointId: selected.id,
baseUrl: normalizeSiteApiEndpointBaseUrl(selected.url),
configuredEndpointCount: endpoints.length,
endpoint: selected,
};
}
export async function resolveSiteApiBaseUrl(
site: SiteRow,
now?: string | Date,
): Promise<string | null> {
const target = await selectSiteApiEndpointTarget(site, now);
return target?.baseUrl || null;
}
export async function requireSiteApiBaseUrl(
site: SiteRow,
now?: string | Date,
): Promise<string> {
const baseUrl = await resolveSiteApiBaseUrl(site, now);
if (baseUrl) return baseUrl;
throw new Error('当前站点的 AI 请求地址均不可用');
}
export async function recordSiteApiEndpointFailure(
endpointId: number,
input: SiteApiEndpointFailureInput,
now?: string | Date,
): Promise<RecordedSiteApiEndpointFailure> {
const nowIso = toIsoTimestamp(now);
const disposition = classifySiteApiEndpointFailure(input);
const cooldownUntil = disposition.retryable
? new Date(Date.parse(nowIso) + SITE_API_ENDPOINT_COOLDOWN_MS).toISOString()
: null;
await db.update(schema.siteApiEndpoints).set({
cooldownUntil,
lastFailedAt: nowIso,
lastFailureReason: disposition.failureReason,
updatedAt: nowIso,
}).where(eq(schema.siteApiEndpoints.id, endpointId)).run();
return {
...disposition,
cooldownUntil,
};
}
export async function recordSiteApiEndpointSuccess(
endpointId: number,
now?: string | Date,
): Promise<void> {
const nowIso = toIsoTimestamp(now);
await db.update(schema.siteApiEndpoints).set({
cooldownUntil: null,
lastSelectedAt: nowIso,
lastFailureReason: null,
updatedAt: nowIso,
}).where(eq(schema.siteApiEndpoints.id, endpointId)).run();
}
export async function runWithSiteApiEndpointPool<T>(
site: SiteRow,
operation: (target: SiteApiEndpointTarget) => Promise<T>,
): Promise<T> {
const attemptedEndpointIds = new Set<number>();
let lastError: unknown;
while (true) {
const target = await selectSiteApiEndpointTarget(site);
if (!target) {
if (lastError) throw lastError;
throw new Error('当前站点的 AI 请求地址均不可用');
}
if (target.endpointId && attemptedEndpointIds.has(target.endpointId)) {
if (lastError) throw lastError;
throw new Error('当前站点的 AI 请求地址均不可用');
}
try {
const result = await operation(target);
if (target.endpointId) {
try {
await recordSiteApiEndpointSuccess(target.endpointId);
} catch (error) {
console.warn('[siteApiEndpointService] failed to record endpoint success', error);
}
}
return result;
} catch (error) {
lastError = error;
if (!target.endpointId) {
throw error;
}
const recordedFailure = await recordSiteApiEndpointFailure(target.endpointId, {
status: error instanceof SiteApiEndpointRequestError ? error.status : undefined,
message: error instanceof Error ? error.message : String(error ?? ''),
error,
});
if (!recordedFailure.rotateToNextEndpoint) {
throw error;
}
attemptedEndpointIds.add(target.endpointId);
}
}
}
src/web/pages/Sites.tsx
+219 -6
View File
@@ -20,11 +20,14 @@ import { shouldIgnoreRowSelectionClick } from './helpers/rowSelection.js';
import { resolveInitialConnectionSegment } from './helpers/defaultConnectionSegment.js';
import {
buildSiteSaveAction,
emptySiteApiEndpoint,
emptySiteCustomHeader,
emptySiteForm,
serializeSiteApiEndpoints,
serializeSiteCustomHeaders,
siteFormFromSite,
type SiteEditorState,
type SiteApiEndpointField,
type SiteForm,
} from './helpers/sitesEditor.js';
import {
@@ -59,12 +62,33 @@ type SiteRow = {
totalBalance?: number;
subscriptionSummary?: SiteSubscriptionSummary | null;
createdAt?: string;
apiEndpoints?: Array<{
id?: number;
url: string;
enabled?: boolean;
sortOrder?: number;
cooldownUntil?: string | null;
lastFailureReason?: string | null;
}>;
};
function hasConfiguredCustomHeaders(customHeaders?: string | null): boolean {
return typeof customHeaders === 'string' && customHeaders.trim().length > 0;
}
function getConfiguredSiteApiEndpoints(site?: Pick<SiteRow, 'apiEndpoints'> | null) {
return Array.isArray(site?.apiEndpoints)
? site.apiEndpoints.filter((item) => typeof item?.url === 'string' && item.url.trim())
: [];
}
function buildSiteApiEndpointSummary(site?: Pick<SiteRow, 'apiEndpoints'> | null): string {
const endpoints = getConfiguredSiteApiEndpoints(site);
if (endpoints.length <= 0) return '跟随主站点 URL';
const enabledCount = endpoints.filter((item) => item.enabled !== false).length;
return `${enabledCount}/${endpoints.length} 条启用`;
}
function formatUsd(value?: number | null): string {
return `$${(value || 0).toFixed(2)}`;
}
@@ -203,7 +227,27 @@ export default function Sites() {
const [sortMode, setSortMode] = useState<SortMode>('custom');
const [highlightSiteId, setHighlightSiteId] = useState<number | null>(null);
const [editor, setEditor] = useState<SiteEditorState | null>(null);
const [form, setForm] = useState<SiteForm>(emptySiteForm());
const apiEndpointDraftIdRef = useRef(0);
const createApiEndpointDraftId = () => {
apiEndpointDraftIdRef.current += 1;
return `site-api-endpoint-draft-${apiEndpointDraftIdRef.current}`;
};
const createEmptyApiEndpointRow = (): SiteApiEndpointField => ({
...emptySiteApiEndpoint(),
draftId: createApiEndpointDraftId(),
});
const hydrateSiteForm = (value: SiteForm): SiteForm => {
const sourceRows = value.apiEndpoints.length > 0 ? value.apiEndpoints : [createEmptyApiEndpointRow()];
return {
...value,
apiEndpoints: sourceRows.map((endpoint) => ({
...endpoint,
draftId: endpoint.draftId || createApiEndpointDraftId(),
})),
};
};
const createEmptySiteForm = (): SiteForm => hydrateSiteForm(emptySiteForm());
const [form, setForm] = useState<SiteForm>(() => createEmptySiteForm());
const [detecting, setDetecting] = useState(false);
const [saving, setSaving] = useState(false);
const [deleting, setDeleting] = useState<number | null>(null);
@@ -364,7 +408,7 @@ export default function Sites() {
const closeEditor = () => {
setEditor(null);
setForm(emptySiteForm());
setForm(createEmptySiteForm());
setSelectedInitializationPresetId(null);
};
@@ -381,14 +425,14 @@ export default function Sites() {
return;
}
setEditor({ mode: 'add' });
setForm(emptySiteForm());
setForm(createEmptySiteForm());
setSelectedInitializationPresetId(null);
scrollToEditorTop();
};
const openEdit = (site: SiteRow) => {
setEditor({ mode: 'edit', editingSiteId: site.id });
setForm(siteFormFromSite(site));
setForm(hydrateSiteForm(siteFormFromSite(site)));
setSelectedInitializationPresetId(detectSiteInitializationPreset(site.url, site.platform)?.id || null);
scrollToEditorTop();
// Load disabled models and discovered models independently so a best-effort
@@ -471,6 +515,11 @@ export default function Sites() {
toast.error(serializedCustomHeaders.error || '自定义请求头格式不正确');
return;
}
const serializedApiEndpoints = serializeSiteApiEndpoints(form.apiEndpoints);
if (!serializedApiEndpoints.valid) {
toast.error(serializedApiEndpoints.error || 'AI 请求地址格式不正确');
return;
}
const payload = {
name: form.name.trim(),
@@ -480,6 +529,7 @@ export default function Sites() {
initializationPresetId: selectedInitializationPresetId,
proxyUrl: form.proxyUrl.trim(),
useSystemProxy: !!form.useSystemProxy,
apiEndpoints: serializedApiEndpoints.apiEndpoints,
customHeaders: serializedCustomHeaders.customHeaders,
globalWeight: Number(parsedGlobalWeight.toFixed(3)),
};
@@ -551,6 +601,48 @@ export default function Sites() {
});
};
const updateApiEndpointRow = (index: number, patch: Partial<SiteApiEndpointField>) => {
setForm((prev) => ({
...prev,
apiEndpoints: prev.apiEndpoints.map((item, itemIndex) => (
itemIndex === index
? { ...item, ...patch }
: item
)),
}));
};
const addApiEndpointRow = () => {
setForm((prev) => ({
...prev,
apiEndpoints: [...prev.apiEndpoints, createEmptyApiEndpointRow()],
}));
};
const removeApiEndpointRow = (index: number) => {
setForm((prev) => {
const nextEndpoints = prev.apiEndpoints.filter((_, itemIndex) => itemIndex !== index);
return {
...prev,
apiEndpoints: nextEndpoints.length > 0 ? nextEndpoints : [createEmptyApiEndpointRow()],
};
});
};
const moveApiEndpointRow = (index: number, direction: 'up' | 'down') => {
setForm((prev) => {
const targetIndex = direction === 'up' ? index - 1 : index + 1;
if (targetIndex < 0 || targetIndex >= prev.apiEndpoints.length) return prev;
const nextEndpoints = [...prev.apiEndpoints];
const [current] = nextEndpoints.splice(index, 1);
nextEndpoints.splice(targetIndex, 0, current);
return {
...prev,
apiEndpoints: nextEndpoints,
};
});
};
const handleSiteCreatedChoice = (choice: 'session' | 'apikey' | 'later') => {
if (!createdSiteForChoice) return;
@@ -935,7 +1027,8 @@ export default function Sites() {
/>
<div style={{ display: 'flex', gap: 8, flexDirection: isMobile ? 'column' : 'row' }}>
<input
placeholder="站点 URL (例如 https://api.example.com)"
data-testid="site-primary-url-input"
placeholder="站点 URL（面板/登录/签到地址，如 https://nih.cc）"
value={form.url}
onChange={(e) => setForm((prev) => ({ ...prev, url: e.target.value }))}
onBlur={() => {
@@ -1008,6 +1101,100 @@ export default function Sites() {
</div>
</div>
)}
<div style={{ fontSize: 12, color: 'var(--color-text-muted)', lineHeight: 1.6 }}>
主站点 URL 用于登录、签到、面板接口和系统访问令牌管理；如果 AI 请求地址和主站点不同，请在下面的 AI 请求地址池里填写。
</div>
<div
style={{
display: 'flex',
flexDirection: 'column',
gap: 10,
padding: 12,
border: '1px solid var(--color-border)',
borderRadius: 'var(--radius-sm)',
background: 'color-mix(in srgb, var(--color-surface) 82%, transparent)',
}}
>
<div style={{ display: 'flex', justifyContent: 'space-between', alignItems: 'center', gap: 8, flexWrap: 'wrap' }}>
<div style={{ fontSize: 13, fontWeight: 600 }}>
AI 请求地址池
</div>
<button
type="button"
onClick={addApiEndpointRow}
className="btn btn-ghost"
style={{ border: '1px solid var(--color-border)' }}
>
+ 添加 AI 地址
</button>
</div>
<div style={{ fontSize: 12, color: 'var(--color-text-muted)', lineHeight: 1.7 }}>
这里只用于 `/v1/*`、模型发现和 API Key 验证。不填时默认跟随主站点 URL；多条地址会按列表顺序参与轮询，禁用的地址不会参与调度。
</div>
{form.apiEndpoints.map((endpoint, index) => (
<div
key={endpoint.draftId || `site-api-endpoint-draft-${index}`}
style={{
display: 'flex',
flexDirection: 'column',
gap: 8,
padding: 10,
border: '1px solid var(--color-border-light)',
borderRadius: 'var(--radius-sm)',
background: 'var(--color-bg)',
}}
>
<div style={{ display: 'flex', gap: 8, flexDirection: isMobile ? 'column' : 'row', alignItems: isMobile ? 'stretch' : 'center' }}>
<input
placeholder="AI 请求地址（如 https://api.nih.cc）"
value={endpoint.url}
onChange={(e) => updateApiEndpointRow(index, { url: e.target.value })}
style={{ ...formInputStyle, flex: 1, fontFamily: 'var(--font-mono)' }}
/>
<label style={{ display: 'flex', alignItems: 'center', gap: 6, fontSize: 12, color: 'var(--color-text-secondary)' }}>
<input
type="checkbox"
checked={endpoint.enabled !== false}
onChange={(e) => updateApiEndpointRow(index, { enabled: e.target.checked })}
/>
启用
</label>
</div>
<div style={{ display: 'flex', gap: 8, flexWrap: 'wrap', justifyContent: 'space-between', alignItems: 'center' }}>
<div style={{ display: 'flex', gap: 8, flexWrap: 'wrap', fontSize: 11, color: 'var(--color-text-muted)' }}>
<span>顺序 #{index + 1}</span>
{endpoint.cooldownUntil ? <span>冷却至 {formatDateTimeLocal(endpoint.cooldownUntil)}</span> : null}
{endpoint.lastFailureReason ? <span>最近失败: {endpoint.lastFailureReason}</span> : null}
</div>
<div style={{ display: 'flex', gap: 8, flexWrap: 'wrap' }}>
<button
type="button"
onClick={() => moveApiEndpointRow(index, 'up')}
disabled={index === 0}
className="btn btn-link btn-link-muted"
>
上移
</button>
<button
type="button"
onClick={() => moveApiEndpointRow(index, 'down')}
disabled={index >= form.apiEndpoints.length - 1}
className="btn btn-link btn-link-muted"
>
下移
</button>
<button
type="button"
onClick={() => removeApiEndpointRow(index)}
className="btn btn-link btn-link-danger"
>
删除
</button>
</div>
</div>
</div>
))}
</div>
<div
style={{
display: 'flex',
@@ -1218,7 +1405,7 @@ export default function Sites() {
style={formInputStyle}
/>
<div style={{ fontSize: 12, color: 'var(--color-text-muted)' }}>
填写后优先使用站点代理；留空则使用系统代理或直连(取决于设置开关状态)。
这里只是 HTTP/SOCKS 代理地址，不是上游 AI API 地址。填写后优先使用站点代理；留空则使用系统代理或直连(取决于设置开关状态)。
</div>
</div>
<label style={{
@@ -1368,6 +1555,29 @@ export default function Sites() {
</a>
) : '-'}
/>
<MobileField
label="AI 请求地址"
stacked
value={(
<div style={{ display: 'flex', flexDirection: 'column', gap: 4 }}>
<span>{buildSiteApiEndpointSummary(site)}</span>
{getConfiguredSiteApiEndpoints(site).map((endpoint, endpointIndex) => (
<span
key={`mobile-site-endpoint-${site.id}-${endpoint.id ?? endpointIndex}`}
style={{
fontSize: 11,
fontFamily: 'var(--font-mono)',
color: endpoint.enabled === false ? 'var(--color-text-muted)' : 'var(--color-text-secondary)',
wordBreak: 'break-all',
}}
>
{endpoint.url}
{endpoint.enabled === false ? '（已禁用）' : ''}
</span>
))}
</div>
)}
/>
<MobileField
label="系统代理"
value={(
@@ -1505,6 +1715,9 @@ export default function Sites() {
自定义头
</span>
) : null}
<span className={`badge ${getConfiguredSiteApiEndpoints(site).length > 0 ? 'badge-warning' : 'badge-muted'}`} style={{ fontSize: 11 }}>
AI 地址: {buildSiteApiEndpointSummary(site)}
</span>
</div>
</td>
<td className="sites-url-cell" style={{ maxWidth: 300 }}>
src/web/pages/helpers/sitesEditor.test.ts
+55
View File
@@ -1,8 +1,10 @@
import { describe, expect, it } from 'vitest';
import {
buildSiteSaveAction,
emptySiteApiEndpoint,
emptySiteCustomHeader,
emptySiteForm,
serializeSiteApiEndpoints,
serializeSiteCustomHeaders,
siteFormFromSite,
} from './sitesEditor.js';
@@ -17,6 +19,10 @@ describe('buildSiteSaveAction', () => {
externalCheckinUrl: 'https://checkin.a.example.com',
platform: 'new-api',
proxyUrl: 'socks5://127.0.0.1:1080',
apiEndpoints: [
{ url: 'https://api-a.example.com', enabled: true, sortOrder: 0 },
{ url: 'https://api-b.example.com', enabled: false, sortOrder: 1 },
],
customHeaders: '{"x-site-token":"alpha"}',
useSystemProxy: false,
globalWeight: 1.2,
@@ -31,6 +37,10 @@ describe('buildSiteSaveAction', () => {
externalCheckinUrl: 'https://checkin.a.example.com',
platform: 'new-api',
proxyUrl: 'socks5://127.0.0.1:1080',
apiEndpoints: [
{ url: 'https://api-a.example.com', enabled: true, sortOrder: 0 },
{ url: 'https://api-b.example.com', enabled: false, sortOrder: 1 },
],
customHeaders: '{"x-site-token":"alpha"}',
useSystemProxy: false,
globalWeight: 1.2,
@@ -48,6 +58,7 @@ describe('buildSiteSaveAction', () => {
platform: 'one-api',
proxyUrl: '',
useSystemProxy: true,
apiEndpoints: [],
customHeaders: '',
globalWeight: 0.8,
},
@@ -63,6 +74,7 @@ describe('buildSiteSaveAction', () => {
platform: 'one-api',
proxyUrl: '',
useSystemProxy: true,
apiEndpoints: [],
customHeaders: '',
globalWeight: 0.8,
},
@@ -80,6 +92,7 @@ describe('buildSiteSaveAction', () => {
platform: '',
proxyUrl: '',
useSystemProxy: false,
apiEndpoints: [],
customHeaders: '',
globalWeight: 1,
},
@@ -94,6 +107,14 @@ describe('buildSiteSaveAction', () => {
externalCheckinUrl: null,
platform: 'new-api',
proxyUrl: 'http://127.0.0.1:8080',
apiEndpoints: [
{
url: 'https://api.example.com',
enabled: false,
cooldownUntil: '2026-04-01T00:05:00.000Z',
lastFailureReason: 'HTTP 502',
},
],
customHeaders: '{"x-site-token":"alpha"}',
globalWeight: 1,
apiKey: 'sk-legacy-site-key',
@@ -101,11 +122,20 @@ describe('buildSiteSaveAction', () => {
expect(emptySiteForm()).not.toHaveProperty('apiKey');
expect(emptySiteForm().customHeaders).toEqual([emptySiteCustomHeader()]);
expect(emptySiteForm().apiEndpoints).toEqual([emptySiteApiEndpoint()]);
expect(emptySiteForm().proxyUrl).toBe('');
expect(siteFormFromSite(legacySite)).not.toHaveProperty('apiKey');
expect(siteFormFromSite({
proxyUrl: 'http://127.0.0.1:8080',
}).proxyUrl).toBe('http://127.0.0.1:8080');
expect(siteFormFromSite(legacySite).apiEndpoints).toEqual([
{
url: 'https://api.example.com',
enabled: false,
cooldownUntil: '2026-04-01T00:05:00.000Z',
lastFailureReason: 'HTTP 502',
},
]);
});
it('parses custom headers json into key value rows', () => {
@@ -139,4 +169,29 @@ describe('buildSiteSaveAction', () => {
error: '请求头 "authorization" 重复了',
});
});
it('serializes api endpoint rows into ordered payloads', () => {
expect(serializeSiteApiEndpoints([
{ url: 'https://api-a.example.com/', enabled: true },
{ url: 'https://api-b.example.com', enabled: false },
emptySiteApiEndpoint(),
])).toEqual({
valid: true,
apiEndpoints: [
{ url: 'https://api-a.example.com', enabled: true, sortOrder: 0 },
{ url: 'https://api-b.example.com', enabled: false, sortOrder: 1 },
],
});
});
it('rejects duplicate api endpoints after normalization', () => {
expect(serializeSiteApiEndpoints([
{ url: 'https://api.example.com/', enabled: true },
{ url: 'https://api.example.com', enabled: true },
])).toEqual({
valid: false,
apiEndpoints: [],
error: 'AI 请求地址 "https://api.example.com" 重复了',
});
});
});
src/web/pages/helpers/sitesEditor.ts
+112 -1
View File
@@ -3,6 +3,14 @@ export type SiteCustomHeaderField = {
value: string;
};
export type SiteApiEndpointField = {
draftId?: string;
url: string;
enabled: boolean;
cooldownUntil?: string | null;
lastFailureReason?: string | null;
};
export type SiteForm = {
name: string;
url: string;
@@ -10,6 +18,7 @@ export type SiteForm = {
platform: string;
proxyUrl: string;
useSystemProxy: boolean;
apiEndpoints: SiteApiEndpointField[];
customHeaders: SiteCustomHeaderField[];
globalWeight: string;
};
@@ -26,6 +35,11 @@ export type SiteSavePayload = {
initializationPresetId?: string | null;
proxyUrl: string;
useSystemProxy: boolean;
apiEndpoints: Array<{
url: string;
enabled: boolean;
sortOrder: number;
}>;
customHeaders: string;
globalWeight: number;
};
@@ -38,6 +52,15 @@ export function emptySiteCustomHeader(): SiteCustomHeaderField {
return { key: '', value: '' };
}
export function emptySiteApiEndpoint(): SiteApiEndpointField {
return {
url: '',
enabled: true,
cooldownUntil: null,
lastFailureReason: null,
};
}
function ensureSiteCustomHeaderRows(rows: SiteCustomHeaderField[]): SiteCustomHeaderField[] {
return rows.length > 0 ? rows : [emptySiteCustomHeader()];
}
@@ -50,11 +73,16 @@ export function emptySiteForm(): SiteForm {
platform: '',
proxyUrl: '',
useSystemProxy: false,
apiEndpoints: [emptySiteApiEndpoint()],
customHeaders: [emptySiteCustomHeader()],
globalWeight: '1',
};
}
function ensureSiteApiEndpointRows(rows: SiteApiEndpointField[]): SiteApiEndpointField[] {
return rows.length > 0 ? rows : [emptySiteApiEndpoint()];
}
function parseCustomHeadersForEditor(raw: unknown): SiteCustomHeaderField[] {
if (typeof raw !== 'string') {
return ensureSiteCustomHeaderRows([]);
@@ -80,10 +108,35 @@ function parseCustomHeadersForEditor(raw: unknown): SiteCustomHeaderField[] {
}
}
export function siteFormFromSite(site: Partial<Omit<SiteForm, 'customHeaders' | 'globalWeight' | 'externalCheckinUrl' | 'proxyUrl' | 'useSystemProxy'>> & {
function parseApiEndpointsForEditor(raw: unknown): SiteApiEndpointField[] {
if (!Array.isArray(raw)) {
return ensureSiteApiEndpointRows([]);
}
const rows: SiteApiEndpointField[] = [];
for (const item of raw) {
if (!item || typeof item !== 'object' || Array.isArray(item)) continue;
const row = item as Record<string, unknown>;
rows.push({
url: typeof row.url === 'string' ? row.url : '',
enabled: row.enabled !== false,
cooldownUntil: typeof row.cooldownUntil === 'string' ? row.cooldownUntil : null,
lastFailureReason: typeof row.lastFailureReason === 'string' ? row.lastFailureReason : null,
});
}
return ensureSiteApiEndpointRows(rows);
}
export function siteFormFromSite(site: Partial<Omit<SiteForm, 'apiEndpoints' | 'customHeaders' | 'globalWeight' | 'externalCheckinUrl' | 'proxyUrl' | 'useSystemProxy'>> & {
externalCheckinUrl?: string | null;
proxyUrl?: string | null;
useSystemProxy?: boolean | null;
apiEndpoints?: Array<{
url?: string | null;
enabled?: boolean | null;
cooldownUntil?: string | null;
lastFailureReason?: string | null;
}> | null;
customHeaders?: string | null;
globalWeight?: number | string | null;
}): SiteForm {
@@ -96,11 +149,27 @@ export function siteFormFromSite(site: Partial<Omit<SiteForm, 'customHeaders' |
platform: site.platform ?? '',
proxyUrl: site.proxyUrl ?? '',
useSystemProxy: !!site.useSystemProxy,
apiEndpoints: parseApiEndpointsForEditor(site.apiEndpoints),
customHeaders: parseCustomHeadersForEditor(site.customHeaders),
globalWeight,
};
}
// Keep this in sync with normalizeSiteApiEndpointBaseUrl in
// src/server/services/siteApiEndpointService.ts.
function normalizeSiteApiEndpointUrl(raw: string): string {
const trimmed = raw.trim();
if (!trimmed) return '';
try {
const parsed = new URL(trimmed);
parsed.search = '';
parsed.hash = '';
return parsed.toString().replace(/\/+$/, '');
} catch {
return trimmed.replace(/\/+$/, '');
}
}
export function serializeSiteCustomHeaders(fields: SiteCustomHeaderField[]): {
valid: boolean;
customHeaders: string;
@@ -131,6 +200,48 @@ export function serializeSiteCustomHeaders(fields: SiteCustomHeaderField[]): {
};
}
export function serializeSiteApiEndpoints(fields: SiteApiEndpointField[]): {
valid: boolean;
apiEndpoints: Array<{
url: string;
enabled: boolean;
sortOrder: number;
}>;
error?: string;
} {
const apiEndpoints: Array<{
url: string;
enabled: boolean;
sortOrder: number;
}> = [];
const seen = new Set<string>();
for (const field of fields) {
const rawUrl = field.url.trim();
if (!rawUrl) continue;
const normalizedUrl = normalizeSiteApiEndpointUrl(rawUrl);
if (!normalizedUrl) continue;
if (seen.has(normalizedUrl)) {
return {
valid: false,
apiEndpoints: [],
error: `AI 请求地址 "${normalizedUrl}" 重复了`,
};
}
seen.add(normalizedUrl);
apiEndpoints.push({
url: normalizedUrl || rawUrl,
enabled: field.enabled !== false,
sortOrder: apiEndpoints.length,
});
}
return {
valid: true,
apiEndpoints,
};
}
export function buildSiteSaveAction(editor: SiteEditorState, form: SiteSavePayload): SiteSaveAction {
if (editor.mode === 'edit') {
if (!Number.isFinite(editor.editingSiteId)) {
src/web/pages/sites.create-redirect.test.tsx
+12 -17
View File
@@ -33,6 +33,13 @@ async function flushMicrotasks() {
});
}
function findPrimarySiteUrlInput(root: ReactTestRenderer) {
return root.root.find((node) => (
node.type === 'input'
&& node.props['data-testid'] === 'site-primary-url-input'
));
}
function LocationProbe() {
const location = useLocation();
return <div>{`${location.pathname}${location.search}`}</div>;
@@ -78,7 +85,7 @@ async function createSiteAndClickModalChoice(
const nameInput = root.root.find((node) => node.type === 'input' && node.props.placeholder === '站点名称');
const urlInput = root.root.find((node) => (
node.type === 'input'
&& node.props.placeholder === '站点 URL (例如 https://api.example.com)'
&& node.props['data-testid'] === 'site-primary-url-input'
));
const selects = root.root.findAllByType(ModernSelect);
const platformSelect = selects.at(-1);
@@ -307,10 +314,7 @@ describe('Sites create redirect', () => {
});
await flushMicrotasks();
const urlInput = root.root.find((node) => (
node.type === 'input'
&& node.props.placeholder === '站点 URL (例如 https://api.example.com)'
));
const urlInput = findPrimarySiteUrlInput(root);
const selects = root.root.findAllByType(ModernSelect);
const platformSelect = selects.at(-1);
@@ -363,10 +367,7 @@ describe('Sites create redirect', () => {
});
await flushMicrotasks();
const urlInput = root.root.find((node) => (
node.type === 'input'
&& node.props.placeholder === '站点 URL (例如 https://api.example.com)'
));
const urlInput = findPrimarySiteUrlInput(root);
const platformSelect = root.root.findAllByType(ModernSelect).at(-1);
await act(async () => {
@@ -419,10 +420,7 @@ describe('Sites create redirect', () => {
});
await flushMicrotasks();
const urlInput = root.root.find((node) => (
node.type === 'input'
&& node.props.placeholder === '站点 URL (例如 https://api.example.com)'
));
const urlInput = findPrimarySiteUrlInput(root);
await act(async () => {
urlInput.props.onChange({ target: { value: 'https://coding.dashscope.aliyuncs.com/v1' } });
@@ -531,10 +529,7 @@ describe('Sites create redirect', () => {
// Fill form
const nameInput = root.root.find((node) => node.type === 'input' && node.props.placeholder === '站点名称');
const urlInput = root.root.find((node) => (
node.type === 'input'
&& node.props.placeholder === '站点 URL (例如 https://api.example.com)'
));
const urlInput = findPrimarySiteUrlInput(root);
const selects = root.root.findAllByType(ModernSelect);
const platformSelect = selects.at(-1);
const saveButton = root.root.find((node) => (