[codex] Add route cooldown controls (#365)

* Add route cooldown controls

* Address route cooldown review comments

src/server/routes/api/settings.events.test.ts

@@ -65,6 +65,7 @@ describe('settings and auth events', () => {
     (config as any).proxyDebugRetentionHours = 24;
     (config as any).proxyDebugMaxBodyBytes = 262144;
     config.routingFallbackUnitCost = 1;
+    (config as any).tokenRouterFailureCooldownMaxSec = 30 * 24 * 60 * 60;
     (config as any).disableCrossProtocolFallback = false;
     (config as any).telegramEnabled = false;
     (config as any).telegramApiBaseUrl = 'https://api.telegram.org';
@@ -461,6 +462,52 @@ describe('settings and auth events', () => {
     expect(runtime.routingFallbackUnitCost).toBe(0.25);
   });
 
+  it('persists and returns token router failure cooldown cap from runtime settings', async () => {
+    const updateResponse = await app.inject({
+      method: 'PUT',
+      url: '/api/settings/runtime',
+      payload: {
+        tokenRouterFailureCooldownMaxSec: 2 * 24 * 60 * 60,
+      },
+    });
+
+    expect(updateResponse.statusCode).toBe(200);
+    const updated = updateResponse.json() as { tokenRouterFailureCooldownMaxSec?: number };
+    expect(updated.tokenRouterFailureCooldownMaxSec).toBe(2 * 24 * 60 * 60);
+    expect((config as any).tokenRouterFailureCooldownMaxSec).toBe(2 * 24 * 60 * 60);
+
+    const saved = await db.select().from(schema.settings).where(eq(schema.settings.key, 'token_router_failure_cooldown_max_sec')).get();
+    expect(saved?.value).toBe(JSON.stringify(2 * 24 * 60 * 60));
+
+    const getResponse = await app.inject({
+      method: 'GET',
+      url: '/api/settings/runtime',
+    });
+    expect(getResponse.statusCode).toBe(200);
+    const runtime = getResponse.json() as { tokenRouterFailureCooldownMaxSec?: number };
+    expect(runtime.tokenRouterFailureCooldownMaxSec).toBe(2 * 24 * 60 * 60);
+  });
+
+  it('clamps token router failure cooldown cap to the supported ceiling', async () => {
+    const ninetyDaysSec = 90 * 24 * 60 * 60;
+    const thirtyDaysSec = 30 * 24 * 60 * 60;
+    const updateResponse = await app.inject({
+      method: 'PUT',
+      url: '/api/settings/runtime',
+      payload: {
+        tokenRouterFailureCooldownMaxSec: ninetyDaysSec,
+      },
+    });
+
+    expect(updateResponse.statusCode).toBe(200);
+    const updated = updateResponse.json() as { tokenRouterFailureCooldownMaxSec?: number };
+    expect(updated.tokenRouterFailureCooldownMaxSec).toBe(thirtyDaysSec);
+    expect((config as any).tokenRouterFailureCooldownMaxSec).toBe(thirtyDaysSec);
+
+    const saved = await db.select().from(schema.settings).where(eq(schema.settings.key, 'token_router_failure_cooldown_max_sec')).get();
+    expect(saved?.value).toBe(JSON.stringify(thirtyDaysSec));
+  });
+
   it('persists and returns disable cross protocol fallback from runtime settings', async () => {
     const updateResponse = await app.inject({
       method: 'PUT',

src/server/routes/api/settings.ts

@@ -1,7 +1,7 @@
 import { FastifyInstance } from 'fastify';
 import cron from 'node-cron';
 import { fetch } from 'undici';
-import { config } from '../../config.js';
+import { config, normalizeTokenRouterFailureCooldownMaxSec } from '../../config.js';
 import { db, runtimeDbDialect, schema } from '../../db/index.js';
 import { upsertSetting } from '../../db/upsertSetting.js';
 import * as routeRefreshWorkflow from '../../services/routeRefreshWorkflow.js';
@@ -90,6 +90,7 @@ interface RuntimeSettingsBody {
   notifyCooldownSec?: number;
   adminIpAllowlist?: string[] | string;
   routingFallbackUnitCost?: number;
+  tokenRouterFailureCooldownMaxSec?: number;
   routingWeights?: Partial<RoutingWeights>;
   proxyErrorKeywords?: string[] | string;
   proxyEmptyContentFailEnabled?: boolean;
@@ -665,6 +666,12 @@ function applyImportedSettingToRuntime(key: string, value: unknown) {
       config.routingFallbackUnitCost = Math.max(1e-6, n);
       return;
     }
+    case 'token_router_failure_cooldown_max_sec': {
+      const normalized = normalizeTokenRouterFailureCooldownMaxSec(value);
+      if (normalized == null) return;
+      config.tokenRouterFailureCooldownMaxSec = normalized;
+      return;
+    }
     default:
       return;
   }
@@ -694,6 +701,7 @@ function getRuntimeSettingsResponse(currentAdminIp = '') {
     proxyDebugRetentionHours: config.proxyDebugRetentionHours,
     proxyDebugMaxBodyBytes: config.proxyDebugMaxBodyBytes,
     routingFallbackUnitCost: config.routingFallbackUnitCost,
+    tokenRouterFailureCooldownMaxSec: config.tokenRouterFailureCooldownMaxSec,
     routingWeights: config.routingWeights,
     webhookUrl: config.webhookUrl,
     barkUrl: config.barkUrl,
@@ -1585,6 +1593,18 @@ export async function settingsRoutes(app: FastifyInstance) {
       upsertSetting('routing_fallback_unit_cost', normalized);
     }
 
+    if (body.tokenRouterFailureCooldownMaxSec !== undefined) {
+      const normalized = normalizeTokenRouterFailureCooldownMaxSec(body.tokenRouterFailureCooldownMaxSec);
+      if (normalized == null) {
+        return reply.code(400).send({ success: false, message: '路由失败冷却上限必须是大于 0 的数字（秒）' });
+      }
+      if (normalized !== config.tokenRouterFailureCooldownMaxSec) {
+        changedLabels.push(`路由失败冷却上限（${config.tokenRouterFailureCooldownMaxSec}s -> ${normalized}s）`);
+      }
+      config.tokenRouterFailureCooldownMaxSec = normalized;
+      upsertSetting('token_router_failure_cooldown_max_sec', normalized);
+    }
+
     if (changedLabels.length > 0) {
       let eventType: 'checkin' | 'balance' | 'proxy' | 'status' | 'token' = 'status';
       if (changedLabels.length === 1) {

src/server/routes/api/tokens.cooldown-clear.test.ts

@@ -0,0 +1,281 @@
+import Fastify, { type FastifyInstance } from 'fastify';
+import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest';
+import { mkdtempSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { eq } from 'drizzle-orm';
+
+type DbModule = typeof import('../../db/index.js');
+
+describe('POST /api/routes/:id/cooldown/clear', () => {
+  let app: FastifyInstance;
+  let db: DbModule['db'];
+  let schema: DbModule['schema'];
+  let dataDir = '';
+  let seedId = 0;
+
+  const nextId = () => {
+    seedId += 1;
+    return seedId;
+  };
+
+  const seedAccountWithToken = async () => {
+    const id = nextId();
+    const site = await db.insert(schema.sites).values({
+      name: `cooldown-site-${id}`,
+      url: `https://cooldown-site-${id}.example.com`,
+      platform: 'new-api',
+      status: 'active',
+    }).returning().get();
+
+    const account = await db.insert(schema.accounts).values({
+      siteId: site.id,
+      username: `cooldown-user-${id}`,
+      accessToken: `cooldown-access-token-${id}`,
+      status: 'active',
+    }).returning().get();
+
+    const token = await db.insert(schema.accountTokens).values({
+      accountId: account.id,
+      name: `cooldown-token-${id}`,
+      token: `sk-cooldown-token-${id}`,
+      enabled: true,
+      isDefault: true,
+    }).returning().get();
+
+    return { site, account, token };
+  };
+
+  beforeAll(async () => {
+    dataDir = mkdtempSync(join(tmpdir(), 'metapi-route-cooldown-clear-'));
+    process.env.DATA_DIR = dataDir;
+
+    await import('../../db/migrate.js');
+    const dbModule = await import('../../db/index.js');
+    const routesModule = await import('./tokens.js');
+
+    db = dbModule.db;
+    schema = dbModule.schema;
+
+    app = Fastify();
+    await app.register(routesModule.tokensRoutes);
+  });
+
+  beforeEach(async () => {
+    await db.delete(schema.routeGroupSources).run();
+    await db.delete(schema.routeChannels).run();
+    await db.delete(schema.tokenRoutes).run();
+    await db.delete(schema.accountTokens).run();
+    await db.delete(schema.accounts).run();
+    await db.delete(schema.sites).run();
+    seedId = 0;
+  });
+
+  afterAll(async () => {
+    await app.close();
+    delete process.env.DATA_DIR;
+  });
+
+  it('clears cooldown and failure counters for a direct route', async () => {
+    const seeded = await seedAccountWithToken();
+    const route = await db.insert(schema.tokenRoutes).values({
+      modelPattern: 'gpt-4o-mini',
+      enabled: true,
+    }).returning().get();
+
+    const channel = await db.insert(schema.routeChannels).values({
+      routeId: route.id,
+      accountId: seeded.account.id,
+      tokenId: seeded.token.id,
+      priority: 0,
+      weight: 10,
+      enabled: true,
+      failCount: 8,
+      lastFailAt: '2026-04-01T00:00:00.000Z',
+      consecutiveFailCount: 2,
+      cooldownLevel: 3,
+      cooldownUntil: '2099-01-01T00:00:00.000Z',
+    }).returning().get();
+
+    const response = await app.inject({
+      method: 'POST',
+      url: `/api/routes/${route.id}/cooldown/clear`,
+    });
+
+    expect(response.statusCode).toBe(200);
+    expect(response.json()).toMatchObject({
+      success: true,
+      clearedChannels: 1,
+    });
+
+    const refreshed = await db.select().from(schema.routeChannels)
+      .where(eq(schema.routeChannels.id, channel.id))
+      .get();
+
+    expect(refreshed).toMatchObject({
+      failCount: 0,
+      lastFailAt: null,
+      consecutiveFailCount: 0,
+      cooldownLevel: 0,
+      cooldownUntil: null,
+    });
+  });
+
+  it('clears cooldown for source-route channels exposed by explicit groups', async () => {
+    const seeded = await seedAccountWithToken();
+    const sourceRoute = await db.insert(schema.tokenRoutes).values({
+      modelPattern: 'claude-sonnet-4-5',
+      enabled: true,
+    }).returning().get();
+
+    const groupRoute = await db.insert(schema.tokenRoutes).values({
+      modelPattern: 'claude-opus-4-6',
+      displayName: 'claude-opus-4-6',
+      routeMode: 'explicit_group',
+      enabled: true,
+    }).returning().get();
+
+    await db.insert(schema.routeGroupSources).values({
+      groupRouteId: groupRoute.id,
+      sourceRouteId: sourceRoute.id,
+    }).run();
+
+    const channel = await db.insert(schema.routeChannels).values({
+      routeId: sourceRoute.id,
+      accountId: seeded.account.id,
+      tokenId: seeded.token.id,
+      sourceModel: 'claude-sonnet-4-5',
+      priority: 0,
+      weight: 10,
+      enabled: true,
+      failCount: 5,
+      lastFailAt: '2026-04-01T00:00:00.000Z',
+      consecutiveFailCount: 1,
+      cooldownLevel: 2,
+      cooldownUntil: '2099-01-01T00:00:00.000Z',
+    }).returning().get();
+
+    const response = await app.inject({
+      method: 'POST',
+      url: `/api/routes/${groupRoute.id}/cooldown/clear`,
+    });
+
+    expect(response.statusCode).toBe(200);
+    expect(response.json()).toMatchObject({
+      success: true,
+      clearedChannels: 1,
+    });
+
+    const refreshed = await db.select().from(schema.routeChannels)
+      .where(eq(schema.routeChannels.id, channel.id))
+      .get();
+
+    expect(refreshed).toMatchObject({
+      failCount: 0,
+      lastFailAt: null,
+      consecutiveFailCount: 0,
+      cooldownLevel: 0,
+      cooldownUntil: null,
+    });
+  });
+
+  it('only clears cooldown for explicit-group source routes that are enabled exact routes', async () => {
+    const seeded = await seedAccountWithToken();
+    const visibleSourceRoute = await db.insert(schema.tokenRoutes).values({
+      modelPattern: 'gpt-5.4',
+      enabled: true,
+    }).returning().get();
+    const disabledSourceRoute = await db.insert(schema.tokenRoutes).values({
+      modelPattern: 'gpt-4.1',
+      enabled: false,
+    }).returning().get();
+    const wildcardSourceRoute = await db.insert(schema.tokenRoutes).values({
+      modelPattern: 'gpt-*',
+      enabled: true,
+    }).returning().get();
+
+    const groupRoute = await db.insert(schema.tokenRoutes).values({
+      modelPattern: 'gpt-clear-group',
+      displayName: 'gpt-clear-group',
+      routeMode: 'explicit_group',
+      enabled: true,
+    }).returning().get();
+
+    await db.insert(schema.routeGroupSources).values([
+      { groupRouteId: groupRoute.id, sourceRouteId: visibleSourceRoute.id },
+      { groupRouteId: groupRoute.id, sourceRouteId: disabledSourceRoute.id },
+      { groupRouteId: groupRoute.id, sourceRouteId: wildcardSourceRoute.id },
+    ]).run();
+
+    const visibleChannel = await db.insert(schema.routeChannels).values({
+      routeId: visibleSourceRoute.id,
+      accountId: seeded.account.id,
+      tokenId: seeded.token.id,
+      priority: 0,
+      weight: 10,
+      enabled: true,
+      failCount: 4,
+      lastFailAt: '2026-04-01T00:00:00.000Z',
+      consecutiveFailCount: 1,
+      cooldownLevel: 2,
+      cooldownUntil: '2099-01-01T00:00:00.000Z',
+    }).returning().get();
+    const disabledChannel = await db.insert(schema.routeChannels).values({
+      routeId: disabledSourceRoute.id,
+      accountId: seeded.account.id,
+      tokenId: seeded.token.id,
+      priority: 0,
+      weight: 10,
+      enabled: true,
+      failCount: 6,
+      lastFailAt: '2026-04-01T00:00:00.000Z',
+      consecutiveFailCount: 2,
+      cooldownLevel: 3,
+      cooldownUntil: '2099-01-01T00:00:00.000Z',
+    }).returning().get();
+    const wildcardChannel = await db.insert(schema.routeChannels).values({
+      routeId: wildcardSourceRoute.id,
+      accountId: seeded.account.id,
+      tokenId: seeded.token.id,
+      priority: 0,
+      weight: 10,
+      enabled: true,
+      failCount: 7,
+      lastFailAt: '2026-04-01T00:00:00.000Z',
+      consecutiveFailCount: 2,
+      cooldownLevel: 3,
+      cooldownUntil: '2099-01-01T00:00:00.000Z',
+    }).returning().get();
+
+    const response = await app.inject({
+      method: 'POST',
+      url: `/api/routes/${groupRoute.id}/cooldown/clear`,
+    });
+
+    expect(response.statusCode).toBe(200);
+    expect(response.json()).toMatchObject({
+      success: true,
+      clearedChannels: 1,
+    });
+
+    const refreshedVisible = await db.select().from(schema.routeChannels)
+      .where(eq(schema.routeChannels.id, visibleChannel.id))
+      .get();
+    const refreshedDisabled = await db.select().from(schema.routeChannels)
+      .where(eq(schema.routeChannels.id, disabledChannel.id))
+      .get();
+    const refreshedWildcard = await db.select().from(schema.routeChannels)
+      .where(eq(schema.routeChannels.id, wildcardChannel.id))
+      .get();
+
+    expect(refreshedVisible).toMatchObject({
+      failCount: 0,
+      lastFailAt: null,
+      consecutiveFailCount: 0,
+      cooldownLevel: 0,
+      cooldownUntil: null,
+    });
+    expect(refreshedDisabled?.cooldownUntil).toBe('2099-01-01T00:00:00.000Z');
+    expect(refreshedWildcard?.cooldownUntil).toBe('2099-01-01T00:00:00.000Z');
+  });
+});

src/server/routes/api/tokens.ts

@@ -20,6 +20,7 @@ import {
   parseRouteDecisionSnapshot,
   saveRouteDecisionSnapshots,
 } from '../../services/routeDecisionSnapshotStore.js';
+import { clearRouteCooldown } from '../../services/routeCooldownService.js';
 import { normalizeTokenRouteMode, type RouteMode } from '../../../shared/tokenRouteContract.js';
 import {
   parseRouteChannelBatchCreatePayload,
@@ -768,8 +769,17 @@ export async function tokensRoutes(app: FastifyInstance) {
     const channelsByRoute = await fetchChannelsForRouteRows([route]);
     return channelsByRoute.get(routeId) || [];
   });
-
-  // Batch add channels to a route
+
+  app.post<{ Params: { id: string } }>('/api/routes/:id/cooldown/clear', async (request, reply) => {
+    const routeId = parseInt(request.params.id, 10);
+    const result = await clearRouteCooldown(routeId);
+    if (!result) {
+      return reply.code(404).send({ success: false, message: '路由不存在' });
+    }
+    return result;
+  });
+
+  // Batch add channels to a route
   app.post<{ Params: { id: string }; Body: unknown }>('/api/routes/:id/channels/batch', async (request, reply) => {
     const parsedBody = parseRouteChannelBatchCreatePayload(request.body);
     if (!parsedBody.success) {