[codex] Add route cooldown controls (#365)

* Add route cooldown controls

* Address route cooldown review comments

src/server/config.ts

@@ -7,6 +7,7 @@ const DEFAULT_CODEX_CLIENT_ID = 'app_EMoamEEZ73f0CkXaXp7hrann';
 const DEFAULT_CLAUDE_CLIENT_ID = '9d1c250a-e61b-44d9-88ed-5944d1962f5e';
 const DEFAULT_GEMINI_CLI_CLIENT_ID = '681255809395-oo8ft2oprdrnp9e3aqf6av3hmdib135j.apps.googleusercontent.com';
 const DEFAULT_GEMINI_CLI_CLIENT_SECRET = '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';
+export const TOKEN_ROUTER_FAILURE_COOLDOWN_MAX_SEC_CEILING = 30 * 24 * 60 * 60;
 
 function parseBoolean(value: string | undefined, fallback = false): boolean {
   if (value === undefined) return fallback;
@@ -49,6 +50,15 @@ function parseDbType(value: string | undefined): 'sqlite' | 'mysql' | 'postgres'
   return 'sqlite';
 }
 
+export function normalizeTokenRouterFailureCooldownMaxSec(value: unknown): number | null {
+  const normalized = Number(value);
+  if (!Number.isFinite(normalized) || normalized <= 0) return null;
+  return Math.min(
+    TOKEN_ROUTER_FAILURE_COOLDOWN_MAX_SEC_CEILING,
+    Math.max(1, Math.trunc(normalized)),
+  );
+}
+
 function parseListenHost(env: NodeJS.ProcessEnv): string {
   return (env.HOST || '0.0.0.0').trim() || '0.0.0.0';
 }
@@ -108,6 +118,9 @@ export function buildConfig(env: NodeJS.ProcessEnv) {
     dbSsl: parseBoolean(env.DB_SSL, false),
     requestBodyLimit: DEFAULT_REQUEST_BODY_LIMIT,
     routingFallbackUnitCost: Math.max(1e-6, parseNumber(env.ROUTING_FALLBACK_UNIT_COST, 1)),
+    tokenRouterFailureCooldownMaxSec: normalizeTokenRouterFailureCooldownMaxSec(
+      parseNumber(env.TOKEN_ROUTER_FAILURE_COOLDOWN_MAX_SEC, TOKEN_ROUTER_FAILURE_COOLDOWN_MAX_SEC_CEILING),
+    ) ?? TOKEN_ROUTER_FAILURE_COOLDOWN_MAX_SEC_CEILING,
     tokenRouterCacheTtlMs: Math.max(100, Math.trunc(parseNumber(env.TOKEN_ROUTER_CACHE_TTL_MS, 1_500))),
     proxyMaxChannelAttempts: Math.max(1, Math.trunc(parseNumber(env.PROXY_MAX_CHANNEL_ATTEMPTS, 3))),
     proxyStickySessionEnabled: parseBoolean(env.PROXY_STICKY_SESSION_ENABLED, true),

src/server/index.ts

@@ -1,7 +1,11 @@
 import Fastify from 'fastify';
 import cors from '@fastify/cors';
 import fastifyStatic from '@fastify/static';
-import { buildFastifyOptions, config } from './config.js';
+import {
+  buildFastifyOptions,
+  config,
+  normalizeTokenRouterFailureCooldownMaxSec,
+} from './config.js';
 import { normalizePayloadRulesConfig } from './services/payloadRules.js';
 import { authMiddleware } from './middleware/auth.js';
 import { sitesRoutes } from './routes/api/sites.js';
@@ -287,6 +291,12 @@ function applyRuntimeSettings(settingsMap: Map<string, string>) {
     config.routingFallbackUnitCost = Math.max(1e-6, routingFallbackUnitCost);
   }
 
+  const tokenRouterFailureCooldownMaxSec = parseSettingFromMap<number>(settingsMap, 'token_router_failure_cooldown_max_sec');
+  const normalizedFailureCooldownMaxSec = normalizeTokenRouterFailureCooldownMaxSec(tokenRouterFailureCooldownMaxSec);
+  if (normalizedFailureCooldownMaxSec != null) {
+    config.tokenRouterFailureCooldownMaxSec = normalizedFailureCooldownMaxSec;
+  }
+
   const webhookUrl = parseSettingFromMap<string>(settingsMap, 'webhook_url');
   if (typeof webhookUrl === 'string') config.webhookUrl = webhookUrl;
 

src/server/routes/api/settings.events.test.ts

@@ -65,6 +65,7 @@ describe('settings and auth events', () => {
     (config as any).proxyDebugRetentionHours = 24;
     (config as any).proxyDebugMaxBodyBytes = 262144;
     config.routingFallbackUnitCost = 1;
+    (config as any).tokenRouterFailureCooldownMaxSec = 30 * 24 * 60 * 60;
     (config as any).disableCrossProtocolFallback = false;
     (config as any).telegramEnabled = false;
     (config as any).telegramApiBaseUrl = 'https://api.telegram.org';
@@ -461,6 +462,52 @@ describe('settings and auth events', () => {
     expect(runtime.routingFallbackUnitCost).toBe(0.25);
   });
 
+  it('persists and returns token router failure cooldown cap from runtime settings', async () => {
+    const updateResponse = await app.inject({
+      method: 'PUT',
+      url: '/api/settings/runtime',
+      payload: {
+        tokenRouterFailureCooldownMaxSec: 2 * 24 * 60 * 60,
+      },
+    });
+
+    expect(updateResponse.statusCode).toBe(200);
+    const updated = updateResponse.json() as { tokenRouterFailureCooldownMaxSec?: number };
+    expect(updated.tokenRouterFailureCooldownMaxSec).toBe(2 * 24 * 60 * 60);
+    expect((config as any).tokenRouterFailureCooldownMaxSec).toBe(2 * 24 * 60 * 60);
+
+    const saved = await db.select().from(schema.settings).where(eq(schema.settings.key, 'token_router_failure_cooldown_max_sec')).get();
+    expect(saved?.value).toBe(JSON.stringify(2 * 24 * 60 * 60));
+
+    const getResponse = await app.inject({
+      method: 'GET',
+      url: '/api/settings/runtime',
+    });
+    expect(getResponse.statusCode).toBe(200);
+    const runtime = getResponse.json() as { tokenRouterFailureCooldownMaxSec?: number };
+    expect(runtime.tokenRouterFailureCooldownMaxSec).toBe(2 * 24 * 60 * 60);
+  });
+
+  it('clamps token router failure cooldown cap to the supported ceiling', async () => {
+    const ninetyDaysSec = 90 * 24 * 60 * 60;
+    const thirtyDaysSec = 30 * 24 * 60 * 60;
+    const updateResponse = await app.inject({
+      method: 'PUT',
+      url: '/api/settings/runtime',
+      payload: {
+        tokenRouterFailureCooldownMaxSec: ninetyDaysSec,
+      },
+    });
+
+    expect(updateResponse.statusCode).toBe(200);
+    const updated = updateResponse.json() as { tokenRouterFailureCooldownMaxSec?: number };
+    expect(updated.tokenRouterFailureCooldownMaxSec).toBe(thirtyDaysSec);
+    expect((config as any).tokenRouterFailureCooldownMaxSec).toBe(thirtyDaysSec);
+
+    const saved = await db.select().from(schema.settings).where(eq(schema.settings.key, 'token_router_failure_cooldown_max_sec')).get();
+    expect(saved?.value).toBe(JSON.stringify(thirtyDaysSec));
+  });
+
   it('persists and returns disable cross protocol fallback from runtime settings', async () => {
     const updateResponse = await app.inject({
       method: 'PUT',

src/server/routes/api/settings.ts

@@ -1,7 +1,7 @@
 import { FastifyInstance } from 'fastify';
 import cron from 'node-cron';
 import { fetch } from 'undici';
-import { config } from '../../config.js';
+import { config, normalizeTokenRouterFailureCooldownMaxSec } from '../../config.js';
 import { db, runtimeDbDialect, schema } from '../../db/index.js';
 import { upsertSetting } from '../../db/upsertSetting.js';
 import * as routeRefreshWorkflow from '../../services/routeRefreshWorkflow.js';
@@ -90,6 +90,7 @@ interface RuntimeSettingsBody {
   notifyCooldownSec?: number;
   adminIpAllowlist?: string[] | string;
   routingFallbackUnitCost?: number;
+  tokenRouterFailureCooldownMaxSec?: number;
   routingWeights?: Partial<RoutingWeights>;
   proxyErrorKeywords?: string[] | string;
   proxyEmptyContentFailEnabled?: boolean;
@@ -665,6 +666,12 @@ function applyImportedSettingToRuntime(key: string, value: unknown) {
       config.routingFallbackUnitCost = Math.max(1e-6, n);
       return;
     }
+    case 'token_router_failure_cooldown_max_sec': {
+      const normalized = normalizeTokenRouterFailureCooldownMaxSec(value);
+      if (normalized == null) return;
+      config.tokenRouterFailureCooldownMaxSec = normalized;
+      return;
+    }
     default:
       return;
   }
@@ -694,6 +701,7 @@ function getRuntimeSettingsResponse(currentAdminIp = '') {
     proxyDebugRetentionHours: config.proxyDebugRetentionHours,
     proxyDebugMaxBodyBytes: config.proxyDebugMaxBodyBytes,
     routingFallbackUnitCost: config.routingFallbackUnitCost,
+    tokenRouterFailureCooldownMaxSec: config.tokenRouterFailureCooldownMaxSec,
     routingWeights: config.routingWeights,
     webhookUrl: config.webhookUrl,
     barkUrl: config.barkUrl,
@@ -1585,6 +1593,18 @@ export async function settingsRoutes(app: FastifyInstance) {
       upsertSetting('routing_fallback_unit_cost', normalized);
     }
 
+    if (body.tokenRouterFailureCooldownMaxSec !== undefined) {
+      const normalized = normalizeTokenRouterFailureCooldownMaxSec(body.tokenRouterFailureCooldownMaxSec);
+      if (normalized == null) {
+        return reply.code(400).send({ success: false, message: '路由失败冷却上限必须是大于 0 的数字（秒）' });
+      }
+      if (normalized !== config.tokenRouterFailureCooldownMaxSec) {
+        changedLabels.push(`路由失败冷却上限（${config.tokenRouterFailureCooldownMaxSec}s -> ${normalized}s）`);
+      }
+      config.tokenRouterFailureCooldownMaxSec = normalized;
+      upsertSetting('token_router_failure_cooldown_max_sec', normalized);
+    }
+
     if (changedLabels.length > 0) {
       let eventType: 'checkin' | 'balance' | 'proxy' | 'status' | 'token' = 'status';
       if (changedLabels.length === 1) {

src/server/routes/api/tokens.cooldown-clear.test.ts

@@ -0,0 +1,281 @@
+import Fastify, { type FastifyInstance } from 'fastify';
+import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest';
+import { mkdtempSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { eq } from 'drizzle-orm';
+
+type DbModule = typeof import('../../db/index.js');
+
+describe('POST /api/routes/:id/cooldown/clear', () => {
+  let app: FastifyInstance;
+  let db: DbModule['db'];
+  let schema: DbModule['schema'];
+  let dataDir = '';
+  let seedId = 0;
+
+  const nextId = () => {
+    seedId += 1;
+    return seedId;
+  };
+
+  const seedAccountWithToken = async () => {
+    const id = nextId();
+    const site = await db.insert(schema.sites).values({
+      name: `cooldown-site-${id}`,
+      url: `https://cooldown-site-${id}.example.com`,
+      platform: 'new-api',
+      status: 'active',
+    }).returning().get();
+
+    const account = await db.insert(schema.accounts).values({
+      siteId: site.id,
+      username: `cooldown-user-${id}`,
+      accessToken: `cooldown-access-token-${id}`,
+      status: 'active',
+    }).returning().get();
+
+    const token = await db.insert(schema.accountTokens).values({
+      accountId: account.id,
+      name: `cooldown-token-${id}`,
+      token: `sk-cooldown-token-${id}`,
+      enabled: true,
+      isDefault: true,
+    }).returning().get();
+
+    return { site, account, token };
+  };
+
+  beforeAll(async () => {
+    dataDir = mkdtempSync(join(tmpdir(), 'metapi-route-cooldown-clear-'));
+    process.env.DATA_DIR = dataDir;
+
+    await import('../../db/migrate.js');
+    const dbModule = await import('../../db/index.js');
+    const routesModule = await import('./tokens.js');
+
+    db = dbModule.db;
+    schema = dbModule.schema;
+
+    app = Fastify();
+    await app.register(routesModule.tokensRoutes);
+  });
+
+  beforeEach(async () => {
+    await db.delete(schema.routeGroupSources).run();
+    await db.delete(schema.routeChannels).run();
+    await db.delete(schema.tokenRoutes).run();
+    await db.delete(schema.accountTokens).run();
+    await db.delete(schema.accounts).run();
+    await db.delete(schema.sites).run();
+    seedId = 0;
+  });
+
+  afterAll(async () => {
+    await app.close();
+    delete process.env.DATA_DIR;
+  });
+
+  it('clears cooldown and failure counters for a direct route', async () => {
+    const seeded = await seedAccountWithToken();
+    const route = await db.insert(schema.tokenRoutes).values({
+      modelPattern: 'gpt-4o-mini',
+      enabled: true,
+    }).returning().get();
+
+    const channel = await db.insert(schema.routeChannels).values({
+      routeId: route.id,
+      accountId: seeded.account.id,
+      tokenId: seeded.token.id,
+      priority: 0,
+      weight: 10,
+      enabled: true,
+      failCount: 8,
+      lastFailAt: '2026-04-01T00:00:00.000Z',
+      consecutiveFailCount: 2,
+      cooldownLevel: 3,
+      cooldownUntil: '2099-01-01T00:00:00.000Z',
+    }).returning().get();
+
+    const response = await app.inject({
+      method: 'POST',
+      url: `/api/routes/${route.id}/cooldown/clear`,
+    });
+
+    expect(response.statusCode).toBe(200);
+    expect(response.json()).toMatchObject({
+      success: true,
+      clearedChannels: 1,
+    });
+
+    const refreshed = await db.select().from(schema.routeChannels)
+      .where(eq(schema.routeChannels.id, channel.id))
+      .get();
+
+    expect(refreshed).toMatchObject({
+      failCount: 0,
+      lastFailAt: null,
+      consecutiveFailCount: 0,
+      cooldownLevel: 0,
+      cooldownUntil: null,
+    });
+  });
+
+  it('clears cooldown for source-route channels exposed by explicit groups', async () => {
+    const seeded = await seedAccountWithToken();
+    const sourceRoute = await db.insert(schema.tokenRoutes).values({
+      modelPattern: 'claude-sonnet-4-5',
+      enabled: true,
+    }).returning().get();
+
+    const groupRoute = await db.insert(schema.tokenRoutes).values({
+      modelPattern: 'claude-opus-4-6',
+      displayName: 'claude-opus-4-6',
+      routeMode: 'explicit_group',
+      enabled: true,
+    }).returning().get();
+
+    await db.insert(schema.routeGroupSources).values({
+      groupRouteId: groupRoute.id,
+      sourceRouteId: sourceRoute.id,
+    }).run();
+
+    const channel = await db.insert(schema.routeChannels).values({
+      routeId: sourceRoute.id,
+      accountId: seeded.account.id,
+      tokenId: seeded.token.id,
+      sourceModel: 'claude-sonnet-4-5',
+      priority: 0,
+      weight: 10,
+      enabled: true,
+      failCount: 5,
+      lastFailAt: '2026-04-01T00:00:00.000Z',
+      consecutiveFailCount: 1,
+      cooldownLevel: 2,
+      cooldownUntil: '2099-01-01T00:00:00.000Z',
+    }).returning().get();
+
+    const response = await app.inject({
+      method: 'POST',
+      url: `/api/routes/${groupRoute.id}/cooldown/clear`,
+    });
+
+    expect(response.statusCode).toBe(200);
+    expect(response.json()).toMatchObject({
+      success: true,
+      clearedChannels: 1,
+    });
+
+    const refreshed = await db.select().from(schema.routeChannels)
+      .where(eq(schema.routeChannels.id, channel.id))
+      .get();
+
+    expect(refreshed).toMatchObject({
+      failCount: 0,
+      lastFailAt: null,
+      consecutiveFailCount: 0,
+      cooldownLevel: 0,
+      cooldownUntil: null,
+    });
+  });
+
+  it('only clears cooldown for explicit-group source routes that are enabled exact routes', async () => {
+    const seeded = await seedAccountWithToken();
+    const visibleSourceRoute = await db.insert(schema.tokenRoutes).values({
+      modelPattern: 'gpt-5.4',
+      enabled: true,
+    }).returning().get();
+    const disabledSourceRoute = await db.insert(schema.tokenRoutes).values({
+      modelPattern: 'gpt-4.1',
+      enabled: false,
+    }).returning().get();
+    const wildcardSourceRoute = await db.insert(schema.tokenRoutes).values({
+      modelPattern: 'gpt-*',
+      enabled: true,
+    }).returning().get();
+
+    const groupRoute = await db.insert(schema.tokenRoutes).values({
+      modelPattern: 'gpt-clear-group',
+      displayName: 'gpt-clear-group',
+      routeMode: 'explicit_group',
+      enabled: true,
+    }).returning().get();
+
+    await db.insert(schema.routeGroupSources).values([
+      { groupRouteId: groupRoute.id, sourceRouteId: visibleSourceRoute.id },
+      { groupRouteId: groupRoute.id, sourceRouteId: disabledSourceRoute.id },
+      { groupRouteId: groupRoute.id, sourceRouteId: wildcardSourceRoute.id },
+    ]).run();
+
+    const visibleChannel = await db.insert(schema.routeChannels).values({
+      routeId: visibleSourceRoute.id,
+      accountId: seeded.account.id,
+      tokenId: seeded.token.id,
+      priority: 0,
+      weight: 10,
+      enabled: true,
+      failCount: 4,
+      lastFailAt: '2026-04-01T00:00:00.000Z',
+      consecutiveFailCount: 1,
+      cooldownLevel: 2,
+      cooldownUntil: '2099-01-01T00:00:00.000Z',
+    }).returning().get();
+    const disabledChannel = await db.insert(schema.routeChannels).values({
+      routeId: disabledSourceRoute.id,
+      accountId: seeded.account.id,
+      tokenId: seeded.token.id,
+      priority: 0,
+      weight: 10,
+      enabled: true,
+      failCount: 6,
+      lastFailAt: '2026-04-01T00:00:00.000Z',
+      consecutiveFailCount: 2,
+      cooldownLevel: 3,
+      cooldownUntil: '2099-01-01T00:00:00.000Z',
+    }).returning().get();
+    const wildcardChannel = await db.insert(schema.routeChannels).values({
+      routeId: wildcardSourceRoute.id,
+      accountId: seeded.account.id,
+      tokenId: seeded.token.id,
+      priority: 0,
+      weight: 10,
+      enabled: true,
+      failCount: 7,
+      lastFailAt: '2026-04-01T00:00:00.000Z',
+      consecutiveFailCount: 2,
+      cooldownLevel: 3,
+      cooldownUntil: '2099-01-01T00:00:00.000Z',
+    }).returning().get();
+
+    const response = await app.inject({
+      method: 'POST',
+      url: `/api/routes/${groupRoute.id}/cooldown/clear`,
+    });
+
+    expect(response.statusCode).toBe(200);
+    expect(response.json()).toMatchObject({
+      success: true,
+      clearedChannels: 1,
+    });
+
+    const refreshedVisible = await db.select().from(schema.routeChannels)
+      .where(eq(schema.routeChannels.id, visibleChannel.id))
+      .get();
+    const refreshedDisabled = await db.select().from(schema.routeChannels)
+      .where(eq(schema.routeChannels.id, disabledChannel.id))
+      .get();
+    const refreshedWildcard = await db.select().from(schema.routeChannels)
+      .where(eq(schema.routeChannels.id, wildcardChannel.id))
+      .get();
+
+    expect(refreshedVisible).toMatchObject({
+      failCount: 0,
+      lastFailAt: null,
+      consecutiveFailCount: 0,
+      cooldownLevel: 0,
+      cooldownUntil: null,
+    });
+    expect(refreshedDisabled?.cooldownUntil).toBe('2099-01-01T00:00:00.000Z');
+    expect(refreshedWildcard?.cooldownUntil).toBe('2099-01-01T00:00:00.000Z');
+  });
+});

src/server/routes/api/tokens.ts

@@ -20,6 +20,7 @@ import {
   parseRouteDecisionSnapshot,
   saveRouteDecisionSnapshots,
 } from '../../services/routeDecisionSnapshotStore.js';
+import { clearRouteCooldown } from '../../services/routeCooldownService.js';
 import { normalizeTokenRouteMode, type RouteMode } from '../../../shared/tokenRouteContract.js';
 import {
   parseRouteChannelBatchCreatePayload,
@@ -768,8 +769,17 @@ export async function tokensRoutes(app: FastifyInstance) {
     const channelsByRoute = await fetchChannelsForRouteRows([route]);
     return channelsByRoute.get(routeId) || [];
   });
-
-  // Batch add channels to a route
+
+  app.post<{ Params: { id: string } }>('/api/routes/:id/cooldown/clear', async (request, reply) => {
+    const routeId = parseInt(request.params.id, 10);
+    const result = await clearRouteCooldown(routeId);
+    if (!result) {
+      return reply.code(404).send({ success: false, message: '路由不存在' });
+    }
+    return result;
+  });
+
+  // Batch add channels to a route
   app.post<{ Params: { id: string }; Body: unknown }>('/api/routes/:id/channels/batch', async (request, reply) => {
     const parsedBody = parseRouteChannelBatchCreatePayload(request.body);
     if (!parsedBody.success) {

src/server/services/routeCooldownService.ts

@@ -0,0 +1,142 @@
+import { eq, inArray } from 'drizzle-orm';
+import { db, schema } from '../db/index.js';
+import {
+  clearRouteDecisionSnapshot,
+  clearRouteDecisionSnapshots,
+} from './routeDecisionSnapshotStore.js';
+import { tokenRouter } from './tokenRouter.js';
+import { normalizeTokenRouteMode, type RouteMode } from '../../shared/tokenRouteContract.js';
+
+type RouteRow = typeof schema.tokenRoutes.$inferSelect & {
+  routeMode: RouteMode;
+  sourceRouteIds: number[];
+};
+
+function isExactModelPattern(modelPattern: string): boolean {
+  const normalized = modelPattern.trim();
+  if (!normalized) return false;
+  if (normalized.toLowerCase().startsWith('re:')) return false;
+  return !/[\*\?]/.test(normalized);
+}
+
+function isExplicitGroupRoute(route: Pick<RouteRow, 'routeMode'> | Pick<typeof schema.tokenRoutes.$inferSelect, 'routeMode'>): boolean {
+  return normalizeTokenRouteMode(route.routeMode) === 'explicit_group';
+}
+
+function normalizeSourceRouteIds(sourceRouteIds: number[]): number[] {
+  return Array.from(new Set(
+    sourceRouteIds
+      .filter((routeId): routeId is number => Number.isFinite(routeId) && routeId > 0)
+      .map((routeId) => Math.trunc(routeId)),
+  ));
+}
+
+async function loadRouteSourceIdsMap(routeIds: number[]): Promise<Map<number, number[]>> {
+  const normalizedRouteIds = normalizeSourceRouteIds(routeIds);
+  if (normalizedRouteIds.length === 0) return new Map();
+
+  const rows = await db.select().from(schema.routeGroupSources)
+    .where(inArray(schema.routeGroupSources.groupRouteId, normalizedRouteIds))
+    .all();
+  const sourceRouteIdsByRouteId = new Map<number, number[]>();
+  for (const row of rows) {
+    if (!sourceRouteIdsByRouteId.has(row.groupRouteId)) {
+      sourceRouteIdsByRouteId.set(row.groupRouteId, []);
+    }
+    sourceRouteIdsByRouteId.get(row.groupRouteId)!.push(row.sourceRouteId);
+  }
+  for (const [routeId, sourceRouteIds] of sourceRouteIdsByRouteId.entries()) {
+    sourceRouteIdsByRouteId.set(routeId, normalizeSourceRouteIds(sourceRouteIds));
+  }
+  return sourceRouteIdsByRouteId;
+}
+
+function decorateRoutesWithSources(
+  routes: Array<typeof schema.tokenRoutes.$inferSelect>,
+  sourceRouteIdsByRouteId: Map<number, number[]>,
+): RouteRow[] {
+  return routes.map((route) => ({
+    ...route,
+    routeMode: normalizeTokenRouteMode(route.routeMode),
+    sourceRouteIds: sourceRouteIdsByRouteId.get(route.id) ?? [],
+  }));
+}
+
+async function getRouteWithSources(routeId: number): Promise<RouteRow | null> {
+  const route = await db.select().from(schema.tokenRoutes).where(eq(schema.tokenRoutes.id, routeId)).get();
+  if (!route) return null;
+  const sourceRouteIdsByRouteId = await loadRouteSourceIdsMap([routeId]);
+  return decorateRoutesWithSources([route], sourceRouteIdsByRouteId)[0] ?? null;
+}
+
+async function resolveCooldownClearRouteIds(route: RouteRow): Promise<number[]> {
+  if (!isExplicitGroupRoute(route)) {
+    return [route.id];
+  }
+
+  const sourceRouteIds = normalizeSourceRouteIds(route.sourceRouteIds);
+  if (sourceRouteIds.length === 0) return [];
+
+  const sourceRoutes = await db.select({
+    id: schema.tokenRoutes.id,
+    modelPattern: schema.tokenRoutes.modelPattern,
+    routeMode: schema.tokenRoutes.routeMode,
+    enabled: schema.tokenRoutes.enabled,
+  }).from(schema.tokenRoutes)
+    .where(inArray(schema.tokenRoutes.id, sourceRouteIds))
+    .all();
+
+  return sourceRoutes
+    .filter((sourceRoute) => (
+      sourceRoute.enabled
+      && normalizeTokenRouteMode(sourceRoute.routeMode) !== 'explicit_group'
+      && isExactModelPattern(sourceRoute.modelPattern)
+    ))
+    .map((sourceRoute) => sourceRoute.id);
+}
+
+async function clearDependentExplicitGroupSnapshotsBySourceRouteIds(sourceRouteIds: number[]): Promise<void> {
+  const normalizedSourceRouteIds = normalizeSourceRouteIds(sourceRouteIds);
+  if (normalizedSourceRouteIds.length === 0) return;
+
+  const rows = await db.select({ groupRouteId: schema.routeGroupSources.groupRouteId })
+    .from(schema.routeGroupSources)
+    .where(inArray(schema.routeGroupSources.sourceRouteId, normalizedSourceRouteIds))
+    .all();
+  const dependentRouteIds: number[] = Array.from(new Set(
+    rows
+      .map((row) => row.groupRouteId)
+      .filter((routeId): routeId is number => Number.isFinite(routeId) && routeId > 0),
+  ));
+  if (dependentRouteIds.length === 0) return;
+  await clearRouteDecisionSnapshots(dependentRouteIds);
+}
+
+export async function clearRouteCooldown(routeId: number): Promise<{ success: true; clearedChannels: number } | null> {
+  const route = await getRouteWithSources(routeId);
+  if (!route) return null;
+
+  const actualRouteIds = await resolveCooldownClearRouteIds(route);
+  const channelRows: Array<{ id: number; routeId: number }> = actualRouteIds.length > 0
+    ? await db.select({
+      id: schema.routeChannels.id,
+      routeId: schema.routeChannels.routeId,
+    }).from(schema.routeChannels)
+      .where(inArray(schema.routeChannels.routeId, actualRouteIds))
+      .all()
+    : [];
+
+  const affectedRouteIds = Array.from(new Set(channelRows.map((row) => row.routeId)));
+  const clearedChannels = await tokenRouter.clearChannelFailureState(channelRows.map((row) => row.id));
+
+  await clearRouteDecisionSnapshot(route.id);
+  if (affectedRouteIds.length > 0) {
+    await clearRouteDecisionSnapshots(affectedRouteIds);
+    await clearDependentExplicitGroupSnapshotsBySourceRouteIds(affectedRouteIds);
+  }
+
+  return {
+    success: true,
+    clearedChannels,
+  };
+}

src/server/services/tokenRouter.cache.test.ts

@@ -13,10 +13,12 @@ describe('TokenRouter runtime cache', () => {
   let schema: DbModule['schema'];
   let TokenRouter: TokenRouterModule['TokenRouter'];
   let invalidateTokenRouterCache: TokenRouterModule['invalidateTokenRouterCache'];
+  let isChannelRecentlyFailed: TokenRouterModule['isChannelRecentlyFailed'];
   let resetSiteRuntimeHealthState: TokenRouterModule['resetSiteRuntimeHealthState'];
   let config: ConfigModule['config'];
   let dataDir = '';
   let originalCacheTtlMs = 0;
+  let originalFailureCooldownMaxSec = 0;
 
   beforeAll(async () => {
     dataDir = mkdtempSync(join(tmpdir(), 'metapi-token-router-cache-'));
@@ -30,9 +32,11 @@ describe('TokenRouter runtime cache', () => {
     schema = dbModule.schema;
     TokenRouter = tokenRouterModule.TokenRouter;
     invalidateTokenRouterCache = tokenRouterModule.invalidateTokenRouterCache;
+    isChannelRecentlyFailed = tokenRouterModule.isChannelRecentlyFailed;
     resetSiteRuntimeHealthState = tokenRouterModule.resetSiteRuntimeHealthState;
     config = configModule.config;
     originalCacheTtlMs = config.tokenRouterCacheTtlMs;
+    originalFailureCooldownMaxSec = config.tokenRouterFailureCooldownMaxSec;
   });
 
   beforeEach(async () => {
@@ -43,12 +47,14 @@ describe('TokenRouter runtime cache', () => {
     await db.delete(schema.accounts).run();
     await db.delete(schema.sites).run();
     config.tokenRouterCacheTtlMs = 60_000;
+    config.tokenRouterFailureCooldownMaxSec = originalFailureCooldownMaxSec;
     invalidateTokenRouterCache();
     resetSiteRuntimeHealthState();
   });
 
   afterAll(() => {
     config.tokenRouterCacheTtlMs = originalCacheTtlMs;
+    config.tokenRouterFailureCooldownMaxSec = originalFailureCooldownMaxSec;
     invalidateTokenRouterCache();
     resetSiteRuntimeHealthState();
     delete process.env.DATA_DIR;
@@ -175,6 +181,73 @@ describe('TokenRouter runtime cache', () => {
     expect(thirdCooldownMs).toBeLessThanOrEqual(35_000);
   });
 
+  it('caps generic failure cooldowns at the configured maximum', async () => {
+    config.tokenRouterFailureCooldownMaxSec = 20;
+
+    const site = await db.insert(schema.sites).values({
+      name: 'capped-cooldown-site',
+      url: 'https://capped-cooldown.example.com',
+      platform: 'new-api',
+      status: 'active',
+    }).returning().get();
+
+    const account = await db.insert(schema.accounts).values({
+      siteId: site.id,
+      username: 'capped-cooldown-user',
+      accessToken: 'capped-cooldown-access-token',
+      apiToken: 'capped-cooldown-api-token',
+      status: 'active',
+    }).returning().get();
+
+    const token = await db.insert(schema.accountTokens).values({
+      accountId: account.id,
+      name: 'capped-cooldown-token',
+      token: 'sk-capped-cooldown-token',
+      enabled: true,
+      isDefault: true,
+    }).returning().get();
+
+    const route = await db.insert(schema.tokenRoutes).values({
+      modelPattern: 'gpt-4o-mini',
+      routingStrategy: 'weighted',
+      enabled: true,
+    }).returning().get();
+
+    const channel = await db.insert(schema.routeChannels).values({
+      routeId: route.id,
+      accountId: account.id,
+      tokenId: token.id,
+      priority: 0,
+      weight: 10,
+      enabled: true,
+    }).returning().get();
+
+    const router = new TokenRouter();
+
+    await router.recordFailure(channel.id);
+    await router.recordFailure(channel.id);
+
+    const startedAt = Date.now();
+    await router.recordFailure(channel.id);
+
+    const record = await db.select().from(schema.routeChannels)
+      .where(eq(schema.routeChannels.id, channel.id))
+      .get();
+    const cooldownMs = Date.parse(String(record?.cooldownUntil || '')) - startedAt;
+
+    expect(cooldownMs).toBeGreaterThanOrEqual(17_000);
+    expect(cooldownMs).toBeLessThanOrEqual(23_000);
+    const recentFailureCheckAt = Date.now();
+    expect(isChannelRecentlyFailed({
+      failCount: 3,
+      lastFailAt: new Date(recentFailureCheckAt - 19_000).toISOString(),
+    }, recentFailureCheckAt)).toBe(true);
+    expect(isChannelRecentlyFailed({
+      failCount: 3,
+      lastFailAt: new Date(recentFailureCheckAt - 21_000).toISOString(),
+    }, recentFailureCheckAt)).toBe(false);
+  });
+
   it('uses codex oauth reset hints for usage-limit cooldowns', async () => {
     const site = await db.insert(schema.sites).values({
       name: 'codex-oauth-site',

src/server/services/tokenRouter.selection.test.ts

@@ -594,6 +594,88 @@ describe('TokenRouter selection scoring', () => {
     expect((recoveredCandidateB?.probability || 0)).toBeLessThan(70);
   });
 
+  it('clears persisted runtime breaker state when channel cooldown is manually cleared', async () => {
+    config.routingWeights = {
+      baseWeightFactor: 1,
+      valueScoreFactor: 0,
+      costWeight: 0,
+      balanceWeight: 0,
+      usageWeight: 0,
+    };
+
+    const route = await createRoute('gpt-5.4');
+
+    const siteA = await createSite('clear-breaker-a');
+    const accountA = await createAccount(siteA.id, 'clear-breaker-user-a');
+    const tokenA = await createToken(accountA.id, 'clear-breaker-token-a');
+    const channelA = await db.insert(schema.routeChannels).values({
+      routeId: route.id,
+      accountId: accountA.id,
+      tokenId: tokenA.id,
+      priority: 0,
+      weight: 10,
+      enabled: true,
+    }).returning().get();
+
+    const siteB = await createSite('clear-breaker-b');
+    const accountB = await createAccount(siteB.id, 'clear-breaker-user-b');
+    const tokenB = await createToken(accountB.id, 'clear-breaker-token-b');
+    const channelB = await db.insert(schema.routeChannels).values({
+      routeId: route.id,
+      accountId: accountB.id,
+      tokenId: tokenB.id,
+      priority: 0,
+      weight: 10,
+      enabled: true,
+    }).returning().get();
+
+    const router = new TokenRouter();
+    for (let index = 0; index < 3; index += 1) {
+      await router.recordFailure(channelA.id, {
+        status: 502,
+        errorText: 'Bad gateway',
+        modelName: 'gpt-5.4',
+      });
+    }
+    await db.update(schema.routeChannels).set({
+      cooldownUntil: null,
+      lastFailAt: null,
+      failCount: 0,
+      consecutiveFailCount: 0,
+      cooldownLevel: 0,
+    }).where(eq(schema.routeChannels.id, channelA.id)).run();
+    invalidateTokenRouterCache();
+
+    let decision = await router.explainSelection('gpt-5.4');
+    const breakerCandidateA = decision.candidates.find((candidate) => candidate.channelId === channelA.id);
+    const breakerCandidateB = decision.candidates.find((candidate) => candidate.channelId === channelB.id);
+    expect(breakerCandidateA?.reason || '').toContain('熔断');
+    expect((breakerCandidateA?.probability || 0)).toBe(0);
+    expect((breakerCandidateB?.probability || 0)).toBe(100);
+
+    await router.clearChannelFailureState([channelA.id]);
+    resetSiteRuntimeHealthState();
+    invalidateTokenRouterCache();
+
+    const refreshedChannel = await db.select().from(schema.routeChannels)
+      .where(eq(schema.routeChannels.id, channelA.id))
+      .get();
+    expect(refreshedChannel).toMatchObject({
+      failCount: 0,
+      lastFailAt: null,
+      consecutiveFailCount: 0,
+      cooldownLevel: 0,
+      cooldownUntil: null,
+    });
+
+    decision = await router.explainSelection('gpt-5.4');
+    const recoveredCandidateA = decision.candidates.find((candidate) => candidate.channelId === channelA.id);
+    const recoveredCandidateB = decision.candidates.find((candidate) => candidate.channelId === channelB.id);
+    expect(recoveredCandidateA?.reason || '').not.toContain('熔断');
+    expect((recoveredCandidateA?.probability || 0)).toBeGreaterThan(30);
+    expect((recoveredCandidateB?.probability || 0)).toBeLessThan(70);
+  });
+
   it('does not open a site breaker for repeated timeout validation errors', async () => {
     config.routingWeights = {
       baseWeightFactor: 1,

src/server/services/tokenRouter.ts

@@ -1,7 +1,11 @@
 import { and, eq, inArray, isNull } from 'drizzle-orm';
 import { db, schema } from '../db/index.js';
 import { upsertSetting } from '../db/upsertSetting.js';
-import { config } from '../config.js';
+import {
+  config,
+  normalizeTokenRouterFailureCooldownMaxSec,
+  TOKEN_ROUTER_FAILURE_COOLDOWN_MAX_SEC_CEILING,
+} from '../config.js';
 import { getCachedModelRoutingReferenceCost, refreshModelPricingCatalog } from './modelPricingService.js';
 import { RETRYABLE_TIMEOUT_PATTERNS } from './proxyRetryPolicy.js';
 import {
@@ -203,6 +207,21 @@ function resolveFailureBackoffSec(failCount?: number | null): number {
   return Math.min(FAILURE_BACKOFF_BASE_SEC * fibonacciNumber(normalizedFailCount), MAX_FAILURE_BACKOFF_SEC);
 }
 
+function resolveConfiguredFailureCooldownMaxMs(): number {
+  const normalized = normalizeTokenRouterFailureCooldownMaxSec(config.tokenRouterFailureCooldownMaxSec)
+    ?? TOKEN_ROUTER_FAILURE_COOLDOWN_MAX_SEC_CEILING;
+  return Math.max(1_000, normalized * 1000);
+}
+
+function clampFailureCooldownMs(cooldownMs: number): number {
+  const normalized = Math.max(0, Math.trunc(cooldownMs));
+  return Math.min(normalized, resolveConfiguredFailureCooldownMaxMs());
+}
+
+function resolveEffectiveFailureCooldownMs(failCount?: number | null): number {
+  return clampFailureCooldownMs(resolveFailureBackoffSec(failCount) * 1000);
+}
+
 function resolveRoundRobinCooldownSec(level: number): number {
   const normalizedLevel = Math.max(0, Math.min(ROUND_ROBIN_COOLDOWN_LEVELS_SEC.length - 1, Math.trunc(level)));
   return ROUND_ROBIN_COOLDOWN_LEVELS_SEC[normalizedLevel] ?? 0;
@@ -695,6 +714,45 @@ export async function flushSiteRuntimeHealthPersistence(): Promise<void> {
   }
 }
 
+function clearRuntimeHealthStatesForChannels(rows: Array<{
+  siteId: number;
+  sourceModel: string | null;
+  routeModelPattern: string;
+}>): boolean {
+  let changed = false;
+  const modelKeysBySiteId = new Map<number, Set<string>>();
+
+  for (const row of rows) {
+    if (siteRuntimeHealthStates.delete(row.siteId)) {
+      changed = true;
+    }
+
+    const resolvedModelName = normalizeChannelSourceModel(row.sourceModel)
+      || (isExactRouteModelPattern(row.routeModelPattern) ? row.routeModelPattern.trim() : '');
+    const modelKey = normalizeModelAlias(resolvedModelName);
+    if (!modelKey) continue;
+    if (!modelKeysBySiteId.has(row.siteId)) {
+      modelKeysBySiteId.set(row.siteId, new Set());
+    }
+    modelKeysBySiteId.get(row.siteId)!.add(modelKey);
+  }
+
+  for (const [siteId, modelKeys] of modelKeysBySiteId.entries()) {
+    const modelStates = siteModelRuntimeHealthStates.get(siteId);
+    if (!modelStates) continue;
+    for (const modelKey of modelKeys) {
+      if (modelStates.delete(modelKey)) {
+        changed = true;
+      }
+    }
+    if (modelStates.size === 0) {
+      siteModelRuntimeHealthStates.delete(siteId);
+    }
+  }
+
+  return changed;
+}
+
 export function getSiteRuntimeHealthMultiplier(siteId: number, nowMs = Date.now()): number {
   const state = siteRuntimeHealthStates.get(siteId);
   return getRuntimeHealthMultiplier(state, nowMs);
@@ -919,15 +977,16 @@ export function isChannelRecentlyFailed(
   nowMs = Date.now(),
   avoidSec = resolveFailureBackoffSec(channel.failCount),
 ): boolean {
-  if (avoidSec <= 0) return false;
-  if ((channel.failCount ?? 0) <= 0) return false;
-  if (!channel.lastFailAt) return false;
-
-  const failTs = Date.parse(channel.lastFailAt);
-  if (Number.isNaN(failTs)) return false;
-
-  return nowMs - failTs < avoidSec * 1000;
-}
+  const avoidMs = clampFailureCooldownMs(avoidSec * 1000);
+  if (avoidMs <= 0) return false;
+  if ((channel.failCount ?? 0) <= 0) return false;
+  if (!channel.lastFailAt) return false;
+
+  const failTs = Date.parse(channel.lastFailAt);
+  if (Number.isNaN(failTs)) return false;
+
+  return nowMs - failTs < avoidMs;
+}
 
 export function filterRecentlyFailedCandidates<T extends { channel: FailureAwareChannel }>(
   candidates: T[],
@@ -1771,6 +1830,44 @@ export class TokenRouter {
     recordSiteRuntimeSuccess(account.siteId, latencyMs, modelName);
   }
 
+  /**
+   * Clear persisted failure and cooldown state for the given channels.
+   */
+  async clearChannelFailureState(channelIds: number[]): Promise<number> {
+    const normalizedChannelIds = Array.from(new Set(
+      channelIds
+        .filter((channelId): channelId is number => Number.isFinite(channelId) && channelId > 0)
+        .map((channelId) => Math.trunc(channelId)),
+    ));
+    if (normalizedChannelIds.length === 0) return 0;
+
+    await ensureSiteRuntimeHealthStateLoaded();
+    const runtimeHealthRows = await db.select({
+      siteId: schema.accounts.siteId,
+      sourceModel: schema.routeChannels.sourceModel,
+      routeModelPattern: schema.tokenRoutes.modelPattern,
+    }).from(schema.routeChannels)
+      .innerJoin(schema.accounts, eq(schema.routeChannels.accountId, schema.accounts.id))
+      .innerJoin(schema.tokenRoutes, eq(schema.routeChannels.routeId, schema.tokenRoutes.id))
+      .where(inArray(schema.routeChannels.id, normalizedChannelIds))
+      .all();
+
+    const result = await db.update(schema.routeChannels).set({
+      failCount: 0,
+      lastFailAt: null,
+      consecutiveFailCount: 0,
+      cooldownLevel: 0,
+      cooldownUntil: null,
+    }).where(inArray(schema.routeChannels.id, normalizedChannelIds)).run();
+
+    if (clearRuntimeHealthStatesForChannels(runtimeHealthRows)) {
+      await persistSiteRuntimeHealthState();
+    }
+
+    invalidateTokenRouterCache();
+    return Number(result?.changes || normalizedChannelIds.length);
+  }
+
   /**
    * Record failure and set cooldown.
    */
@@ -1810,12 +1907,13 @@ export class TokenRouter {
       if (consecutiveFailCount >= ROUND_ROBIN_FAILURE_THRESHOLD) {
         cooldownLevel = Math.min(cooldownLevel + 1, ROUND_ROBIN_COOLDOWN_LEVELS_SEC.length - 1);
         const cooldownSec = resolveRoundRobinCooldownSec(cooldownLevel);
-        cooldownUntil = cooldownSec > 0 ? new Date(nowMs + cooldownSec * 1000).toISOString() : null;
+        cooldownUntil = cooldownSec > 0
+          ? new Date(nowMs + clampFailureCooldownMs(cooldownSec * 1000)).toISOString()
+          : null;
         consecutiveFailCount = 0;
       }
     } else {
-      const cooldownSec = resolveFailureBackoffSec(failCount);
-      cooldownUntil = new Date(nowMs + cooldownSec * 1000).toISOString();
+      cooldownUntil = new Date(nowMs + resolveEffectiveFailureCooldownMs(failCount)).toISOString();
       consecutiveFailCount = 0;
       cooldownLevel = 0;
     }