[codex] Add route cooldown controls (#365)

* Add route cooldown controls

* Address route cooldown review comments

src/server/services/routeCooldownService.ts

@@ -0,0 +1,142 @@
+import { eq, inArray } from 'drizzle-orm';
+import { db, schema } from '../db/index.js';
+import {
+  clearRouteDecisionSnapshot,
+  clearRouteDecisionSnapshots,
+} from './routeDecisionSnapshotStore.js';
+import { tokenRouter } from './tokenRouter.js';
+import { normalizeTokenRouteMode, type RouteMode } from '../../shared/tokenRouteContract.js';
+
+type RouteRow = typeof schema.tokenRoutes.$inferSelect & {
+  routeMode: RouteMode;
+  sourceRouteIds: number[];
+};
+
+function isExactModelPattern(modelPattern: string): boolean {
+  const normalized = modelPattern.trim();
+  if (!normalized) return false;
+  if (normalized.toLowerCase().startsWith('re:')) return false;
+  return !/[\*\?]/.test(normalized);
+}
+
+function isExplicitGroupRoute(route: Pick<RouteRow, 'routeMode'> | Pick<typeof schema.tokenRoutes.$inferSelect, 'routeMode'>): boolean {
+  return normalizeTokenRouteMode(route.routeMode) === 'explicit_group';
+}
+
+function normalizeSourceRouteIds(sourceRouteIds: number[]): number[] {
+  return Array.from(new Set(
+    sourceRouteIds
+      .filter((routeId): routeId is number => Number.isFinite(routeId) && routeId > 0)
+      .map((routeId) => Math.trunc(routeId)),
+  ));
+}
+
+async function loadRouteSourceIdsMap(routeIds: number[]): Promise<Map<number, number[]>> {
+  const normalizedRouteIds = normalizeSourceRouteIds(routeIds);
+  if (normalizedRouteIds.length === 0) return new Map();
+
+  const rows = await db.select().from(schema.routeGroupSources)
+    .where(inArray(schema.routeGroupSources.groupRouteId, normalizedRouteIds))
+    .all();
+  const sourceRouteIdsByRouteId = new Map<number, number[]>();
+  for (const row of rows) {
+    if (!sourceRouteIdsByRouteId.has(row.groupRouteId)) {
+      sourceRouteIdsByRouteId.set(row.groupRouteId, []);
+    }
+    sourceRouteIdsByRouteId.get(row.groupRouteId)!.push(row.sourceRouteId);
+  }
+  for (const [routeId, sourceRouteIds] of sourceRouteIdsByRouteId.entries()) {
+    sourceRouteIdsByRouteId.set(routeId, normalizeSourceRouteIds(sourceRouteIds));
+  }
+  return sourceRouteIdsByRouteId;
+}
+
+function decorateRoutesWithSources(
+  routes: Array<typeof schema.tokenRoutes.$inferSelect>,
+  sourceRouteIdsByRouteId: Map<number, number[]>,
+): RouteRow[] {
+  return routes.map((route) => ({
+    ...route,
+    routeMode: normalizeTokenRouteMode(route.routeMode),
+    sourceRouteIds: sourceRouteIdsByRouteId.get(route.id) ?? [],
+  }));
+}
+
+async function getRouteWithSources(routeId: number): Promise<RouteRow | null> {
+  const route = await db.select().from(schema.tokenRoutes).where(eq(schema.tokenRoutes.id, routeId)).get();
+  if (!route) return null;
+  const sourceRouteIdsByRouteId = await loadRouteSourceIdsMap([routeId]);
+  return decorateRoutesWithSources([route], sourceRouteIdsByRouteId)[0] ?? null;
+}
+
+async function resolveCooldownClearRouteIds(route: RouteRow): Promise<number[]> {
+  if (!isExplicitGroupRoute(route)) {
+    return [route.id];
+  }
+
+  const sourceRouteIds = normalizeSourceRouteIds(route.sourceRouteIds);
+  if (sourceRouteIds.length === 0) return [];
+
+  const sourceRoutes = await db.select({
+    id: schema.tokenRoutes.id,
+    modelPattern: schema.tokenRoutes.modelPattern,
+    routeMode: schema.tokenRoutes.routeMode,
+    enabled: schema.tokenRoutes.enabled,
+  }).from(schema.tokenRoutes)
+    .where(inArray(schema.tokenRoutes.id, sourceRouteIds))
+    .all();
+
+  return sourceRoutes
+    .filter((sourceRoute) => (
+      sourceRoute.enabled
+      && normalizeTokenRouteMode(sourceRoute.routeMode) !== 'explicit_group'
+      && isExactModelPattern(sourceRoute.modelPattern)
+    ))
+    .map((sourceRoute) => sourceRoute.id);
+}
+
+async function clearDependentExplicitGroupSnapshotsBySourceRouteIds(sourceRouteIds: number[]): Promise<void> {
+  const normalizedSourceRouteIds = normalizeSourceRouteIds(sourceRouteIds);
+  if (normalizedSourceRouteIds.length === 0) return;
+
+  const rows = await db.select({ groupRouteId: schema.routeGroupSources.groupRouteId })
+    .from(schema.routeGroupSources)
+    .where(inArray(schema.routeGroupSources.sourceRouteId, normalizedSourceRouteIds))
+    .all();
+  const dependentRouteIds: number[] = Array.from(new Set(
+    rows
+      .map((row) => row.groupRouteId)
+      .filter((routeId): routeId is number => Number.isFinite(routeId) && routeId > 0),
+  ));
+  if (dependentRouteIds.length === 0) return;
+  await clearRouteDecisionSnapshots(dependentRouteIds);
+}
+
+export async function clearRouteCooldown(routeId: number): Promise<{ success: true; clearedChannels: number } | null> {
+  const route = await getRouteWithSources(routeId);
+  if (!route) return null;
+
+  const actualRouteIds = await resolveCooldownClearRouteIds(route);
+  const channelRows: Array<{ id: number; routeId: number }> = actualRouteIds.length > 0
+    ? await db.select({
+      id: schema.routeChannels.id,
+      routeId: schema.routeChannels.routeId,
+    }).from(schema.routeChannels)
+      .where(inArray(schema.routeChannels.routeId, actualRouteIds))
+      .all()
+    : [];
+
+  const affectedRouteIds = Array.from(new Set(channelRows.map((row) => row.routeId)));
+  const clearedChannels = await tokenRouter.clearChannelFailureState(channelRows.map((row) => row.id));
+
+  await clearRouteDecisionSnapshot(route.id);
+  if (affectedRouteIds.length > 0) {
+    await clearRouteDecisionSnapshots(affectedRouteIds);
+    await clearDependentExplicitGroupSnapshotsBySourceRouteIds(affectedRouteIds);
+  }
+
+  return {
+    success: true,
+    clearedChannels,
+  };
+}

src/server/services/tokenRouter.cache.test.ts

@@ -13,10 +13,12 @@ describe('TokenRouter runtime cache', () => {
   let schema: DbModule['schema'];
   let TokenRouter: TokenRouterModule['TokenRouter'];
   let invalidateTokenRouterCache: TokenRouterModule['invalidateTokenRouterCache'];
+  let isChannelRecentlyFailed: TokenRouterModule['isChannelRecentlyFailed'];
   let resetSiteRuntimeHealthState: TokenRouterModule['resetSiteRuntimeHealthState'];
   let config: ConfigModule['config'];
   let dataDir = '';
   let originalCacheTtlMs = 0;
+  let originalFailureCooldownMaxSec = 0;
 
   beforeAll(async () => {
     dataDir = mkdtempSync(join(tmpdir(), 'metapi-token-router-cache-'));
@@ -30,9 +32,11 @@ describe('TokenRouter runtime cache', () => {
     schema = dbModule.schema;
     TokenRouter = tokenRouterModule.TokenRouter;
     invalidateTokenRouterCache = tokenRouterModule.invalidateTokenRouterCache;
+    isChannelRecentlyFailed = tokenRouterModule.isChannelRecentlyFailed;
     resetSiteRuntimeHealthState = tokenRouterModule.resetSiteRuntimeHealthState;
     config = configModule.config;
     originalCacheTtlMs = config.tokenRouterCacheTtlMs;
+    originalFailureCooldownMaxSec = config.tokenRouterFailureCooldownMaxSec;
   });
 
   beforeEach(async () => {
@@ -43,12 +47,14 @@ describe('TokenRouter runtime cache', () => {
     await db.delete(schema.accounts).run();
     await db.delete(schema.sites).run();
     config.tokenRouterCacheTtlMs = 60_000;
+    config.tokenRouterFailureCooldownMaxSec = originalFailureCooldownMaxSec;
     invalidateTokenRouterCache();
     resetSiteRuntimeHealthState();
   });
 
   afterAll(() => {
     config.tokenRouterCacheTtlMs = originalCacheTtlMs;
+    config.tokenRouterFailureCooldownMaxSec = originalFailureCooldownMaxSec;
     invalidateTokenRouterCache();
     resetSiteRuntimeHealthState();
     delete process.env.DATA_DIR;
@@ -175,6 +181,73 @@ describe('TokenRouter runtime cache', () => {
     expect(thirdCooldownMs).toBeLessThanOrEqual(35_000);
   });
 
+  it('caps generic failure cooldowns at the configured maximum', async () => {
+    config.tokenRouterFailureCooldownMaxSec = 20;
+
+    const site = await db.insert(schema.sites).values({
+      name: 'capped-cooldown-site',
+      url: 'https://capped-cooldown.example.com',
+      platform: 'new-api',
+      status: 'active',
+    }).returning().get();
+
+    const account = await db.insert(schema.accounts).values({
+      siteId: site.id,
+      username: 'capped-cooldown-user',
+      accessToken: 'capped-cooldown-access-token',
+      apiToken: 'capped-cooldown-api-token',
+      status: 'active',
+    }).returning().get();
+
+    const token = await db.insert(schema.accountTokens).values({
+      accountId: account.id,
+      name: 'capped-cooldown-token',
+      token: 'sk-capped-cooldown-token',
+      enabled: true,
+      isDefault: true,
+    }).returning().get();
+
+    const route = await db.insert(schema.tokenRoutes).values({
+      modelPattern: 'gpt-4o-mini',
+      routingStrategy: 'weighted',
+      enabled: true,
+    }).returning().get();
+
+    const channel = await db.insert(schema.routeChannels).values({
+      routeId: route.id,
+      accountId: account.id,
+      tokenId: token.id,
+      priority: 0,
+      weight: 10,
+      enabled: true,
+    }).returning().get();
+
+    const router = new TokenRouter();
+
+    await router.recordFailure(channel.id);
+    await router.recordFailure(channel.id);
+
+    const startedAt = Date.now();
+    await router.recordFailure(channel.id);
+
+    const record = await db.select().from(schema.routeChannels)
+      .where(eq(schema.routeChannels.id, channel.id))
+      .get();
+    const cooldownMs = Date.parse(String(record?.cooldownUntil || '')) - startedAt;
+
+    expect(cooldownMs).toBeGreaterThanOrEqual(17_000);
+    expect(cooldownMs).toBeLessThanOrEqual(23_000);
+    const recentFailureCheckAt = Date.now();
+    expect(isChannelRecentlyFailed({
+      failCount: 3,
+      lastFailAt: new Date(recentFailureCheckAt - 19_000).toISOString(),
+    }, recentFailureCheckAt)).toBe(true);
+    expect(isChannelRecentlyFailed({
+      failCount: 3,
+      lastFailAt: new Date(recentFailureCheckAt - 21_000).toISOString(),
+    }, recentFailureCheckAt)).toBe(false);
+  });
+
   it('uses codex oauth reset hints for usage-limit cooldowns', async () => {
     const site = await db.insert(schema.sites).values({
       name: 'codex-oauth-site',

src/server/services/tokenRouter.selection.test.ts

@@ -594,6 +594,88 @@ describe('TokenRouter selection scoring', () => {
     expect((recoveredCandidateB?.probability || 0)).toBeLessThan(70);
   });
 
+  it('clears persisted runtime breaker state when channel cooldown is manually cleared', async () => {
+    config.routingWeights = {
+      baseWeightFactor: 1,
+      valueScoreFactor: 0,
+      costWeight: 0,
+      balanceWeight: 0,
+      usageWeight: 0,
+    };
+
+    const route = await createRoute('gpt-5.4');
+
+    const siteA = await createSite('clear-breaker-a');
+    const accountA = await createAccount(siteA.id, 'clear-breaker-user-a');
+    const tokenA = await createToken(accountA.id, 'clear-breaker-token-a');
+    const channelA = await db.insert(schema.routeChannels).values({
+      routeId: route.id,
+      accountId: accountA.id,
+      tokenId: tokenA.id,
+      priority: 0,
+      weight: 10,
+      enabled: true,
+    }).returning().get();
+
+    const siteB = await createSite('clear-breaker-b');
+    const accountB = await createAccount(siteB.id, 'clear-breaker-user-b');
+    const tokenB = await createToken(accountB.id, 'clear-breaker-token-b');
+    const channelB = await db.insert(schema.routeChannels).values({
+      routeId: route.id,
+      accountId: accountB.id,
+      tokenId: tokenB.id,
+      priority: 0,
+      weight: 10,
+      enabled: true,
+    }).returning().get();
+
+    const router = new TokenRouter();
+    for (let index = 0; index < 3; index += 1) {
+      await router.recordFailure(channelA.id, {
+        status: 502,
+        errorText: 'Bad gateway',
+        modelName: 'gpt-5.4',
+      });
+    }
+    await db.update(schema.routeChannels).set({
+      cooldownUntil: null,
+      lastFailAt: null,
+      failCount: 0,
+      consecutiveFailCount: 0,
+      cooldownLevel: 0,
+    }).where(eq(schema.routeChannels.id, channelA.id)).run();
+    invalidateTokenRouterCache();
+
+    let decision = await router.explainSelection('gpt-5.4');
+    const breakerCandidateA = decision.candidates.find((candidate) => candidate.channelId === channelA.id);
+    const breakerCandidateB = decision.candidates.find((candidate) => candidate.channelId === channelB.id);
+    expect(breakerCandidateA?.reason || '').toContain('熔断');
+    expect((breakerCandidateA?.probability || 0)).toBe(0);
+    expect((breakerCandidateB?.probability || 0)).toBe(100);
+
+    await router.clearChannelFailureState([channelA.id]);
+    resetSiteRuntimeHealthState();
+    invalidateTokenRouterCache();
+
+    const refreshedChannel = await db.select().from(schema.routeChannels)
+      .where(eq(schema.routeChannels.id, channelA.id))
+      .get();
+    expect(refreshedChannel).toMatchObject({
+      failCount: 0,
+      lastFailAt: null,
+      consecutiveFailCount: 0,
+      cooldownLevel: 0,
+      cooldownUntil: null,
+    });
+
+    decision = await router.explainSelection('gpt-5.4');
+    const recoveredCandidateA = decision.candidates.find((candidate) => candidate.channelId === channelA.id);
+    const recoveredCandidateB = decision.candidates.find((candidate) => candidate.channelId === channelB.id);
+    expect(recoveredCandidateA?.reason || '').not.toContain('熔断');
+    expect((recoveredCandidateA?.probability || 0)).toBeGreaterThan(30);
+    expect((recoveredCandidateB?.probability || 0)).toBeLessThan(70);
+  });
+
   it('does not open a site breaker for repeated timeout validation errors', async () => {
     config.routingWeights = {
       baseWeightFactor: 1,

src/server/services/tokenRouter.ts

@@ -1,7 +1,11 @@
 import { and, eq, inArray, isNull } from 'drizzle-orm';
 import { db, schema } from '../db/index.js';
 import { upsertSetting } from '../db/upsertSetting.js';
-import { config } from '../config.js';
+import {
+  config,
+  normalizeTokenRouterFailureCooldownMaxSec,
+  TOKEN_ROUTER_FAILURE_COOLDOWN_MAX_SEC_CEILING,
+} from '../config.js';
 import { getCachedModelRoutingReferenceCost, refreshModelPricingCatalog } from './modelPricingService.js';
 import { RETRYABLE_TIMEOUT_PATTERNS } from './proxyRetryPolicy.js';
 import {
@@ -203,6 +207,21 @@ function resolveFailureBackoffSec(failCount?: number | null): number {
   return Math.min(FAILURE_BACKOFF_BASE_SEC * fibonacciNumber(normalizedFailCount), MAX_FAILURE_BACKOFF_SEC);
 }
 
+function resolveConfiguredFailureCooldownMaxMs(): number {
+  const normalized = normalizeTokenRouterFailureCooldownMaxSec(config.tokenRouterFailureCooldownMaxSec)
+    ?? TOKEN_ROUTER_FAILURE_COOLDOWN_MAX_SEC_CEILING;
+  return Math.max(1_000, normalized * 1000);
+}
+
+function clampFailureCooldownMs(cooldownMs: number): number {
+  const normalized = Math.max(0, Math.trunc(cooldownMs));
+  return Math.min(normalized, resolveConfiguredFailureCooldownMaxMs());
+}
+
+function resolveEffectiveFailureCooldownMs(failCount?: number | null): number {
+  return clampFailureCooldownMs(resolveFailureBackoffSec(failCount) * 1000);
+}
+
 function resolveRoundRobinCooldownSec(level: number): number {
   const normalizedLevel = Math.max(0, Math.min(ROUND_ROBIN_COOLDOWN_LEVELS_SEC.length - 1, Math.trunc(level)));
   return ROUND_ROBIN_COOLDOWN_LEVELS_SEC[normalizedLevel] ?? 0;
@@ -695,6 +714,45 @@ export async function flushSiteRuntimeHealthPersistence(): Promise<void> {
   }
 }
 
+function clearRuntimeHealthStatesForChannels(rows: Array<{
+  siteId: number;
+  sourceModel: string | null;
+  routeModelPattern: string;
+}>): boolean {
+  let changed = false;
+  const modelKeysBySiteId = new Map<number, Set<string>>();
+
+  for (const row of rows) {
+    if (siteRuntimeHealthStates.delete(row.siteId)) {
+      changed = true;
+    }
+
+    const resolvedModelName = normalizeChannelSourceModel(row.sourceModel)
+      || (isExactRouteModelPattern(row.routeModelPattern) ? row.routeModelPattern.trim() : '');
+    const modelKey = normalizeModelAlias(resolvedModelName);
+    if (!modelKey) continue;
+    if (!modelKeysBySiteId.has(row.siteId)) {
+      modelKeysBySiteId.set(row.siteId, new Set());
+    }
+    modelKeysBySiteId.get(row.siteId)!.add(modelKey);
+  }
+
+  for (const [siteId, modelKeys] of modelKeysBySiteId.entries()) {
+    const modelStates = siteModelRuntimeHealthStates.get(siteId);
+    if (!modelStates) continue;
+    for (const modelKey of modelKeys) {
+      if (modelStates.delete(modelKey)) {
+        changed = true;
+      }
+    }
+    if (modelStates.size === 0) {
+      siteModelRuntimeHealthStates.delete(siteId);
+    }
+  }
+
+  return changed;
+}
+
 export function getSiteRuntimeHealthMultiplier(siteId: number, nowMs = Date.now()): number {
   const state = siteRuntimeHealthStates.get(siteId);
   return getRuntimeHealthMultiplier(state, nowMs);
@@ -919,15 +977,16 @@ export function isChannelRecentlyFailed(
   nowMs = Date.now(),
   avoidSec = resolveFailureBackoffSec(channel.failCount),
 ): boolean {
-  if (avoidSec <= 0) return false;
-  if ((channel.failCount ?? 0) <= 0) return false;
-  if (!channel.lastFailAt) return false;
-
-  const failTs = Date.parse(channel.lastFailAt);
-  if (Number.isNaN(failTs)) return false;
-
-  return nowMs - failTs < avoidSec * 1000;
-}
+  const avoidMs = clampFailureCooldownMs(avoidSec * 1000);
+  if (avoidMs <= 0) return false;
+  if ((channel.failCount ?? 0) <= 0) return false;
+  if (!channel.lastFailAt) return false;
+
+  const failTs = Date.parse(channel.lastFailAt);
+  if (Number.isNaN(failTs)) return false;
+
+  return nowMs - failTs < avoidMs;
+}
 
 export function filterRecentlyFailedCandidates<T extends { channel: FailureAwareChannel }>(
   candidates: T[],
@@ -1771,6 +1830,44 @@ export class TokenRouter {
     recordSiteRuntimeSuccess(account.siteId, latencyMs, modelName);
   }
 
+  /**
+   * Clear persisted failure and cooldown state for the given channels.
+   */
+  async clearChannelFailureState(channelIds: number[]): Promise<number> {
+    const normalizedChannelIds = Array.from(new Set(
+      channelIds
+        .filter((channelId): channelId is number => Number.isFinite(channelId) && channelId > 0)
+        .map((channelId) => Math.trunc(channelId)),
+    ));
+    if (normalizedChannelIds.length === 0) return 0;
+
+    await ensureSiteRuntimeHealthStateLoaded();
+    const runtimeHealthRows = await db.select({
+      siteId: schema.accounts.siteId,
+      sourceModel: schema.routeChannels.sourceModel,
+      routeModelPattern: schema.tokenRoutes.modelPattern,
+    }).from(schema.routeChannels)
+      .innerJoin(schema.accounts, eq(schema.routeChannels.accountId, schema.accounts.id))
+      .innerJoin(schema.tokenRoutes, eq(schema.routeChannels.routeId, schema.tokenRoutes.id))
+      .where(inArray(schema.routeChannels.id, normalizedChannelIds))
+      .all();
+
+    const result = await db.update(schema.routeChannels).set({
+      failCount: 0,
+      lastFailAt: null,
+      consecutiveFailCount: 0,
+      cooldownLevel: 0,
+      cooldownUntil: null,
+    }).where(inArray(schema.routeChannels.id, normalizedChannelIds)).run();
+
+    if (clearRuntimeHealthStatesForChannels(runtimeHealthRows)) {
+      await persistSiteRuntimeHealthState();
+    }
+
+    invalidateTokenRouterCache();
+    return Number(result?.changes || normalizedChannelIds.length);
+  }
+
   /**
    * Record failure and set cooldown.
    */
@@ -1810,12 +1907,13 @@ export class TokenRouter {
       if (consecutiveFailCount >= ROUND_ROBIN_FAILURE_THRESHOLD) {
         cooldownLevel = Math.min(cooldownLevel + 1, ROUND_ROBIN_COOLDOWN_LEVELS_SEC.length - 1);
         const cooldownSec = resolveRoundRobinCooldownSec(cooldownLevel);
-        cooldownUntil = cooldownSec > 0 ? new Date(nowMs + cooldownSec * 1000).toISOString() : null;
+        cooldownUntil = cooldownSec > 0
+          ? new Date(nowMs + clampFailureCooldownMs(cooldownSec * 1000)).toISOString()
+          : null;
         consecutiveFailCount = 0;
       }
     } else {
-      const cooldownSec = resolveFailureBackoffSec(failCount);
-      cooldownUntil = new Date(nowMs + cooldownSec * 1000).toISOString();
+      cooldownUntil = new Date(nowMs + resolveEffectiveFailureCooldownMs(failCount)).toISOString();
       consecutiveFailCount = 0;
       cooldownLevel = 0;
     }