fix: stabilize Codex continuation and proxy retry recovery / 修复 Codex 续传与代理重试稳定性 (#257)

* wip: preserve local changes before upstream merge

* Improve proxy routing stability and add stable-first strategy

* Sync explicit-group routing strategy to eligible source routes

* Fix proxy retry and runtime health review issues

* fix: make proxy channel retry attempts configurable

* fix: harden proxy retry recovery and bookkeeping

* fix: address pr 257 review comments

---------

Co-authored-by: Cita <juricek.chen@gmail.com>

src/server/services/platforms/sub2api.subscription-summary.test.ts

@@ -99,4 +99,42 @@ describe('Sub2ApiAdapter subscription summary parsing', () => {
       ],
     });
   });
+
+  it('preserves explicit zero activeCount from summary payload', async () => {
+    const adapter = new MockSub2ApiAdapter({
+      '/api/v1/auth/me': {
+        code: 0,
+        message: 'success',
+        data: { id: 1, username: 'demo', email: 'demo@example.com', balance: 8.5 },
+      },
+      '/api/v1/subscriptions/summary': {
+        code: 0,
+        message: 'success',
+        data: {
+          active_count: 0,
+          total_used_usd: 0,
+          subscriptions: [
+            {
+              id: 11,
+              group_name: 'Expired',
+              status: 'expired',
+            },
+          ],
+        },
+      },
+    });
+
+    const balance = await adapter.getBalance('https://sub2api.example.com', 'jwt-token');
+    expect(balance.subscriptionSummary).toEqual({
+      activeCount: 0,
+      totalUsedUsd: 0,
+      subscriptions: [
+        {
+          id: 11,
+          groupName: 'Expired',
+          status: 'expired',
+        },
+      ],
+    });
+  });
 });

src/server/services/platforms/sub2api.ts

@@ -37,6 +37,15 @@ export class Sub2ApiAdapter extends BasePlatformAdapter {
     return undefined;
   }
 
+  private parseNonNegativeInteger(raw: unknown): number | undefined {
+    if (typeof raw === 'number' && Number.isFinite(raw) && raw >= 0) return Math.trunc(raw);
+    if (typeof raw === 'string') {
+      const parsed = Number.parseInt(raw.trim(), 10);
+      if (!Number.isNaN(parsed) && parsed >= 0) return parsed;
+    }
+    return undefined;
+  }
+
   private parseNonNegativeNumber(raw: unknown): number | undefined {
     if (typeof raw === 'number' && Number.isFinite(raw) && raw >= 0) {
       return this.roundCurrency(raw);
@@ -178,7 +187,7 @@ export class Sub2ApiAdapter extends BasePlatformAdapter {
       ? payload as Record<string, unknown>
       : {};
     const subscriptions = this.parseSubscriptionItems(payload);
-    const activeCount = this.parsePositiveInteger(body.active_count ?? body.activeCount);
+    const activeCount = this.parseNonNegativeInteger(body.active_count ?? body.activeCount);
     const totalUsedUsd = this.parseNonNegativeNumber(body.total_used_usd ?? body.totalUsedUsd);
     const inferredUsedUsd = subscriptions.reduce((sum, item) => sum + (item.monthlyUsedUsd || 0), 0);
 

src/server/services/proxyChannelRetry.test.ts

@@ -0,0 +1,37 @@
+import { afterEach, describe, expect, it } from 'vitest';
+import { buildConfig, config } from '../config.js';
+import {
+  canRetryProxyChannel,
+  getProxyMaxChannelAttempts,
+  getProxyMaxChannelRetries,
+} from './proxyChannelRetry.js';
+
+const originalProxyMaxChannelAttempts = config.proxyMaxChannelAttempts;
+
+afterEach(() => {
+  config.proxyMaxChannelAttempts = originalProxyMaxChannelAttempts;
+});
+
+describe('proxyChannelRetry', () => {
+  it('parses proxy max channel attempts from config with a safer default', () => {
+    expect(buildConfig({} as NodeJS.ProcessEnv).proxyMaxChannelAttempts).toBe(3);
+    expect(buildConfig({ PROXY_MAX_CHANNEL_ATTEMPTS: '3' } as NodeJS.ProcessEnv).proxyMaxChannelAttempts).toBe(3);
+  });
+
+  it('derives retry budget from total channel attempts', () => {
+    config.proxyMaxChannelAttempts = 5;
+
+    expect(getProxyMaxChannelAttempts()).toBe(5);
+    expect(getProxyMaxChannelRetries()).toBe(4);
+    expect(canRetryProxyChannel(3)).toBe(true);
+    expect(canRetryProxyChannel(4)).toBe(false);
+  });
+
+  it('clamps invalid runtime config to at least one channel attempt', () => {
+    config.proxyMaxChannelAttempts = 0;
+
+    expect(getProxyMaxChannelAttempts()).toBe(1);
+    expect(getProxyMaxChannelRetries()).toBe(0);
+    expect(canRetryProxyChannel(0)).toBe(false);
+  });
+});

src/server/services/proxyChannelRetry.ts

@@ -0,0 +1,14 @@
+import { config } from '../config.js';
+
+export function getProxyMaxChannelAttempts(): number {
+  const attempts = Math.trunc(config.proxyMaxChannelAttempts || 0);
+  return attempts > 0 ? attempts : 1;
+}
+
+export function getProxyMaxChannelRetries(): number {
+  return Math.max(0, getProxyMaxChannelAttempts() - 1);
+}
+
+export function canRetryProxyChannel(retryCount: number): boolean {
+  return retryCount < getProxyMaxChannelRetries();
+}

src/server/services/proxyRetryPolicy.test.ts

@@ -43,4 +43,13 @@ describe('proxyRetryPolicy', () => {
       shouldRetryProxyRequest(400, 'Unsupported legacy protocol: /v1/chat/completions is not supported. Please use /v1/responses.'),
     ).toBe(true);
   });
+
+  it('does not retry client-side timeout validation errors', () => {
+    expect(
+      shouldRetryProxyRequest(400, '{"error":{"message":"timeout must be <= 60"}}'),
+    ).toBe(false);
+    expect(
+      shouldRetryProxyRequest(400, '{"error":{"message":"invalid timeout parameter"}}'),
+    ).toBe(false);
+  });
 });

src/server/services/proxyRetryPolicy.ts

@@ -14,6 +14,10 @@ const MODEL_UNSUPPORTED_PATTERNS: RegExp[] = [
   /you\s+do\s+not\s+have\s+access\s+to\s+the\s+model/i,
 ];
 
+export const RETRYABLE_TIMEOUT_PATTERNS: RegExp[] = [
+  /(request timed out|connection timed out|read timeout|\btimed out\b)/i,
+];
+
 const RETRYABLE_CHANNEL_LOCAL_PATTERNS: RegExp[] = [
   /unsupported\s+legacy\s+protocol/i,
   /please\s+use\s+\/v1\/responses/i,
@@ -34,7 +38,7 @@ const RETRYABLE_CHANNEL_LOCAL_PATTERNS: RegExp[] = [
   /gateway\s+time-?out/i,
   /service\s+unavailable/i,
   /cpu\s+overloaded/i,
-  /timeout/i,
+  ...RETRYABLE_TIMEOUT_PATTERNS,
 ];
 
 const NON_RETRYABLE_REQUEST_PATTERNS: RegExp[] = [

src/server/services/tokenRouter.selection.test.ts

@@ -546,6 +546,67 @@ describe('TokenRouter selection scoring', () => {
     expect((recoveredCandidateB?.probability || 0)).toBeLessThan(70);
   });
 
+  it('does not open a site breaker for repeated timeout validation errors', async () => {
+    config.routingWeights = {
+      baseWeightFactor: 1,
+      valueScoreFactor: 0,
+      costWeight: 0,
+      balanceWeight: 0,
+      usageWeight: 0,
+    };
+
+    const route = await createRoute('gpt-5.4');
+
+    const siteA = await createSite('timeout-validation-a');
+    const accountA = await createAccount(siteA.id, 'timeout-validation-user-a');
+    const tokenA = await createToken(accountA.id, 'timeout-validation-token-a');
+    const channelA = await db.insert(schema.routeChannels).values({
+      routeId: route.id,
+      accountId: accountA.id,
+      tokenId: tokenA.id,
+      priority: 0,
+      weight: 10,
+      enabled: true,
+    }).returning().get();
+
+    const siteB = await createSite('timeout-validation-b');
+    const accountB = await createAccount(siteB.id, 'timeout-validation-user-b');
+    const tokenB = await createToken(accountB.id, 'timeout-validation-token-b');
+    const channelB = await db.insert(schema.routeChannels).values({
+      routeId: route.id,
+      accountId: accountB.id,
+      tokenId: tokenB.id,
+      priority: 0,
+      weight: 10,
+      enabled: true,
+    }).returning().get();
+
+    const router = new TokenRouter();
+    for (let index = 0; index < 3; index += 1) {
+      await router.recordFailure(channelA.id, {
+        status: 400,
+        errorText: 'invalid timeout parameter',
+      });
+    }
+    await db.update(schema.routeChannels).set({
+      cooldownUntil: null,
+      lastFailAt: null,
+      failCount: 0,
+    }).where(eq(schema.routeChannels.id, channelA.id)).run();
+    invalidateTokenRouterCache();
+
+    const decision = await router.explainSelection('gpt-5.4');
+    const candidateA = decision.candidates.find((candidate) => candidate.channelId === channelA.id);
+    const candidateB = decision.candidates.find((candidate) => candidate.channelId === channelB.id);
+
+    expect(candidateA).toBeTruthy();
+    expect(candidateB).toBeTruthy();
+    expect(candidateA?.reason || '').not.toContain('站点熔断');
+    expect(candidateB?.reason || '').not.toContain('站点熔断');
+    expect(decision.summary.join(' ')).not.toContain('站点熔断避让');
+    expect((candidateA?.probability || 0)).toBeGreaterThan(0);
+  });
+
   it('uses persisted site success and latency history to prefer historically healthier sites', async () => {
     config.routingWeights = {
       baseWeightFactor: 1,

src/server/services/tokenRouter.ts

@@ -3,6 +3,7 @@ import { db, schema } from '../db/index.js';
 import { upsertSetting } from '../db/upsertSetting.js';
 import { config } from '../config.js';
 import { getCachedModelRoutingReferenceCost, refreshModelPricingCatalog } from './modelPricingService.js';
+import { RETRYABLE_TIMEOUT_PATTERNS } from './proxyRetryPolicy.js';
 import {
   normalizeRouteRoutingStrategy,
   type RouteRoutingStrategy,
@@ -133,8 +134,7 @@ const SITE_VALIDATION_FAILURE_PATTERNS: RegExp[] = [
 const SITE_TRANSIENT_FAILURE_PATTERNS: RegExp[] = [
   /bad\s+gateway/i,
   /gateway\s+time-?out/i,
-  /timed?\s*out/i,
-  /timeout/i,
+  ...RETRYABLE_TIMEOUT_PATTERNS,
   /service\s+unavailable/i,
   /temporar(?:y|ily)\s+unavailable/i,
   /cpu\s+overloaded/i,
@@ -505,7 +505,9 @@ function scheduleSiteRuntimeHealthPersistence(): void {
   if (siteRuntimeHealthSaveTimer) return;
   siteRuntimeHealthSaveTimer = setTimeout(() => {
     siteRuntimeHealthSaveTimer = null;
-    void persistSiteRuntimeHealthState();
+    void persistSiteRuntimeHealthState().catch((error) => {
+      console.error('Failed to persist site runtime health state', error);
+    });
   }, SITE_RUNTIME_HEALTH_PERSIST_DEBOUNCE_MS);
 }
 
@@ -560,8 +562,11 @@ async function ensureSiteRuntimeHealthStateLoaded(): Promise<void> {
     siteRuntimeHealthLoadPromise = (async () => {
       try {
         await loadSiteRuntimeHealthStateFromSettings();
-      } finally {
         siteRuntimeHealthLoaded = true;
+      } catch (error) {
+        console.warn('Failed to restore site runtime health state from settings', error);
+        siteRuntimeHealthLoadPromise = null;
+        siteRuntimeHealthLoaded = false;
       }
     })();
   }