fix: stabilize Codex continuation and proxy retry recovery / 修复 Codex 续传与代理重试稳定性 (#257)

* wip: preserve local changes before upstream merge

* Improve proxy routing stability and add stable-first strategy

* Sync explicit-group routing strategy to eligible source routes

* Fix proxy retry and runtime health review issues

* fix: make proxy channel retry attempts configurable

* fix: harden proxy retry recovery and bookkeeping

* fix: address pr 257 review comments

---------

Co-authored-by: Cita <juricek.chen@gmail.com>

src/server/routes/api/tokens.route-update-rebuild.test.ts

@@ -244,6 +244,110 @@ describe('PUT /api/routes/:id route rebuild', () => {
     ]));
   });
 
+  it('syncs explicit-group routing strategy to unique source routes', async () => {
+    await seedAccountWithToken('claude-opus-4-5');
+    await seedAccountWithToken('claude-sonnet-4-5');
+
+    const exactRouteA = await db.insert(schema.tokenRoutes).values({
+      modelPattern: 'claude-opus-4-5',
+      enabled: true,
+      routingStrategy: 'weighted',
+    }).returning().get();
+    const exactRouteB = await db.insert(schema.tokenRoutes).values({
+      modelPattern: 'claude-sonnet-4-5',
+      enabled: true,
+      routingStrategy: 'weighted',
+    }).returning().get();
+
+    const createResponse = await app.inject({
+      method: 'POST',
+      url: '/api/routes',
+      payload: {
+        routeMode: 'explicit_group',
+        displayName: 'claude-opus-4-6',
+        sourceRouteIds: [exactRouteA.id, exactRouteB.id],
+        routingStrategy: 'stable_first',
+      },
+    });
+
+    expect(createResponse.statusCode).toBe(200);
+
+    const refreshedRouteA = await db.select().from(schema.tokenRoutes)
+      .where(eq(schema.tokenRoutes.id, exactRouteA.id))
+      .get();
+    const refreshedRouteB = await db.select().from(schema.tokenRoutes)
+      .where(eq(schema.tokenRoutes.id, exactRouteB.id))
+      .get();
+
+    expect(refreshedRouteA?.routingStrategy).toBe('stable_first');
+    expect(refreshedRouteB?.routingStrategy).toBe('stable_first');
+
+    const groupRouteId = (createResponse.json() as { id: number }).id;
+    const updateResponse = await app.inject({
+      method: 'PUT',
+      url: `/api/routes/${groupRouteId}`,
+      payload: {
+        routingStrategy: 'round_robin',
+      },
+    });
+
+    expect(updateResponse.statusCode).toBe(200);
+
+    const updatedRouteA = await db.select().from(schema.tokenRoutes)
+      .where(eq(schema.tokenRoutes.id, exactRouteA.id))
+      .get();
+    const updatedRouteB = await db.select().from(schema.tokenRoutes)
+      .where(eq(schema.tokenRoutes.id, exactRouteB.id))
+      .get();
+
+    expect(updatedRouteA?.routingStrategy).toBe('round_robin');
+    expect(updatedRouteB?.routingStrategy).toBe('round_robin');
+  });
+
+  it('does not overwrite source routes shared by another explicit-group', async () => {
+    await seedAccountWithToken('claude-opus-4-5');
+
+    const sharedSourceRoute = await db.insert(schema.tokenRoutes).values({
+      modelPattern: 'claude-opus-4-5',
+      enabled: true,
+      routingStrategy: 'weighted',
+    }).returning().get();
+
+    const firstGroupResponse = await app.inject({
+      method: 'POST',
+      url: '/api/routes',
+      payload: {
+        routeMode: 'explicit_group',
+        displayName: 'claude-opus-4-6',
+        sourceRouteIds: [sharedSourceRoute.id],
+        routingStrategy: 'stable_first',
+      },
+    });
+    expect(firstGroupResponse.statusCode).toBe(200);
+
+    await db.update(schema.tokenRoutes).set({
+      routingStrategy: 'weighted',
+    }).where(eq(schema.tokenRoutes.id, sharedSourceRoute.id)).run();
+
+    const secondGroupResponse = await app.inject({
+      method: 'POST',
+      url: '/api/routes',
+      payload: {
+        routeMode: 'explicit_group',
+        displayName: 'claude-opus-4-6-alt',
+        sourceRouteIds: [sharedSourceRoute.id],
+        routingStrategy: 'round_robin',
+      },
+    });
+    expect(secondGroupResponse.statusCode).toBe(200);
+
+    const refreshedSharedRoute = await db.select().from(schema.tokenRoutes)
+      .where(eq(schema.tokenRoutes.id, sharedSourceRoute.id))
+      .get();
+
+    expect(refreshedSharedRoute?.routingStrategy).toBe('weighted');
+  });
+
   it('fills missing sourceModel from source exact routes when loading explicit-group channels', async () => {
     const sourceA = await seedAccountWithToken('deepseek-chat');
     const sourceB = await seedAccountWithToken('deepseek-reasoner');

src/server/routes/api/tokens.ts

@@ -6,7 +6,11 @@ import {
   ACCOUNT_TOKEN_VALUE_STATUS_READY,
   isUsableAccountToken,
 } from '../../services/accountTokenService.js';
-import { normalizeRouteRoutingStrategy } from '../../services/routeRoutingStrategy.js';
+import {
+  DEFAULT_ROUTE_ROUTING_STRATEGY,
+  normalizeRouteRoutingStrategy,
+  type RouteRoutingStrategy,
+} from '../../services/routeRoutingStrategy.js';
 import { invalidateTokenRouterCache, matchesModelPattern, tokenRouter } from '../../services/tokenRouter.js';
 import { startBackgroundTask } from '../../services/backgroundTaskService.js';
 import {
@@ -133,6 +137,68 @@ async function replaceRouteSourceRouteIds(routeId: number, sourceRouteIds: numbe
   ).run();
 }
 
+async function syncExplicitGroupSourceRouteStrategies(input: {
+  groupRouteId: number;
+  sourceRouteIds: number[];
+  targetStrategy: RouteRoutingStrategy;
+  previousStrategy?: RouteRoutingStrategy | null;
+}): Promise<number[]> {
+  const normalizedSourceRouteIds = Array.from(new Set(
+    input.sourceRouteIds.filter((routeId): routeId is number => Number.isFinite(routeId) && routeId > 0),
+  ));
+  if (normalizedSourceRouteIds.length === 0) return [];
+
+  const [sourceRoutes, sourceGroupRows] = await Promise.all([
+    db.select().from(schema.tokenRoutes)
+      .where(inArray(schema.tokenRoutes.id, normalizedSourceRouteIds))
+      .all(),
+    db.select({
+      groupRouteId: schema.routeGroupSources.groupRouteId,
+      sourceRouteId: schema.routeGroupSources.sourceRouteId,
+    }).from(schema.routeGroupSources)
+      .where(inArray(schema.routeGroupSources.sourceRouteId, normalizedSourceRouteIds))
+      .all(),
+  ]);
+
+  const otherGroupRefsBySourceRouteId = new Map<number, Set<number>>();
+  for (const row of sourceGroupRows) {
+    if (row.groupRouteId === input.groupRouteId) continue;
+    if (!otherGroupRefsBySourceRouteId.has(row.sourceRouteId)) {
+      otherGroupRefsBySourceRouteId.set(row.sourceRouteId, new Set());
+    }
+    otherGroupRefsBySourceRouteId.get(row.sourceRouteId)!.add(row.groupRouteId);
+  }
+
+  const previousStrategy = input.previousStrategy
+    ? normalizeRouteRoutingStrategy(input.previousStrategy)
+    : null;
+  const updatableRouteIds: number[] = [];
+  for (const route of sourceRoutes) {
+    if (normalizeRouteMode(route.routeMode) === 'explicit_group') continue;
+    if (!isExactModelPattern(route.modelPattern)) continue;
+    if ((otherGroupRefsBySourceRouteId.get(route.id)?.size || 0) > 0) continue;
+
+    const currentStrategy = normalizeRouteRoutingStrategy(route.routingStrategy);
+    const shouldSync = (
+      currentStrategy === DEFAULT_ROUTE_ROUTING_STRATEGY
+      || currentStrategy === input.targetStrategy
+      || (previousStrategy !== null && currentStrategy === previousStrategy)
+    );
+    if (!shouldSync) continue;
+    if (currentStrategy === input.targetStrategy) continue;
+    updatableRouteIds.push(route.id);
+  }
+
+  if (updatableRouteIds.length === 0) return [];
+
+  await db.update(schema.tokenRoutes).set({
+    routingStrategy: input.targetStrategy,
+    updatedAt: new Date().toISOString(),
+  }).where(inArray(schema.tokenRoutes.id, updatableRouteIds)).run();
+
+  return updatableRouteIds;
+}
+
 async function clearDependentExplicitGroupSnapshotsBySourceRouteIds(sourceRouteIds: number[]): Promise<void> {
   const normalizedSourceRouteIds = Array.from(new Set(
     sourceRouteIds.filter((routeId): routeId is number => Number.isFinite(routeId) && routeId > 0),
@@ -894,6 +960,7 @@ export async function tokensRoutes(app: FastifyInstance) {
     const routeMode = normalizeRouteMode(body.routeMode);
     const displayName = typeof body.displayName === 'string' ? body.displayName.trim() : '';
     const sourceRouteIds = normalizeSourceRouteIdsInput(body.sourceRouteIds);
+    const normalizedRoutingStrategy = normalizeRouteRoutingStrategy(body.routingStrategy);
     const modelPattern = routeMode === 'explicit_group'
       ? displayName
       : (typeof body.modelPattern === 'string' ? body.modelPattern.trim() : '');
@@ -916,7 +983,7 @@ export async function tokensRoutes(app: FastifyInstance) {
       displayIcon: body.displayIcon,
       routeMode,
       modelMapping: body.modelMapping,
-      routingStrategy: normalizeRouteRoutingStrategy(body.routingStrategy),
+      routingStrategy: normalizedRoutingStrategy,
       enabled: body.enabled ?? true,
     }).run();
     const routeId = Number(insertedRoute.lastInsertRowid || 0);
@@ -930,6 +997,15 @@ export async function tokensRoutes(app: FastifyInstance) {
 
     if (routeMode === 'explicit_group') {
       await replaceRouteSourceRouteIds(route.id, sourceRouteIds);
+      const syncedRouteIds = await syncExplicitGroupSourceRouteStrategies({
+        groupRouteId: route.id,
+        sourceRouteIds,
+        targetStrategy: normalizedRoutingStrategy,
+      });
+      if (syncedRouteIds.length > 0) {
+        await clearRouteDecisionSnapshots(syncedRouteIds);
+        await clearDependentExplicitGroupSnapshotsBySourceRouteIds(syncedRouteIds);
+      }
     } else {
       await populateRouteChannelsByModelPattern(route.id, modelPattern);
     }
@@ -954,6 +1030,8 @@ export async function tokensRoutes(app: FastifyInstance) {
     let nextModelPattern = existingRoute.modelPattern;
     let nextDisplayName = existingRoute.displayName ?? '';
     let nextSourceRouteIds = existingRoute.sourceRouteIds;
+    const previousRoutingStrategy = normalizeRouteRoutingStrategy(existingRoute.routingStrategy);
+    let nextRoutingStrategy = previousRoutingStrategy;
 
     if (body.displayName !== undefined) {
       nextDisplayName = String(body.displayName || '').trim();
@@ -978,7 +1056,10 @@ export async function tokensRoutes(app: FastifyInstance) {
       updates.modelPattern = nextModelPattern;
     }
     if (body.modelMapping !== undefined) updates.modelMapping = body.modelMapping;
-    if (body.routingStrategy !== undefined) updates.routingStrategy = normalizeRouteRoutingStrategy(body.routingStrategy);
+    if (body.routingStrategy !== undefined) {
+      nextRoutingStrategy = normalizeRouteRoutingStrategy(body.routingStrategy);
+      updates.routingStrategy = nextRoutingStrategy;
+    }
     if (body.enabled !== undefined) updates.enabled = body.enabled;
     if (body.routeMode !== undefined) updates.routeMode = routeMode;
     updates.updatedAt = new Date().toISOString();
@@ -987,6 +1068,19 @@ export async function tokensRoutes(app: FastifyInstance) {
     if (routeMode === 'explicit_group' && body.sourceRouteIds !== undefined) {
       await replaceRouteSourceRouteIds(id, nextSourceRouteIds);
     }
+    const shouldSyncExplicitGroupSources = (
+      routeMode === 'explicit_group'
+      && (body.routingStrategy !== undefined || body.sourceRouteIds !== undefined)
+    );
+    let syncedSourceRouteIds: number[] = [];
+    if (shouldSyncExplicitGroupSources) {
+      syncedSourceRouteIds = await syncExplicitGroupSourceRouteStrategies({
+        groupRouteId: id,
+        sourceRouteIds: nextSourceRouteIds,
+        targetStrategy: nextRoutingStrategy,
+        previousStrategy: previousRoutingStrategy,
+      });
+    }
     const modelPatternChanged = nextModelPattern !== existingRoute.modelPattern;
     const routeBehaviorChanged = modelPatternChanged
       || (routeMode === 'explicit_group' && body.sourceRouteIds !== undefined)
@@ -1000,6 +1094,10 @@ export async function tokensRoutes(app: FastifyInstance) {
       await clearRouteDecisionSnapshot(id);
       await clearDependentExplicitGroupSnapshotsBySourceRouteIds([id]);
     }
+    if (syncedSourceRouteIds.length > 0) {
+      await clearRouteDecisionSnapshots(syncedSourceRouteIds);
+      await clearDependentExplicitGroupSnapshotsBySourceRouteIds(syncedSourceRouteIds);
+    }
     invalidateTokenRouterCache();
     return await getRouteWithSources(id);
   });