[codex] fix expired connection health recovery (#393)

* fix expired connection health recovery

* address review comments and test regressions

src/server/routes/api/accounts.apikey-recovery.test.ts

@@ -0,0 +1,216 @@
+import Fastify, { type FastifyInstance } from 'fastify';
+import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
+import { mkdtempSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { eq } from 'drizzle-orm';
+
+const getModelsMock = vi.fn();
+
+vi.mock('../../services/platforms/index.js', () => ({
+  getAdapter: () => ({
+    getModels: (...args: unknown[]) => getModelsMock(...args),
+  }),
+}));
+
+type DbModule = typeof import('../../db/index.js');
+
+describe('accounts api key recovery', { timeout: 15_000 }, () => {
+  let app: FastifyInstance;
+  let db: DbModule['db'];
+  let schema: DbModule['schema'];
+  let dataDir = '';
+
+  beforeAll(async () => {
+    dataDir = mkdtempSync(join(tmpdir(), 'metapi-accounts-apikey-recovery-'));
+    process.env.DATA_DIR = dataDir;
+
+    await import('../../db/migrate.js');
+    const dbModule = await import('../../db/index.js');
+    const routesModule = await import('./accounts.js');
+    db = dbModule.db;
+    schema = dbModule.schema;
+
+    app = Fastify();
+    await app.register(routesModule.accountsRoutes);
+  });
+
+  beforeEach(async () => {
+    getModelsMock.mockReset();
+
+    await db.delete(schema.proxyLogs).run();
+    await db.delete(schema.checkinLogs).run();
+    await db.delete(schema.routeChannels).run();
+    await db.delete(schema.tokenRoutes).run();
+    await db.delete(schema.tokenModelAvailability).run();
+    await db.delete(schema.modelAvailability).run();
+    await db.delete(schema.accountTokens).run();
+    await db.delete(schema.siteApiEndpoints).run();
+    await db.delete(schema.accounts).run();
+    await db.delete(schema.sites).run();
+  });
+
+  afterAll(async () => {
+    await app.close();
+    delete process.env.DATA_DIR;
+  });
+
+  it('reactivates an expired API key connection after editing in a working replacement key', async () => {
+    getModelsMock.mockResolvedValueOnce(['gpt-4.1']);
+
+    const site = await db.insert(schema.sites).values({
+      name: 'Recovery Site',
+      url: 'https://recovery.example.com',
+      platform: 'new-api',
+      status: 'active',
+    }).returning().get();
+
+    const account = await db.insert(schema.accounts).values({
+      siteId: site.id,
+      username: 'expired-apikey-user',
+      accessToken: '',
+      apiToken: 'sk-old-expired-key',
+      status: 'expired',
+      checkinEnabled: false,
+      extraConfig: JSON.stringify({ credentialMode: 'apikey' }),
+    }).returning().get();
+
+    const response = await app.inject({
+      method: 'PUT',
+      url: `/api/accounts/${account.id}`,
+      payload: {
+        username: 'expired-apikey-user',
+        status: 'expired',
+        checkinEnabled: false,
+        accessToken: '',
+        apiToken: 'sk-new-valid-key',
+      },
+    });
+
+    expect(response.statusCode).toBe(200);
+    expect(response.json()).toMatchObject({
+      id: account.id,
+      status: 'active',
+      apiToken: 'sk-new-valid-key',
+    });
+
+    const latest = await db.select().from(schema.accounts).where(eq(schema.accounts.id, account.id)).get();
+    expect(latest).toMatchObject({
+      id: account.id,
+      status: 'active',
+      apiToken: 'sk-new-valid-key',
+    });
+
+    const availabilityRows = await db.select().from(schema.modelAvailability)
+      .where(eq(schema.modelAvailability.accountId, account.id))
+      .all();
+    expect(availabilityRows.map((row) => row.modelName)).toContain('gpt-4.1');
+
+    const routeChannels = await db.select().from(schema.routeChannels).all();
+    expect(routeChannels.some((channel) => channel.accountId === account.id)).toBe(true);
+  });
+
+  it('keeps an expired API key connection pinned when only status is edited', async () => {
+    const site = await db.insert(schema.sites).values({
+      name: 'Pinned Site',
+      url: 'https://pinned.example.com',
+      platform: 'new-api',
+      status: 'active',
+    }).returning().get();
+
+    const account = await db.insert(schema.accounts).values({
+      siteId: site.id,
+      username: 'status-only-user',
+      accessToken: '',
+      apiToken: 'sk-still-expired',
+      status: 'expired',
+      checkinEnabled: false,
+      extraConfig: JSON.stringify({ credentialMode: 'apikey' }),
+    }).returning().get();
+
+    const response = await app.inject({
+      method: 'PUT',
+      url: `/api/accounts/${account.id}`,
+      payload: {
+        username: 'status-only-user-edited',
+        status: 'active',
+        checkinEnabled: false,
+      },
+    });
+
+    expect(response.statusCode).toBe(200);
+    expect(response.json()).toMatchObject({
+      id: account.id,
+      username: 'status-only-user-edited',
+      status: 'expired',
+    });
+    expect(getModelsMock).not.toHaveBeenCalled();
+  });
+
+  it('keeps the new api key but preserves expired status and previous availability when recovery fails', async () => {
+    getModelsMock.mockRejectedValueOnce(new Error('upstream unavailable'));
+
+    const site = await db.insert(schema.sites).values({
+      name: 'Recovery Failure Site',
+      url: 'https://recovery-failure.example.com',
+      platform: 'new-api',
+      status: 'active',
+    }).returning().get();
+
+    const account = await db.insert(schema.accounts).values({
+      siteId: site.id,
+      username: 'expired-apikey-user',
+      accessToken: '',
+      apiToken: 'sk-old-expired-key',
+      status: 'expired',
+      checkinEnabled: false,
+      extraConfig: JSON.stringify({ credentialMode: 'apikey' }),
+    }).returning().get();
+
+    await db.insert(schema.modelAvailability).values({
+      accountId: account.id,
+      modelName: 'gpt-4.1',
+      available: true,
+      latencyMs: 100,
+      checkedAt: '2026-04-01T10:00:00.000Z',
+    }).run();
+
+    const response = await app.inject({
+      method: 'PUT',
+      url: `/api/accounts/${account.id}`,
+      payload: {
+        username: 'expired-apikey-user',
+        status: 'expired',
+        checkinEnabled: false,
+        accessToken: '',
+        apiToken: 'sk-new-invalid-key',
+      },
+    });
+
+    expect(response.statusCode).toBe(200);
+    expect(response.json()).toMatchObject({
+      id: account.id,
+      status: 'expired',
+      apiToken: 'sk-new-invalid-key',
+    });
+
+    const latest = await db.select().from(schema.accounts).where(eq(schema.accounts.id, account.id)).get();
+    expect(latest).toMatchObject({
+      id: account.id,
+      status: 'expired',
+      apiToken: 'sk-new-invalid-key',
+    });
+
+    const availabilityRows = await db.select().from(schema.modelAvailability)
+      .where(eq(schema.modelAvailability.accountId, account.id))
+      .all();
+    expect(availabilityRows).toHaveLength(1);
+    expect(availabilityRows[0]).toMatchObject({
+      accountId: account.id,
+      modelName: 'gpt-4.1',
+    });
+
+    const routeChannels = await db.select().from(schema.routeChannels).all();
+    expect(routeChannels.some((channel) => channel.accountId === account.id)).toBe(false);
+  });
+});

src/server/routes/api/accounts.ts

@@ -20,6 +20,7 @@ import {
   type AccountCredentialMode,
 } from '../../services/accountExtraConfig.js';
 import { encryptAccountPassword } from '../../services/accountCredentialService.js';
+import { applyAccountUpdateWorkflow } from '../../services/accountUpdateWorkflow.js';
 import { startBackgroundTask } from '../../services/backgroundTaskService.js';
 import { parseCheckinRewardAmount } from '../../services/checkinRewardParser.js';
 import { estimateRewardWithTodayIncomeFallback } from '../../services/todayIncomeRewardService.js';
@@ -1392,33 +1393,45 @@ export async function accountsRoutes(app: FastifyInstance) {
       });
     }
 
-    updates.updatedAt = new Date().toISOString();
-    await db.update(schema.accounts).set(updates).where(eq(schema.accounts.id, id)).run();
-
     const nextAccessToken = typeof updates.accessToken === 'string' ? updates.accessToken : account.accessToken;
+    const nextApiToken = Object.prototype.hasOwnProperty.call(updates, 'apiToken')
+      ? updates.apiToken
+      : account.apiToken;
     const nextExtraConfig = typeof updates.extraConfig === 'string' ? updates.extraConfig : account.extraConfig;
     const explicitNextMode = getCredentialModeFromExtraConfig(nextExtraConfig);
     const nextCredentialMode =
       explicitNextMode && explicitNextMode !== 'auto'
         ? explicitNextMode
         : (hasSessionTokenValue(nextAccessToken) ? 'session' : 'apikey');
+    const nextStatus = typeof updates.status === 'string' && updates.status.trim()
+      ? updates.status.trim()
+      : (account.status || 'active');
     const needsModelRefresh =
       Object.prototype.hasOwnProperty.call(body, 'accessToken')
       || Object.prototype.hasOwnProperty.call(body, 'apiToken')
       || Object.prototype.hasOwnProperty.call(body, 'extraConfig')
       || Object.prototype.hasOwnProperty.call(body, 'proxyUrl')
       || wantsManagedSub2ApiAuthPatch;
+    const isExpiredApiKeyAccount =
+      account.status === 'expired'
+      && nextCredentialMode === 'apikey'
+      && nextStatus !== 'disabled';
+    const shouldAttemptExpiredApiKeyRecovery =
+      isExpiredApiKeyAccount
+      && needsModelRefresh;
 
-    await convergeAccountMutation({
+    const { account: updatedAccount } = await applyAccountUpdateWorkflow({
       accountId: id,
-      preferredApiToken: nextCredentialMode !== 'apikey' ? updates.apiToken : null,
+      updates,
-      defaultTokenSource: 'manual',
+      preferredApiToken: nextCredentialMode !== 'apikey' ? nextApiToken : null,
       refreshModels: needsModelRefresh,
-      rebuildRoutes: true,
+      preserveExpiredStatus: isExpiredApiKeyAccount,
+      allowInactiveModelRefresh: shouldAttemptExpiredApiKeyRecovery,
+      reactivateAfterSuccessfulModelRefresh: shouldAttemptExpiredApiKeyRecovery,
       continueOnError: true,
     });
 
-    return await db.select().from(schema.accounts).where(eq(schema.accounts.id, id)).get();
+    return updatedAccount;
   });
 
   // Delete an account

src/server/services/accountHealthService.test.ts

@@ -24,17 +24,61 @@ describe('accountHealthService', () => {
     const health = buildRuntimeHealthForAccount({
       accountStatus: 'expired',
       siteStatus: 'active',
-      extraConfig: null,
+      extraConfig: JSON.stringify({ credentialMode: 'apikey' }),
       sessionCapable: false,
     });
 
     expect(health).toMatchObject({
-      state: 'unknown',
+      state: 'unhealthy',
-      source: 'none',
+      source: 'auth',
     });
+    expect(health.reason).toContain('连接已过期');
     expect(health.reason).not.toContain('访问令牌已过期');
   });
 
+  it('does not show proxy-only expired accounts as healthy just because models were discovered before expiry', () => {
+    const health = buildRuntimeHealthForAccount({
+      accountStatus: 'expired',
+      siteStatus: 'active',
+      extraConfig: JSON.stringify({
+        credentialMode: 'apikey',
+        runtimeHealth: {
+          state: 'healthy',
+          reason: '模型探测成功',
+          source: 'model-discovery',
+          checkedAt: '2026-04-01T10:00:00.000Z',
+        },
+      }),
+      sessionCapable: false,
+      hasDiscoveredModels: true,
+    });
+
+    expect(health).toMatchObject({
+      state: 'unhealthy',
+      source: 'auth',
+    });
+    expect(health.reason).toContain('连接已过期');
+  });
+
+  it('uses a generic expired-credential message for oauth-backed accounts', () => {
+    const health = buildRuntimeHealthForAccount({
+      accountStatus: 'expired',
+      siteStatus: 'active',
+      extraConfig: JSON.stringify({
+        oauth: {
+          provider: 'codex',
+        },
+      }),
+      sessionCapable: false,
+    });
+
+    expect(health).toMatchObject({
+      state: 'unhealthy',
+      source: 'auth',
+      reason: '连接凭证已过期，请更新凭证',
+    });
+  });
+
   it('ignores stored auth failures for proxy-only accounts', () => {
     const health = buildRuntimeHealthForAccount({
       accountStatus: 'active',

src/server/services/accountHealthService.ts

@@ -1,6 +1,10 @@
 import { eq } from 'drizzle-orm';
 import { db, schema } from '../db/index.js';
-import { mergeAccountExtraConfig } from './accountExtraConfig.js';
+import {
+  getCredentialModeFromExtraConfig,
+  hasOauthProvider,
+  mergeAccountExtraConfig,
+} from './accountExtraConfig.js';
 
 export type RuntimeHealthState = 'healthy' | 'unhealthy' | 'degraded' | 'unknown' | 'disabled';
 
@@ -115,10 +119,18 @@ export function buildRuntimeHealthForAccount(input: {
     };
   }
 
-  if (accountStatus === 'expired' && input.sessionCapable !== false) {
+  if (accountStatus === 'expired') {
+    const credentialMode = getCredentialModeFromExtraConfig(input.extraConfig);
+    const usesOauthCredential = hasOauthProvider({ extraConfig: input.extraConfig });
     return {
       state: 'unhealthy',
-      reason: '访问令牌已过期',
+      reason: usesOauthCredential
+        ? '连接凭证已过期，请更新凭证'
+        : credentialMode === 'apikey'
+          ? '连接已过期，请更新 API Key'
+          : (credentialMode === 'session'
+            ? '访问令牌已过期'
+            : '连接凭证已过期，请更新凭证'),
       source: 'auth',
       checkedAt: null,
     };

src/server/services/accountMutationWorkflow.test.ts

@@ -120,6 +120,19 @@ describe('accountMutationWorkflow', () => {
     expect(result.refreshedModels).toBe(false);
   });
 
+  it('passes allowInactive to model refresh when recovery explicitly requests it', async () => {
+    refreshModelsForAccountMock.mockResolvedValue({ accountId: 5, refreshed: true, status: 'success' });
+
+    const { convergeAccountMutation } = await import('./accountMutationWorkflow.js');
+    await convergeAccountMutation({
+      accountId: 5,
+      refreshModels: true,
+      allowInactiveModelRefresh: true,
+    });
+
+    expect(refreshModelsForAccountMock).toHaveBeenCalledWith(5, { allowInactive: true });
+  });
+
   it('refreshes model coverage in batches and maps failures', async () => {
     refreshModelsForAccountMock
       .mockResolvedValueOnce({ accountId: 1, refreshed: true, status: 'success' })

src/server/services/accountMutationWorkflow.ts

@@ -32,6 +32,7 @@ export async function convergeAccountMutation(input: {
   upstreamTokens?: UpstreamTokenLike[];
   refreshBalance?: boolean;
   refreshModels?: boolean;
+  allowInactiveModelRefresh?: boolean;
   rebuildRoutes?: boolean;
   continueOnError?: boolean;
 }): Promise<{
@@ -111,7 +112,11 @@ export async function convergeAccountMutation(input: {
   }
 
   if (input.refreshModels) {
-    const modelRefreshResult = await runStep(() => refreshModelsForAccount(input.accountId));
+    const modelRefreshResult = await runStep(() => (
+      input.allowInactiveModelRefresh
+        ? refreshModelsForAccount(input.accountId, { allowInactive: true })
+        : refreshModelsForAccount(input.accountId)
+    ));
     if (modelRefreshResult) {
       result.modelRefreshResult = modelRefreshResult;
       result.refreshedModels = modelRefreshResult.refreshed === true;

src/server/services/accountUpdateWorkflow.ts

@@ -0,0 +1,65 @@
+import { eq } from 'drizzle-orm';
+import { db, schema } from '../db/index.js';
+import {
+  convergeAccountMutation,
+  rebuildRoutesBestEffort,
+} from './accountMutationWorkflow.js';
+
+type AccountUpdateWorkflowInput = {
+  accountId: number;
+  updates: Partial<typeof schema.accounts.$inferInsert>;
+  preferredApiToken?: string | null;
+  refreshModels: boolean;
+  preserveExpiredStatus?: boolean;
+  allowInactiveModelRefresh?: boolean;
+  reactivateAfterSuccessfulModelRefresh?: boolean;
+  continueOnError?: boolean;
+};
+
+export async function applyAccountUpdateWorkflow(input: AccountUpdateWorkflowInput) {
+  const persistedUpdates: Partial<typeof schema.accounts.$inferInsert> = {
+    ...input.updates,
+    ...(input.preserveExpiredStatus ? { status: 'expired' } : {}),
+    updatedAt: new Date().toISOString(),
+  };
+
+  await db.update(schema.accounts)
+    .set(persistedUpdates)
+    .where(eq(schema.accounts.id, input.accountId))
+    .run();
+
+  const convergence = await convergeAccountMutation({
+    accountId: input.accountId,
+    preferredApiToken: input.preferredApiToken,
+    defaultTokenSource: 'manual',
+    refreshModels: input.refreshModels,
+    allowInactiveModelRefresh: input.allowInactiveModelRefresh,
+    rebuildRoutes: false,
+    continueOnError: input.continueOnError,
+  });
+
+  if (
+    input.reactivateAfterSuccessfulModelRefresh
+    && convergence.modelRefreshResult?.status === 'success'
+  ) {
+    await db.update(schema.accounts)
+      .set({
+        status: 'active',
+        updatedAt: new Date().toISOString(),
+      })
+      .where(eq(schema.accounts.id, input.accountId))
+      .run();
+  }
+
+  await rebuildRoutesBestEffort();
+
+  const account = await db.select()
+    .from(schema.accounts)
+    .where(eq(schema.accounts.id, input.accountId))
+    .get();
+
+  return {
+    account,
+    convergence,
+  };
+}

src/web/pages/Accounts.tsx

@@ -597,6 +597,13 @@ export default function Accounts() {
   };
 
   const resolveRuntimeHealth = (account: any) => {
+    if (account.status === 'expired') {
+      return {
+        ...runtimeHealthMap.unhealthy,
+        label: '已过期',
+        reason: account.runtimeHealth?.reason || '连接凭证已过期，请更新凭证',
+      };
+    }
     const capabilities = resolveAccountCapabilities(account);
     const fallbackState = account.status === 'disabled' || account.site?.status === 'disabled'
       ? 'disabled'

src/web/pages/accounts.proxy-only-expired.test.tsx

@@ -51,6 +51,10 @@ describe('Accounts proxy-only expired state', () => {
         accessToken: 'api-key-only-token',
         status: 'expired',
         checkinEnabled: false,
+        runtimeHealth: {
+          state: 'unhealthy',
+          reason: '连接已过期，请更新 API Key',
+        },
         capabilities: {
           canCheckin: false,
           canRefreshBalance: false,
@@ -77,8 +81,18 @@ describe('Accounts proxy-only expired state', () => {
 
       const rendered = JSON.stringify(root.toJSON());
       expect(rendered).not.toContain('仅代理');
+      expect(rendered).toContain('已过期');
+      expect(rendered).toContain('连接已过期，请更新 API Key');
       expect(rendered).not.toContain('访问令牌已过期');
 
+      const badgeTexts = root.root.findAll((node) => (
+        node.type === 'span'
+        && typeof node.props.className === 'string'
+        && node.props.className.includes('badge')
+      )).map((node) => collectText(node).trim());
+      expect(badgeTexts).toContain('已过期');
+      expect(badgeTexts).not.toContain('健康');
+
       const actionTexts = root.root.findAll((node) => node.type === 'button').map((node) => collectText(node));
       expect(actionTexts).not.toContain('重新绑定');
     } finally {

src/web/pages/accounts.refresh-reload-on-error.test.tsx

@@ -105,7 +105,7 @@ describe('Accounts refresh action', () => {
 
       expect(apiMock.getAccounts).toHaveBeenCalledTimes(2);
       const rendered = JSON.stringify(root.toJSON());
-      expect(rendered).toContain('异常');
+      expect(rendered).toContain('已过期');
     } finally {
       root?.unmount();
     }