feat: classify model refresh failures and update health

2026-03-10 16:02:45 +08:00
parent 3acba86edf
commit e1b4452c82
2 changed files with 117 additions and 65 deletions
@@ -79,7 +79,9 @@ describe('refreshModelsForAccount credential discovery', () => {
    expect(result).toMatchObject({
      accountId: account.id,
      refreshed: true,
+      status: 'success',
      modelCount: 2,
+      modelsPreview: ['claude-sonnet-4-5-20250929', 'claude-opus-4-6'],
      tokenScanned: 0,
      discoveredByCredential: true,
    });
@@ -7,10 +7,28 @@ import { invalidateTokenRouterCache } from './tokenRouter.js';
 import { setAccountRuntimeHealth } from './accountHealthService.js';
 import { clearAllRouteDecisionSnapshots } from './routeDecisionSnapshotStore.js';

-const API_TOKEN_DISCOVERY_TIMEOUT_MS = 8_000;
-const MODEL_DISCOVERY_TIMEOUT_MS = 12_000;
-const MODEL_REFRESH_BATCH_SIZE = 3;
-
+const API_TOKEN_DISCOVERY_TIMEOUT_MS = 8_000;
+const MODEL_DISCOVERY_TIMEOUT_MS = 12_000;
+const MODEL_REFRESH_BATCH_SIZE = 3;
+
+type ModelRefreshErrorCode = 'timeout' | 'unauthorized' | 'empty_models' | 'unknown';
+
+function classifyModelDiscoveryError(message: string): ModelRefreshErrorCode {
+  const lowered = message.toLowerCase();
+  if (lowered.includes('timeout') || lowered.includes('timed out') || lowered.includes('请求超时')) return 'timeout';
+  if (lowered.includes('http 401') || lowered.includes('http 403')
+    || lowered.includes('unauthorized') || lowered.includes('invalid')
+    || lowered.includes('无权') || lowered.includes('未提供令牌')) return 'unauthorized';
+  return 'unknown';
+}
+
+function buildModelFailureMessage(code: ModelRefreshErrorCode, fallback?: string) {
+  if (code === 'timeout') return '模型获取失败（请求超时）';
+  if (code === 'unauthorized') return '模型获取失败，API Key 已无效';
+  if (code === 'empty_models') return '模型获取失败：未获取到可用模型';
+  return fallback || '模型获取失败';
+}
+
 function isSiteDisabled(status?: string | null): boolean {
  return (status || 'active') === 'disabled';
 }
@@ -120,11 +138,16 @@ export async function refreshModelsForAccount(accountId: number) {
    }
  }

-  const accountModels = new Set<string>();
-  const modelLatency = new Map<string, number | null>();
-  let scannedTokenCount = 0;
-  let discoveredByCredential = false;
-  const attemptedCredentials = new Set<string>();
+  const accountModels = new Set<string>();
+  const modelLatency = new Map<string, number | null>();
+  let scannedTokenCount = 0;
+  let discoveredByCredential = false;
+  const attemptedCredentials = new Set<string>();
+  const failureMessages: string[] = [];
+  const recordFailure = (err: unknown) => {
+    const message = (err as { message?: string })?.message || String(err || '');
+    if (message) failureMessages.push(message);
+  };

  const mergeDiscoveredModels = (models: string[], latencyMs: number | null) => {
    for (const modelName of models) {
@@ -146,19 +169,20 @@ export async function refreshModelsForAccount(accountId: number) {
    attemptedCredentials.add(credential);

    const startedAt = Date.now();
-    let models: string[] = [];
-    try {
-      models = normalizeModels(
-        await withTimeout(
-          () => adapter.getModels(site.url, credential, platformUserId),
-          MODEL_DISCOVERY_TIMEOUT_MS,
-          `model discovery timeout (${Math.round(MODEL_DISCOVERY_TIMEOUT_MS / 1000)}s)`,
-        ),
-      );
-    } catch {
-      models = [];
-    }
-    if (models.length === 0) return;
+    let models: string[] = [];
+    try {
+      models = normalizeModels(
+        await withTimeout(
+          () => adapter.getModels(site.url, credential, platformUserId),
+          MODEL_DISCOVERY_TIMEOUT_MS,
+          `model discovery timeout (${Math.round(MODEL_DISCOVERY_TIMEOUT_MS / 1000)}s)`,
+        ),
+      );
+    } catch (err) {
+      recordFailure(err);
+      models = [];
+    }
+    if (models.length === 0) return;
    discoveredByCredential = true;
    const latencyMs = Date.now() - startedAt;
    mergeDiscoveredModels(models, latencyMs);
@@ -171,19 +195,20 @@ export async function refreshModelsForAccount(accountId: number) {

  for (const token of enabledTokens) {
    const startedAt = Date.now();
-    let models: string[] = [];
-
-    try {
-      models = normalizeModels(
-        await withTimeout(
-          () => adapter.getModels(site.url, token.token, platformUserId),
-          MODEL_DISCOVERY_TIMEOUT_MS,
-          `model discovery timeout (${Math.round(MODEL_DISCOVERY_TIMEOUT_MS / 1000)}s)`,
-        ),
-      );
-    } catch {
-      models = [];
-    }
+    let models: string[] = [];
+
+    try {
+      models = normalizeModels(
+        await withTimeout(
+          () => adapter.getModels(site.url, token.token, platformUserId),
+          MODEL_DISCOVERY_TIMEOUT_MS,
+          `model discovery timeout (${Math.round(MODEL_DISCOVERY_TIMEOUT_MS / 1000)}s)`,
+        ),
+      );
+    } catch (err) {
+      recordFailure(err);
+      models = [];
+    }

    if (models.length === 0) continue;

@@ -204,37 +229,62 @@ export async function refreshModelsForAccount(accountId: number) {
    mergeDiscoveredModels(models, latencyMs);
  }

-  if (accountModels.size > 0) {
-    const checkedAt = new Date().toISOString();
-    await db.insert(schema.modelAvailability).values(
-      Array.from(accountModels).map((modelName) => ({
-        accountId: account.id,
-        modelName,
-        available: true,
-        latencyMs: modelLatency.get(modelName) ?? null,
-        checkedAt,
-      })),
-    ).run();
-
-    if (isApiKeyConnection(account)) {
-      await setAccountRuntimeHealth(account.id, {
-        state: 'healthy',
-        reason: '模型探测成功',
-        source: 'model-discovery',
-        checkedAt,
-      });
-    }
+  if (accountModels.size === 0) {
+    const firstMessage = failureMessages[0] || '';
+    const errorCode = firstMessage ? classifyModelDiscoveryError(firstMessage) : 'empty_models';
+    const errorMessage = buildModelFailureMessage(errorCode, firstMessage);
+    await setAccountRuntimeHealth(account.id, {
+      state: 'unhealthy',
+      reason: errorMessage,
+      source: 'model-discovery',
+      checkedAt: new Date().toISOString(),
+    });
+    return {
+      accountId,
+      refreshed: true,
+      status: 'failed',
+      errorCode,
+      errorMessage,
+      modelCount: 0,
+      modelsPreview: [],
+      tokenScanned: scannedTokenCount,
+      discoveredByCredential,
+      discoveredApiToken: !!discoveredApiToken,
+    };
  }
-
-  return {
-    accountId,
-    refreshed: true,
-    modelCount: accountModels.size,
-    tokenScanned: scannedTokenCount,
-    discoveredByCredential,
-    discoveredApiToken: !!discoveredApiToken,
-  };
-}
+
+  const checkedAt = new Date().toISOString();
+  await db.insert(schema.modelAvailability).values(
+    Array.from(accountModels).map((modelName) => ({
+      accountId: account.id,
+      modelName,
+      available: true,
+      latencyMs: modelLatency.get(modelName) ?? null,
+      checkedAt,
+    })),
+  ).run();
+
+  if (isApiKeyConnection(account)) {
+    await setAccountRuntimeHealth(account.id, {
+      state: 'healthy',
+      reason: '模型探测成功',
+      source: 'model-discovery',
+      checkedAt,
+    });
+  }
+
+  const modelsPreview = Array.from(accountModels).slice(0, 10);
+  return {
+    accountId,
+    refreshed: true,
+    status: 'success',
+    modelCount: accountModels.size,
+    modelsPreview,
+    tokenScanned: scannedTokenCount,
+    discoveredByCredential,
+    discoveredApiToken: !!discoveredApiToken,
+  };
+}

 async function refreshModelsForAllActiveAccounts() {
  const accounts = await db.select({ id: schema.accounts.id }).from(schema.accounts)