feat: classify model refresh failures and update health

This commit is contained in:
cita
2026-03-10 16:02:45 +08:00
parent 3acba86edf
commit e1b4452c82
2 changed files with 117 additions and 65 deletions
@@ -79,7 +79,9 @@ describe('refreshModelsForAccount credential discovery', () => {
expect(result).toMatchObject({ expect(result).toMatchObject({
accountId: account.id, accountId: account.id,
refreshed: true, refreshed: true,
status: 'success',
modelCount: 2, modelCount: 2,
modelsPreview: ['claude-sonnet-4-5-20250929', 'claude-opus-4-6'],
tokenScanned: 0, tokenScanned: 0,
discoveredByCredential: true, discoveredByCredential: true,
}); });
+115 -65
View File
@@ -7,10 +7,28 @@ import { invalidateTokenRouterCache } from './tokenRouter.js';
import { setAccountRuntimeHealth } from './accountHealthService.js'; import { setAccountRuntimeHealth } from './accountHealthService.js';
import { clearAllRouteDecisionSnapshots } from './routeDecisionSnapshotStore.js'; import { clearAllRouteDecisionSnapshots } from './routeDecisionSnapshotStore.js';
const API_TOKEN_DISCOVERY_TIMEOUT_MS = 8_000; const API_TOKEN_DISCOVERY_TIMEOUT_MS = 8_000;
const MODEL_DISCOVERY_TIMEOUT_MS = 12_000; const MODEL_DISCOVERY_TIMEOUT_MS = 12_000;
const MODEL_REFRESH_BATCH_SIZE = 3; const MODEL_REFRESH_BATCH_SIZE = 3;
type ModelRefreshErrorCode = 'timeout' | 'unauthorized' | 'empty_models' | 'unknown';
function classifyModelDiscoveryError(message: string): ModelRefreshErrorCode {
const lowered = message.toLowerCase();
if (lowered.includes('timeout') || lowered.includes('timed out') || lowered.includes('请求超时')) return 'timeout';
if (lowered.includes('http 401') || lowered.includes('http 403')
|| lowered.includes('unauthorized') || lowered.includes('invalid')
|| lowered.includes('无权') || lowered.includes('未提供令牌')) return 'unauthorized';
return 'unknown';
}
function buildModelFailureMessage(code: ModelRefreshErrorCode, fallback?: string) {
if (code === 'timeout') return '模型获取失败(请求超时)';
if (code === 'unauthorized') return '模型获取失败,API Key 已无效';
if (code === 'empty_models') return '模型获取失败:未获取到可用模型';
return fallback || '模型获取失败';
}
function isSiteDisabled(status?: string | null): boolean { function isSiteDisabled(status?: string | null): boolean {
return (status || 'active') === 'disabled'; return (status || 'active') === 'disabled';
} }
@@ -120,11 +138,16 @@ export async function refreshModelsForAccount(accountId: number) {
} }
} }
const accountModels = new Set<string>(); const accountModels = new Set<string>();
const modelLatency = new Map<string, number | null>(); const modelLatency = new Map<string, number | null>();
let scannedTokenCount = 0; let scannedTokenCount = 0;
let discoveredByCredential = false; let discoveredByCredential = false;
const attemptedCredentials = new Set<string>(); const attemptedCredentials = new Set<string>();
const failureMessages: string[] = [];
const recordFailure = (err: unknown) => {
const message = (err as { message?: string })?.message || String(err || '');
if (message) failureMessages.push(message);
};
const mergeDiscoveredModels = (models: string[], latencyMs: number | null) => { const mergeDiscoveredModels = (models: string[], latencyMs: number | null) => {
for (const modelName of models) { for (const modelName of models) {
@@ -146,19 +169,20 @@ export async function refreshModelsForAccount(accountId: number) {
attemptedCredentials.add(credential); attemptedCredentials.add(credential);
const startedAt = Date.now(); const startedAt = Date.now();
let models: string[] = []; let models: string[] = [];
try { try {
models = normalizeModels( models = normalizeModels(
await withTimeout( await withTimeout(
() => adapter.getModels(site.url, credential, platformUserId), () => adapter.getModels(site.url, credential, platformUserId),
MODEL_DISCOVERY_TIMEOUT_MS, MODEL_DISCOVERY_TIMEOUT_MS,
`model discovery timeout (${Math.round(MODEL_DISCOVERY_TIMEOUT_MS / 1000)}s)`, `model discovery timeout (${Math.round(MODEL_DISCOVERY_TIMEOUT_MS / 1000)}s)`,
), ),
); );
} catch { } catch (err) {
models = []; recordFailure(err);
} models = [];
if (models.length === 0) return; }
if (models.length === 0) return;
discoveredByCredential = true; discoveredByCredential = true;
const latencyMs = Date.now() - startedAt; const latencyMs = Date.now() - startedAt;
mergeDiscoveredModels(models, latencyMs); mergeDiscoveredModels(models, latencyMs);
@@ -171,19 +195,20 @@ export async function refreshModelsForAccount(accountId: number) {
for (const token of enabledTokens) { for (const token of enabledTokens) {
const startedAt = Date.now(); const startedAt = Date.now();
let models: string[] = []; let models: string[] = [];
try { try {
models = normalizeModels( models = normalizeModels(
await withTimeout( await withTimeout(
() => adapter.getModels(site.url, token.token, platformUserId), () => adapter.getModels(site.url, token.token, platformUserId),
MODEL_DISCOVERY_TIMEOUT_MS, MODEL_DISCOVERY_TIMEOUT_MS,
`model discovery timeout (${Math.round(MODEL_DISCOVERY_TIMEOUT_MS / 1000)}s)`, `model discovery timeout (${Math.round(MODEL_DISCOVERY_TIMEOUT_MS / 1000)}s)`,
), ),
); );
} catch { } catch (err) {
models = []; recordFailure(err);
} models = [];
}
if (models.length === 0) continue; if (models.length === 0) continue;
@@ -204,37 +229,62 @@ export async function refreshModelsForAccount(accountId: number) {
mergeDiscoveredModels(models, latencyMs); mergeDiscoveredModels(models, latencyMs);
} }
if (accountModels.size > 0) { if (accountModels.size === 0) {
const checkedAt = new Date().toISOString(); const firstMessage = failureMessages[0] || '';
await db.insert(schema.modelAvailability).values( const errorCode = firstMessage ? classifyModelDiscoveryError(firstMessage) : 'empty_models';
Array.from(accountModels).map((modelName) => ({ const errorMessage = buildModelFailureMessage(errorCode, firstMessage);
accountId: account.id, await setAccountRuntimeHealth(account.id, {
modelName, state: 'unhealthy',
available: true, reason: errorMessage,
latencyMs: modelLatency.get(modelName) ?? null, source: 'model-discovery',
checkedAt, checkedAt: new Date().toISOString(),
})), });
).run(); return {
accountId,
if (isApiKeyConnection(account)) { refreshed: true,
await setAccountRuntimeHealth(account.id, { status: 'failed',
state: 'healthy', errorCode,
reason: '模型探测成功', errorMessage,
source: 'model-discovery', modelCount: 0,
checkedAt, modelsPreview: [],
}); tokenScanned: scannedTokenCount,
} discoveredByCredential,
discoveredApiToken: !!discoveredApiToken,
};
} }
return { const checkedAt = new Date().toISOString();
accountId, await db.insert(schema.modelAvailability).values(
refreshed: true, Array.from(accountModels).map((modelName) => ({
modelCount: accountModels.size, accountId: account.id,
tokenScanned: scannedTokenCount, modelName,
discoveredByCredential, available: true,
discoveredApiToken: !!discoveredApiToken, latencyMs: modelLatency.get(modelName) ?? null,
}; checkedAt,
} })),
).run();
if (isApiKeyConnection(account)) {
await setAccountRuntimeHealth(account.id, {
state: 'healthy',
reason: '模型探测成功',
source: 'model-discovery',
checkedAt,
});
}
const modelsPreview = Array.from(accountModels).slice(0, 10);
return {
accountId,
refreshed: true,
status: 'success',
modelCount: accountModels.size,
modelsPreview,
tokenScanned: scannedTokenCount,
discoveredByCredential,
discoveredApiToken: !!discoveredApiToken,
};
}
async function refreshModelsForAllActiveAccounts() { async function refreshModelsForAllActiveAccounts() {
const accounts = await db.select({ id: schema.accounts.id }).from(schema.accounts) const accounts = await db.select({ id: schema.accounts.id }).from(schema.accounts)