feat: classify model refresh failures and update health
This commit is contained in:
@@ -79,7 +79,9 @@ describe('refreshModelsForAccount credential discovery', () => {
|
|||||||
expect(result).toMatchObject({
|
expect(result).toMatchObject({
|
||||||
accountId: account.id,
|
accountId: account.id,
|
||||||
refreshed: true,
|
refreshed: true,
|
||||||
|
status: 'success',
|
||||||
modelCount: 2,
|
modelCount: 2,
|
||||||
|
modelsPreview: ['claude-sonnet-4-5-20250929', 'claude-opus-4-6'],
|
||||||
tokenScanned: 0,
|
tokenScanned: 0,
|
||||||
discoveredByCredential: true,
|
discoveredByCredential: true,
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -7,10 +7,28 @@ import { invalidateTokenRouterCache } from './tokenRouter.js';
|
|||||||
import { setAccountRuntimeHealth } from './accountHealthService.js';
|
import { setAccountRuntimeHealth } from './accountHealthService.js';
|
||||||
import { clearAllRouteDecisionSnapshots } from './routeDecisionSnapshotStore.js';
|
import { clearAllRouteDecisionSnapshots } from './routeDecisionSnapshotStore.js';
|
||||||
|
|
||||||
const API_TOKEN_DISCOVERY_TIMEOUT_MS = 8_000;
|
const API_TOKEN_DISCOVERY_TIMEOUT_MS = 8_000;
|
||||||
const MODEL_DISCOVERY_TIMEOUT_MS = 12_000;
|
const MODEL_DISCOVERY_TIMEOUT_MS = 12_000;
|
||||||
const MODEL_REFRESH_BATCH_SIZE = 3;
|
const MODEL_REFRESH_BATCH_SIZE = 3;
|
||||||
|
|
||||||
|
type ModelRefreshErrorCode = 'timeout' | 'unauthorized' | 'empty_models' | 'unknown';
|
||||||
|
|
||||||
|
function classifyModelDiscoveryError(message: string): ModelRefreshErrorCode {
|
||||||
|
const lowered = message.toLowerCase();
|
||||||
|
if (lowered.includes('timeout') || lowered.includes('timed out') || lowered.includes('请求超时')) return 'timeout';
|
||||||
|
if (lowered.includes('http 401') || lowered.includes('http 403')
|
||||||
|
|| lowered.includes('unauthorized') || lowered.includes('invalid')
|
||||||
|
|| lowered.includes('无权') || lowered.includes('未提供令牌')) return 'unauthorized';
|
||||||
|
return 'unknown';
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildModelFailureMessage(code: ModelRefreshErrorCode, fallback?: string) {
|
||||||
|
if (code === 'timeout') return '模型获取失败(请求超时)';
|
||||||
|
if (code === 'unauthorized') return '模型获取失败,API Key 已无效';
|
||||||
|
if (code === 'empty_models') return '模型获取失败:未获取到可用模型';
|
||||||
|
return fallback || '模型获取失败';
|
||||||
|
}
|
||||||
|
|
||||||
function isSiteDisabled(status?: string | null): boolean {
|
function isSiteDisabled(status?: string | null): boolean {
|
||||||
return (status || 'active') === 'disabled';
|
return (status || 'active') === 'disabled';
|
||||||
}
|
}
|
||||||
@@ -120,11 +138,16 @@ export async function refreshModelsForAccount(accountId: number) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const accountModels = new Set<string>();
|
const accountModels = new Set<string>();
|
||||||
const modelLatency = new Map<string, number | null>();
|
const modelLatency = new Map<string, number | null>();
|
||||||
let scannedTokenCount = 0;
|
let scannedTokenCount = 0;
|
||||||
let discoveredByCredential = false;
|
let discoveredByCredential = false;
|
||||||
const attemptedCredentials = new Set<string>();
|
const attemptedCredentials = new Set<string>();
|
||||||
|
const failureMessages: string[] = [];
|
||||||
|
const recordFailure = (err: unknown) => {
|
||||||
|
const message = (err as { message?: string })?.message || String(err || '');
|
||||||
|
if (message) failureMessages.push(message);
|
||||||
|
};
|
||||||
|
|
||||||
const mergeDiscoveredModels = (models: string[], latencyMs: number | null) => {
|
const mergeDiscoveredModels = (models: string[], latencyMs: number | null) => {
|
||||||
for (const modelName of models) {
|
for (const modelName of models) {
|
||||||
@@ -146,19 +169,20 @@ export async function refreshModelsForAccount(accountId: number) {
|
|||||||
attemptedCredentials.add(credential);
|
attemptedCredentials.add(credential);
|
||||||
|
|
||||||
const startedAt = Date.now();
|
const startedAt = Date.now();
|
||||||
let models: string[] = [];
|
let models: string[] = [];
|
||||||
try {
|
try {
|
||||||
models = normalizeModels(
|
models = normalizeModels(
|
||||||
await withTimeout(
|
await withTimeout(
|
||||||
() => adapter.getModels(site.url, credential, platformUserId),
|
() => adapter.getModels(site.url, credential, platformUserId),
|
||||||
MODEL_DISCOVERY_TIMEOUT_MS,
|
MODEL_DISCOVERY_TIMEOUT_MS,
|
||||||
`model discovery timeout (${Math.round(MODEL_DISCOVERY_TIMEOUT_MS / 1000)}s)`,
|
`model discovery timeout (${Math.round(MODEL_DISCOVERY_TIMEOUT_MS / 1000)}s)`,
|
||||||
),
|
),
|
||||||
);
|
);
|
||||||
} catch {
|
} catch (err) {
|
||||||
models = [];
|
recordFailure(err);
|
||||||
}
|
models = [];
|
||||||
if (models.length === 0) return;
|
}
|
||||||
|
if (models.length === 0) return;
|
||||||
discoveredByCredential = true;
|
discoveredByCredential = true;
|
||||||
const latencyMs = Date.now() - startedAt;
|
const latencyMs = Date.now() - startedAt;
|
||||||
mergeDiscoveredModels(models, latencyMs);
|
mergeDiscoveredModels(models, latencyMs);
|
||||||
@@ -171,19 +195,20 @@ export async function refreshModelsForAccount(accountId: number) {
|
|||||||
|
|
||||||
for (const token of enabledTokens) {
|
for (const token of enabledTokens) {
|
||||||
const startedAt = Date.now();
|
const startedAt = Date.now();
|
||||||
let models: string[] = [];
|
let models: string[] = [];
|
||||||
|
|
||||||
try {
|
try {
|
||||||
models = normalizeModels(
|
models = normalizeModels(
|
||||||
await withTimeout(
|
await withTimeout(
|
||||||
() => adapter.getModels(site.url, token.token, platformUserId),
|
() => adapter.getModels(site.url, token.token, platformUserId),
|
||||||
MODEL_DISCOVERY_TIMEOUT_MS,
|
MODEL_DISCOVERY_TIMEOUT_MS,
|
||||||
`model discovery timeout (${Math.round(MODEL_DISCOVERY_TIMEOUT_MS / 1000)}s)`,
|
`model discovery timeout (${Math.round(MODEL_DISCOVERY_TIMEOUT_MS / 1000)}s)`,
|
||||||
),
|
),
|
||||||
);
|
);
|
||||||
} catch {
|
} catch (err) {
|
||||||
models = [];
|
recordFailure(err);
|
||||||
}
|
models = [];
|
||||||
|
}
|
||||||
|
|
||||||
if (models.length === 0) continue;
|
if (models.length === 0) continue;
|
||||||
|
|
||||||
@@ -204,37 +229,62 @@ export async function refreshModelsForAccount(accountId: number) {
|
|||||||
mergeDiscoveredModels(models, latencyMs);
|
mergeDiscoveredModels(models, latencyMs);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (accountModels.size > 0) {
|
if (accountModels.size === 0) {
|
||||||
const checkedAt = new Date().toISOString();
|
const firstMessage = failureMessages[0] || '';
|
||||||
await db.insert(schema.modelAvailability).values(
|
const errorCode = firstMessage ? classifyModelDiscoveryError(firstMessage) : 'empty_models';
|
||||||
Array.from(accountModels).map((modelName) => ({
|
const errorMessage = buildModelFailureMessage(errorCode, firstMessage);
|
||||||
accountId: account.id,
|
await setAccountRuntimeHealth(account.id, {
|
||||||
modelName,
|
state: 'unhealthy',
|
||||||
available: true,
|
reason: errorMessage,
|
||||||
latencyMs: modelLatency.get(modelName) ?? null,
|
source: 'model-discovery',
|
||||||
checkedAt,
|
checkedAt: new Date().toISOString(),
|
||||||
})),
|
});
|
||||||
).run();
|
return {
|
||||||
|
accountId,
|
||||||
if (isApiKeyConnection(account)) {
|
refreshed: true,
|
||||||
await setAccountRuntimeHealth(account.id, {
|
status: 'failed',
|
||||||
state: 'healthy',
|
errorCode,
|
||||||
reason: '模型探测成功',
|
errorMessage,
|
||||||
source: 'model-discovery',
|
modelCount: 0,
|
||||||
checkedAt,
|
modelsPreview: [],
|
||||||
});
|
tokenScanned: scannedTokenCount,
|
||||||
}
|
discoveredByCredential,
|
||||||
|
discoveredApiToken: !!discoveredApiToken,
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
return {
|
const checkedAt = new Date().toISOString();
|
||||||
accountId,
|
await db.insert(schema.modelAvailability).values(
|
||||||
refreshed: true,
|
Array.from(accountModels).map((modelName) => ({
|
||||||
modelCount: accountModels.size,
|
accountId: account.id,
|
||||||
tokenScanned: scannedTokenCount,
|
modelName,
|
||||||
discoveredByCredential,
|
available: true,
|
||||||
discoveredApiToken: !!discoveredApiToken,
|
latencyMs: modelLatency.get(modelName) ?? null,
|
||||||
};
|
checkedAt,
|
||||||
}
|
})),
|
||||||
|
).run();
|
||||||
|
|
||||||
|
if (isApiKeyConnection(account)) {
|
||||||
|
await setAccountRuntimeHealth(account.id, {
|
||||||
|
state: 'healthy',
|
||||||
|
reason: '模型探测成功',
|
||||||
|
source: 'model-discovery',
|
||||||
|
checkedAt,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
const modelsPreview = Array.from(accountModels).slice(0, 10);
|
||||||
|
return {
|
||||||
|
accountId,
|
||||||
|
refreshed: true,
|
||||||
|
status: 'success',
|
||||||
|
modelCount: accountModels.size,
|
||||||
|
modelsPreview,
|
||||||
|
tokenScanned: scannedTokenCount,
|
||||||
|
discoveredByCredential,
|
||||||
|
discoveredApiToken: !!discoveredApiToken,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
async function refreshModelsForAllActiveAccounts() {
|
async function refreshModelsForAllActiveAccounts() {
|
||||||
const accounts = await db.select({ id: schema.accounts.id }).from(schema.accounts)
|
const accounts = await db.select({ id: schema.accounts.id }).from(schema.accounts)
|
||||||
|
|||||||
Reference in New Issue
Block a user