forked from boos/BoosAPI
feat: classify model refresh failures and update health
This commit is contained in:
@@ -79,7 +79,9 @@ describe('refreshModelsForAccount credential discovery', () => {
|
|||||||
expect(result).toMatchObject({
|
expect(result).toMatchObject({
|
||||||
accountId: account.id,
|
accountId: account.id,
|
||||||
refreshed: true,
|
refreshed: true,
|
||||||
|
status: 'success',
|
||||||
modelCount: 2,
|
modelCount: 2,
|
||||||
|
modelsPreview: ['claude-sonnet-4-5-20250929', 'claude-opus-4-6'],
|
||||||
tokenScanned: 0,
|
tokenScanned: 0,
|
||||||
discoveredByCredential: true,
|
discoveredByCredential: true,
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -11,6 +11,24 @@ const API_TOKEN_DISCOVERY_TIMEOUT_MS = 8_000;
|
|||||||
const MODEL_DISCOVERY_TIMEOUT_MS = 12_000;
|
const MODEL_DISCOVERY_TIMEOUT_MS = 12_000;
|
||||||
const MODEL_REFRESH_BATCH_SIZE = 3;
|
const MODEL_REFRESH_BATCH_SIZE = 3;
|
||||||
|
|
||||||
|
type ModelRefreshErrorCode = 'timeout' | 'unauthorized' | 'empty_models' | 'unknown';
|
||||||
|
|
||||||
|
function classifyModelDiscoveryError(message: string): ModelRefreshErrorCode {
|
||||||
|
const lowered = message.toLowerCase();
|
||||||
|
if (lowered.includes('timeout') || lowered.includes('timed out') || lowered.includes('请求超时')) return 'timeout';
|
||||||
|
if (lowered.includes('http 401') || lowered.includes('http 403')
|
||||||
|
|| lowered.includes('unauthorized') || lowered.includes('invalid')
|
||||||
|
|| lowered.includes('无权') || lowered.includes('未提供令牌')) return 'unauthorized';
|
||||||
|
return 'unknown';
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildModelFailureMessage(code: ModelRefreshErrorCode, fallback?: string) {
|
||||||
|
if (code === 'timeout') return '模型获取失败(请求超时)';
|
||||||
|
if (code === 'unauthorized') return '模型获取失败,API Key 已无效';
|
||||||
|
if (code === 'empty_models') return '模型获取失败:未获取到可用模型';
|
||||||
|
return fallback || '模型获取失败';
|
||||||
|
}
|
||||||
|
|
||||||
function isSiteDisabled(status?: string | null): boolean {
|
function isSiteDisabled(status?: string | null): boolean {
|
||||||
return (status || 'active') === 'disabled';
|
return (status || 'active') === 'disabled';
|
||||||
}
|
}
|
||||||
@@ -125,6 +143,11 @@ export async function refreshModelsForAccount(accountId: number) {
|
|||||||
let scannedTokenCount = 0;
|
let scannedTokenCount = 0;
|
||||||
let discoveredByCredential = false;
|
let discoveredByCredential = false;
|
||||||
const attemptedCredentials = new Set<string>();
|
const attemptedCredentials = new Set<string>();
|
||||||
|
const failureMessages: string[] = [];
|
||||||
|
const recordFailure = (err: unknown) => {
|
||||||
|
const message = (err as { message?: string })?.message || String(err || '');
|
||||||
|
if (message) failureMessages.push(message);
|
||||||
|
};
|
||||||
|
|
||||||
const mergeDiscoveredModels = (models: string[], latencyMs: number | null) => {
|
const mergeDiscoveredModels = (models: string[], latencyMs: number | null) => {
|
||||||
for (const modelName of models) {
|
for (const modelName of models) {
|
||||||
@@ -155,7 +178,8 @@ export async function refreshModelsForAccount(accountId: number) {
|
|||||||
`model discovery timeout (${Math.round(MODEL_DISCOVERY_TIMEOUT_MS / 1000)}s)`,
|
`model discovery timeout (${Math.round(MODEL_DISCOVERY_TIMEOUT_MS / 1000)}s)`,
|
||||||
),
|
),
|
||||||
);
|
);
|
||||||
} catch {
|
} catch (err) {
|
||||||
|
recordFailure(err);
|
||||||
models = [];
|
models = [];
|
||||||
}
|
}
|
||||||
if (models.length === 0) return;
|
if (models.length === 0) return;
|
||||||
@@ -181,7 +205,8 @@ export async function refreshModelsForAccount(accountId: number) {
|
|||||||
`model discovery timeout (${Math.round(MODEL_DISCOVERY_TIMEOUT_MS / 1000)}s)`,
|
`model discovery timeout (${Math.round(MODEL_DISCOVERY_TIMEOUT_MS / 1000)}s)`,
|
||||||
),
|
),
|
||||||
);
|
);
|
||||||
} catch {
|
} catch (err) {
|
||||||
|
recordFailure(err);
|
||||||
models = [];
|
models = [];
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -204,32 +229,57 @@ export async function refreshModelsForAccount(accountId: number) {
|
|||||||
mergeDiscoveredModels(models, latencyMs);
|
mergeDiscoveredModels(models, latencyMs);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (accountModels.size > 0) {
|
if (accountModels.size === 0) {
|
||||||
const checkedAt = new Date().toISOString();
|
const firstMessage = failureMessages[0] || '';
|
||||||
await db.insert(schema.modelAvailability).values(
|
const errorCode = firstMessage ? classifyModelDiscoveryError(firstMessage) : 'empty_models';
|
||||||
Array.from(accountModels).map((modelName) => ({
|
const errorMessage = buildModelFailureMessage(errorCode, firstMessage);
|
||||||
accountId: account.id,
|
await setAccountRuntimeHealth(account.id, {
|
||||||
modelName,
|
state: 'unhealthy',
|
||||||
available: true,
|
reason: errorMessage,
|
||||||
latencyMs: modelLatency.get(modelName) ?? null,
|
source: 'model-discovery',
|
||||||
checkedAt,
|
checkedAt: new Date().toISOString(),
|
||||||
})),
|
});
|
||||||
).run();
|
return {
|
||||||
|
accountId,
|
||||||
if (isApiKeyConnection(account)) {
|
refreshed: true,
|
||||||
await setAccountRuntimeHealth(account.id, {
|
status: 'failed',
|
||||||
state: 'healthy',
|
errorCode,
|
||||||
reason: '模型探测成功',
|
errorMessage,
|
||||||
source: 'model-discovery',
|
modelCount: 0,
|
||||||
checkedAt,
|
modelsPreview: [],
|
||||||
});
|
tokenScanned: scannedTokenCount,
|
||||||
}
|
discoveredByCredential,
|
||||||
|
discoveredApiToken: !!discoveredApiToken,
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const checkedAt = new Date().toISOString();
|
||||||
|
await db.insert(schema.modelAvailability).values(
|
||||||
|
Array.from(accountModels).map((modelName) => ({
|
||||||
|
accountId: account.id,
|
||||||
|
modelName,
|
||||||
|
available: true,
|
||||||
|
latencyMs: modelLatency.get(modelName) ?? null,
|
||||||
|
checkedAt,
|
||||||
|
})),
|
||||||
|
).run();
|
||||||
|
|
||||||
|
if (isApiKeyConnection(account)) {
|
||||||
|
await setAccountRuntimeHealth(account.id, {
|
||||||
|
state: 'healthy',
|
||||||
|
reason: '模型探测成功',
|
||||||
|
source: 'model-discovery',
|
||||||
|
checkedAt,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
const modelsPreview = Array.from(accountModels).slice(0, 10);
|
||||||
return {
|
return {
|
||||||
accountId,
|
accountId,
|
||||||
refreshed: true,
|
refreshed: true,
|
||||||
|
status: 'success',
|
||||||
modelCount: accountModels.size,
|
modelCount: accountModels.size,
|
||||||
|
modelsPreview,
|
||||||
tokenScanned: scannedTokenCount,
|
tokenScanned: scannedTokenCount,
|
||||||
discoveredByCredential,
|
discoveredByCredential,
|
||||||
discoveredApiToken: !!discoveredApiToken,
|
discoveredApiToken: !!discoveredApiToken,
|
||||||
|
|||||||
Reference in New Issue
Block a user