feat(server): add proactive channel probes and load-aware stable routing / 服务端新增主动探活与负载感知稳定路由 (#330)
* wip: preserve local changes before upstream merge * Improve proxy routing stability and add stable-first strategy * Sync explicit-group routing strategy to eligible source routes * Fix proxy retry and runtime health review issues * fix: make proxy channel retry attempts configurable * fix: harden proxy retry recovery and bookkeeping * fix: address pr 257 review comments * feat(proxy): add sticky session channel coordination * feat(server): add proactive channel probes and load-aware stable routing * test(server): stabilize session load routing selection assertion * fix(proxy): harden codex forwarding stability * fix(router): rotate stable-first channels by healthy order * fix(server): address probe review feedback * fix(server): address probe review follow-ups * fix(ci): address CodeQL and core test failures * feat(server): refine stable-first routing recovery and observation * Finish PR 330 settings and review fixes * Fix chat site endpoint CI mock * Address latest PR 330 review comments --------- Co-authored-by: Cita <juricek.chen@gmail.com>
This commit is contained in:
Bainily
@@ -301,6 +301,21 @@ export function getRunningTaskByDedupeKey(key: string): BackgroundTask | null {
|
||||
return task;
|
||||
}
|
||||
|
||||
export async function waitForBackgroundTaskCompletion(taskId: string, pollIntervalMs = 25): Promise<BackgroundTask | null> {
|
||||
const safePollIntervalMs = Math.max(5, Math.trunc(pollIntervalMs || 0));
|
||||
while (true) {
|
||||
const task = getBackgroundTask(taskId);
|
||||
if (!task) return null;
|
||||
if (task.status !== 'pending' && task.status !== 'running') {
|
||||
return task;
|
||||
}
|
||||
await new Promise<void>((resolve) => {
|
||||
const timer = setTimeout(resolve, safePollIntervalMs);
|
||||
timer.unref?.();
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export function summarizeCheckinResults(results: Array<{ result?: any }>): { total: number; success: number; skipped: number; failed: number } {
|
||||
const summary = { total: results.length, success: 0, skipped: 0, failed: 0 };
|
||||
for (const item of results) {
|
||||
|
||||
@@ -0,0 +1,348 @@
|
||||
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { mkdtempSync, rmSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { eq } from 'drizzle-orm';
|
||||
|
||||
const probeRuntimeModelMock = vi.fn();
|
||||
|
||||
vi.mock('./runtimeModelProbe.js', () => ({
|
||||
probeRuntimeModel: (...args: unknown[]) => probeRuntimeModelMock(...args),
|
||||
}));
|
||||
|
||||
type DbModule = typeof import('../db/index.js');
|
||||
type RecoveryModule = typeof import('./channelRecoveryProbeService.js');
|
||||
type CoordinatorModule = typeof import('./proxyChannelCoordinator.js');
|
||||
type ConfigModule = typeof import('../config.js');
|
||||
type TokenRouterModule = typeof import('./tokenRouter.js');
|
||||
|
||||
describe('channelRecoveryProbeService', () => {
|
||||
let db: DbModule['db'];
|
||||
let schema: DbModule['schema'];
|
||||
let runChannelRecoveryProbeSweep: RecoveryModule['runChannelRecoveryProbeSweep'];
|
||||
let resetChannelRecoveryProbeState: RecoveryModule['resetChannelRecoveryProbeState'];
|
||||
let proxyChannelCoordinator: CoordinatorModule['proxyChannelCoordinator'];
|
||||
let resetProxyChannelCoordinatorState: CoordinatorModule['resetProxyChannelCoordinatorState'];
|
||||
let invalidateTokenRouterCache: TokenRouterModule['invalidateTokenRouterCache'];
|
||||
let resetSiteRuntimeHealthState: TokenRouterModule['resetSiteRuntimeHealthState'];
|
||||
let config: ConfigModule['config'];
|
||||
let dataDir = '';
|
||||
let originalDataDir: string | undefined;
|
||||
let originalConcurrencyLimit = 0;
|
||||
|
||||
beforeAll(async () => {
|
||||
dataDir = mkdtempSync(join(tmpdir(), 'metapi-channel-recovery-probe-'));
|
||||
originalDataDir = process.env.DATA_DIR;
|
||||
process.env.DATA_DIR = dataDir;
|
||||
|
||||
await import('../db/migrate.js');
|
||||
const dbModule = await import('../db/index.js');
|
||||
const recoveryModule = await import('./channelRecoveryProbeService.js');
|
||||
const coordinatorModule = await import('./proxyChannelCoordinator.js');
|
||||
const configModule = await import('../config.js');
|
||||
const tokenRouterModule = await import('./tokenRouter.js');
|
||||
|
||||
db = dbModule.db;
|
||||
schema = dbModule.schema;
|
||||
runChannelRecoveryProbeSweep = recoveryModule.runChannelRecoveryProbeSweep;
|
||||
resetChannelRecoveryProbeState = recoveryModule.resetChannelRecoveryProbeState;
|
||||
proxyChannelCoordinator = coordinatorModule.proxyChannelCoordinator;
|
||||
resetProxyChannelCoordinatorState = coordinatorModule.resetProxyChannelCoordinatorState;
|
||||
invalidateTokenRouterCache = tokenRouterModule.invalidateTokenRouterCache;
|
||||
resetSiteRuntimeHealthState = tokenRouterModule.resetSiteRuntimeHealthState;
|
||||
config = configModule.config;
|
||||
originalConcurrencyLimit = config.proxySessionChannelConcurrencyLimit;
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
probeRuntimeModelMock.mockReset();
|
||||
probeRuntimeModelMock.mockResolvedValue({
|
||||
status: 'supported',
|
||||
latencyMs: 320,
|
||||
reason: 'probe succeeded',
|
||||
});
|
||||
config.proxySessionChannelConcurrencyLimit = 1;
|
||||
resetChannelRecoveryProbeState();
|
||||
resetProxyChannelCoordinatorState();
|
||||
invalidateTokenRouterCache();
|
||||
resetSiteRuntimeHealthState();
|
||||
|
||||
await db.delete(schema.routeChannels).run();
|
||||
await db.delete(schema.tokenRoutes).run();
|
||||
await db.delete(schema.settings).run();
|
||||
await db.delete(schema.accountTokens).run();
|
||||
await db.delete(schema.accounts).run();
|
||||
await db.delete(schema.sites).run();
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
config.proxySessionChannelConcurrencyLimit = originalConcurrencyLimit;
|
||||
resetChannelRecoveryProbeState();
|
||||
resetProxyChannelCoordinatorState();
|
||||
invalidateTokenRouterCache();
|
||||
resetSiteRuntimeHealthState();
|
||||
rmSync(dataDir, { recursive: true, force: true });
|
||||
if (originalDataDir === undefined) {
|
||||
delete process.env.DATA_DIR;
|
||||
} else {
|
||||
process.env.DATA_DIR = originalDataDir;
|
||||
}
|
||||
});
|
||||
|
||||
it('clears cooldown markers when a background probe succeeds', async () => {
|
||||
const site = await db.insert(schema.sites).values({
|
||||
name: 'recovery-site',
|
||||
url: 'https://recovery-site.example.com',
|
||||
platform: 'new-api',
|
||||
status: 'active',
|
||||
}).returning().get();
|
||||
|
||||
const account = await db.insert(schema.accounts).values({
|
||||
siteId: site.id,
|
||||
username: 'recovery-user',
|
||||
accessToken: 'access-recovery',
|
||||
apiToken: 'sk-recovery',
|
||||
status: 'active',
|
||||
}).returning().get();
|
||||
|
||||
const token = await db.insert(schema.accountTokens).values({
|
||||
accountId: account.id,
|
||||
name: 'default',
|
||||
token: 'sk-recovery-token',
|
||||
enabled: true,
|
||||
isDefault: true,
|
||||
}).returning().get();
|
||||
|
||||
const route = await db.insert(schema.tokenRoutes).values({
|
||||
modelPattern: 'gpt-5.4',
|
||||
enabled: true,
|
||||
}).returning().get();
|
||||
|
||||
const channel = await db.insert(schema.routeChannels).values({
|
||||
routeId: route.id,
|
||||
accountId: account.id,
|
||||
tokenId: token.id,
|
||||
enabled: true,
|
||||
cooldownUntil: new Date(Date.now() + 10 * 60 * 1000).toISOString(),
|
||||
lastFailAt: new Date().toISOString(),
|
||||
consecutiveFailCount: 2,
|
||||
cooldownLevel: 1,
|
||||
}).returning().get();
|
||||
|
||||
await runChannelRecoveryProbeSweep();
|
||||
|
||||
expect(probeRuntimeModelMock).toHaveBeenCalledTimes(1);
|
||||
expect(probeRuntimeModelMock.mock.calls[0]?.[0]).toMatchObject({
|
||||
modelName: 'gpt-5.4',
|
||||
tokenValue: 'sk-recovery-token',
|
||||
});
|
||||
|
||||
const refreshed = await db.select().from(schema.routeChannels)
|
||||
.where(eq(schema.routeChannels.id, channel.id))
|
||||
.get();
|
||||
expect(refreshed?.cooldownUntil).toBeNull();
|
||||
expect(refreshed?.lastFailAt).toBeNull();
|
||||
expect(refreshed?.consecutiveFailCount).toBe(0);
|
||||
expect(refreshed?.cooldownLevel).toBe(0);
|
||||
});
|
||||
|
||||
it('also probes active leased channels in the background', async () => {
|
||||
const site = await db.insert(schema.sites).values({
|
||||
name: 'active-site',
|
||||
url: 'https://active-site.example.com',
|
||||
platform: 'new-api',
|
||||
status: 'active',
|
||||
}).returning().get();
|
||||
|
||||
const account = await db.insert(schema.accounts).values({
|
||||
siteId: site.id,
|
||||
username: 'active-user',
|
||||
accessToken: 'access-active',
|
||||
apiToken: 'sk-active',
|
||||
status: 'active',
|
||||
extraConfig: JSON.stringify({
|
||||
credentialMode: 'session',
|
||||
}),
|
||||
}).returning().get();
|
||||
|
||||
const token = await db.insert(schema.accountTokens).values({
|
||||
accountId: account.id,
|
||||
name: 'default',
|
||||
token: 'sk-active-token',
|
||||
enabled: true,
|
||||
isDefault: true,
|
||||
}).returning().get();
|
||||
|
||||
const route = await db.insert(schema.tokenRoutes).values({
|
||||
modelPattern: 'gpt-5.2',
|
||||
enabled: true,
|
||||
}).returning().get();
|
||||
|
||||
const channel = await db.insert(schema.routeChannels).values({
|
||||
routeId: route.id,
|
||||
accountId: account.id,
|
||||
tokenId: token.id,
|
||||
enabled: true,
|
||||
}).returning().get();
|
||||
|
||||
const lease = await proxyChannelCoordinator.acquireChannelLease({
|
||||
channelId: channel.id,
|
||||
accountExtraConfig: account.extraConfig,
|
||||
});
|
||||
expect(lease.status).toBe('acquired');
|
||||
if (lease.status !== 'acquired') return;
|
||||
|
||||
await runChannelRecoveryProbeSweep();
|
||||
|
||||
expect(probeRuntimeModelMock).toHaveBeenCalledTimes(1);
|
||||
expect(probeRuntimeModelMock.mock.calls[0]?.[0]).toMatchObject({
|
||||
modelName: 'gpt-5.2',
|
||||
tokenValue: 'sk-active-token',
|
||||
});
|
||||
|
||||
lease.lease.release();
|
||||
});
|
||||
|
||||
it('skips provider-directed quota cooldown channels during recovery sweeps', async () => {
|
||||
const site = await db.insert(schema.sites).values({
|
||||
name: 'quota-site',
|
||||
url: 'https://quota-site.example.com',
|
||||
platform: 'codex',
|
||||
status: 'active',
|
||||
}).returning().get();
|
||||
|
||||
const account = await db.insert(schema.accounts).values({
|
||||
siteId: site.id,
|
||||
username: 'quota-user',
|
||||
accessToken: 'access-quota',
|
||||
apiToken: 'sk-quota',
|
||||
status: 'active',
|
||||
}).returning().get();
|
||||
|
||||
const quotaToken = await db.insert(schema.accountTokens).values({
|
||||
accountId: account.id,
|
||||
name: 'quota-token',
|
||||
token: 'sk-quota-token',
|
||||
enabled: true,
|
||||
isDefault: false,
|
||||
}).returning().get();
|
||||
|
||||
const retryToken = await db.insert(schema.accountTokens).values({
|
||||
accountId: account.id,
|
||||
name: 'retry-token',
|
||||
token: 'sk-retry-token',
|
||||
enabled: true,
|
||||
isDefault: false,
|
||||
}).returning().get();
|
||||
|
||||
const route = await db.insert(schema.tokenRoutes).values({
|
||||
modelPattern: 'gpt-5.4',
|
||||
enabled: true,
|
||||
}).returning().get();
|
||||
|
||||
await db.insert(schema.routeChannels).values([
|
||||
{
|
||||
routeId: route.id,
|
||||
accountId: account.id,
|
||||
tokenId: quotaToken.id,
|
||||
enabled: true,
|
||||
cooldownUntil: new Date(Date.now() + 10 * 60 * 1000).toISOString(),
|
||||
lastFailAt: new Date().toISOString(),
|
||||
failCount: 0,
|
||||
consecutiveFailCount: 0,
|
||||
cooldownLevel: 0,
|
||||
},
|
||||
{
|
||||
routeId: route.id,
|
||||
accountId: account.id,
|
||||
tokenId: retryToken.id,
|
||||
enabled: true,
|
||||
cooldownUntil: new Date(Date.now() + 10 * 60 * 1000).toISOString(),
|
||||
lastFailAt: new Date().toISOString(),
|
||||
failCount: 2,
|
||||
consecutiveFailCount: 0,
|
||||
cooldownLevel: 0,
|
||||
},
|
||||
]).run();
|
||||
|
||||
await runChannelRecoveryProbeSweep();
|
||||
|
||||
expect(probeRuntimeModelMock).toHaveBeenCalledTimes(1);
|
||||
expect(probeRuntimeModelMock.mock.calls[0]?.[0]).toMatchObject({
|
||||
tokenValue: 'sk-retry-token',
|
||||
modelName: 'gpt-5.4',
|
||||
});
|
||||
});
|
||||
|
||||
it('prioritizes never-probed active channels before reprobing recently started ones', async () => {
|
||||
const site = await db.insert(schema.sites).values({
|
||||
name: 'priority-site',
|
||||
url: 'https://priority-site.example.com',
|
||||
platform: 'new-api',
|
||||
status: 'active',
|
||||
}).returning().get();
|
||||
|
||||
const leases: Array<{ release: () => void }> = [];
|
||||
try {
|
||||
for (let index = 1; index <= 5; index += 1) {
|
||||
const account = await db.insert(schema.accounts).values({
|
||||
siteId: site.id,
|
||||
username: `priority-user-${index}`,
|
||||
accessToken: `access-priority-${index}`,
|
||||
apiToken: `sk-priority-${index}`,
|
||||
status: 'active',
|
||||
extraConfig: JSON.stringify({
|
||||
credentialMode: 'session',
|
||||
}),
|
||||
}).returning().get();
|
||||
|
||||
const token = await db.insert(schema.accountTokens).values({
|
||||
accountId: account.id,
|
||||
name: `token-${index}`,
|
||||
token: `sk-priority-token-${index}`,
|
||||
enabled: true,
|
||||
isDefault: true,
|
||||
}).returning().get();
|
||||
|
||||
const route = await db.insert(schema.tokenRoutes).values({
|
||||
modelPattern: `gpt-5.4-${index}`,
|
||||
enabled: true,
|
||||
}).returning().get();
|
||||
|
||||
const channel = await db.insert(schema.routeChannels).values({
|
||||
routeId: route.id,
|
||||
accountId: account.id,
|
||||
tokenId: token.id,
|
||||
enabled: true,
|
||||
}).returning().get();
|
||||
|
||||
const lease = await proxyChannelCoordinator.acquireChannelLease({
|
||||
channelId: channel.id,
|
||||
accountExtraConfig: account.extraConfig,
|
||||
});
|
||||
expect(lease.status).toBe('acquired');
|
||||
if (lease.status === 'acquired') {
|
||||
leases.push(lease.lease);
|
||||
}
|
||||
}
|
||||
|
||||
const startedAt = Date.UTC(2026, 3, 1, 0, 0, 0);
|
||||
await runChannelRecoveryProbeSweep(startedAt);
|
||||
|
||||
expect(probeRuntimeModelMock).toHaveBeenCalledTimes(4);
|
||||
expect(probeRuntimeModelMock.mock.calls.map((call) => call[0]?.tokenValue)).not.toContain('sk-priority-token-5');
|
||||
|
||||
probeRuntimeModelMock.mockClear();
|
||||
|
||||
await runChannelRecoveryProbeSweep(startedAt + 5 * 60 * 1000);
|
||||
|
||||
expect(probeRuntimeModelMock).toHaveBeenCalledTimes(4);
|
||||
const secondSweepTokens = probeRuntimeModelMock.mock.calls.map((call) => call[0]?.tokenValue);
|
||||
expect(secondSweepTokens).toContain('sk-priority-token-5');
|
||||
} finally {
|
||||
for (const lease of leases) {
|
||||
lease.release();
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,299 @@
|
||||
import { and, eq, gt, inArray, isNotNull } from 'drizzle-orm';
|
||||
import { db, schema } from '../db/index.js';
|
||||
import { isUsableAccountToken } from './accountTokenService.js';
|
||||
import { getOauthInfoFromAccount } from './oauth/oauthAccount.js';
|
||||
import { proxyChannelCoordinator } from './proxyChannelCoordinator.js';
|
||||
import { probeRuntimeModel } from './runtimeModelProbe.js';
|
||||
import { tokenRouter } from './tokenRouter.js';
|
||||
import { isExactTokenRouteModelPattern } from '../../shared/tokenRoutePatterns.js';
|
||||
|
||||
type RecoveryProbeSource = 'cooldown' | 'active';
|
||||
|
||||
type RecoveryProbeCandidate = {
|
||||
source: RecoveryProbeSource;
|
||||
channelId: number;
|
||||
modelName: string;
|
||||
tokenValue: string;
|
||||
account: typeof schema.accounts.$inferSelect;
|
||||
site: typeof schema.sites.$inferSelect;
|
||||
};
|
||||
|
||||
const CHANNEL_RECOVERY_SWEEP_INTERVAL_MS = 30_000;
|
||||
const CHANNEL_RECOVERY_PROBE_TIMEOUT_MS = 12_000;
|
||||
// Keep recovery probes conservative so they do not look like bulk health checks to upstream providers.
|
||||
const CHANNEL_RECOVERY_PROBE_CONCURRENCY = 1;
|
||||
const CHANNEL_RECOVERY_MAX_BATCH = 4;
|
||||
const CHANNEL_RECOVERY_COOLDOWN_RECHECK_MS = 30_000;
|
||||
const CHANNEL_RECOVERY_ACTIVE_RECHECK_MS = 5 * 60_000;
|
||||
|
||||
let recoveryProbeSchedulerTimer: ReturnType<typeof setInterval> | null = null;
|
||||
let recoveryProbeSweepInFlight: Promise<void> | null = null;
|
||||
const recoveryProbeInFlightKeys = new Set<string>();
|
||||
const recoveryProbeLastStartedAtByKey = new Map<string, number>();
|
||||
|
||||
function shouldUnrefTimer(timer: ReturnType<typeof setTimeout> | ReturnType<typeof setInterval>) {
|
||||
if (typeof (timer as { unref?: () => void }).unref === 'function') {
|
||||
(timer as { unref: () => void }).unref();
|
||||
}
|
||||
}
|
||||
|
||||
function buildRecoveryProbeKey(channelId: number, modelName: string): string {
|
||||
return `${channelId}:${String(modelName || '').trim().toLowerCase()}`;
|
||||
}
|
||||
|
||||
function resolveRecoveryProbeWindowMs(source: RecoveryProbeSource): number {
|
||||
return source === 'cooldown'
|
||||
? CHANNEL_RECOVERY_COOLDOWN_RECHECK_MS
|
||||
: CHANNEL_RECOVERY_ACTIVE_RECHECK_MS;
|
||||
}
|
||||
|
||||
function resolveProbeModelName(row: {
|
||||
route_channels: typeof schema.routeChannels.$inferSelect;
|
||||
token_routes: typeof schema.tokenRoutes.$inferSelect;
|
||||
}): string {
|
||||
const sourceModel = String(row.route_channels.sourceModel || '').trim();
|
||||
if (sourceModel) return sourceModel;
|
||||
const routeModelPattern = String(row.token_routes.modelPattern || '').trim();
|
||||
return isExactTokenRouteModelPattern(routeModelPattern) ? routeModelPattern : '';
|
||||
}
|
||||
|
||||
function resolveProbeTokenValue(row: {
|
||||
route_channels: typeof schema.routeChannels.$inferSelect;
|
||||
accounts: typeof schema.accounts.$inferSelect;
|
||||
account_tokens: typeof schema.accountTokens.$inferSelect | null;
|
||||
}): string | null {
|
||||
if (typeof row.route_channels.tokenId === 'number' && row.route_channels.tokenId > 0) {
|
||||
if (!row.account_tokens || !isUsableAccountToken(row.account_tokens)) return null;
|
||||
const tokenValue = String(row.account_tokens.token || '').trim();
|
||||
return tokenValue || null;
|
||||
}
|
||||
|
||||
if (getOauthInfoFromAccount(row.accounts)) {
|
||||
const accessToken = String(row.accounts.accessToken || '').trim();
|
||||
return accessToken || null;
|
||||
}
|
||||
|
||||
const fallbackApiToken = String(row.accounts.apiToken || '').trim();
|
||||
return fallbackApiToken || null;
|
||||
}
|
||||
|
||||
function isProviderDirectedCooldown(row: {
|
||||
route_channels: typeof schema.routeChannels.$inferSelect;
|
||||
}): boolean {
|
||||
return !!row.route_channels.cooldownUntil
|
||||
&& (row.route_channels.failCount ?? 0) <= 0
|
||||
&& (row.route_channels.consecutiveFailCount ?? 0) <= 0
|
||||
&& (row.route_channels.cooldownLevel ?? 0) <= 0;
|
||||
}
|
||||
|
||||
async function mapWithConcurrency<T>(
|
||||
items: T[],
|
||||
concurrency: number,
|
||||
worker: (item: T, index: number) => Promise<void>,
|
||||
): Promise<void> {
|
||||
const safeConcurrency = Math.max(1, Math.min(items.length || 1, Math.trunc(concurrency || 1)));
|
||||
let nextIndex = 0;
|
||||
const runWorker = async () => {
|
||||
while (true) {
|
||||
const currentIndex = nextIndex;
|
||||
nextIndex += 1;
|
||||
if (currentIndex >= items.length) return;
|
||||
await worker(items[currentIndex] as T, currentIndex);
|
||||
}
|
||||
};
|
||||
|
||||
await Promise.all(Array.from({ length: safeConcurrency }, () => runWorker()));
|
||||
}
|
||||
|
||||
async function loadCoolingProbeCandidates(nowIso: string): Promise<RecoveryProbeCandidate[]> {
|
||||
const rows = await db.select()
|
||||
.from(schema.routeChannels)
|
||||
.innerJoin(schema.accounts, eq(schema.routeChannels.accountId, schema.accounts.id))
|
||||
.innerJoin(schema.sites, eq(schema.accounts.siteId, schema.sites.id))
|
||||
.innerJoin(schema.tokenRoutes, eq(schema.routeChannels.routeId, schema.tokenRoutes.id))
|
||||
.leftJoin(schema.accountTokens, eq(schema.routeChannels.tokenId, schema.accountTokens.id))
|
||||
.where(and(
|
||||
eq(schema.routeChannels.enabled, true),
|
||||
eq(schema.accounts.status, 'active'),
|
||||
eq(schema.sites.status, 'active'),
|
||||
isNotNull(schema.routeChannels.cooldownUntil),
|
||||
gt(schema.routeChannels.cooldownUntil, nowIso),
|
||||
))
|
||||
.all();
|
||||
|
||||
return rows.flatMap((row) => {
|
||||
if (isProviderDirectedCooldown(row)) return [];
|
||||
const modelName = resolveProbeModelName(row);
|
||||
const tokenValue = resolveProbeTokenValue(row);
|
||||
if (!modelName || !tokenValue) return [];
|
||||
return [{
|
||||
source: 'cooldown' as const,
|
||||
channelId: row.route_channels.id,
|
||||
modelName,
|
||||
tokenValue,
|
||||
account: row.accounts,
|
||||
site: row.sites,
|
||||
}];
|
||||
});
|
||||
}
|
||||
|
||||
async function loadActiveProbeCandidates(activeChannelIds: number[]): Promise<RecoveryProbeCandidate[]> {
|
||||
if (activeChannelIds.length <= 0) return [];
|
||||
|
||||
const rows = await db.select()
|
||||
.from(schema.routeChannels)
|
||||
.innerJoin(schema.accounts, eq(schema.routeChannels.accountId, schema.accounts.id))
|
||||
.innerJoin(schema.sites, eq(schema.accounts.siteId, schema.sites.id))
|
||||
.innerJoin(schema.tokenRoutes, eq(schema.routeChannels.routeId, schema.tokenRoutes.id))
|
||||
.leftJoin(schema.accountTokens, eq(schema.routeChannels.tokenId, schema.accountTokens.id))
|
||||
.where(and(
|
||||
eq(schema.routeChannels.enabled, true),
|
||||
eq(schema.accounts.status, 'active'),
|
||||
eq(schema.sites.status, 'active'),
|
||||
inArray(schema.routeChannels.id, activeChannelIds),
|
||||
))
|
||||
.all();
|
||||
|
||||
return rows.flatMap((row) => {
|
||||
const modelName = resolveProbeModelName(row);
|
||||
const tokenValue = resolveProbeTokenValue(row);
|
||||
if (!modelName || !tokenValue) return [];
|
||||
return [{
|
||||
source: 'active' as const,
|
||||
channelId: row.route_channels.id,
|
||||
modelName,
|
||||
tokenValue,
|
||||
account: row.accounts,
|
||||
site: row.sites,
|
||||
}];
|
||||
});
|
||||
}
|
||||
|
||||
function mergeRecoveryProbeCandidates(candidates: RecoveryProbeCandidate[]): RecoveryProbeCandidate[] {
|
||||
const merged = new Map<number, RecoveryProbeCandidate>();
|
||||
for (const candidate of candidates) {
|
||||
const existing = merged.get(candidate.channelId);
|
||||
if (!existing || (existing.source === 'active' && candidate.source === 'cooldown')) {
|
||||
merged.set(candidate.channelId, candidate);
|
||||
}
|
||||
}
|
||||
return Array.from(merged.values());
|
||||
}
|
||||
|
||||
function shouldProbeCandidate(candidate: RecoveryProbeCandidate, nowMs: number): boolean {
|
||||
const key = buildRecoveryProbeKey(candidate.channelId, candidate.modelName);
|
||||
if (recoveryProbeInFlightKeys.has(key)) return false;
|
||||
const lastStartedAt = recoveryProbeLastStartedAtByKey.get(key) ?? 0;
|
||||
return (nowMs - lastStartedAt) >= resolveRecoveryProbeWindowMs(candidate.source);
|
||||
}
|
||||
|
||||
function compareRecoveryProbeCandidatePriority(left: RecoveryProbeCandidate, right: RecoveryProbeCandidate): number {
|
||||
const leftKey = buildRecoveryProbeKey(left.channelId, left.modelName);
|
||||
const rightKey = buildRecoveryProbeKey(right.channelId, right.modelName);
|
||||
const leftLastStartedAt = recoveryProbeLastStartedAtByKey.get(leftKey);
|
||||
const rightLastStartedAt = recoveryProbeLastStartedAtByKey.get(rightKey);
|
||||
|
||||
if (leftLastStartedAt == null && rightLastStartedAt == null) {
|
||||
return left.channelId - right.channelId;
|
||||
}
|
||||
if (leftLastStartedAt == null) return -1;
|
||||
if (rightLastStartedAt == null) return 1;
|
||||
if (leftLastStartedAt !== rightLastStartedAt) {
|
||||
return leftLastStartedAt - rightLastStartedAt;
|
||||
}
|
||||
return left.channelId - right.channelId;
|
||||
}
|
||||
|
||||
async function runRecoveryProbeCandidate(candidate: RecoveryProbeCandidate, nowMs: number): Promise<void> {
|
||||
const key = buildRecoveryProbeKey(candidate.channelId, candidate.modelName);
|
||||
recoveryProbeInFlightKeys.add(key);
|
||||
recoveryProbeLastStartedAtByKey.set(key, nowMs);
|
||||
try {
|
||||
const result = await probeRuntimeModel({
|
||||
site: candidate.site,
|
||||
account: candidate.account,
|
||||
modelName: candidate.modelName,
|
||||
tokenValue: candidate.tokenValue,
|
||||
timeoutMs: CHANNEL_RECOVERY_PROBE_TIMEOUT_MS,
|
||||
});
|
||||
if (result.status === 'supported') {
|
||||
await tokenRouter.recordProbeSuccess(
|
||||
candidate.channelId,
|
||||
result.latencyMs ?? 0,
|
||||
candidate.modelName,
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn(`[channel-recovery-probe] channel ${candidate.channelId} probe failed`, error);
|
||||
} finally {
|
||||
recoveryProbeInFlightKeys.delete(key);
|
||||
}
|
||||
}
|
||||
|
||||
export async function runChannelRecoveryProbeSweep(nowMs = Date.now()): Promise<void> {
|
||||
if (recoveryProbeSweepInFlight) {
|
||||
await recoveryProbeSweepInFlight;
|
||||
return;
|
||||
}
|
||||
|
||||
recoveryProbeSweepInFlight = (async () => {
|
||||
const nowIso = new Date(nowMs).toISOString();
|
||||
const activeChannelIds = proxyChannelCoordinator.getActiveChannelIds();
|
||||
const [coolingCandidates, activeCandidates] = await Promise.all([
|
||||
loadCoolingProbeCandidates(nowIso),
|
||||
loadActiveProbeCandidates(activeChannelIds),
|
||||
]);
|
||||
|
||||
const merged = mergeRecoveryProbeCandidates([
|
||||
...coolingCandidates,
|
||||
...activeCandidates,
|
||||
]);
|
||||
const dueCandidates = merged
|
||||
.filter((candidate) => shouldProbeCandidate(candidate, nowMs))
|
||||
.sort(compareRecoveryProbeCandidatePriority)
|
||||
.slice(0, CHANNEL_RECOVERY_MAX_BATCH);
|
||||
if (dueCandidates.length <= 0) return;
|
||||
|
||||
await mapWithConcurrency(
|
||||
dueCandidates,
|
||||
CHANNEL_RECOVERY_PROBE_CONCURRENCY,
|
||||
async (candidate) => runRecoveryProbeCandidate(candidate, nowMs),
|
||||
);
|
||||
})().finally(() => {
|
||||
recoveryProbeSweepInFlight = null;
|
||||
});
|
||||
|
||||
await recoveryProbeSweepInFlight;
|
||||
}
|
||||
|
||||
export function startChannelRecoveryProbeScheduler(intervalMs = CHANNEL_RECOVERY_SWEEP_INTERVAL_MS) {
|
||||
stopChannelRecoveryProbeScheduler();
|
||||
const safeIntervalMs = Math.max(10_000, Math.trunc(intervalMs || 0));
|
||||
recoveryProbeSchedulerTimer = setInterval(() => {
|
||||
void runChannelRecoveryProbeSweep().catch((error) => {
|
||||
console.warn('[channel-recovery-probe] background sweep failed', error);
|
||||
});
|
||||
}, safeIntervalMs);
|
||||
shouldUnrefTimer(recoveryProbeSchedulerTimer);
|
||||
void runChannelRecoveryProbeSweep().catch((error) => {
|
||||
console.warn('[channel-recovery-probe] initial sweep failed', error);
|
||||
});
|
||||
return {
|
||||
enabled: true,
|
||||
intervalMs: safeIntervalMs,
|
||||
};
|
||||
}
|
||||
|
||||
export function stopChannelRecoveryProbeScheduler() {
|
||||
if (recoveryProbeSchedulerTimer) {
|
||||
clearInterval(recoveryProbeSchedulerTimer);
|
||||
recoveryProbeSchedulerTimer = null;
|
||||
}
|
||||
}
|
||||
|
||||
export function resetChannelRecoveryProbeState() {
|
||||
stopChannelRecoveryProbeScheduler();
|
||||
recoveryProbeSweepInFlight = null;
|
||||
recoveryProbeInFlightKeys.clear();
|
||||
recoveryProbeLastStartedAtByKey.clear();
|
||||
}
|
||||
@@ -0,0 +1,507 @@
|
||||
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { mkdtempSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { eq } from 'drizzle-orm';
|
||||
|
||||
const resolveUpstreamEndpointCandidatesMock = vi.fn();
|
||||
const buildUpstreamEndpointRequestMock = vi.fn();
|
||||
const dispatchRuntimeRequestMock = vi.fn();
|
||||
const resolveChannelProxyUrlMock = vi.fn();
|
||||
const withSiteRecordProxyRequestInitMock = vi.fn();
|
||||
const rebuildRoutesOnlyMock = vi.fn();
|
||||
|
||||
vi.mock('./upstreamEndpointRuntime.js', () => ({
|
||||
resolveUpstreamEndpointCandidates: (...args: unknown[]) => resolveUpstreamEndpointCandidatesMock(...args),
|
||||
buildUpstreamEndpointRequest: (...args: unknown[]) => buildUpstreamEndpointRequestMock(...args),
|
||||
}));
|
||||
|
||||
vi.mock('./runtimeDispatch.js', () => ({
|
||||
dispatchRuntimeRequest: (...args: unknown[]) => dispatchRuntimeRequestMock(...args),
|
||||
}));
|
||||
|
||||
vi.mock('./siteProxy.js', () => ({
|
||||
resolveChannelProxyUrl: (...args: unknown[]) => resolveChannelProxyUrlMock(...args),
|
||||
withSiteRecordProxyRequestInit: (...args: unknown[]) => withSiteRecordProxyRequestInitMock(...args),
|
||||
}));
|
||||
|
||||
vi.mock('./routeRefreshWorkflow.js', () => ({
|
||||
rebuildRoutesOnly: (...args: unknown[]) => rebuildRoutesOnlyMock(...args),
|
||||
}));
|
||||
|
||||
type DbModule = typeof import('../db/index.js');
|
||||
type ProbeModule = typeof import('./modelAvailabilityProbeService.js');
|
||||
|
||||
describe('modelAvailabilityProbeService', () => {
|
||||
let db: DbModule['db'];
|
||||
let schema: DbModule['schema'];
|
||||
let executeModelAvailabilityProbe: ProbeModule['executeModelAvailabilityProbe'];
|
||||
let resetProbeExecutionState: ProbeModule['__resetModelAvailabilityProbeExecutionStateForTests'];
|
||||
let dataDir = '';
|
||||
let originalDataDir: string | undefined;
|
||||
|
||||
beforeAll(async () => {
|
||||
dataDir = mkdtempSync(join(tmpdir(), 'metapi-model-probe-'));
|
||||
originalDataDir = process.env.DATA_DIR;
|
||||
process.env.DATA_DIR = dataDir;
|
||||
|
||||
await import('../db/migrate.js');
|
||||
const dbModule = await import('../db/index.js');
|
||||
const probeModule = await import('./modelAvailabilityProbeService.js');
|
||||
|
||||
db = dbModule.db;
|
||||
schema = dbModule.schema;
|
||||
executeModelAvailabilityProbe = probeModule.executeModelAvailabilityProbe;
|
||||
resetProbeExecutionState = probeModule.__resetModelAvailabilityProbeExecutionStateForTests;
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
resolveUpstreamEndpointCandidatesMock.mockReset();
|
||||
buildUpstreamEndpointRequestMock.mockReset();
|
||||
dispatchRuntimeRequestMock.mockReset();
|
||||
resolveChannelProxyUrlMock.mockReset();
|
||||
withSiteRecordProxyRequestInitMock.mockReset();
|
||||
rebuildRoutesOnlyMock.mockReset();
|
||||
rebuildRoutesOnlyMock.mockResolvedValue(undefined);
|
||||
resetProbeExecutionState();
|
||||
|
||||
resolveUpstreamEndpointCandidatesMock.mockResolvedValue(['chat']);
|
||||
buildUpstreamEndpointRequestMock.mockImplementation((input: { modelName?: string; endpoint?: string }) => ({
|
||||
path: '/v1/chat/completions',
|
||||
headers: {
|
||||
'content-type': 'application/json',
|
||||
},
|
||||
body: {
|
||||
model: input.modelName || 'unknown-model',
|
||||
endpoint: input.endpoint || 'chat',
|
||||
},
|
||||
runtime: {
|
||||
executor: 'default',
|
||||
modelName: input.modelName || 'unknown-model',
|
||||
stream: false,
|
||||
},
|
||||
}));
|
||||
dispatchRuntimeRequestMock.mockImplementation(async (input: { request?: { body?: { model?: string } } }) => {
|
||||
const modelName = String(input?.request?.body?.model || '');
|
||||
if (modelName === 'gpt-ghost') {
|
||||
return new Response(JSON.stringify({
|
||||
error: {
|
||||
message: 'The model `gpt-ghost` does not exist',
|
||||
},
|
||||
}), {
|
||||
status: 404,
|
||||
headers: {
|
||||
'content-type': 'application/json',
|
||||
},
|
||||
});
|
||||
}
|
||||
return new Response(JSON.stringify({
|
||||
id: 'resp_probe',
|
||||
model: modelName,
|
||||
choices: [
|
||||
{
|
||||
message: {
|
||||
role: 'assistant',
|
||||
content: 'OK',
|
||||
},
|
||||
},
|
||||
],
|
||||
}), {
|
||||
status: 200,
|
||||
headers: {
|
||||
'content-type': 'application/json',
|
||||
},
|
||||
});
|
||||
});
|
||||
resolveChannelProxyUrlMock.mockReturnValue(null);
|
||||
withSiteRecordProxyRequestInitMock.mockImplementation((_site: unknown, init: unknown) => init);
|
||||
|
||||
await db.delete(schema.routeChannels).run();
|
||||
await db.delete(schema.tokenRoutes).run();
|
||||
await db.delete(schema.tokenModelAvailability).run();
|
||||
await db.delete(schema.modelAvailability).run();
|
||||
await db.delete(schema.accountTokens).run();
|
||||
await db.delete(schema.accounts).run();
|
||||
await db.delete(schema.sites).run();
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
if (originalDataDir === undefined) {
|
||||
delete process.env.DATA_DIR;
|
||||
} else {
|
||||
process.env.DATA_DIR = originalDataDir;
|
||||
}
|
||||
});
|
||||
|
||||
it('marks supported rows true and unsupported rows false after probing', async () => {
|
||||
const site = await db.insert(schema.sites).values({
|
||||
name: 'probe-site',
|
||||
url: 'https://probe.example.com',
|
||||
platform: 'new-api',
|
||||
status: 'active',
|
||||
}).returning().get();
|
||||
|
||||
const account = await db.insert(schema.accounts).values({
|
||||
siteId: site.id,
|
||||
username: 'probe-user',
|
||||
accessToken: '',
|
||||
apiToken: 'sk-account-probe',
|
||||
status: 'active',
|
||||
extraConfig: JSON.stringify({
|
||||
credentialMode: 'apikey',
|
||||
}),
|
||||
}).returning().get();
|
||||
|
||||
const token = await db.insert(schema.accountTokens).values({
|
||||
accountId: account.id,
|
||||
name: 'default',
|
||||
token: 'sk-token-probe',
|
||||
enabled: true,
|
||||
isDefault: true,
|
||||
valueStatus: 'ready',
|
||||
}).returning().get();
|
||||
|
||||
await db.insert(schema.modelAvailability).values([
|
||||
{
|
||||
accountId: account.id,
|
||||
modelName: 'gpt-real',
|
||||
available: true,
|
||||
checkedAt: '2026-03-20T00:00:00.000Z',
|
||||
},
|
||||
]).run();
|
||||
|
||||
await db.insert(schema.tokenModelAvailability).values([
|
||||
{
|
||||
tokenId: token.id,
|
||||
modelName: 'gpt-ghost',
|
||||
available: true,
|
||||
checkedAt: '2026-03-20T00:00:00.000Z',
|
||||
},
|
||||
]).run();
|
||||
|
||||
const result = await executeModelAvailabilityProbe({
|
||||
accountId: account.id,
|
||||
rebuildRoutes: false,
|
||||
});
|
||||
|
||||
expect(result.summary).toMatchObject({
|
||||
totalAccounts: 1,
|
||||
scanned: 2,
|
||||
supported: 1,
|
||||
unsupported: 1,
|
||||
inconclusive: 0,
|
||||
skippedModels: 0,
|
||||
updatedRows: 2,
|
||||
rebuiltRoutes: false,
|
||||
});
|
||||
|
||||
const accountRows = await db.select().from(schema.modelAvailability)
|
||||
.where(eq(schema.modelAvailability.accountId, account.id))
|
||||
.all();
|
||||
expect(accountRows).toHaveLength(1);
|
||||
expect(accountRows[0]?.available).toBe(true);
|
||||
|
||||
const tokenRows = await db.select().from(schema.tokenModelAvailability)
|
||||
.where(eq(schema.tokenModelAvailability.tokenId, token.id))
|
||||
.all();
|
||||
expect(tokenRows).toHaveLength(1);
|
||||
expect(tokenRows[0]?.available).toBe(false);
|
||||
});
|
||||
|
||||
it('skips probing obvious non-conversation models', async () => {
|
||||
const site = await db.insert(schema.sites).values({
|
||||
name: 'probe-site',
|
||||
url: 'https://probe.example.com',
|
||||
platform: 'new-api',
|
||||
status: 'active',
|
||||
}).returning().get();
|
||||
|
||||
const account = await db.insert(schema.accounts).values({
|
||||
siteId: site.id,
|
||||
username: 'probe-user',
|
||||
accessToken: '',
|
||||
apiToken: 'sk-account-probe',
|
||||
status: 'active',
|
||||
extraConfig: JSON.stringify({
|
||||
credentialMode: 'apikey',
|
||||
}),
|
||||
}).returning().get();
|
||||
|
||||
await db.insert(schema.modelAvailability).values([
|
||||
{
|
||||
accountId: account.id,
|
||||
modelName: 'text-embedding-3-large',
|
||||
available: true,
|
||||
checkedAt: '2026-03-20T00:00:00.000Z',
|
||||
},
|
||||
]).run();
|
||||
|
||||
const result = await executeModelAvailabilityProbe({
|
||||
accountId: account.id,
|
||||
rebuildRoutes: false,
|
||||
});
|
||||
|
||||
expect(result.summary).toMatchObject({
|
||||
totalAccounts: 1,
|
||||
scanned: 1,
|
||||
supported: 0,
|
||||
unsupported: 0,
|
||||
inconclusive: 0,
|
||||
skippedModels: 1,
|
||||
updatedRows: 0,
|
||||
});
|
||||
expect(dispatchRuntimeRequestMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('does not mark a model unavailable when probe result is inconclusive', async () => {
|
||||
dispatchRuntimeRequestMock.mockImplementation(async (input: { request?: { body?: { model?: string } } }) => {
|
||||
const modelName = String(input?.request?.body?.model || '');
|
||||
if (modelName === 'gpt-flaky') {
|
||||
return new Response(JSON.stringify({
|
||||
error: {
|
||||
message: 'upstream temporary failure',
|
||||
},
|
||||
}), {
|
||||
status: 502,
|
||||
headers: {
|
||||
'content-type': 'application/json',
|
||||
},
|
||||
});
|
||||
}
|
||||
return new Response(JSON.stringify({
|
||||
id: 'resp_probe',
|
||||
model: modelName,
|
||||
choices: [
|
||||
{
|
||||
message: {
|
||||
role: 'assistant',
|
||||
content: 'OK',
|
||||
},
|
||||
},
|
||||
],
|
||||
}), {
|
||||
status: 200,
|
||||
headers: {
|
||||
'content-type': 'application/json',
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
const site = await db.insert(schema.sites).values({
|
||||
name: 'probe-site',
|
||||
url: 'https://probe.example.com',
|
||||
platform: 'new-api',
|
||||
status: 'active',
|
||||
}).returning().get();
|
||||
|
||||
const account = await db.insert(schema.accounts).values({
|
||||
siteId: site.id,
|
||||
username: 'probe-user',
|
||||
accessToken: '',
|
||||
apiToken: 'sk-account-probe',
|
||||
status: 'active',
|
||||
extraConfig: JSON.stringify({
|
||||
credentialMode: 'apikey',
|
||||
}),
|
||||
}).returning().get();
|
||||
|
||||
const checkedAt = '2026-03-20T00:00:00.000Z';
|
||||
await db.insert(schema.modelAvailability).values([
|
||||
{
|
||||
accountId: account.id,
|
||||
modelName: 'gpt-flaky',
|
||||
available: true,
|
||||
checkedAt,
|
||||
},
|
||||
]).run();
|
||||
|
||||
const result = await executeModelAvailabilityProbe({
|
||||
accountId: account.id,
|
||||
rebuildRoutes: false,
|
||||
});
|
||||
|
||||
expect(result.summary).toMatchObject({
|
||||
totalAccounts: 1,
|
||||
scanned: 1,
|
||||
supported: 0,
|
||||
unsupported: 0,
|
||||
inconclusive: 1,
|
||||
skippedModels: 0,
|
||||
updatedRows: 0,
|
||||
});
|
||||
|
||||
const accountRows = await db.select().from(schema.modelAvailability)
|
||||
.where(eq(schema.modelAvailability.accountId, account.id))
|
||||
.all();
|
||||
expect(accountRows).toHaveLength(1);
|
||||
expect(accountRows[0]?.available).toBe(true);
|
||||
expect(accountRows[0]?.checkedAt).toBe(checkedAt);
|
||||
});
|
||||
|
||||
it('does not rebuild routes when probe only refreshes checkedAt and latency', async () => {
|
||||
const site = await db.insert(schema.sites).values({
|
||||
name: 'stable-site',
|
||||
url: 'https://stable.example.com',
|
||||
platform: 'new-api',
|
||||
status: 'active',
|
||||
}).returning().get();
|
||||
|
||||
const account = await db.insert(schema.accounts).values({
|
||||
siteId: site.id,
|
||||
username: 'stable-user',
|
||||
accessToken: '',
|
||||
apiToken: 'sk-stable',
|
||||
status: 'active',
|
||||
}).returning().get();
|
||||
|
||||
await db.insert(schema.modelAvailability).values({
|
||||
accountId: account.id,
|
||||
modelName: 'gpt-stable',
|
||||
available: true,
|
||||
checkedAt: '2026-03-20T00:00:00.000Z',
|
||||
latencyMs: 123,
|
||||
}).run();
|
||||
|
||||
const result = await executeModelAvailabilityProbe({
|
||||
accountId: account.id,
|
||||
rebuildRoutes: true,
|
||||
});
|
||||
|
||||
expect(result.summary.updatedRows).toBe(1);
|
||||
expect(result.summary.rebuiltRoutes).toBe(false);
|
||||
expect(rebuildRoutesOnlyMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('rebuilds routes when probe flips availability state', async () => {
|
||||
const site = await db.insert(schema.sites).values({
|
||||
name: 'changed-site',
|
||||
url: 'https://changed.example.com',
|
||||
platform: 'new-api',
|
||||
status: 'active',
|
||||
}).returning().get();
|
||||
|
||||
const account = await db.insert(schema.accounts).values({
|
||||
siteId: site.id,
|
||||
username: 'changed-user',
|
||||
accessToken: '',
|
||||
apiToken: 'sk-changed',
|
||||
status: 'active',
|
||||
}).returning().get();
|
||||
|
||||
await db.insert(schema.modelAvailability).values({
|
||||
accountId: account.id,
|
||||
modelName: 'gpt-changed',
|
||||
available: false,
|
||||
checkedAt: '2026-03-20T00:00:00.000Z',
|
||||
}).run();
|
||||
|
||||
const result = await executeModelAvailabilityProbe({
|
||||
accountId: account.id,
|
||||
rebuildRoutes: true,
|
||||
});
|
||||
|
||||
expect(result.summary.updatedRows).toBe(1);
|
||||
expect(result.summary.rebuiltRoutes).toBe(true);
|
||||
expect(rebuildRoutesOnlyMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('does not count inactive accounts in probe results', async () => {
|
||||
const site = await db.insert(schema.sites).values({
|
||||
name: 'inactive-site',
|
||||
url: 'https://inactive.example.com',
|
||||
platform: 'new-api',
|
||||
status: 'active',
|
||||
}).returning().get();
|
||||
|
||||
const account = await db.insert(schema.accounts).values({
|
||||
siteId: site.id,
|
||||
username: 'inactive-user',
|
||||
accessToken: '',
|
||||
apiToken: 'sk-inactive',
|
||||
status: 'disabled',
|
||||
}).returning().get();
|
||||
|
||||
await db.insert(schema.modelAvailability).values({
|
||||
accountId: account.id,
|
||||
modelName: 'gpt-inactive',
|
||||
available: true,
|
||||
checkedAt: '2026-03-20T00:00:00.000Z',
|
||||
}).run();
|
||||
|
||||
const result = await executeModelAvailabilityProbe({
|
||||
accountId: account.id,
|
||||
rebuildRoutes: false,
|
||||
});
|
||||
|
||||
expect(result.summary.totalAccounts).toBe(0);
|
||||
expect(result.results).toEqual([]);
|
||||
expect(dispatchRuntimeRequestMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('skips an account when another probe already holds the execution lease', async () => {
|
||||
const site = await db.insert(schema.sites).values({
|
||||
name: 'leased-site',
|
||||
url: 'https://leased.example.com',
|
||||
platform: 'new-api',
|
||||
status: 'active',
|
||||
}).returning().get();
|
||||
|
||||
const account = await db.insert(schema.accounts).values({
|
||||
siteId: site.id,
|
||||
username: 'leased-user',
|
||||
accessToken: '',
|
||||
apiToken: 'sk-leased',
|
||||
status: 'active',
|
||||
}).returning().get();
|
||||
|
||||
await db.insert(schema.modelAvailability).values({
|
||||
accountId: account.id,
|
||||
modelName: 'gpt-leased',
|
||||
available: true,
|
||||
checkedAt: '2026-03-20T00:00:00.000Z',
|
||||
}).run();
|
||||
|
||||
let releaseFirstProbe: (() => void) | null = null;
|
||||
const firstProbeStarted = new Promise<void>((resolve) => {
|
||||
dispatchRuntimeRequestMock.mockImplementationOnce(async () => {
|
||||
resolve();
|
||||
await new Promise<void>((release) => {
|
||||
releaseFirstProbe = release;
|
||||
});
|
||||
return new Response(JSON.stringify({
|
||||
id: 'resp_probe',
|
||||
choices: [{ message: { role: 'assistant', content: 'OK' } }],
|
||||
}), {
|
||||
status: 200,
|
||||
headers: { 'content-type': 'application/json' },
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
const firstProbe = executeModelAvailabilityProbe({
|
||||
accountId: account.id,
|
||||
rebuildRoutes: false,
|
||||
});
|
||||
await firstProbeStarted;
|
||||
|
||||
const overlappingProbe = await executeModelAvailabilityProbe({
|
||||
accountId: account.id,
|
||||
rebuildRoutes: false,
|
||||
});
|
||||
|
||||
releaseFirstProbe?.();
|
||||
await firstProbe;
|
||||
|
||||
expect(overlappingProbe.summary).toMatchObject({
|
||||
totalAccounts: 1,
|
||||
skipped: 1,
|
||||
scanned: 0,
|
||||
});
|
||||
expect(overlappingProbe.results[0]).toMatchObject({
|
||||
accountId: account.id,
|
||||
siteId: site.id,
|
||||
status: 'skipped',
|
||||
message: 'model availability probe already running for account',
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,447 @@
|
||||
import { and, asc, eq } from 'drizzle-orm';
|
||||
import { config } from '../config.js';
|
||||
import { db, schema } from '../db/index.js';
|
||||
import { startBackgroundTask } from './backgroundTaskService.js';
|
||||
import { isUsableAccountToken, ACCOUNT_TOKEN_VALUE_STATUS_READY } from './accountTokenService.js';
|
||||
import { probeRuntimeModel } from './runtimeModelProbe.js';
|
||||
import * as routeRefreshWorkflow from './routeRefreshWorkflow.js';
|
||||
|
||||
type ProbeStatus = 'supported' | 'unsupported' | 'inconclusive' | 'skipped';
|
||||
|
||||
type ProbeAccountTarget = {
|
||||
kind: 'account';
|
||||
rowId: number;
|
||||
modelName: string;
|
||||
lastKnownAvailable: boolean;
|
||||
account: typeof schema.accounts.$inferSelect;
|
||||
site: typeof schema.sites.$inferSelect;
|
||||
};
|
||||
|
||||
type ProbeTokenTarget = {
|
||||
kind: 'token';
|
||||
rowId: number;
|
||||
tokenId: number;
|
||||
modelName: string;
|
||||
tokenValue: string;
|
||||
lastKnownAvailable: boolean;
|
||||
account: typeof schema.accounts.$inferSelect;
|
||||
site: typeof schema.sites.$inferSelect;
|
||||
};
|
||||
|
||||
type ProbeTarget = ProbeAccountTarget | ProbeTokenTarget;
|
||||
type ProbeAccountContext = {
|
||||
account: typeof schema.accounts.$inferSelect;
|
||||
site: typeof schema.sites.$inferSelect;
|
||||
};
|
||||
|
||||
export type ModelAvailabilityProbeAccountResult = {
|
||||
accountId: number;
|
||||
siteId: number;
|
||||
status: 'success' | 'failed' | 'skipped';
|
||||
scanned: number;
|
||||
supported: number;
|
||||
unsupported: number;
|
||||
inconclusive: number;
|
||||
skipped: number;
|
||||
updatedRows: number;
|
||||
message: string;
|
||||
};
|
||||
|
||||
export type ModelAvailabilityProbeExecutionResult = {
|
||||
results: ModelAvailabilityProbeAccountResult[];
|
||||
summary: {
|
||||
totalAccounts: number;
|
||||
success: number;
|
||||
failed: number;
|
||||
skipped: number;
|
||||
scanned: number;
|
||||
supported: number;
|
||||
unsupported: number;
|
||||
inconclusive: number;
|
||||
skippedModels: number;
|
||||
updatedRows: number;
|
||||
rebuiltRoutes: boolean;
|
||||
};
|
||||
};
|
||||
|
||||
let probeSchedulerTimer: ReturnType<typeof setInterval> | null = null;
|
||||
const probeAccountLeases = new Set<number>();
|
||||
|
||||
async function mapWithConcurrency<T, R>(
|
||||
items: T[],
|
||||
concurrency: number,
|
||||
worker: (item: T, index: number) => Promise<R>,
|
||||
): Promise<R[]> {
|
||||
const safeConcurrency = Math.max(1, Math.min(items.length || 1, Math.trunc(concurrency || 1)));
|
||||
const results = new Array<R>(items.length);
|
||||
let nextIndex = 0;
|
||||
|
||||
const runWorker = async () => {
|
||||
while (true) {
|
||||
const currentIndex = nextIndex;
|
||||
nextIndex += 1;
|
||||
if (currentIndex >= items.length) return;
|
||||
results[currentIndex] = await worker(items[currentIndex] as T, currentIndex);
|
||||
}
|
||||
};
|
||||
|
||||
await Promise.all(Array.from({ length: safeConcurrency }, () => runWorker()));
|
||||
return results;
|
||||
}
|
||||
|
||||
async function probeSingleTarget(target: ProbeTarget): Promise<{
|
||||
status: ProbeStatus;
|
||||
latencyMs: number | null;
|
||||
reason: string;
|
||||
}> {
|
||||
return await probeRuntimeModel({
|
||||
site: target.site,
|
||||
account: target.account,
|
||||
modelName: target.modelName,
|
||||
timeoutMs: config.modelAvailabilityProbeTimeoutMs,
|
||||
tokenValue: target.kind === 'token' ? target.tokenValue : undefined,
|
||||
});
|
||||
}
|
||||
|
||||
async function updateProbeRow(target: ProbeTarget, status: ProbeStatus, latencyMs: number | null): Promise<{
|
||||
touched: boolean;
|
||||
availabilityChanged: boolean;
|
||||
}> {
|
||||
if (status === 'inconclusive' || status === 'skipped') {
|
||||
return {
|
||||
touched: false,
|
||||
availabilityChanged: false,
|
||||
};
|
||||
}
|
||||
const nextAvailable = status === 'supported';
|
||||
const patch = {
|
||||
available: nextAvailable,
|
||||
latencyMs,
|
||||
checkedAt: new Date().toISOString(),
|
||||
};
|
||||
|
||||
if (target.kind === 'account') {
|
||||
await db.update(schema.modelAvailability)
|
||||
.set(patch)
|
||||
.where(eq(schema.modelAvailability.id, target.rowId))
|
||||
.run();
|
||||
return {
|
||||
touched: true,
|
||||
availabilityChanged: target.lastKnownAvailable !== nextAvailable,
|
||||
};
|
||||
}
|
||||
|
||||
await db.update(schema.tokenModelAvailability)
|
||||
.set(patch)
|
||||
.where(eq(schema.tokenModelAvailability.id, target.rowId))
|
||||
.run();
|
||||
return {
|
||||
touched: true,
|
||||
availabilityChanged: target.lastKnownAvailable !== nextAvailable,
|
||||
};
|
||||
}
|
||||
|
||||
async function loadActiveProbeAccountContext(accountId: number): Promise<ProbeAccountContext | null> {
|
||||
const accountRow = await db.select()
|
||||
.from(schema.accounts)
|
||||
.innerJoin(schema.sites, eq(schema.accounts.siteId, schema.sites.id))
|
||||
.where(eq(schema.accounts.id, accountId))
|
||||
.get();
|
||||
if (!accountRow) return null;
|
||||
if ((accountRow.accounts.status || 'active') !== 'active') return null;
|
||||
if ((accountRow.sites.status || 'active') !== 'active') return null;
|
||||
return {
|
||||
account: accountRow.accounts,
|
||||
site: accountRow.sites,
|
||||
};
|
||||
}
|
||||
|
||||
async function loadProbeTargetsForAccount(context: ProbeAccountContext): Promise<ProbeTarget[]> {
|
||||
const targets: ProbeTarget[] = [];
|
||||
const accountModels = await db.select()
|
||||
.from(schema.modelAvailability)
|
||||
.where(eq(schema.modelAvailability.accountId, context.account.id))
|
||||
.orderBy(asc(schema.modelAvailability.checkedAt))
|
||||
.all();
|
||||
for (const row of accountModels) {
|
||||
if (row.isManual) continue;
|
||||
targets.push({
|
||||
kind: 'account',
|
||||
rowId: row.id,
|
||||
modelName: row.modelName,
|
||||
lastKnownAvailable: !!row.available,
|
||||
account: context.account,
|
||||
site: context.site,
|
||||
});
|
||||
}
|
||||
|
||||
const tokenRows = await db.select()
|
||||
.from(schema.tokenModelAvailability)
|
||||
.innerJoin(schema.accountTokens, eq(schema.tokenModelAvailability.tokenId, schema.accountTokens.id))
|
||||
.where(and(
|
||||
eq(schema.accountTokens.accountId, context.account.id),
|
||||
eq(schema.accountTokens.enabled, true),
|
||||
eq(schema.accountTokens.valueStatus, ACCOUNT_TOKEN_VALUE_STATUS_READY),
|
||||
))
|
||||
.orderBy(asc(schema.tokenModelAvailability.checkedAt))
|
||||
.all();
|
||||
for (const row of tokenRows) {
|
||||
if (!isUsableAccountToken(row.account_tokens)) continue;
|
||||
const tokenValue = String(row.account_tokens.token || '').trim();
|
||||
if (!tokenValue) continue;
|
||||
targets.push({
|
||||
kind: 'token',
|
||||
rowId: row.token_model_availability.id,
|
||||
tokenId: row.account_tokens.id,
|
||||
modelName: row.token_model_availability.modelName,
|
||||
tokenValue,
|
||||
lastKnownAvailable: !!row.token_model_availability.available,
|
||||
account: context.account,
|
||||
site: context.site,
|
||||
});
|
||||
}
|
||||
|
||||
return targets;
|
||||
}
|
||||
|
||||
function tryAcquireProbeAccountLease(accountId: number): boolean {
|
||||
if (!Number.isFinite(accountId) || accountId <= 0) return false;
|
||||
if (probeAccountLeases.has(accountId)) return false;
|
||||
probeAccountLeases.add(accountId);
|
||||
return true;
|
||||
}
|
||||
|
||||
function releaseProbeAccountLease(accountId: number): void {
|
||||
probeAccountLeases.delete(accountId);
|
||||
}
|
||||
|
||||
function buildSkippedProbeAccountResult(input: {
|
||||
accountId: number;
|
||||
siteId: number;
|
||||
message: string;
|
||||
}): ModelAvailabilityProbeAccountResult {
|
||||
return {
|
||||
accountId: input.accountId,
|
||||
siteId: input.siteId,
|
||||
status: 'skipped',
|
||||
scanned: 0,
|
||||
supported: 0,
|
||||
unsupported: 0,
|
||||
inconclusive: 0,
|
||||
skipped: 0,
|
||||
updatedRows: 0,
|
||||
message: input.message,
|
||||
};
|
||||
}
|
||||
|
||||
function summarizeProbeResults(results: ModelAvailabilityProbeAccountResult[], rebuiltRoutes: boolean): ModelAvailabilityProbeExecutionResult {
|
||||
return {
|
||||
results,
|
||||
summary: {
|
||||
totalAccounts: results.length,
|
||||
success: results.filter((item) => item.status === 'success').length,
|
||||
failed: results.filter((item) => item.status === 'failed').length,
|
||||
skipped: results.filter((item) => item.status === 'skipped').length,
|
||||
scanned: results.reduce((sum, item) => sum + item.scanned, 0),
|
||||
supported: results.reduce((sum, item) => sum + item.supported, 0),
|
||||
unsupported: results.reduce((sum, item) => sum + item.unsupported, 0),
|
||||
inconclusive: results.reduce((sum, item) => sum + item.inconclusive, 0),
|
||||
skippedModels: results.reduce((sum, item) => sum + item.skipped, 0),
|
||||
updatedRows: results.reduce((sum, item) => sum + item.updatedRows, 0),
|
||||
rebuiltRoutes,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export async function executeModelAvailabilityProbe(input: {
|
||||
accountId?: number;
|
||||
rebuildRoutes?: boolean;
|
||||
} = {}): Promise<ModelAvailabilityProbeExecutionResult> {
|
||||
const accountIds = input.accountId
|
||||
? [input.accountId]
|
||||
: (await db.select({ id: schema.accounts.id })
|
||||
.from(schema.accounts)
|
||||
.where(eq(schema.accounts.status, 'active'))
|
||||
.all()).map((row) => row.id);
|
||||
|
||||
const results: ModelAvailabilityProbeAccountResult[] = [];
|
||||
let shouldRebuildRoutes = false;
|
||||
|
||||
for (const accountId of accountIds) {
|
||||
const context = await loadActiveProbeAccountContext(accountId);
|
||||
if (!context) {
|
||||
continue;
|
||||
}
|
||||
if (!tryAcquireProbeAccountLease(accountId)) {
|
||||
results.push(buildSkippedProbeAccountResult({
|
||||
accountId,
|
||||
siteId: context.site.id,
|
||||
message: 'model availability probe already running for account',
|
||||
}));
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
const targets = await loadProbeTargetsForAccount(context);
|
||||
if (targets.length <= 0) {
|
||||
results.push(buildSkippedProbeAccountResult({
|
||||
accountId,
|
||||
siteId: context.site.id,
|
||||
message: 'no discovered models to probe',
|
||||
}));
|
||||
continue;
|
||||
}
|
||||
|
||||
let supported = 0;
|
||||
let unsupported = 0;
|
||||
let inconclusive = 0;
|
||||
let skipped = 0;
|
||||
let updatedRows = 0;
|
||||
let failed = false;
|
||||
|
||||
const probeOutcomes = await mapWithConcurrency(
|
||||
targets,
|
||||
config.modelAvailabilityProbeConcurrency,
|
||||
async (target) => {
|
||||
try {
|
||||
const probe = await probeSingleTarget(target);
|
||||
const update = await updateProbeRow(target, probe.status, probe.latencyMs);
|
||||
return {
|
||||
target,
|
||||
probe,
|
||||
touched: update.touched,
|
||||
availabilityChanged: update.availabilityChanged,
|
||||
failed: false,
|
||||
};
|
||||
} catch (error) {
|
||||
console.warn(`[model-probe] account ${accountId} model ${target.modelName} probe failed`, error);
|
||||
return {
|
||||
target,
|
||||
probe: {
|
||||
status: 'inconclusive' as const,
|
||||
latencyMs: null,
|
||||
reason: error instanceof Error ? error.message : 'probe failed',
|
||||
},
|
||||
touched: false,
|
||||
availabilityChanged: false,
|
||||
failed: true,
|
||||
};
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
for (const outcome of probeOutcomes) {
|
||||
if (outcome.probe.status === 'supported') supported += 1;
|
||||
if (outcome.probe.status === 'unsupported') unsupported += 1;
|
||||
if (outcome.probe.status === 'inconclusive') inconclusive += 1;
|
||||
if (outcome.probe.status === 'skipped') skipped += 1;
|
||||
if (outcome.touched) {
|
||||
updatedRows += 1;
|
||||
}
|
||||
if (outcome.availabilityChanged) {
|
||||
shouldRebuildRoutes = true;
|
||||
}
|
||||
if (outcome.failed) {
|
||||
failed = true;
|
||||
}
|
||||
}
|
||||
|
||||
results.push({
|
||||
accountId,
|
||||
siteId: context.site.id,
|
||||
status: failed ? 'failed' : 'success',
|
||||
scanned: targets.length,
|
||||
supported,
|
||||
unsupported,
|
||||
inconclusive,
|
||||
skipped,
|
||||
updatedRows,
|
||||
message: failed
|
||||
? 'model availability probe finished with partial failures'
|
||||
: 'model availability probe finished',
|
||||
});
|
||||
} finally {
|
||||
releaseProbeAccountLease(accountId);
|
||||
}
|
||||
}
|
||||
|
||||
let rebuiltRoutes = false;
|
||||
if (input.rebuildRoutes !== false && shouldRebuildRoutes) {
|
||||
await routeRefreshWorkflow.rebuildRoutesOnly();
|
||||
rebuiltRoutes = true;
|
||||
}
|
||||
|
||||
return summarizeProbeResults(results, rebuiltRoutes);
|
||||
}
|
||||
|
||||
export function buildModelAvailabilityProbeTaskDedupeKey(accountId?: number | null): string {
|
||||
const normalizedAccountId = Number.isFinite(accountId as number) && Number(accountId) > 0
|
||||
? Math.trunc(Number(accountId))
|
||||
: null;
|
||||
return normalizedAccountId
|
||||
? `model-availability-probe-${normalizedAccountId}`
|
||||
: 'model-availability-probe-all';
|
||||
}
|
||||
|
||||
export function queueModelAvailabilityProbeTask(input: {
|
||||
accountId?: number;
|
||||
title?: string;
|
||||
}) {
|
||||
const accountId = Number.isFinite(input.accountId as number) ? Math.trunc(input.accountId as number) : null;
|
||||
const title = input.title || (accountId
|
||||
? `探测模型可用性 #${accountId}`
|
||||
: '探测模型可用性');
|
||||
const dedupeKey = buildModelAvailabilityProbeTaskDedupeKey(accountId);
|
||||
|
||||
return startBackgroundTask(
|
||||
{
|
||||
type: 'model-probe',
|
||||
title,
|
||||
dedupeKey,
|
||||
notifyOnFailure: true,
|
||||
successMessage: (currentTask) => {
|
||||
const summary = (currentTask.result as ModelAvailabilityProbeExecutionResult | undefined)?.summary;
|
||||
if (!summary) return `${title}已完成`;
|
||||
return `${title}完成：探测 ${summary.scanned}，可用 ${summary.supported}，不可用 ${summary.unsupported}，不确定 ${summary.inconclusive}`;
|
||||
},
|
||||
failureMessage: (currentTask) => `${title}失败：${currentTask.error || 'unknown error'}`,
|
||||
},
|
||||
async () => executeModelAvailabilityProbe({
|
||||
accountId: accountId ?? undefined,
|
||||
rebuildRoutes: true,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
export function startModelAvailabilityProbeScheduler(intervalMs = config.modelAvailabilityProbeIntervalMs) {
|
||||
stopModelAvailabilityProbeScheduler();
|
||||
if (!config.modelAvailabilityProbeEnabled) {
|
||||
return {
|
||||
enabled: false,
|
||||
intervalMs: 0,
|
||||
};
|
||||
}
|
||||
|
||||
const safeIntervalMs = Math.max(60_000, Math.trunc(intervalMs || 0));
|
||||
probeSchedulerTimer = setInterval(() => {
|
||||
void queueModelAvailabilityProbeTask({
|
||||
title: '后台模型可用性探测',
|
||||
});
|
||||
}, safeIntervalMs);
|
||||
probeSchedulerTimer.unref?.();
|
||||
return {
|
||||
enabled: true,
|
||||
intervalMs: safeIntervalMs,
|
||||
};
|
||||
}
|
||||
|
||||
export function stopModelAvailabilityProbeScheduler() {
|
||||
if (probeSchedulerTimer) {
|
||||
clearInterval(probeSchedulerTimer);
|
||||
probeSchedulerTimer = null;
|
||||
}
|
||||
}
|
||||
|
||||
export function __resetModelAvailabilityProbeExecutionStateForTests(): void {
|
||||
probeAccountLeases.clear();
|
||||
}
|
||||
@@ -162,4 +162,71 @@ describe('proxyChannelCoordinator', () => {
|
||||
second.lease.release();
|
||||
}
|
||||
});
|
||||
|
||||
it('exposes the set of currently active leased channels', async () => {
|
||||
const lease = await proxyChannelCoordinator.acquireChannelLease({
|
||||
channelId: 23,
|
||||
accountExtraConfig: JSON.stringify({ credentialMode: 'session' }),
|
||||
});
|
||||
expect(lease.status).toBe('acquired');
|
||||
if (lease.status !== 'acquired') return;
|
||||
|
||||
expect(proxyChannelCoordinator.getActiveChannelIds()).toEqual([23]);
|
||||
|
||||
lease.lease.release();
|
||||
expect(proxyChannelCoordinator.getActiveChannelIds()).toEqual([]);
|
||||
});
|
||||
|
||||
it('reports active and waiting load for a guarded session channel', async () => {
|
||||
const first = await proxyChannelCoordinator.acquireChannelLease({
|
||||
channelId: 31,
|
||||
accountExtraConfig: JSON.stringify({ credentialMode: 'session' }),
|
||||
});
|
||||
expect(first.status).toBe('acquired');
|
||||
if (first.status !== 'acquired') return;
|
||||
|
||||
const secondPromise = proxyChannelCoordinator.acquireChannelLease({
|
||||
channelId: 31,
|
||||
accountExtraConfig: JSON.stringify({ credentialMode: 'session' }),
|
||||
});
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
|
||||
expect(proxyChannelCoordinator.getChannelLoadSnapshot({
|
||||
channelId: 31,
|
||||
accountExtraConfig: JSON.stringify({ credentialMode: 'session' }),
|
||||
})).toEqual({
|
||||
channelId: 31,
|
||||
sessionScoped: true,
|
||||
concurrencyLimit: 1,
|
||||
activeLeaseCount: 1,
|
||||
waitingCount: 1,
|
||||
loadRatio: 2,
|
||||
saturated: true,
|
||||
});
|
||||
|
||||
first.lease.release();
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
|
||||
const second = await secondPromise;
|
||||
expect(second.status).toBe('acquired');
|
||||
if (second.status === 'acquired') {
|
||||
second.lease.release();
|
||||
}
|
||||
});
|
||||
|
||||
it('treats structured oauth providers as session-scoped in load snapshots', () => {
|
||||
expect(proxyChannelCoordinator.getChannelLoadSnapshot({
|
||||
channelId: 41,
|
||||
accountExtraConfig: JSON.stringify({ credentialMode: 'session' }),
|
||||
accountOauthProvider: 'codex',
|
||||
})).toEqual({
|
||||
channelId: 41,
|
||||
sessionScoped: true,
|
||||
concurrencyLimit: 1,
|
||||
activeLeaseCount: 0,
|
||||
waitingCount: 0,
|
||||
loadRatio: 0,
|
||||
saturated: false,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -24,6 +24,16 @@ type ChannelRuntimeState = {
|
||||
queue: ChannelWaiter[];
|
||||
};
|
||||
|
||||
export type ProxyChannelLoadSnapshot = {
|
||||
channelId: number;
|
||||
sessionScoped: boolean;
|
||||
concurrencyLimit: number;
|
||||
activeLeaseCount: number;
|
||||
waitingCount: number;
|
||||
loadRatio: number;
|
||||
saturated: boolean;
|
||||
};
|
||||
|
||||
export type ProxyChannelLease = {
|
||||
channelId: number;
|
||||
isActive(): boolean;
|
||||
@@ -104,9 +114,15 @@ function getOrCreateChannelRuntimeState(channelId: number): ChannelRuntimeState
|
||||
return state;
|
||||
}
|
||||
|
||||
function pruneCancelledWaiters(state: ChannelRuntimeState): void {
|
||||
if (state.queue.length <= 0) return;
|
||||
state.queue = state.queue.filter((waiter) => !waiter.cancelled);
|
||||
}
|
||||
|
||||
function maybeDeleteChannelRuntimeState(channelId: number): void {
|
||||
const state = channelRuntimeStates.get(channelId);
|
||||
if (!state) return;
|
||||
pruneCancelledWaiters(state);
|
||||
if (state.activeLeaseIds.size <= 0 && state.queue.every((waiter) => waiter.cancelled)) {
|
||||
channelRuntimeStates.delete(channelId);
|
||||
}
|
||||
@@ -177,6 +193,62 @@ class ProxyChannelCoordinator {
|
||||
stickySessionBindings.delete(normalizedKey);
|
||||
}
|
||||
|
||||
getActiveChannelIds(): number[] {
|
||||
const ids: number[] = [];
|
||||
for (const [channelId, state] of channelRuntimeStates.entries()) {
|
||||
pruneCancelledWaiters(state);
|
||||
if (state.activeLeaseIds.size > 0) {
|
||||
ids.push(channelId);
|
||||
}
|
||||
}
|
||||
return ids;
|
||||
}
|
||||
|
||||
getChannelLoadSnapshot(input: {
|
||||
channelId: number;
|
||||
accountExtraConfig?: string | null;
|
||||
accountOauthProvider?: string | null;
|
||||
}): ProxyChannelLoadSnapshot {
|
||||
const channelId = Math.trunc(input.channelId || 0);
|
||||
const sessionScoped = isSessionScopedChannel({
|
||||
extraConfig: input.accountExtraConfig,
|
||||
oauthProvider: input.accountOauthProvider,
|
||||
});
|
||||
const concurrencyLimit = getChannelConcurrencyLimit({
|
||||
extraConfig: input.accountExtraConfig,
|
||||
oauthProvider: input.accountOauthProvider,
|
||||
});
|
||||
const state = channelId > 0 ? channelRuntimeStates.get(channelId) : null;
|
||||
if (state) {
|
||||
pruneCancelledWaiters(state);
|
||||
}
|
||||
const activeLeaseCount = state?.activeLeaseIds.size ?? 0;
|
||||
const waitingCount = state?.queue.length ?? 0;
|
||||
const denominator = concurrencyLimit > 0 ? concurrencyLimit : 1;
|
||||
return {
|
||||
channelId,
|
||||
sessionScoped,
|
||||
concurrencyLimit,
|
||||
activeLeaseCount,
|
||||
waitingCount,
|
||||
loadRatio: (activeLeaseCount + waitingCount) / denominator,
|
||||
saturated: concurrencyLimit > 0 && activeLeaseCount >= concurrencyLimit,
|
||||
};
|
||||
}
|
||||
|
||||
getChannelLoadSnapshots(input: Array<{
|
||||
channelId: number;
|
||||
accountExtraConfig?: string | null;
|
||||
accountOauthProvider?: string | null;
|
||||
}>): Map<number, ProxyChannelLoadSnapshot> {
|
||||
const snapshots = new Map<number, ProxyChannelLoadSnapshot>();
|
||||
for (const item of input) {
|
||||
const snapshot = this.getChannelLoadSnapshot(item);
|
||||
snapshots.set(snapshot.channelId, snapshot);
|
||||
}
|
||||
return snapshots;
|
||||
}
|
||||
|
||||
async acquireChannelLease(input: {
|
||||
channelId: number;
|
||||
accountExtraConfig?: string | null;
|
||||
@@ -202,6 +274,7 @@ class ProxyChannelCoordinator {
|
||||
}
|
||||
|
||||
const state = getOrCreateChannelRuntimeState(channelId);
|
||||
pruneCancelledWaiters(state);
|
||||
if (state.activeLeaseIds.size < concurrencyLimit) {
|
||||
return {
|
||||
status: 'acquired',
|
||||
@@ -226,6 +299,7 @@ class ProxyChannelCoordinator {
|
||||
waiter.timer = setTimeout(() => {
|
||||
waiter.cancelled = true;
|
||||
waiter.timer = null;
|
||||
pruneCancelledWaiters(state);
|
||||
maybeDeleteChannelRuntimeState(channelId);
|
||||
resolve({
|
||||
status: 'timeout',
|
||||
@@ -285,6 +359,7 @@ class ProxyChannelCoordinator {
|
||||
private drainQueue(channelId: number): void {
|
||||
const state = channelRuntimeStates.get(channelId);
|
||||
if (!state) return;
|
||||
pruneCancelledWaiters(state);
|
||||
const concurrencyLimit = Math.max(0, Math.trunc(config.proxySessionChannelConcurrencyLimit || 0));
|
||||
while (state.activeLeaseIds.size < concurrencyLimit && state.queue.length > 0) {
|
||||
const waiter = state.queue.shift();
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { readRuntimeResponseText } from '../proxy-core/executors/types.js';
|
||||
import type {
|
||||
EndpointAttemptSuccessContext,
|
||||
} from '../routes/proxy/endpointFlow.js';
|
||||
} from '../proxy-core/orchestration/endpointFlow.js';
|
||||
import {
|
||||
finalizeProxyDebugTrace,
|
||||
insertProxyDebugAttempt,
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { shouldRetryProxyRequest } from './proxyRetryPolicy.js';
|
||||
import { shouldAbortSameSiteEndpointFallback, shouldRetryProxyRequest } from './proxyRetryPolicy.js';
|
||||
|
||||
describe('proxyRetryPolicy', () => {
|
||||
it('retries on rate limit and server errors', () => {
|
||||
@@ -52,4 +52,16 @@ describe('proxyRetryPolicy', () => {
|
||||
shouldRetryProxyRequest(400, '{"error":{"message":"invalid timeout parameter"}}'),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it('aborts same-site endpoint fallback on rate-limit and quota responses', () => {
|
||||
expect(
|
||||
shouldAbortSameSiteEndpointFallback(429, '{"error":{"message":"rate limit exceeded"}}'),
|
||||
).toBe(true);
|
||||
expect(
|
||||
shouldAbortSameSiteEndpointFallback(429, '{"error":{"message":"quota exceeded"}}'),
|
||||
).toBe(true);
|
||||
expect(
|
||||
shouldAbortSameSiteEndpointFallback(429, '{"error":{"message":"too many requests"}}'),
|
||||
).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -9,6 +9,7 @@ const MODEL_UNSUPPORTED_PATTERNS: RegExp[] = [
|
||||
/model.*does\s+not\s+exist/i,
|
||||
/no\s+such\s+model/i,
|
||||
/unknown\s+model/i,
|
||||
/unknown\s+provider\s+for\s+model/i,
|
||||
/invalid\s+model/i,
|
||||
/model[_\s-]?not[_\s-]?found/i,
|
||||
/you\s+do\s+not\s+have\s+access\s+to\s+the\s+model/i,
|
||||
@@ -54,6 +55,23 @@ const NON_RETRYABLE_REQUEST_PATTERNS: RegExp[] = [
|
||||
/unsupported\s+media\s+type/i,
|
||||
];
|
||||
|
||||
const SAME_SITE_ENDPOINT_ABORT_PATTERNS: RegExp[] = [
|
||||
/\b429\b/i,
|
||||
/too\s+many\s+requests/i,
|
||||
/rate\s+limit/i,
|
||||
/quota(?:\s+exceeded)?/i,
|
||||
/bad\s+gateway/i,
|
||||
/gateway\s+time-?out/i,
|
||||
/service\s+unavailable/i,
|
||||
/temporar(?:y|ily)\s+unavailable/i,
|
||||
/cpu\s+overloaded/i,
|
||||
/connection\s+reset/i,
|
||||
/connection\s+refused/i,
|
||||
/econnreset/i,
|
||||
/econnrefused/i,
|
||||
...RETRYABLE_TIMEOUT_PATTERNS,
|
||||
];
|
||||
|
||||
function isModelUnsupportedErrorMessage(rawMessage?: string | null): boolean {
|
||||
const text = (rawMessage || '').trim();
|
||||
if (!text) return false;
|
||||
@@ -76,3 +94,10 @@ export function shouldRetryProxyRequest(status: number, upstreamErrorText?: stri
|
||||
if (status === 400 || status === 404 || status === 422) return false;
|
||||
return false;
|
||||
}
|
||||
|
||||
export function shouldAbortSameSiteEndpointFallback(status: number, upstreamErrorText?: string | null): boolean {
|
||||
if (status < 500 && status !== 408 && status !== 429) {
|
||||
return false;
|
||||
}
|
||||
return matchesAnyPattern(SAME_SITE_ENDPOINT_ABORT_PATTERNS, upstreamErrorText);
|
||||
}
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
export { dispatchRuntimeRequest } from '../routes/proxy/runtimeExecutor.js';
|
||||
@@ -0,0 +1,116 @@
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
const resolveUpstreamEndpointCandidatesMock = vi.fn();
|
||||
const buildUpstreamEndpointRequestMock = vi.fn();
|
||||
const dispatchRuntimeRequestMock = vi.fn();
|
||||
const resolveChannelProxyUrlMock = vi.fn();
|
||||
const withSiteRecordProxyRequestInitMock = vi.fn();
|
||||
|
||||
vi.mock('../routes/proxy/upstreamEndpoint.js', () => ({
|
||||
resolveUpstreamEndpointCandidates: (...args: unknown[]) => resolveUpstreamEndpointCandidatesMock(...args),
|
||||
buildUpstreamEndpointRequest: (...args: unknown[]) => buildUpstreamEndpointRequestMock(...args),
|
||||
}));
|
||||
|
||||
vi.mock('../routes/proxy/runtimeExecutor.js', () => ({
|
||||
dispatchRuntimeRequest: (...args: unknown[]) => dispatchRuntimeRequestMock(...args),
|
||||
}));
|
||||
|
||||
vi.mock('./siteProxy.js', () => ({
|
||||
resolveChannelProxyUrl: (...args: unknown[]) => resolveChannelProxyUrlMock(...args),
|
||||
withSiteRecordProxyRequestInit: (...args: unknown[]) => withSiteRecordProxyRequestInitMock(...args),
|
||||
}));
|
||||
|
||||
describe('probeRuntimeModel', () => {
|
||||
const site = {
|
||||
id: 1,
|
||||
name: 'probe-site',
|
||||
url: 'https://probe.example.com',
|
||||
platform: 'new-api',
|
||||
status: 'active',
|
||||
} as any;
|
||||
|
||||
const account = {
|
||||
id: 1,
|
||||
siteId: 1,
|
||||
username: 'probe-user',
|
||||
accessToken: '',
|
||||
apiToken: 'sk-probe',
|
||||
status: 'active',
|
||||
extraConfig: null,
|
||||
} as any;
|
||||
|
||||
beforeEach(() => {
|
||||
vi.resetModules();
|
||||
resolveUpstreamEndpointCandidatesMock.mockReset();
|
||||
buildUpstreamEndpointRequestMock.mockReset();
|
||||
dispatchRuntimeRequestMock.mockReset();
|
||||
resolveChannelProxyUrlMock.mockReset();
|
||||
withSiteRecordProxyRequestInitMock.mockReset();
|
||||
|
||||
buildUpstreamEndpointRequestMock.mockReturnValue({
|
||||
path: '/v1/chat/completions',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: { model: 'gpt-5.4' },
|
||||
runtime: {
|
||||
executor: 'default',
|
||||
modelName: 'gpt-5.4',
|
||||
stream: false,
|
||||
},
|
||||
});
|
||||
resolveChannelProxyUrlMock.mockReturnValue(null);
|
||||
withSiteRecordProxyRequestInitMock.mockImplementation(async (_site: unknown, init: RequestInit) => init);
|
||||
});
|
||||
|
||||
it('returns an inconclusive result instead of throwing when endpoint resolution fails', async () => {
|
||||
resolveUpstreamEndpointCandidatesMock.mockRejectedValue(new Error('resolution failed'));
|
||||
|
||||
const { probeRuntimeModel } = await import('./runtimeModelProbe.js');
|
||||
const result = await probeRuntimeModel({
|
||||
site,
|
||||
account,
|
||||
modelName: 'gpt-5.4',
|
||||
timeoutMs: 10,
|
||||
});
|
||||
|
||||
expect(result.status).toBe('inconclusive');
|
||||
expect(result.reason).toContain('resolution failed');
|
||||
expect(result.latencyMs).not.toBeNull();
|
||||
});
|
||||
|
||||
it('uses the remaining timeout budget for the runtime request phase', async () => {
|
||||
resolveUpstreamEndpointCandidatesMock.mockImplementation(async () => {
|
||||
await new Promise((resolve) => setTimeout(resolve, 15));
|
||||
return ['chat'];
|
||||
});
|
||||
dispatchRuntimeRequestMock.mockImplementation(async (input: {
|
||||
buildInit: (requestUrl: string, request: Record<string, unknown>) => Promise<RequestInit>;
|
||||
request: Record<string, unknown>;
|
||||
targetUrl?: string;
|
||||
}) => {
|
||||
const init = await input.buildInit(input.targetUrl || 'https://probe.example.com/v1/chat/completions', input.request);
|
||||
const signal = init.signal as AbortSignal | undefined;
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const timer = setTimeout(resolve, 40);
|
||||
signal?.addEventListener('abort', () => {
|
||||
clearTimeout(timer);
|
||||
reject(signal.reason ?? new Error('aborted'));
|
||||
}, { once: true });
|
||||
});
|
||||
return new Response(JSON.stringify({ ok: true }), { status: 200 });
|
||||
});
|
||||
|
||||
const { probeRuntimeModel } = await import('./runtimeModelProbe.js');
|
||||
const startedAt = Date.now();
|
||||
const result = await probeRuntimeModel({
|
||||
site,
|
||||
account,
|
||||
modelName: 'gpt-5.4',
|
||||
timeoutMs: 30,
|
||||
});
|
||||
const elapsedMs = Date.now() - startedAt;
|
||||
|
||||
expect(result.status).toBe('inconclusive');
|
||||
expect(result.latencyMs).not.toBeNull();
|
||||
expect(elapsedMs).toBeLessThan(200);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,250 @@
|
||||
import { getOauthInfoFromAccount } from './oauth/oauthAccount.js';
|
||||
import { buildOauthProviderHeaders } from './oauth/service.js';
|
||||
import { resolveChannelProxyUrl, withSiteRecordProxyRequestInit } from './siteProxy.js';
|
||||
import { dispatchRuntimeRequest } from './runtimeDispatch.js';
|
||||
import {
|
||||
buildUpstreamEndpointRequest,
|
||||
resolveUpstreamEndpointCandidates,
|
||||
type UpstreamEndpoint,
|
||||
} from './upstreamEndpointRuntime.js';
|
||||
import { executeEndpointFlow, type BuiltEndpointRequest } from '../proxy-core/orchestration/endpointFlow.js';
|
||||
import type { schema } from '../db/index.js';
|
||||
|
||||
export type RuntimeModelProbeStatus = 'supported' | 'unsupported' | 'inconclusive' | 'skipped';
|
||||
|
||||
export type RuntimeModelProbeResult = {
|
||||
status: RuntimeModelProbeStatus;
|
||||
latencyMs: number | null;
|
||||
reason: string;
|
||||
};
|
||||
|
||||
const NON_CONVERSATION_MODEL_PATTERNS = [
|
||||
/(^|[-_/])embedding(s)?($|[-_/])/i,
|
||||
/(^|[-_/])rerank($|[-_/])/i,
|
||||
/(^|[-_/])moderation($|[-_/])/i,
|
||||
/(^|[-_/])whisper($|[-_/])/i,
|
||||
/(^|[-_/])tts($|[-_/])/i,
|
||||
/(^|[-_/])transcribe|transcription/i,
|
||||
/(^|[-_/])dall-e($|[-_/])/i,
|
||||
/(^|[-_/])imagen($|[-_/])/i,
|
||||
/(^|[-_/])veo($|[-_/])/i,
|
||||
/(^|[-_/])cogvideo($|[-_/])/i,
|
||||
];
|
||||
|
||||
const DEFINITE_UNSUPPORTED_PATTERNS = [
|
||||
/no such model/i,
|
||||
/unknown model/i,
|
||||
/unsupported model/i,
|
||||
/invalid model/i,
|
||||
/model[^]{0,80}(does not exist|not found|not available|unavailable|unsupported|invalid|disabled)/i,
|
||||
/(does not exist|not found|not available|unavailable|unsupported|invalid|disabled)[^]{0,40}model/i,
|
||||
/模型[^]{0,40}(不存在|不可用|不支持|无效|禁用|未开通|未开放)/,
|
||||
/(不存在|不可用|不支持|无效|禁用)[^]{0,20}模型/,
|
||||
/model[^]{0,80}(access denied|permission|forbidden|not allowed)/i,
|
||||
/模型[^]{0,40}(无权限|未授权|禁止访问)/,
|
||||
];
|
||||
|
||||
function isLikelyConversationModel(modelName: string): boolean {
|
||||
const normalized = String(modelName || '').trim();
|
||||
if (!normalized) return false;
|
||||
if (normalized.startsWith('__')) return false;
|
||||
return !NON_CONVERSATION_MODEL_PATTERNS.some((pattern) => pattern.test(normalized));
|
||||
}
|
||||
|
||||
function classifyUnsupportedFailure(status: number, rawErrorText: string): boolean {
|
||||
if (![400, 403, 404, 422].includes(status)) return false;
|
||||
const normalized = String(rawErrorText || '').trim();
|
||||
if (!normalized) return false;
|
||||
return DEFINITE_UNSUPPORTED_PATTERNS.some((pattern) => pattern.test(normalized));
|
||||
}
|
||||
|
||||
function buildProbeBody(modelName: string): Record<string, unknown> {
|
||||
return {
|
||||
model: modelName,
|
||||
messages: [
|
||||
{
|
||||
role: 'user',
|
||||
content: 'Reply with OK.',
|
||||
},
|
||||
],
|
||||
max_tokens: 8,
|
||||
stream: false,
|
||||
};
|
||||
}
|
||||
|
||||
async function withTimeout<T>(fn: () => Promise<T>, timeoutMs: number, timeoutMessage: string): Promise<T> {
|
||||
let timer: ReturnType<typeof setTimeout> | null = null;
|
||||
try {
|
||||
return await Promise.race([
|
||||
fn(),
|
||||
new Promise<T>((_, reject) => {
|
||||
timer = setTimeout(() => reject(new Error(timeoutMessage)), timeoutMs);
|
||||
timer.unref?.();
|
||||
}),
|
||||
]);
|
||||
} finally {
|
||||
if (timer) clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
|
||||
function resolveRemainingTimeoutMs(deadlineAtMs: number, timeoutLabel: string): number {
|
||||
const remainingMs = deadlineAtMs - Date.now();
|
||||
if (remainingMs <= 0) {
|
||||
throw new Error(timeoutLabel);
|
||||
}
|
||||
return remainingMs;
|
||||
}
|
||||
|
||||
export async function probeRuntimeModel(input: {
|
||||
site: typeof schema.sites.$inferSelect;
|
||||
account: typeof schema.accounts.$inferSelect;
|
||||
modelName: string;
|
||||
timeoutMs: number;
|
||||
tokenValue?: string | null;
|
||||
}): Promise<RuntimeModelProbeResult> {
|
||||
if (!isLikelyConversationModel(input.modelName)) {
|
||||
return {
|
||||
status: 'skipped',
|
||||
latencyMs: null,
|
||||
reason: 'skipped non-conversation model probe',
|
||||
};
|
||||
}
|
||||
|
||||
const oauth = getOauthInfoFromAccount(input.account);
|
||||
const tokenValue = String(
|
||||
input.tokenValue
|
||||
|| (oauth ? input.account.accessToken : input.account.apiToken)
|
||||
|| '',
|
||||
).trim();
|
||||
if (!tokenValue) {
|
||||
return {
|
||||
status: 'inconclusive',
|
||||
latencyMs: null,
|
||||
reason: 'missing credential for probe',
|
||||
};
|
||||
}
|
||||
|
||||
const startedAt = Date.now();
|
||||
const deadlineAtMs = startedAt + Math.max(1, input.timeoutMs);
|
||||
try {
|
||||
const endpointCandidates = await withTimeout(
|
||||
() => resolveUpstreamEndpointCandidates(
|
||||
{
|
||||
site: input.site,
|
||||
account: input.account,
|
||||
},
|
||||
input.modelName,
|
||||
'openai',
|
||||
input.modelName,
|
||||
),
|
||||
resolveRemainingTimeoutMs(
|
||||
deadlineAtMs,
|
||||
`runtime model probe candidate resolution timeout (${Math.round(input.timeoutMs / 1000)}s)`,
|
||||
),
|
||||
`runtime model probe candidate resolution timeout (${Math.round(input.timeoutMs / 1000)}s)`,
|
||||
);
|
||||
if (endpointCandidates.length <= 0) {
|
||||
return {
|
||||
status: 'inconclusive',
|
||||
latencyMs: Date.now() - startedAt,
|
||||
reason: 'no compatible probe endpoint candidates',
|
||||
};
|
||||
}
|
||||
|
||||
const providerHeaders = buildOauthProviderHeaders({
|
||||
account: input.account,
|
||||
downstreamHeaders: {},
|
||||
});
|
||||
const openaiBody = buildProbeBody(input.modelName);
|
||||
const channelProxyUrl = resolveChannelProxyUrl(input.site, input.account.extraConfig);
|
||||
const abortController = new AbortController();
|
||||
const remainingExecutionTimeoutMs = resolveRemainingTimeoutMs(
|
||||
deadlineAtMs,
|
||||
`runtime model probe timeout (${Math.round(input.timeoutMs / 1000)}s)`,
|
||||
);
|
||||
const abortTimer = setTimeout(() => {
|
||||
abortController.abort(new Error(`runtime model probe timeout (${Math.round(input.timeoutMs / 1000)}s)`));
|
||||
}, remainingExecutionTimeoutMs);
|
||||
abortTimer.unref?.();
|
||||
|
||||
const buildRequest = (endpoint: UpstreamEndpoint): BuiltEndpointRequest => {
|
||||
const request = buildUpstreamEndpointRequest({
|
||||
endpoint,
|
||||
modelName: input.modelName,
|
||||
stream: false,
|
||||
tokenValue,
|
||||
oauthProvider: oauth?.provider,
|
||||
oauthProjectId: oauth?.projectId,
|
||||
sitePlatform: input.site.platform,
|
||||
siteUrl: input.site.url,
|
||||
openaiBody,
|
||||
downstreamFormat: 'openai',
|
||||
downstreamHeaders: {},
|
||||
providerHeaders,
|
||||
});
|
||||
return {
|
||||
endpoint,
|
||||
path: request.path,
|
||||
headers: request.headers,
|
||||
body: request.body as Record<string, unknown>,
|
||||
runtime: request.runtime,
|
||||
};
|
||||
};
|
||||
const dispatchRequest = async (
|
||||
request: BuiltEndpointRequest,
|
||||
targetUrl: string,
|
||||
) => (
|
||||
dispatchRuntimeRequest({
|
||||
siteUrl: input.site.url,
|
||||
targetUrl,
|
||||
request,
|
||||
buildInit: (_requestUrl, requestForFetch) => withSiteRecordProxyRequestInit(
|
||||
input.site,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: requestForFetch.headers,
|
||||
body: JSON.stringify(requestForFetch.body),
|
||||
signal: abortController.signal,
|
||||
},
|
||||
channelProxyUrl,
|
||||
),
|
||||
})
|
||||
);
|
||||
|
||||
let result: Awaited<ReturnType<typeof executeEndpointFlow>>;
|
||||
try {
|
||||
result = await executeEndpointFlow({
|
||||
siteUrl: input.site.url,
|
||||
proxyUrl: channelProxyUrl,
|
||||
endpointCandidates,
|
||||
buildRequest,
|
||||
dispatchRequest,
|
||||
});
|
||||
} finally {
|
||||
clearTimeout(abortTimer);
|
||||
}
|
||||
const latencyMs = Date.now() - startedAt;
|
||||
|
||||
if (result.ok) {
|
||||
await result.upstream.text().catch(() => undefined);
|
||||
return {
|
||||
status: 'supported',
|
||||
latencyMs,
|
||||
reason: 'probe succeeded',
|
||||
};
|
||||
}
|
||||
|
||||
const rawErrorText = String(result.rawErrText || result.errText || '').trim();
|
||||
return {
|
||||
status: classifyUnsupportedFailure(result.status || 0, rawErrorText) ? 'unsupported' : 'inconclusive',
|
||||
latencyMs,
|
||||
reason: rawErrorText || `probe failed with status ${result.status || 0}`,
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
status: 'inconclusive',
|
||||
latencyMs: Date.now() - startedAt,
|
||||
reason: error instanceof Error ? error.message : 'probe failed',
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -2,7 +2,7 @@ import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest';
|
||||
import { mkdtempSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { eq } from 'drizzle-orm';
|
||||
import { eq, inArray } from 'drizzle-orm';
|
||||
|
||||
type DbModule = typeof import('../db/index.js');
|
||||
type TokenRouterModule = typeof import('./tokenRouter.js');
|
||||
@@ -615,6 +615,90 @@ describe('TokenRouter runtime cache', () => {
|
||||
expect(await router.selectChannel('gpt-4o-mini')).toBeNull();
|
||||
});
|
||||
|
||||
it('clears short-window cooldown on sibling channels after a successful recovery probe', async () => {
|
||||
const site = await db.insert(schema.sites).values({
|
||||
name: 'shared-credential-recovery-site',
|
||||
url: 'https://shared-credential-recovery.example.com',
|
||||
platform: 'codex',
|
||||
status: 'active',
|
||||
}).returning().get();
|
||||
|
||||
const account = await db.insert(schema.accounts).values({
|
||||
siteId: site.id,
|
||||
username: 'shared-credential-recovery-user',
|
||||
accessToken: 'shared-credential-recovery-access-token',
|
||||
status: 'active',
|
||||
oauthProvider: 'codex',
|
||||
extraConfig: JSON.stringify({
|
||||
credentialMode: 'session',
|
||||
oauth: {
|
||||
provider: 'codex',
|
||||
},
|
||||
}),
|
||||
}).returning().get();
|
||||
|
||||
const primaryRoute = await db.insert(schema.tokenRoutes).values({
|
||||
modelPattern: 'gpt-5.4',
|
||||
routingStrategy: 'weighted',
|
||||
enabled: true,
|
||||
}).returning().get();
|
||||
|
||||
const siblingRoute = await db.insert(schema.tokenRoutes).values({
|
||||
modelPattern: 'gpt-4o-mini',
|
||||
routingStrategy: 'weighted',
|
||||
enabled: true,
|
||||
}).returning().get();
|
||||
|
||||
const primaryChannel = await db.insert(schema.routeChannels).values({
|
||||
routeId: primaryRoute.id,
|
||||
accountId: account.id,
|
||||
priority: 0,
|
||||
weight: 10,
|
||||
enabled: true,
|
||||
}).returning().get();
|
||||
|
||||
const siblingChannel = await db.insert(schema.routeChannels).values({
|
||||
routeId: siblingRoute.id,
|
||||
accountId: account.id,
|
||||
priority: 0,
|
||||
weight: 10,
|
||||
enabled: true,
|
||||
}).returning().get();
|
||||
|
||||
const router = new TokenRouter();
|
||||
await router.recordFailure(primaryChannel.id, {
|
||||
status: 429,
|
||||
errorText: JSON.stringify({
|
||||
error: {
|
||||
type: 'usage_limit_reached',
|
||||
message: 'The usage limit has been reached',
|
||||
},
|
||||
}),
|
||||
modelName: 'gpt-5.4',
|
||||
});
|
||||
|
||||
await router.recordProbeSuccess(primaryChannel.id, 180, 'gpt-5.4');
|
||||
|
||||
const refreshedChannels = await db.select().from(schema.routeChannels)
|
||||
.where(inArray(schema.routeChannels.id, [primaryChannel.id, siblingChannel.id]))
|
||||
.all();
|
||||
|
||||
expect(refreshedChannels).toEqual(expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
id: primaryChannel.id,
|
||||
cooldownUntil: null,
|
||||
consecutiveFailCount: 0,
|
||||
cooldownLevel: 0,
|
||||
}),
|
||||
expect.objectContaining({
|
||||
id: siblingChannel.id,
|
||||
cooldownUntil: null,
|
||||
consecutiveFailCount: 0,
|
||||
cooldownLevel: 0,
|
||||
}),
|
||||
]));
|
||||
});
|
||||
|
||||
it('round robins across all available channels regardless of priority', async () => {
|
||||
const site = await db.insert(schema.sites).values({
|
||||
name: 'round-robin-site',
|
||||
|
||||
@@ -7,6 +7,7 @@ import { eq } from 'drizzle-orm';
|
||||
type DbModule = typeof import('../db/index.js');
|
||||
type TokenRouterModule = typeof import('./tokenRouter.js');
|
||||
type ConfigModule = typeof import('../config.js');
|
||||
type ProxyChannelCoordinatorModule = typeof import('./proxyChannelCoordinator.js');
|
||||
|
||||
const mockedCatalogRoutingCost = vi.fn<(
|
||||
input: { siteId: number; accountId: number; modelName: string }
|
||||
@@ -24,14 +25,20 @@ describe('TokenRouter selection scoring', () => {
|
||||
let db: DbModule['db'];
|
||||
let schema: DbModule['schema'];
|
||||
let TokenRouter: TokenRouterModule['TokenRouter'];
|
||||
let tokenRouterTestUtils: TokenRouterModule['__tokenRouterTestUtils'];
|
||||
let invalidateTokenRouterCache: TokenRouterModule['invalidateTokenRouterCache'];
|
||||
let resetSiteRuntimeHealthState: TokenRouterModule['resetSiteRuntimeHealthState'];
|
||||
let flushSiteRuntimeHealthPersistence: TokenRouterModule['flushSiteRuntimeHealthPersistence'];
|
||||
let filterRecentlyFailedCandidates: TokenRouterModule['filterRecentlyFailedCandidates'];
|
||||
let config: ConfigModule['config'];
|
||||
let proxyChannelCoordinator: ProxyChannelCoordinatorModule['proxyChannelCoordinator'];
|
||||
let resetProxyChannelCoordinatorState: ProxyChannelCoordinatorModule['resetProxyChannelCoordinatorState'];
|
||||
let dataDir = '';
|
||||
let idSeed = 0;
|
||||
let originalRoutingWeights: typeof config.routingWeights;
|
||||
let originalRoutingFallbackUnitCost: number;
|
||||
let originalProxySessionChannelConcurrencyLimit: number;
|
||||
let originalProxySessionChannelQueueWaitMs: number;
|
||||
|
||||
const nextId = () => {
|
||||
idSeed += 1;
|
||||
@@ -46,21 +53,30 @@ describe('TokenRouter selection scoring', () => {
|
||||
const dbModule = await import('../db/index.js');
|
||||
const tokenRouterModule = await import('./tokenRouter.js');
|
||||
const configModule = await import('../config.js');
|
||||
const coordinatorModule = await import('./proxyChannelCoordinator.js');
|
||||
db = dbModule.db;
|
||||
schema = dbModule.schema;
|
||||
TokenRouter = tokenRouterModule.TokenRouter;
|
||||
tokenRouterTestUtils = tokenRouterModule.__tokenRouterTestUtils;
|
||||
invalidateTokenRouterCache = tokenRouterModule.invalidateTokenRouterCache;
|
||||
resetSiteRuntimeHealthState = tokenRouterModule.resetSiteRuntimeHealthState;
|
||||
flushSiteRuntimeHealthPersistence = tokenRouterModule.flushSiteRuntimeHealthPersistence;
|
||||
filterRecentlyFailedCandidates = tokenRouterModule.filterRecentlyFailedCandidates;
|
||||
config = configModule.config;
|
||||
proxyChannelCoordinator = coordinatorModule.proxyChannelCoordinator;
|
||||
resetProxyChannelCoordinatorState = coordinatorModule.resetProxyChannelCoordinatorState;
|
||||
originalRoutingWeights = { ...config.routingWeights };
|
||||
originalRoutingFallbackUnitCost = config.routingFallbackUnitCost;
|
||||
originalProxySessionChannelConcurrencyLimit = config.proxySessionChannelConcurrencyLimit;
|
||||
originalProxySessionChannelQueueWaitMs = config.proxySessionChannelQueueWaitMs;
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
idSeed = 0;
|
||||
mockedCatalogRoutingCost.mockReset();
|
||||
mockedCatalogRoutingCost.mockReturnValue(null);
|
||||
config.proxySessionChannelConcurrencyLimit = originalProxySessionChannelConcurrencyLimit;
|
||||
config.proxySessionChannelQueueWaitMs = originalProxySessionChannelQueueWaitMs;
|
||||
await db.delete(schema.routeChannels).run();
|
||||
await db.delete(schema.tokenRoutes).run();
|
||||
await db.delete(schema.settings).run();
|
||||
@@ -69,13 +85,17 @@ describe('TokenRouter selection scoring', () => {
|
||||
await db.delete(schema.sites).run();
|
||||
invalidateTokenRouterCache();
|
||||
resetSiteRuntimeHealthState();
|
||||
resetProxyChannelCoordinatorState();
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
config.routingWeights = { ...originalRoutingWeights };
|
||||
config.routingFallbackUnitCost = originalRoutingFallbackUnitCost;
|
||||
config.proxySessionChannelConcurrencyLimit = originalProxySessionChannelConcurrencyLimit;
|
||||
config.proxySessionChannelQueueWaitMs = originalProxySessionChannelQueueWaitMs;
|
||||
invalidateTokenRouterCache();
|
||||
resetSiteRuntimeHealthState();
|
||||
resetProxyChannelCoordinatorState();
|
||||
delete process.env.DATA_DIR;
|
||||
});
|
||||
|
||||
@@ -96,7 +116,7 @@ describe('TokenRouter selection scoring', () => {
|
||||
}).returning().get();
|
||||
}
|
||||
|
||||
async function createAccount(siteId: number, usernamePrefix: string) {
|
||||
async function createAccount(siteId: number, usernamePrefix: string, options: { extraConfig?: string | null } = {}) {
|
||||
const id = nextId();
|
||||
return await db.insert(schema.accounts).values({
|
||||
siteId,
|
||||
@@ -104,6 +124,7 @@ describe('TokenRouter selection scoring', () => {
|
||||
accessToken: `access-${id}`,
|
||||
apiToken: `sk-${id}`,
|
||||
status: 'active',
|
||||
extraConfig: options.extraConfig ?? null,
|
||||
}).returning().get();
|
||||
}
|
||||
|
||||
@@ -126,7 +147,11 @@ describe('TokenRouter selection scoring', () => {
|
||||
usageWeight: 0,
|
||||
};
|
||||
|
||||
const route = await createRoute('gpt-5.2');
|
||||
const route = await db.insert(schema.tokenRoutes).values({
|
||||
modelPattern: 'gpt-5.2',
|
||||
routingStrategy: 'stable_first',
|
||||
enabled: true,
|
||||
}).returning().get();
|
||||
const site = await createSite('sticky-site');
|
||||
const account = await createAccount(site.id, 'sticky-user');
|
||||
const tokenA = await createToken(account.id, 'sticky-a');
|
||||
@@ -165,6 +190,27 @@ describe('TokenRouter selection scoring', () => {
|
||||
await expect(router.selectPreferredChannel('gpt-5.2', preferredChannel.id)).resolves.toBeNull();
|
||||
});
|
||||
|
||||
it('avoids recently failed candidates by default when healthy alternatives exist', () => {
|
||||
const nowMs = Date.now();
|
||||
const filtered = filterRecentlyFailedCandidates([
|
||||
{
|
||||
channel: {
|
||||
failCount: 3,
|
||||
lastFailAt: new Date(nowMs).toISOString(),
|
||||
},
|
||||
},
|
||||
{
|
||||
channel: {
|
||||
failCount: 0,
|
||||
lastFailAt: null,
|
||||
},
|
||||
},
|
||||
], nowMs);
|
||||
|
||||
expect(filtered).toHaveLength(1);
|
||||
expect(filtered[0]?.channel.failCount).toBe(0);
|
||||
});
|
||||
|
||||
it('normalizes probability across channels on the same site', async () => {
|
||||
config.routingWeights = {
|
||||
baseWeightFactor: 1,
|
||||
@@ -790,6 +836,66 @@ describe('TokenRouter selection scoring', () => {
|
||||
expect(weakCandidate?.reason || '').toContain('成功率=40.0%');
|
||||
});
|
||||
|
||||
it('stable_first ranks recent and fallback success rate ahead of balance-heavy weak sites', async () => {
|
||||
config.routingWeights = {
|
||||
baseWeightFactor: 0,
|
||||
valueScoreFactor: 1,
|
||||
costWeight: 0.1,
|
||||
balanceWeight: 10,
|
||||
usageWeight: 0,
|
||||
};
|
||||
|
||||
const route = await db.insert(schema.tokenRoutes).values({
|
||||
modelPattern: 'gpt-5.4',
|
||||
routingStrategy: 'stable_first',
|
||||
enabled: true,
|
||||
}).returning().get();
|
||||
|
||||
const siteStable = await createSite('stable-rate-front');
|
||||
const accountStable = await createAccount(siteStable.id, 'stable-rate-user-front');
|
||||
const tokenStable = await createToken(accountStable.id, 'stable-rate-token-front');
|
||||
const stableChannel = await db.insert(schema.routeChannels).values({
|
||||
routeId: route.id,
|
||||
accountId: accountStable.id,
|
||||
tokenId: tokenStable.id,
|
||||
priority: 0,
|
||||
weight: 10,
|
||||
enabled: true,
|
||||
successCount: 80,
|
||||
failCount: 4,
|
||||
totalLatencyMs: 80 * 380,
|
||||
}).returning().get();
|
||||
|
||||
const siteWeak = await createSite('stable-rate-back');
|
||||
const accountWeak = await createAccount(siteWeak.id, 'stable-rate-user-back');
|
||||
await db.update(schema.accounts).set({
|
||||
balance: 999999,
|
||||
}).where(eq(schema.accounts.id, accountWeak.id)).run();
|
||||
const tokenWeak = await createToken(accountWeak.id, 'stable-rate-token-back');
|
||||
const weakChannel = await db.insert(schema.routeChannels).values({
|
||||
routeId: route.id,
|
||||
accountId: accountWeak.id,
|
||||
tokenId: tokenWeak.id,
|
||||
priority: 0,
|
||||
weight: 10,
|
||||
enabled: true,
|
||||
successCount: 5,
|
||||
failCount: 21,
|
||||
totalLatencyMs: 5 * 1600,
|
||||
}).returning().get();
|
||||
|
||||
const decision = await new TokenRouter().explainSelection('gpt-5.4');
|
||||
const stableCandidate = decision.candidates.find((candidate) => candidate.channelId === stableChannel.id);
|
||||
const weakCandidate = decision.candidates.find((candidate) => candidate.channelId === weakChannel.id);
|
||||
|
||||
expect(stableCandidate).toBeTruthy();
|
||||
expect(weakCandidate).toBeTruthy();
|
||||
expect(decision.selectedChannelId).toBe(stableChannel.id);
|
||||
expect((stableCandidate?.probability || 0)).toBeGreaterThan(weakCandidate?.probability || 0);
|
||||
expect(stableCandidate?.reason || '').toContain('近期成功率=');
|
||||
expect(weakCandidate?.reason || '').toContain('综合近期成功率=');
|
||||
});
|
||||
|
||||
it('reloads persisted runtime health after in-memory reset', async () => {
|
||||
config.routingWeights = {
|
||||
baseWeightFactor: 1,
|
||||
@@ -856,6 +962,79 @@ describe('TokenRouter selection scoring', () => {
|
||||
expect(candidateA?.reason || '').toContain('运行时健康=');
|
||||
});
|
||||
|
||||
it('keeps a recovered stable_first site behind healthier peers until recent success rebuilds', async () => {
|
||||
config.routingWeights = {
|
||||
baseWeightFactor: 1,
|
||||
valueScoreFactor: 0,
|
||||
costWeight: 0,
|
||||
balanceWeight: 0,
|
||||
usageWeight: 0,
|
||||
};
|
||||
|
||||
const route = await db.insert(schema.tokenRoutes).values({
|
||||
modelPattern: 'gpt-5.3',
|
||||
routingStrategy: 'stable_first',
|
||||
enabled: true,
|
||||
}).returning().get();
|
||||
|
||||
const siteRecovered = await createSite('stable-recovery-a');
|
||||
const accountRecovered = await createAccount(siteRecovered.id, 'stable-recovery-user-a');
|
||||
const tokenRecovered = await createToken(accountRecovered.id, 'stable-recovery-token-a');
|
||||
const recoveredChannel = await db.insert(schema.routeChannels).values({
|
||||
routeId: route.id,
|
||||
accountId: accountRecovered.id,
|
||||
tokenId: tokenRecovered.id,
|
||||
priority: 0,
|
||||
weight: 10,
|
||||
enabled: true,
|
||||
}).returning().get();
|
||||
|
||||
const siteHealthy = await createSite('stable-recovery-b');
|
||||
const accountHealthy = await createAccount(siteHealthy.id, 'stable-recovery-user-b');
|
||||
const tokenHealthy = await createToken(accountHealthy.id, 'stable-recovery-token-b');
|
||||
const healthyChannel = await db.insert(schema.routeChannels).values({
|
||||
routeId: route.id,
|
||||
accountId: accountHealthy.id,
|
||||
tokenId: tokenHealthy.id,
|
||||
priority: 0,
|
||||
weight: 10,
|
||||
enabled: true,
|
||||
}).returning().get();
|
||||
|
||||
const router = new TokenRouter();
|
||||
for (let index = 0; index < 3; index += 1) {
|
||||
await router.recordFailure(recoveredChannel.id, {
|
||||
status: 502,
|
||||
errorText: 'Gateway timeout',
|
||||
modelName: 'gpt-5.3',
|
||||
});
|
||||
}
|
||||
await db.update(schema.routeChannels).set({
|
||||
cooldownUntil: null,
|
||||
lastFailAt: null,
|
||||
failCount: 0,
|
||||
consecutiveFailCount: 0,
|
||||
cooldownLevel: 0,
|
||||
}).where(eq(schema.routeChannels.id, recoveredChannel.id)).run();
|
||||
|
||||
await router.recordSuccess(recoveredChannel.id, 900, 0, 'gpt-5.3');
|
||||
for (let index = 0; index < 4; index += 1) {
|
||||
await router.recordSuccess(healthyChannel.id, 320, 0, 'gpt-5.3');
|
||||
}
|
||||
invalidateTokenRouterCache();
|
||||
|
||||
const preview = await router.previewSelectedChannel('gpt-5.3');
|
||||
const decision = await router.explainSelection('gpt-5.3');
|
||||
const recoveredCandidate = decision.candidates.find((candidate) => candidate.channelId === recoveredChannel.id);
|
||||
const healthyCandidate = decision.candidates.find((candidate) => candidate.channelId === healthyChannel.id);
|
||||
|
||||
expect(preview?.channel.id).toBe(healthyChannel.id);
|
||||
expect(decision.selectedChannelId).toBe(healthyChannel.id);
|
||||
expect((recoveredCandidate?.probability || 0)).toBeLessThan(healthyCandidate?.probability || 0);
|
||||
expect(recoveredCandidate?.reason || '').toContain('近期成功率=');
|
||||
expect(healthyCandidate?.reason || '').toContain('近期成功率=');
|
||||
});
|
||||
|
||||
it('penalizes the failed model more than unrelated models on the same site', async () => {
|
||||
config.routingWeights = {
|
||||
baseWeightFactor: 1,
|
||||
@@ -934,6 +1113,90 @@ describe('TokenRouter selection scoring', () => {
|
||||
expect(gptCandidateA?.reason || '').toContain('模型=');
|
||||
});
|
||||
|
||||
it('treats unknown provider for model as model-scoped degradation instead of opening a site breaker', async () => {
|
||||
config.routingWeights = {
|
||||
baseWeightFactor: 1,
|
||||
valueScoreFactor: 0,
|
||||
costWeight: 0,
|
||||
balanceWeight: 0,
|
||||
usageWeight: 0,
|
||||
};
|
||||
|
||||
const gptRoute = await createRoute('gpt-5.4');
|
||||
const claudeRoute = await createRoute('claude-sonnet-4-6');
|
||||
|
||||
const siteA = await createSite('unknown-provider-a');
|
||||
const accountA = await createAccount(siteA.id, 'unknown-provider-user-a');
|
||||
const tokenA = await createToken(accountA.id, 'unknown-provider-token-a');
|
||||
const gptChannelA = await db.insert(schema.routeChannels).values({
|
||||
routeId: gptRoute.id,
|
||||
accountId: accountA.id,
|
||||
tokenId: tokenA.id,
|
||||
priority: 0,
|
||||
weight: 10,
|
||||
enabled: true,
|
||||
}).returning().get();
|
||||
await db.insert(schema.routeChannels).values({
|
||||
routeId: claudeRoute.id,
|
||||
accountId: accountA.id,
|
||||
tokenId: tokenA.id,
|
||||
priority: 0,
|
||||
weight: 10,
|
||||
enabled: true,
|
||||
}).run();
|
||||
|
||||
const siteB = await createSite('unknown-provider-b');
|
||||
const accountB = await createAccount(siteB.id, 'unknown-provider-user-b');
|
||||
const tokenB = await createToken(accountB.id, 'unknown-provider-token-b');
|
||||
await db.insert(schema.routeChannels).values([
|
||||
{
|
||||
routeId: gptRoute.id,
|
||||
accountId: accountB.id,
|
||||
tokenId: tokenB.id,
|
||||
priority: 0,
|
||||
weight: 10,
|
||||
enabled: true,
|
||||
},
|
||||
{
|
||||
routeId: claudeRoute.id,
|
||||
accountId: accountB.id,
|
||||
tokenId: tokenB.id,
|
||||
priority: 0,
|
||||
weight: 10,
|
||||
enabled: true,
|
||||
},
|
||||
]).run();
|
||||
|
||||
const router = new TokenRouter();
|
||||
for (let index = 0; index < 3; index += 1) {
|
||||
await router.recordFailure(gptChannelA.id, {
|
||||
status: 502,
|
||||
errorText: 'unknown provider for model gpt-5.4',
|
||||
modelName: 'gpt-5.4',
|
||||
});
|
||||
}
|
||||
await db.update(schema.routeChannels).set({
|
||||
cooldownUntil: null,
|
||||
lastFailAt: null,
|
||||
failCount: 0,
|
||||
}).where(eq(schema.routeChannels.id, gptChannelA.id)).run();
|
||||
invalidateTokenRouterCache();
|
||||
|
||||
const gptDecision = await router.explainSelection('gpt-5.4');
|
||||
const claudeDecision = await router.explainSelection('claude-sonnet-4-6');
|
||||
const gptCandidateA = gptDecision.candidates.find((candidate) => candidate.siteName.startsWith('unknown-provider-a'));
|
||||
const claudeCandidateA = claudeDecision.candidates.find((candidate) => candidate.siteName.startsWith('unknown-provider-a'));
|
||||
|
||||
expect(gptCandidateA).toBeTruthy();
|
||||
expect(claudeCandidateA).toBeTruthy();
|
||||
expect(gptDecision.summary.join(' ')).not.toContain('站点熔断避让');
|
||||
expect(claudeDecision.summary.join(' ')).not.toContain('站点熔断避让');
|
||||
expect(gptCandidateA?.reason || '').not.toContain('站点熔断');
|
||||
expect(claudeCandidateA?.reason || '').not.toContain('站点熔断');
|
||||
expect((gptCandidateA?.probability || 0)).toBeLessThan((claudeCandidateA?.probability || 0));
|
||||
expect(gptCandidateA?.reason || '').toContain('模型=');
|
||||
});
|
||||
|
||||
it('stable_first deterministically chooses the healthiest candidate', async () => {
|
||||
config.routingWeights = {
|
||||
baseWeightFactor: 1,
|
||||
@@ -993,4 +1256,282 @@ describe('TokenRouter selection scoring', () => {
|
||||
expect(decision.summary.join(' ')).toContain('稳定优先');
|
||||
expect(decision.selectedChannelId).toBe(channelB.id);
|
||||
});
|
||||
|
||||
it('stable_first rotates across sites that remain inside the stable pool', async () => {
|
||||
config.routingWeights = {
|
||||
baseWeightFactor: 1,
|
||||
valueScoreFactor: 0,
|
||||
costWeight: 0,
|
||||
balanceWeight: 0,
|
||||
usageWeight: 0,
|
||||
};
|
||||
|
||||
const route = await db.insert(schema.tokenRoutes).values({
|
||||
modelPattern: 'gpt-5.4',
|
||||
routingStrategy: 'stable_first',
|
||||
enabled: true,
|
||||
}).returning().get();
|
||||
|
||||
const siteA = await createSite('stable-pool-a');
|
||||
const accountA = await createAccount(siteA.id, 'stable-pool-user-a');
|
||||
const tokenA = await createToken(accountA.id, 'stable-pool-token-a');
|
||||
const channelA = await db.insert(schema.routeChannels).values({
|
||||
routeId: route.id,
|
||||
accountId: accountA.id,
|
||||
tokenId: tokenA.id,
|
||||
priority: 0,
|
||||
weight: 10,
|
||||
enabled: true,
|
||||
}).returning().get();
|
||||
|
||||
const siteB = await createSite('stable-pool-b');
|
||||
const accountB = await createAccount(siteB.id, 'stable-pool-user-b');
|
||||
const tokenB = await createToken(accountB.id, 'stable-pool-token-b');
|
||||
const channelB = await db.insert(schema.routeChannels).values({
|
||||
routeId: route.id,
|
||||
accountId: accountB.id,
|
||||
tokenId: tokenB.id,
|
||||
priority: 0,
|
||||
weight: 9,
|
||||
enabled: true,
|
||||
}).returning().get();
|
||||
|
||||
const router = new TokenRouter();
|
||||
const first = await router.selectChannel('gpt-5.4');
|
||||
const second = await router.selectChannel('gpt-5.4');
|
||||
const third = await router.selectChannel('gpt-5.4');
|
||||
const decision = await router.explainSelection('gpt-5.4');
|
||||
|
||||
expect(first?.channel.id).toBe(channelA.id);
|
||||
expect(second?.channel.id).toBe(channelB.id);
|
||||
expect(third?.channel.id).toBe(channelA.id);
|
||||
expect(decision.summary.join(' ')).toContain('主池站点 2');
|
||||
});
|
||||
|
||||
it('stable_first rotates across healthy sites in configured priority order instead of stopping at the first layer', async () => {
|
||||
config.routingWeights = {
|
||||
baseWeightFactor: 1,
|
||||
valueScoreFactor: 0,
|
||||
costWeight: 0,
|
||||
balanceWeight: 0,
|
||||
usageWeight: 0,
|
||||
};
|
||||
|
||||
const route = await db.insert(schema.tokenRoutes).values({
|
||||
modelPattern: 'gpt-4.1',
|
||||
routingStrategy: 'stable_first',
|
||||
enabled: true,
|
||||
}).returning().get();
|
||||
|
||||
const siteA = await createSite('ordered-stable-a');
|
||||
const accountA = await createAccount(siteA.id, 'ordered-stable-user-a');
|
||||
const tokenA = await createToken(accountA.id, 'ordered-stable-token-a');
|
||||
const channelA = await db.insert(schema.routeChannels).values({
|
||||
routeId: route.id,
|
||||
accountId: accountA.id,
|
||||
tokenId: tokenA.id,
|
||||
priority: 0,
|
||||
weight: 10,
|
||||
enabled: true,
|
||||
}).returning().get();
|
||||
|
||||
const siteB = await createSite('ordered-stable-b');
|
||||
const accountB = await createAccount(siteB.id, 'ordered-stable-user-b');
|
||||
const tokenB = await createToken(accountB.id, 'ordered-stable-token-b');
|
||||
const channelB = await db.insert(schema.routeChannels).values({
|
||||
routeId: route.id,
|
||||
accountId: accountB.id,
|
||||
tokenId: tokenB.id,
|
||||
priority: 4,
|
||||
weight: 10,
|
||||
enabled: true,
|
||||
}).returning().get();
|
||||
|
||||
const siteC = await createSite('ordered-stable-c');
|
||||
const accountC = await createAccount(siteC.id, 'ordered-stable-user-c');
|
||||
const tokenC = await createToken(accountC.id, 'ordered-stable-token-c');
|
||||
const channelC = await db.insert(schema.routeChannels).values({
|
||||
routeId: route.id,
|
||||
accountId: accountC.id,
|
||||
tokenId: tokenC.id,
|
||||
priority: 8,
|
||||
weight: 10,
|
||||
enabled: true,
|
||||
}).returning().get();
|
||||
|
||||
const router = new TokenRouter();
|
||||
const first = await router.selectChannel('gpt-4.1');
|
||||
const second = await router.selectChannel('gpt-4.1');
|
||||
const third = await router.selectChannel('gpt-4.1');
|
||||
const fourth = await router.selectChannel('gpt-4.1');
|
||||
const decision = await router.explainSelection('gpt-4.1');
|
||||
|
||||
expect(first?.channel.id).toBe(channelA.id);
|
||||
expect(second?.channel.id).toBe(channelB.id);
|
||||
expect(third?.channel.id).toBe(channelC.id);
|
||||
expect(fourth?.channel.id).toBe(channelA.id);
|
||||
expect(decision.summary.join(' ')).toContain('按配置顺序轮询站点');
|
||||
});
|
||||
|
||||
it('stable_first gives observation-pool sites occasional real traffic without background probing', async () => {
|
||||
config.routingWeights = {
|
||||
baseWeightFactor: 1,
|
||||
valueScoreFactor: 0,
|
||||
costWeight: 0,
|
||||
balanceWeight: 0,
|
||||
usageWeight: 0,
|
||||
};
|
||||
|
||||
const route = await db.insert(schema.tokenRoutes).values({
|
||||
modelPattern: 'gpt-5.4-probe-free',
|
||||
routingStrategy: 'stable_first',
|
||||
enabled: true,
|
||||
}).returning().get();
|
||||
|
||||
const sitePrimary = await createSite('observation-primary');
|
||||
const accountPrimary = await createAccount(sitePrimary.id, 'observation-user-primary');
|
||||
const tokenPrimary = await createToken(accountPrimary.id, 'observation-token-primary');
|
||||
const primaryChannel = await db.insert(schema.routeChannels).values({
|
||||
routeId: route.id,
|
||||
accountId: accountPrimary.id,
|
||||
tokenId: tokenPrimary.id,
|
||||
priority: 0,
|
||||
weight: 10,
|
||||
enabled: true,
|
||||
successCount: 48,
|
||||
failCount: 0,
|
||||
totalLatencyMs: 48 * 320,
|
||||
}).returning().get();
|
||||
|
||||
const siteObservation = await createSite('observation-candidate');
|
||||
const accountObservation = await createAccount(siteObservation.id, 'observation-user-candidate');
|
||||
const tokenObservation = await createToken(accountObservation.id, 'observation-token-candidate');
|
||||
const observationChannel = await db.insert(schema.routeChannels).values({
|
||||
routeId: route.id,
|
||||
accountId: accountObservation.id,
|
||||
tokenId: tokenObservation.id,
|
||||
priority: 1,
|
||||
weight: 10,
|
||||
enabled: true,
|
||||
}).returning().get();
|
||||
|
||||
const router = new TokenRouter();
|
||||
const selectedChannelIds: number[] = [];
|
||||
for (let index = 0; index < 23; index += 1) {
|
||||
const selected = await router.selectChannel('gpt-5.4-probe-free');
|
||||
selectedChannelIds.push(selected?.channel.id ?? 0);
|
||||
}
|
||||
let decision = await router.explainSelection('gpt-5.4-probe-free');
|
||||
let observationCandidate = decision.candidates.find((candidate) => candidate.channelId === observationChannel.id);
|
||||
let primaryCandidate = decision.candidates.find((candidate) => candidate.channelId === primaryChannel.id);
|
||||
|
||||
expect(observationCandidate?.probability).toBe(100);
|
||||
expect(primaryCandidate?.probability).toBe(0);
|
||||
expect(decision.summary.join(' ')).toContain('本次命中观察池灰度流量');
|
||||
|
||||
const observationSelected = await router.selectChannel('gpt-5.4-probe-free');
|
||||
selectedChannelIds.push(observationSelected?.channel.id ?? 0);
|
||||
|
||||
decision = await router.explainSelection('gpt-5.4-probe-free');
|
||||
observationCandidate = decision.candidates.find((candidate) => candidate.channelId === observationChannel.id);
|
||||
primaryCandidate = decision.candidates.find((candidate) => candidate.channelId === primaryChannel.id);
|
||||
|
||||
expect(selectedChannelIds.filter((channelId) => channelId === observationChannel.id)).toHaveLength(1);
|
||||
expect(selectedChannelIds.filter((channelId) => channelId === primaryChannel.id).length).toBeGreaterThan(20);
|
||||
expect(decision.summary.join(' ')).toContain('观察池站点 1');
|
||||
expect(decision.summary.join(' ')).toContain('还需 23 次主池请求');
|
||||
expect(observationCandidate?.reason || '').toContain('观察池');
|
||||
expect(observationCandidate?.probability).toBe(0);
|
||||
expect(primaryCandidate?.reason || '').toContain('主池');
|
||||
expect((primaryCandidate?.probability || 0)).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('caps the stable_first rotation cache size', () => {
|
||||
invalidateTokenRouterCache();
|
||||
|
||||
for (let index = 0; index < 1200; index += 1) {
|
||||
tokenRouterTestUtils.rememberStableFirstSiteSelectionForKey(`route:${index}`, (index % 7) + 1);
|
||||
}
|
||||
|
||||
expect(tokenRouterTestUtils.getStableFirstRotationCacheSize()).toBeLessThanOrEqual(1024);
|
||||
});
|
||||
|
||||
it('penalizes saturated session-scoped channels using runtime load snapshots', async () => {
|
||||
config.routingWeights = {
|
||||
baseWeightFactor: 1,
|
||||
valueScoreFactor: 0,
|
||||
costWeight: 0,
|
||||
balanceWeight: 0,
|
||||
usageWeight: 0,
|
||||
};
|
||||
config.proxySessionChannelConcurrencyLimit = 1;
|
||||
config.proxySessionChannelQueueWaitMs = 5_000;
|
||||
|
||||
const route = await db.insert(schema.tokenRoutes).values({
|
||||
modelPattern: 'gpt-5.2',
|
||||
routingStrategy: 'stable_first',
|
||||
enabled: true,
|
||||
}).returning().get();
|
||||
const sessionExtraConfig = JSON.stringify({ credentialMode: 'session' });
|
||||
|
||||
const siteBusy = await createSite('runtime-load-busy');
|
||||
const accountBusy = await createAccount(siteBusy.id, 'runtime-load-user-busy', {
|
||||
extraConfig: sessionExtraConfig,
|
||||
});
|
||||
const tokenBusy = await createToken(accountBusy.id, 'runtime-load-token-busy');
|
||||
const channelBusy = await db.insert(schema.routeChannels).values({
|
||||
routeId: route.id,
|
||||
accountId: accountBusy.id,
|
||||
tokenId: tokenBusy.id,
|
||||
priority: 0,
|
||||
weight: 10,
|
||||
enabled: true,
|
||||
}).returning().get();
|
||||
|
||||
const siteFree = await createSite('runtime-load-free');
|
||||
const accountFree = await createAccount(siteFree.id, 'runtime-load-user-free', {
|
||||
extraConfig: sessionExtraConfig,
|
||||
});
|
||||
const tokenFree = await createToken(accountFree.id, 'runtime-load-token-free');
|
||||
const channelFree = await db.insert(schema.routeChannels).values({
|
||||
routeId: route.id,
|
||||
accountId: accountFree.id,
|
||||
tokenId: tokenFree.id,
|
||||
priority: 0,
|
||||
weight: 10,
|
||||
enabled: true,
|
||||
}).returning().get();
|
||||
|
||||
const activeLease = await proxyChannelCoordinator.acquireChannelLease({
|
||||
channelId: channelBusy.id,
|
||||
accountExtraConfig: accountBusy.extraConfig,
|
||||
});
|
||||
expect(activeLease.status).toBe('acquired');
|
||||
if (activeLease.status !== 'acquired') return;
|
||||
|
||||
const queuedLeasePromise = proxyChannelCoordinator.acquireChannelLease({
|
||||
channelId: channelBusy.id,
|
||||
accountExtraConfig: accountBusy.extraConfig,
|
||||
});
|
||||
await Promise.resolve();
|
||||
|
||||
const router = new TokenRouter();
|
||||
const preview = await router.previewSelectedChannel('gpt-5.2');
|
||||
const decision = await router.explainSelection('gpt-5.2');
|
||||
const busyCandidate = decision.candidates.find((candidate) => candidate.channelId === channelBusy.id);
|
||||
const freeCandidate = decision.candidates.find((candidate) => candidate.channelId === channelFree.id);
|
||||
|
||||
expect(preview?.channel.id).toBe(channelFree.id);
|
||||
expect(busyCandidate?.reason || '').toContain('会话负载=');
|
||||
expect(busyCandidate?.reason || '').toContain('活跃=1/1');
|
||||
expect(busyCandidate?.reason || '').toContain('等待=1');
|
||||
expect((busyCandidate?.probability || 0)).toBeLessThan((freeCandidate?.probability || 0));
|
||||
|
||||
activeLease.lease.release();
|
||||
const queuedLease = await queuedLeasePromise;
|
||||
expect(queuedLease.status).toBe('acquired');
|
||||
if (queuedLease.status === 'acquired') {
|
||||
queuedLease.lease.release();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
@@ -81,7 +81,7 @@ describe('filterRecentlyFailedCandidates', () => {
|
||||
expect(result.map((c) => c.id).sort()).toEqual(['a', 'b']);
|
||||
});
|
||||
|
||||
it('keeps candidates unchanged when avoidSec is omitted', () => {
|
||||
it('uses the default recent-failure backoff when avoidSec is omitted', () => {
|
||||
const nowMs = Date.now();
|
||||
const candidates: Candidate[] = [
|
||||
{
|
||||
@@ -101,7 +101,7 @@ describe('filterRecentlyFailedCandidates', () => {
|
||||
];
|
||||
|
||||
const result = filterRecentlyFailedCandidates(candidates, nowMs);
|
||||
expect(result).toEqual(candidates);
|
||||
expect(result.map((candidate) => candidate.id)).toEqual(['healthy']);
|
||||
});
|
||||
|
||||
it('does not penalize stale failures outside the avoidance window', () => {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,6 @@
|
||||
export {
|
||||
buildClaudeCountTokensUpstreamRequest,
|
||||
buildUpstreamEndpointRequest,
|
||||
resolveUpstreamEndpointCandidates,
|
||||
type UpstreamEndpoint,
|
||||
} from '../routes/proxy/upstreamEndpoint.js';
|
||||
@@ -33,6 +33,7 @@ export type EndpointCapabilityProfile = {
|
||||
hasNonImageFileInput: boolean;
|
||||
hasRemoteDocumentUrl: boolean;
|
||||
wantsNativeResponsesReasoning: boolean;
|
||||
wantsContinuationAwareResponses: boolean;
|
||||
};
|
||||
|
||||
type EndpointRuntimeState = {
|
||||
@@ -88,6 +89,7 @@ export function buildEndpointCapabilityProfile(input?: {
|
||||
hasNonImageFileInput?: boolean;
|
||||
conversationFileSummary?: ConversationFileInputSummary;
|
||||
wantsNativeResponsesReasoning?: boolean;
|
||||
wantsContinuationAwareResponses?: boolean;
|
||||
};
|
||||
}): EndpointCapabilityProfile {
|
||||
const conversationFileSummary = input?.requestCapabilities?.conversationFileSummary;
|
||||
@@ -107,6 +109,7 @@ export function buildEndpointCapabilityProfile(input?: {
|
||||
conversationFileSummary?.hasRemoteDocumentUrl === true
|
||||
),
|
||||
wantsNativeResponsesReasoning: input?.requestCapabilities?.wantsNativeResponsesReasoning === true,
|
||||
wantsContinuationAwareResponses: input?.requestCapabilities?.wantsContinuationAwareResponses === true,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -131,6 +134,7 @@ function buildEndpointRuntimeStateKey(input: {
|
||||
capabilityProfile.hasNonImageFileInput ? 'files' : 'nofiles',
|
||||
capabilityProfile.hasRemoteDocumentUrl ? 'remoteurl' : 'noremoteurl',
|
||||
capabilityProfile.wantsNativeResponsesReasoning ? 'reasoning' : 'noreasoning',
|
||||
capabilityProfile.wantsContinuationAwareResponses ? 'continuation' : 'nocontinuation',
|
||||
].join(':');
|
||||
}
|
||||
|
||||
@@ -247,6 +251,7 @@ export function getUpstreamEndpointRuntimeStateSnapshot(input: {
|
||||
hasNonImageFileInput?: boolean;
|
||||
conversationFileSummary?: ConversationFileInputSummary;
|
||||
wantsNativeResponsesReasoning?: boolean;
|
||||
wantsContinuationAwareResponses?: boolean;
|
||||
};
|
||||
}) {
|
||||
const capabilityProfile = buildEndpointCapabilityProfile({
|
||||
@@ -349,6 +354,7 @@ export function recordUpstreamEndpointSuccess(input: {
|
||||
hasNonImageFileInput?: boolean;
|
||||
conversationFileSummary?: ConversationFileInputSummary;
|
||||
wantsNativeResponsesReasoning?: boolean;
|
||||
wantsContinuationAwareResponses?: boolean;
|
||||
};
|
||||
}): UpstreamEndpointRuntimeMemoryWrite | null {
|
||||
const capabilityProfile = buildEndpointCapabilityProfile({
|
||||
@@ -390,6 +396,7 @@ export function recordUpstreamEndpointFailure(input: {
|
||||
hasNonImageFileInput?: boolean;
|
||||
conversationFileSummary?: ConversationFileInputSummary;
|
||||
wantsNativeResponsesReasoning?: boolean;
|
||||
wantsContinuationAwareResponses?: boolean;
|
||||
};
|
||||
}): UpstreamEndpointRuntimeMemoryWrite | null {
|
||||
const capabilityProfile = buildEndpointCapabilityProfile({
|
||||
|
||||
Reference in New Issue
Block a user