feat(server): add proactive channel probes and load-aware stable routing / 服务端新增主动探活与负载感知稳定路由 (#330)
* wip: preserve local changes before upstream merge * Improve proxy routing stability and add stable-first strategy * Sync explicit-group routing strategy to eligible source routes * Fix proxy retry and runtime health review issues * fix: make proxy channel retry attempts configurable * fix: harden proxy retry recovery and bookkeeping * fix: address pr 257 review comments * feat(proxy): add sticky session channel coordination * feat(server): add proactive channel probes and load-aware stable routing * test(server): stabilize session load routing selection assertion * fix(proxy): harden codex forwarding stability * fix(router): rotate stable-first channels by healthy order * fix(server): address probe review feedback * fix(server): address probe review follow-ups * fix(ci): address CodeQL and core test failures * feat(server): refine stable-first routing recovery and observation * Finish PR 330 settings and review fixes * Fix chat site endpoint CI mock * Address latest PR 330 review comments --------- Co-authored-by: Cita <juricek.chen@gmail.com>
This commit is contained in:
Bainily
@@ -0,0 +1,106 @@
|
||||
import Fastify, { type FastifyInstance } from 'fastify';
|
||||
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { mkdtempSync, rmSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { eq } from 'drizzle-orm';
|
||||
|
||||
const { startModelAvailabilityProbeSchedulerMock, stopModelAvailabilityProbeSchedulerMock } = vi.hoisted(() => ({
|
||||
startModelAvailabilityProbeSchedulerMock: vi.fn(),
|
||||
stopModelAvailabilityProbeSchedulerMock: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock('../../services/modelAvailabilityProbeService.js', () => ({
|
||||
startModelAvailabilityProbeScheduler: startModelAvailabilityProbeSchedulerMock,
|
||||
stopModelAvailabilityProbeScheduler: stopModelAvailabilityProbeSchedulerMock,
|
||||
}));
|
||||
|
||||
type ConfigModule = typeof import('../../config.js');
|
||||
type DbModule = typeof import('../../db/index.js');
|
||||
|
||||
describe('settings model availability probe runtime setting', () => {
|
||||
let app: FastifyInstance;
|
||||
let config: ConfigModule['config'];
|
||||
let db: DbModule['db'];
|
||||
let schema: DbModule['schema'];
|
||||
let dataDir = '';
|
||||
|
||||
beforeAll(async () => {
|
||||
dataDir = mkdtempSync(join(tmpdir(), 'metapi-settings-model-probe-'));
|
||||
process.env.DATA_DIR = dataDir;
|
||||
|
||||
await import('../../db/migrate.js');
|
||||
const dbModule = await import('../../db/index.js');
|
||||
const configModule = await import('../../config.js');
|
||||
const settingsRoutesModule = await import('./settings.js');
|
||||
|
||||
db = dbModule.db;
|
||||
schema = dbModule.schema;
|
||||
config = configModule.config;
|
||||
|
||||
app = Fastify();
|
||||
await app.register(settingsRoutesModule.settingsRoutes);
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
await db.delete(schema.settings).run();
|
||||
config.modelAvailabilityProbeEnabled = false;
|
||||
startModelAvailabilityProbeSchedulerMock.mockReset();
|
||||
stopModelAvailabilityProbeSchedulerMock.mockReset();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await app.close();
|
||||
rmSync(dataDir, { recursive: true, force: true });
|
||||
delete process.env.DATA_DIR;
|
||||
});
|
||||
|
||||
it('persists enabling the model availability probe and starts the scheduler', async () => {
|
||||
const updateResponse = await app.inject({
|
||||
method: 'PUT',
|
||||
url: '/api/settings/runtime',
|
||||
payload: {
|
||||
modelAvailabilityProbeEnabled: true,
|
||||
},
|
||||
});
|
||||
|
||||
expect(updateResponse.statusCode).toBe(200);
|
||||
const updated = updateResponse.json() as { modelAvailabilityProbeEnabled?: boolean };
|
||||
expect(updated.modelAvailabilityProbeEnabled).toBe(true);
|
||||
expect(config.modelAvailabilityProbeEnabled).toBe(true);
|
||||
expect(startModelAvailabilityProbeSchedulerMock).toHaveBeenCalledTimes(1);
|
||||
expect(stopModelAvailabilityProbeSchedulerMock).not.toHaveBeenCalled();
|
||||
|
||||
const saved = await db.select().from(schema.settings).where(eq(schema.settings.key, 'model_availability_probe_enabled')).get();
|
||||
expect(saved?.value).toBe(JSON.stringify(true));
|
||||
|
||||
const getResponse = await app.inject({
|
||||
method: 'GET',
|
||||
url: '/api/settings/runtime',
|
||||
});
|
||||
expect(getResponse.statusCode).toBe(200);
|
||||
expect((getResponse.json() as { modelAvailabilityProbeEnabled?: boolean }).modelAvailabilityProbeEnabled).toBe(true);
|
||||
});
|
||||
|
||||
it('persists disabling the model availability probe and stops the scheduler', async () => {
|
||||
config.modelAvailabilityProbeEnabled = true;
|
||||
|
||||
const updateResponse = await app.inject({
|
||||
method: 'PUT',
|
||||
url: '/api/settings/runtime',
|
||||
payload: {
|
||||
modelAvailabilityProbeEnabled: false,
|
||||
},
|
||||
});
|
||||
|
||||
expect(updateResponse.statusCode).toBe(200);
|
||||
const updated = updateResponse.json() as { modelAvailabilityProbeEnabled?: boolean };
|
||||
expect(updated.modelAvailabilityProbeEnabled).toBe(false);
|
||||
expect(config.modelAvailabilityProbeEnabled).toBe(false);
|
||||
expect(stopModelAvailabilityProbeSchedulerMock).toHaveBeenCalledTimes(1);
|
||||
expect(startModelAvailabilityProbeSchedulerMock).not.toHaveBeenCalled();
|
||||
|
||||
const saved = await db.select().from(schema.settings).where(eq(schema.settings.key, 'model_availability_probe_enabled')).get();
|
||||
expect(saved?.value).toBe(JSON.stringify(false));
|
||||
});
|
||||
});
|
||||
@@ -40,12 +40,17 @@ import { invalidateSiteProxyCache, normalizeSiteProxyUrl, withExplicitProxyReque
|
||||
import { performFactoryReset } from '../../services/factoryResetService.js';
|
||||
import { normalizeLogCleanupRetentionDays } from '../../services/logCleanupService.js';
|
||||
import { stopProxyLogRetentionService } from '../../services/proxyLogRetentionService.js';
|
||||
import {
|
||||
startModelAvailabilityProbeScheduler,
|
||||
stopModelAvailabilityProbeScheduler,
|
||||
} from '../../services/modelAvailabilityProbeService.js';
|
||||
|
||||
type RoutingWeights = typeof config.routingWeights;
|
||||
|
||||
interface RuntimeSettingsBody {
|
||||
proxyToken?: string;
|
||||
systemProxyUrl?: string;
|
||||
modelAvailabilityProbeEnabled?: boolean;
|
||||
codexUpstreamWebsocketEnabled?: boolean;
|
||||
disableCrossProtocolFallback?: boolean;
|
||||
proxySessionChannelConcurrencyLimit?: number;
|
||||
@@ -404,6 +409,16 @@ function applyImportedSettingToRuntime(key: string, value: unknown) {
|
||||
config.systemProxyUrl = normalizeSiteProxyUrl(value) || '';
|
||||
return;
|
||||
}
|
||||
case 'model_availability_probe_enabled': {
|
||||
if (typeof value !== 'boolean') return;
|
||||
config.modelAvailabilityProbeEnabled = value;
|
||||
if (value) {
|
||||
startModelAvailabilityProbeScheduler();
|
||||
} else {
|
||||
stopModelAvailabilityProbeScheduler();
|
||||
}
|
||||
return;
|
||||
}
|
||||
case 'codex_upstream_websocket_enabled': {
|
||||
if (typeof value !== 'boolean') return;
|
||||
config.codexUpstreamWebsocketEnabled = value;
|
||||
@@ -687,6 +702,7 @@ function getRuntimeSettingsResponse(currentAdminIp = '') {
|
||||
logCleanupUsageLogsEnabled: config.logCleanupUsageLogsEnabled,
|
||||
logCleanupProgramLogsEnabled: config.logCleanupProgramLogsEnabled,
|
||||
logCleanupRetentionDays: config.logCleanupRetentionDays,
|
||||
modelAvailabilityProbeEnabled: config.modelAvailabilityProbeEnabled,
|
||||
codexUpstreamWebsocketEnabled: config.codexUpstreamWebsocketEnabled,
|
||||
disableCrossProtocolFallback: config.disableCrossProtocolFallback,
|
||||
proxySessionChannelConcurrencyLimit: config.proxySessionChannelConcurrencyLimit,
|
||||
@@ -1101,6 +1117,29 @@ export async function settingsRoutes(app: FastifyInstance) {
|
||||
invalidateSiteProxyCache();
|
||||
}
|
||||
|
||||
if (body.modelAvailabilityProbeEnabled !== undefined) {
|
||||
let nextValue = false;
|
||||
try {
|
||||
nextValue = parseBooleanFlag(body.modelAvailabilityProbeEnabled, '批量测活开关');
|
||||
} catch (err: any) {
|
||||
return reply.code(400).send({
|
||||
success: false,
|
||||
message: err?.message || '批量测活开关格式无效',
|
||||
});
|
||||
}
|
||||
|
||||
if (nextValue !== config.modelAvailabilityProbeEnabled) {
|
||||
changedLabels.push(nextValue ? '开启批量测活' : '关闭批量测活');
|
||||
}
|
||||
await upsertSetting('model_availability_probe_enabled', nextValue);
|
||||
config.modelAvailabilityProbeEnabled = nextValue;
|
||||
if (nextValue) {
|
||||
startModelAvailabilityProbeScheduler();
|
||||
} else {
|
||||
stopModelAvailabilityProbeScheduler();
|
||||
}
|
||||
}
|
||||
|
||||
if (body.codexUpstreamWebsocketEnabled !== undefined) {
|
||||
let nextValue = false;
|
||||
try {
|
||||
|
||||
@@ -0,0 +1,157 @@
|
||||
import Fastify, { type FastifyInstance } from 'fastify';
|
||||
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { mkdtempSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
|
||||
const buildModelAvailabilityProbeTaskDedupeKeyMock = vi.fn();
|
||||
const queueModelAvailabilityProbeTaskMock = vi.fn();
|
||||
const getBackgroundTaskMock = vi.fn();
|
||||
const getRunningTaskByDedupeKeyMock = vi.fn();
|
||||
const waitForBackgroundTaskCompletionMock = vi.fn();
|
||||
|
||||
vi.mock('../../services/modelAvailabilityProbeService.js', async () => {
|
||||
const actual = await vi.importActual<typeof import('../../services/modelAvailabilityProbeService.js')>('../../services/modelAvailabilityProbeService.js');
|
||||
return {
|
||||
...actual,
|
||||
buildModelAvailabilityProbeTaskDedupeKey: (...args: unknown[]) => buildModelAvailabilityProbeTaskDedupeKeyMock(...args),
|
||||
queueModelAvailabilityProbeTask: (...args: unknown[]) => queueModelAvailabilityProbeTaskMock(...args),
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock('../../services/backgroundTaskService.js', async () => {
|
||||
const actual = await vi.importActual<typeof import('../../services/backgroundTaskService.js')>('../../services/backgroundTaskService.js');
|
||||
return {
|
||||
...actual,
|
||||
getBackgroundTask: (...args: unknown[]) => getBackgroundTaskMock(...args),
|
||||
getRunningTaskByDedupeKey: (...args: unknown[]) => getRunningTaskByDedupeKeyMock(...args),
|
||||
waitForBackgroundTaskCompletion: (...args: unknown[]) => waitForBackgroundTaskCompletionMock(...args),
|
||||
};
|
||||
});
|
||||
|
||||
describe('/api/models/probe', () => {
|
||||
let app: FastifyInstance;
|
||||
let dataDir = '';
|
||||
let originalDataDir: string | undefined;
|
||||
|
||||
beforeAll(async () => {
|
||||
dataDir = mkdtempSync(join(tmpdir(), 'metapi-stats-model-probe-'));
|
||||
originalDataDir = process.env.DATA_DIR;
|
||||
process.env.DATA_DIR = dataDir;
|
||||
|
||||
await import('../../db/migrate.js');
|
||||
const routesModule = await import('./stats.js');
|
||||
app = Fastify();
|
||||
await app.register(routesModule.statsRoutes);
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
buildModelAvailabilityProbeTaskDedupeKeyMock.mockReset();
|
||||
queueModelAvailabilityProbeTaskMock.mockReset();
|
||||
getBackgroundTaskMock.mockReset();
|
||||
getRunningTaskByDedupeKeyMock.mockReset();
|
||||
waitForBackgroundTaskCompletionMock.mockReset();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await app.close();
|
||||
if (originalDataDir === undefined) {
|
||||
delete process.env.DATA_DIR;
|
||||
} else {
|
||||
process.env.DATA_DIR = originalDataDir;
|
||||
}
|
||||
});
|
||||
|
||||
it('rejects non-object request bodies instead of defaulting to a full probe', async () => {
|
||||
const response = await app.inject({
|
||||
method: 'POST',
|
||||
url: '/api/models/probe',
|
||||
headers: {
|
||||
'content-type': 'application/json',
|
||||
},
|
||||
payload: '"oops"',
|
||||
});
|
||||
|
||||
expect(response.statusCode).toBe(400);
|
||||
expect(queueModelAvailabilityProbeTaskMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('rejects loosely formatted account ids instead of truncating them', async () => {
|
||||
const response = await app.inject({
|
||||
method: 'POST',
|
||||
url: '/api/models/probe',
|
||||
payload: {
|
||||
accountId: '1e2',
|
||||
},
|
||||
});
|
||||
|
||||
expect(response.statusCode).toBe(400);
|
||||
expect(queueModelAvailabilityProbeTaskMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('rejects account ids that exceed the safe integer range', async () => {
|
||||
const response = await app.inject({
|
||||
method: 'POST',
|
||||
url: '/api/models/probe',
|
||||
payload: {
|
||||
accountId: '9007199254740993',
|
||||
},
|
||||
});
|
||||
|
||||
expect(response.statusCode).toBe(400);
|
||||
expect(queueModelAvailabilityProbeTaskMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('reuses the running deduped task for wait=true requests', async () => {
|
||||
buildModelAvailabilityProbeTaskDedupeKeyMock.mockReturnValue('model-availability-probe-7');
|
||||
getRunningTaskByDedupeKeyMock.mockReturnValue({
|
||||
id: 'task-7',
|
||||
status: 'running',
|
||||
});
|
||||
const completedTask = {
|
||||
id: 'task-7',
|
||||
status: 'succeeded',
|
||||
result: {
|
||||
results: [],
|
||||
summary: {
|
||||
totalAccounts: 1,
|
||||
success: 1,
|
||||
failed: 0,
|
||||
skipped: 0,
|
||||
scanned: 3,
|
||||
supported: 2,
|
||||
unsupported: 1,
|
||||
inconclusive: 0,
|
||||
skippedModels: 0,
|
||||
updatedRows: 1,
|
||||
rebuiltRoutes: true,
|
||||
},
|
||||
},
|
||||
};
|
||||
waitForBackgroundTaskCompletionMock.mockResolvedValue(completedTask);
|
||||
|
||||
const response = await app.inject({
|
||||
method: 'POST',
|
||||
url: '/api/models/probe',
|
||||
payload: {
|
||||
accountId: 7,
|
||||
wait: true,
|
||||
},
|
||||
});
|
||||
|
||||
expect(response.statusCode).toBe(200);
|
||||
expect(queueModelAvailabilityProbeTaskMock).not.toHaveBeenCalled();
|
||||
expect(waitForBackgroundTaskCompletionMock).toHaveBeenCalledWith('task-7');
|
||||
expect(response.json()).toMatchObject({
|
||||
success: true,
|
||||
reused: true,
|
||||
jobId: 'task-7',
|
||||
summary: {
|
||||
totalAccounts: 1,
|
||||
scanned: 3,
|
||||
supported: 2,
|
||||
unsupported: 1,
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -6,8 +6,18 @@ import { refreshModelsForAccount } from '../../services/modelService.js';
|
||||
import * as routeRefreshWorkflow from '../../services/routeRefreshWorkflow.js';
|
||||
import { buildModelAnalysis } from '../../services/modelAnalysisService.js';
|
||||
import { fallbackTokenCost, fetchModelPricingCatalog } from '../../services/modelPricingService.js';
|
||||
import {
|
||||
buildModelAvailabilityProbeTaskDedupeKey,
|
||||
queueModelAvailabilityProbeTask,
|
||||
type ModelAvailabilityProbeExecutionResult,
|
||||
} from '../../services/modelAvailabilityProbeService.js';
|
||||
import { getUpstreamModelDescriptionsCached } from '../../services/upstreamModelDescriptionService.js';
|
||||
import { getRunningTaskByDedupeKey, startBackgroundTask } from '../../services/backgroundTaskService.js';
|
||||
import {
|
||||
getBackgroundTask,
|
||||
getRunningTaskByDedupeKey,
|
||||
startBackgroundTask,
|
||||
waitForBackgroundTaskCompletion,
|
||||
} from '../../services/backgroundTaskService.js';
|
||||
import { parseCheckinRewardAmount } from '../../services/checkinRewardParser.js';
|
||||
import { estimateRewardWithTodayIncomeFallback } from '../../services/todayIncomeRewardService.js';
|
||||
import {
|
||||
@@ -39,6 +49,10 @@ function parseBooleanFlag(raw?: string): boolean {
|
||||
return normalized === '1' || normalized === 'true' || normalized === 'yes';
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return !!value && typeof value === 'object' && !Array.isArray(value);
|
||||
}
|
||||
|
||||
const MODELS_MARKETPLACE_BASE_TTL_MS = 15_000;
|
||||
const MODELS_MARKETPLACE_PRICING_TTL_MS = 90_000;
|
||||
const limitModelTokenCandidatesRead = createRateLimitGuard({
|
||||
@@ -1462,6 +1476,92 @@ export async function statsRoutes(app: FastifyInstance) {
|
||||
return { success: true, refresh, rebuild };
|
||||
});
|
||||
|
||||
app.post<{ Body?: { accountId?: number; wait?: boolean } }>('/api/models/probe', async (request, reply) => {
|
||||
const requestBody = request.body;
|
||||
if (requestBody !== undefined && !isRecord(requestBody)) {
|
||||
return reply.code(400).send({ success: false, message: '请求体必须是对象' });
|
||||
}
|
||||
|
||||
const rawAccountId = requestBody?.accountId as unknown;
|
||||
const normalizedAccountId = rawAccountId === undefined || rawAccountId === null
|
||||
? ''
|
||||
: String(rawAccountId).trim();
|
||||
const hasAccountId = normalizedAccountId !== '';
|
||||
const parsedAccountId = hasAccountId && /^[1-9]\d*$/.test(normalizedAccountId)
|
||||
? Number(normalizedAccountId)
|
||||
: undefined;
|
||||
const accountId = parsedAccountId !== undefined && Number.isSafeInteger(parsedAccountId)
|
||||
? parsedAccountId
|
||||
: undefined;
|
||||
const wait = requestBody?.wait === true;
|
||||
|
||||
if (hasAccountId && accountId === undefined) {
|
||||
return reply.code(400).send({ success: false, message: '账号 ID 无效' });
|
||||
}
|
||||
|
||||
if (wait) {
|
||||
const taskTitle = accountId ? `探测模型可用性 #${accountId}` : '探测全部模型可用性';
|
||||
const dedupeKey = buildModelAvailabilityProbeTaskDedupeKey(accountId);
|
||||
const runningTask = getRunningTaskByDedupeKey(dedupeKey);
|
||||
const { task, reused } = runningTask
|
||||
? { task: runningTask, reused: true }
|
||||
: queueModelAvailabilityProbeTask({
|
||||
accountId,
|
||||
title: taskTitle,
|
||||
});
|
||||
const completedTask = await waitForBackgroundTaskCompletion(task.id);
|
||||
if (!completedTask) {
|
||||
return reply.code(500).send({ success: false, message: '模型可用性探测任务不存在或已过期' });
|
||||
}
|
||||
if (completedTask.status === 'failed') {
|
||||
return reply.code(500).send({
|
||||
success: false,
|
||||
reused,
|
||||
jobId: completedTask.id,
|
||||
status: completedTask.status,
|
||||
message: completedTask.error || '模型可用性探测失败',
|
||||
});
|
||||
}
|
||||
const result = completedTask.result as ModelAvailabilityProbeExecutionResult | null;
|
||||
if (!result) {
|
||||
return reply.code(500).send({
|
||||
success: false,
|
||||
reused,
|
||||
jobId: completedTask.id,
|
||||
status: completedTask.status,
|
||||
message: '模型可用性探测结果为空',
|
||||
});
|
||||
}
|
||||
if (accountId && result.summary.totalAccounts === 0) {
|
||||
return reply.code(404).send({ success: false, message: '账号不存在' });
|
||||
}
|
||||
return {
|
||||
success: true,
|
||||
reused,
|
||||
jobId: completedTask.id,
|
||||
status: completedTask.status,
|
||||
...result,
|
||||
};
|
||||
}
|
||||
|
||||
const taskTitle = accountId ? `探测模型可用性 #${accountId}` : '探测全部模型可用性';
|
||||
const { task, reused } = queueModelAvailabilityProbeTask({
|
||||
accountId,
|
||||
title: taskTitle,
|
||||
});
|
||||
|
||||
return reply.code(202).send({
|
||||
success: true,
|
||||
queued: true,
|
||||
reused,
|
||||
jobId: task.id,
|
||||
status: task.status,
|
||||
message: reused
|
||||
? '模型可用性探测任务进行中，请稍后查看任务列表'
|
||||
: '已开始模型可用性探测，请稍后查看任务列表',
|
||||
});
|
||||
});
|
||||
|
||||
// Site distribution – per-site aggregate data
|
||||
app.get('/api/stats/site-distribution', async () => {
|
||||
const accountRows = await db.select({
|
||||
|
||||
Reference in New Issue
Block a user