feat(tests): add comprehensive tests for TokenRouter and related components
- Implement tests for TokenRouter downstream policy, covering allowedRouteIds and site weight multipliers. - Add tests for TokenRouter patterns and model mapping, including regex pattern matching and model mapping precedence. - Create tests for BrandIcon component to verify brand detection logic. - Develop tests for Accounts page, focusing on rebind panel behavior and refresh action on error. - Introduce fuzzy search utility tests to validate substring and subsequence matching. - Implement navigation focus helpers tests for building and parsing focus paths. - Add progressive render tests to ensure correct initial and next visible counts. - Create route model candidates index tests to validate route candidate indexing and matching logic.
This commit is contained in:
cita
@@ -448,6 +448,89 @@ describe('chat proxy stream behavior', () => {
|
||||
expect(forwardedBody.thinking).toEqual({ type: 'enabled', budget_tokens: 1024 });
|
||||
});
|
||||
|
||||
it('normalizes Claude thinking adaptive type for legacy upstreams on /v1/messages', async () => {
|
||||
fetchMock.mockResolvedValue(new Response(JSON.stringify({
|
||||
id: 'msg_headers_adaptive',
|
||||
type: 'message',
|
||||
model: 'claude-opus-4-6',
|
||||
content: [{ type: 'text', text: 'ok' }],
|
||||
stop_reason: 'end_turn',
|
||||
}), {
|
||||
status: 200,
|
||||
headers: { 'content-type': 'application/json' },
|
||||
}));
|
||||
|
||||
const response = await app.inject({
|
||||
method: 'POST',
|
||||
url: '/v1/messages',
|
||||
payload: {
|
||||
model: 'claude-opus-4-6',
|
||||
max_tokens: 256,
|
||||
messages: [{ role: 'user', content: 'hello' }],
|
||||
thinking: { type: 'adaptive', budget_tokens: 1024 },
|
||||
},
|
||||
});
|
||||
|
||||
expect(response.statusCode).toBe(200);
|
||||
|
||||
const [_targetUrl, options] = fetchMock.mock.calls[0] as [string, any];
|
||||
const forwardedBody = JSON.parse(options.body);
|
||||
expect(forwardedBody.thinking).toEqual({ type: 'enabled', budget_tokens: 1024 });
|
||||
});
|
||||
|
||||
it('retries /v1/messages with normalized Claude body when upstream says messages is required', async () => {
|
||||
fetchMock
|
||||
.mockResolvedValueOnce(new Response(JSON.stringify({
|
||||
error: {
|
||||
type: '<nil>',
|
||||
message: 'messages is required',
|
||||
},
|
||||
}), {
|
||||
status: 400,
|
||||
headers: { 'content-type': 'application/json' },
|
||||
}))
|
||||
.mockResolvedValueOnce(new Response(JSON.stringify({
|
||||
id: 'msg_retry_ok',
|
||||
type: 'message',
|
||||
model: 'claude-opus-4-6',
|
||||
content: [{ type: 'text', text: 'ok after normalized fallback' }],
|
||||
stop_reason: 'end_turn',
|
||||
}), {
|
||||
status: 200,
|
||||
headers: { 'content-type': 'application/json' },
|
||||
}));
|
||||
|
||||
const response = await app.inject({
|
||||
method: 'POST',
|
||||
url: '/v1/messages',
|
||||
payload: {
|
||||
model: 'claude-opus-4-6',
|
||||
max_tokens: 256,
|
||||
messages: [
|
||||
{
|
||||
role: 'user',
|
||||
content: [
|
||||
{ type: 'text', text: 'hello' },
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
expect(response.statusCode).toBe(200);
|
||||
expect(fetchMock.mock.calls.length).toBe(2);
|
||||
|
||||
const [_firstUrl, firstOptions] = fetchMock.mock.calls[0] as [string, any];
|
||||
const [_secondUrl, secondOptions] = fetchMock.mock.calls[1] as [string, any];
|
||||
const firstBody = JSON.parse(firstOptions.body);
|
||||
const secondBody = JSON.parse(secondOptions.body);
|
||||
|
||||
expect(Array.isArray(firstBody.messages)).toBe(true);
|
||||
expect(Array.isArray(secondBody.messages)).toBe(true);
|
||||
expect(typeof secondBody.messages[0]?.content).toBe('string');
|
||||
expect(secondBody.messages[0]?.content).toContain('hello');
|
||||
});
|
||||
|
||||
it('passes through Claude tool_use SSE events on /v1/messages for CLI tool execution', async () => {
|
||||
const encoder = new TextEncoder();
|
||||
const upstreamBody = new ReadableStream<Uint8Array>({
|
||||
@@ -805,6 +888,133 @@ describe('chat proxy stream behavior', () => {
|
||||
expect(targetUrl).toContain('/v1/messages');
|
||||
});
|
||||
|
||||
it('forces openai platform to use /v1/chat/completions for claude downstream requests', async () => {
|
||||
selectChannelMock.mockReturnValue({
|
||||
channel: { id: 11, routeId: 22 },
|
||||
site: { name: 'openai-site', url: 'https://api.openai.com', platform: 'openai' },
|
||||
account: { id: 33, username: 'demo-user' },
|
||||
tokenName: 'default',
|
||||
tokenValue: 'sk-openai',
|
||||
actualModel: 'gpt-4o-mini',
|
||||
});
|
||||
|
||||
fetchMock.mockResolvedValue(new Response(JSON.stringify({
|
||||
id: 'chatcmpl-openai-for-claude-downstream',
|
||||
object: 'chat.completion',
|
||||
created: 1_706_000_003,
|
||||
model: 'gpt-4o-mini',
|
||||
choices: [{
|
||||
index: 0,
|
||||
message: { role: 'assistant', content: 'openai endpoint selected' },
|
||||
finish_reason: 'stop',
|
||||
}],
|
||||
usage: { prompt_tokens: 4, completion_tokens: 3, total_tokens: 7 },
|
||||
}), {
|
||||
status: 200,
|
||||
headers: { 'content-type': 'application/json' },
|
||||
}));
|
||||
|
||||
const response = await app.inject({
|
||||
method: 'POST',
|
||||
url: '/v1/messages',
|
||||
payload: {
|
||||
model: 'gpt-4o-mini',
|
||||
max_tokens: 128,
|
||||
messages: [{ role: 'user', content: 'hello' }],
|
||||
},
|
||||
});
|
||||
|
||||
expect(response.statusCode).toBe(200);
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||
const [targetUrl] = fetchMock.mock.calls[0] as [string, any];
|
||||
expect(targetUrl).toContain('/v1/chat/completions');
|
||||
expect(targetUrl).not.toContain('/v1/messages');
|
||||
});
|
||||
|
||||
it('forces claude platform to use /v1/messages with x-api-key auth for openai downstream requests', async () => {
|
||||
selectChannelMock.mockReturnValue({
|
||||
channel: { id: 11, routeId: 22 },
|
||||
site: { name: 'claude-site', url: 'https://api.anthropic.com', platform: 'claude' },
|
||||
account: { id: 33, username: 'demo-user' },
|
||||
tokenName: 'default',
|
||||
tokenValue: 'sk-claude',
|
||||
actualModel: 'claude-sonnet-4-5-20250929',
|
||||
});
|
||||
|
||||
fetchMock.mockResolvedValue(new Response(JSON.stringify({
|
||||
id: 'msg_claude_upstream',
|
||||
type: 'message',
|
||||
model: 'claude-sonnet-4-5-20250929',
|
||||
content: [{ type: 'text', text: 'claude endpoint selected' }],
|
||||
stop_reason: 'end_turn',
|
||||
usage: { input_tokens: 5, output_tokens: 2, total_tokens: 7 },
|
||||
}), {
|
||||
status: 200,
|
||||
headers: { 'content-type': 'application/json' },
|
||||
}));
|
||||
|
||||
const response = await app.inject({
|
||||
method: 'POST',
|
||||
url: '/v1/chat/completions',
|
||||
payload: {
|
||||
model: 'claude-sonnet-4-5-20250929',
|
||||
stream: false,
|
||||
messages: [{ role: 'user', content: 'hello' }],
|
||||
},
|
||||
});
|
||||
|
||||
expect(response.statusCode).toBe(200);
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||
const [targetUrl, options] = fetchMock.mock.calls[0] as [string, any];
|
||||
expect(targetUrl).toContain('/v1/messages');
|
||||
expect(options.headers['x-api-key']).toBe('sk-claude');
|
||||
expect(options.headers['anthropic-version']).toBeTruthy();
|
||||
expect(options.headers.Authorization).toBeUndefined();
|
||||
});
|
||||
|
||||
it('routes gemini platform to OpenAI-compatible upstream endpoint path', async () => {
|
||||
selectChannelMock.mockReturnValue({
|
||||
channel: { id: 11, routeId: 22 },
|
||||
site: { name: 'gemini-site', url: 'https://generativelanguage.googleapis.com', platform: 'gemini' },
|
||||
account: { id: 33, username: 'demo-user' },
|
||||
tokenName: 'default',
|
||||
tokenValue: 'gemini-key',
|
||||
actualModel: 'gemini-2.5-flash',
|
||||
});
|
||||
|
||||
fetchMock.mockResolvedValue(new Response(JSON.stringify({
|
||||
id: 'chatcmpl-gemini-openai-compat',
|
||||
object: 'chat.completion',
|
||||
created: 1_706_000_004,
|
||||
model: 'gemini-2.5-flash',
|
||||
choices: [{
|
||||
index: 0,
|
||||
message: { role: 'assistant', content: 'gemini endpoint selected' },
|
||||
finish_reason: 'stop',
|
||||
}],
|
||||
usage: { prompt_tokens: 6, completion_tokens: 4, total_tokens: 10 },
|
||||
}), {
|
||||
status: 200,
|
||||
headers: { 'content-type': 'application/json' },
|
||||
}));
|
||||
|
||||
const response = await app.inject({
|
||||
method: 'POST',
|
||||
url: '/v1/chat/completions',
|
||||
payload: {
|
||||
model: 'gemini-2.5-flash',
|
||||
stream: false,
|
||||
messages: [{ role: 'user', content: 'hello' }],
|
||||
},
|
||||
});
|
||||
|
||||
expect(response.statusCode).toBe(200);
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||
const [targetUrl, options] = fetchMock.mock.calls[0] as [string, any];
|
||||
expect(targetUrl).toContain('/v1beta/openai/chat/completions');
|
||||
expect(options.headers.Authorization).toBe('Bearer gemini-key');
|
||||
});
|
||||
|
||||
it('chooses /v1/messages upstream when catalog indicates messages-only endpoint support', async () => {
|
||||
fetchModelPricingCatalogMock.mockResolvedValue({
|
||||
models: [
|
||||
|
||||
@@ -9,6 +9,7 @@ import { estimateProxyCost } from '../../services/modelPricingService.js';
|
||||
import { shouldRetryProxyRequest } from '../../services/proxyRetryPolicy.js';
|
||||
import { resolveProxyUsageWithSelfLogFallback } from '../../services/proxyUsageFallbackService.js';
|
||||
import { mergeProxyUsage, parseProxyUsage } from '../../services/proxyUsageParser.js';
|
||||
import { withExplicitProxyRequestInit } from '../../services/siteProxy.js';
|
||||
import {
|
||||
type DownstreamFormat,
|
||||
createStreamTransformContext,
|
||||
@@ -27,6 +28,11 @@ import {
|
||||
isEndpointDowngradeError,
|
||||
resolveUpstreamEndpointCandidates,
|
||||
} from './upstreamEndpoint.js';
|
||||
import {
|
||||
ensureModelAllowedForDownstreamKey,
|
||||
getDownstreamRoutingPolicy,
|
||||
recordDownstreamCostUsage,
|
||||
} from './downstreamPolicy.js';
|
||||
|
||||
const MAX_RETRIES = 2;
|
||||
const CLAUDE_SSE_EVENT_NAMES = new Set([
|
||||
@@ -44,6 +50,18 @@ function withUpstreamPath(path: string, message: string): string {
|
||||
return `[upstream:${path}] ${message}`;
|
||||
}
|
||||
|
||||
function shouldRetryClaudeMessagesWithNormalizedBody(
|
||||
downstreamFormat: DownstreamFormat,
|
||||
endpointPath: string,
|
||||
status: number,
|
||||
upstreamErrorText: string,
|
||||
): boolean {
|
||||
if (downstreamFormat !== 'claude') return false;
|
||||
if (!endpointPath.includes('/v1/messages')) return false;
|
||||
if (status < 400 || status >= 500) return false;
|
||||
return /messages\s+is\s+required/i.test(upstreamErrorText);
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return !!value && typeof value === 'object';
|
||||
}
|
||||
@@ -116,18 +134,20 @@ async function handleChatProxyRequest(
|
||||
}
|
||||
|
||||
const { requestedModel, isStream, upstreamBody, claudeOriginalBody } = parsedRequest.value!;
|
||||
if (!ensureModelAllowedForDownstreamKey(request, reply, requestedModel)) return;
|
||||
const downstreamPolicy = getDownstreamRoutingPolicy(request);
|
||||
|
||||
const excludeChannelIds: number[] = [];
|
||||
let retryCount = 0;
|
||||
|
||||
while (retryCount <= MAX_RETRIES) {
|
||||
let selected = retryCount === 0
|
||||
? tokenRouter.selectChannel(requestedModel)
|
||||
: tokenRouter.selectNextChannel(requestedModel, excludeChannelIds);
|
||||
? tokenRouter.selectChannel(requestedModel, downstreamPolicy)
|
||||
: tokenRouter.selectNextChannel(requestedModel, excludeChannelIds, downstreamPolicy);
|
||||
|
||||
if (!selected && retryCount === 0) {
|
||||
await refreshModelsAndRebuildRoutes();
|
||||
selected = tokenRouter.selectChannel(requestedModel);
|
||||
selected = tokenRouter.selectChannel(requestedModel, downstreamPolicy);
|
||||
}
|
||||
|
||||
if (!selected) {
|
||||
@@ -165,6 +185,8 @@ async function handleChatProxyRequest(
|
||||
modelName,
|
||||
stream: isStream,
|
||||
tokenValue: selected.tokenValue,
|
||||
sitePlatform: selected.site.platform,
|
||||
siteUrl: selected.site.url,
|
||||
openaiBody: upstreamBody,
|
||||
downstreamFormat,
|
||||
claudeOriginalBody,
|
||||
@@ -174,11 +196,11 @@ async function handleChatProxyRequest(
|
||||
const targetUrl = `${selected.site.url}${endpointRequest.path}`;
|
||||
startTime = Date.now();
|
||||
|
||||
const response = await fetch(targetUrl, {
|
||||
const response = await fetch(targetUrl, withExplicitProxyRequestInit(selected.site.proxyUrl, {
|
||||
method: 'POST',
|
||||
headers: endpointRequest.headers,
|
||||
body: JSON.stringify(endpointRequest.body),
|
||||
});
|
||||
}));
|
||||
|
||||
if (response.ok) {
|
||||
upstream = response;
|
||||
@@ -187,6 +209,42 @@ async function handleChatProxyRequest(
|
||||
}
|
||||
|
||||
const rawErrText = await response.text().catch(() => 'unknown error');
|
||||
|
||||
if (shouldRetryClaudeMessagesWithNormalizedBody(downstreamFormat, endpointRequest.path, response.status, rawErrText)) {
|
||||
const normalizedClaudeRequest = buildUpstreamEndpointRequest({
|
||||
endpoint: endpointCandidates[endpointIndex],
|
||||
modelName,
|
||||
stream: isStream,
|
||||
tokenValue: selected.tokenValue,
|
||||
sitePlatform: selected.site.platform,
|
||||
siteUrl: selected.site.url,
|
||||
openaiBody: upstreamBody,
|
||||
downstreamFormat,
|
||||
claudeOriginalBody: undefined,
|
||||
downstreamHeaders: request.headers as Record<string, unknown>,
|
||||
});
|
||||
const normalizedTargetUrl = `${selected.site.url}${normalizedClaudeRequest.path}`;
|
||||
const normalizedResponse = await fetch(normalizedTargetUrl, withExplicitProxyRequestInit(selected.site.proxyUrl, {
|
||||
method: 'POST',
|
||||
headers: normalizedClaudeRequest.headers,
|
||||
body: JSON.stringify(normalizedClaudeRequest.body),
|
||||
}));
|
||||
|
||||
if (normalizedResponse.ok) {
|
||||
upstream = normalizedResponse;
|
||||
successfulUpstreamPath = normalizedClaudeRequest.path;
|
||||
break;
|
||||
}
|
||||
|
||||
const normalizedErrText = await normalizedResponse.text().catch(() => 'unknown error');
|
||||
finalStatus = normalizedResponse.status;
|
||||
finalErrText = withUpstreamPath(
|
||||
normalizedClaudeRequest.path,
|
||||
`normalized-claude-body fallback failed: ${normalizedErrText}`,
|
||||
);
|
||||
break;
|
||||
}
|
||||
|
||||
const errText = withUpstreamPath(endpointRequest.path, rawErrText);
|
||||
const shouldDowngradeEndpoint = (
|
||||
endpointIndex < endpointCandidates.length - 1
|
||||
@@ -332,6 +390,7 @@ async function handleChatProxyRequest(
|
||||
}
|
||||
|
||||
tokenRouter.recordSuccess(selected.channel.id, latency, estimatedCost);
|
||||
recordDownstreamCostUsage(request, estimatedCost);
|
||||
logProxy(
|
||||
selected,
|
||||
requestedModel,
|
||||
@@ -470,6 +529,7 @@ async function handleChatProxyRequest(
|
||||
}
|
||||
|
||||
tokenRouter.recordSuccess(selected.channel.id, latency, estimatedCost);
|
||||
recordDownstreamCostUsage(request, estimatedCost);
|
||||
logProxy(
|
||||
selected,
|
||||
requestedModel,
|
||||
@@ -530,6 +590,7 @@ async function handleChatProxyRequest(
|
||||
}
|
||||
|
||||
tokenRouter.recordSuccess(selected.channel.id, latency, estimatedCost);
|
||||
recordDownstreamCostUsage(request, estimatedCost);
|
||||
logProxy(
|
||||
selected,
|
||||
requestedModel,
|
||||
|
||||
@@ -9,30 +9,34 @@ import { estimateProxyCost } from '../../services/modelPricingService.js';
|
||||
import { shouldRetryProxyRequest } from '../../services/proxyRetryPolicy.js';
|
||||
import { resolveProxyUsageWithSelfLogFallback } from '../../services/proxyUsageFallbackService.js';
|
||||
import { mergeProxyUsage, parseProxyUsage, pullSseDataEvents } from '../../services/proxyUsageParser.js';
|
||||
import { ensureModelAllowedForDownstreamKey, getDownstreamRoutingPolicy, recordDownstreamCostUsage } from './downstreamPolicy.js';
|
||||
import { withExplicitProxyRequestInit } from '../../services/siteProxy.js';
|
||||
|
||||
const MAX_RETRIES = 2;
|
||||
|
||||
export async function completionsProxyRoute(app: FastifyInstance) {
|
||||
app.post('/v1/completions', async (request: FastifyRequest, reply: FastifyReply) => {
|
||||
const body = request.body as any;
|
||||
const requestedModel = body?.model;
|
||||
if (!requestedModel) {
|
||||
return reply.code(400).send({ error: { message: 'model is required', type: 'invalid_request_error' } });
|
||||
}
|
||||
|
||||
const isStream = body.stream === true;
|
||||
const requestedModel = body?.model;
|
||||
if (!requestedModel) {
|
||||
return reply.code(400).send({ error: { message: 'model is required', type: 'invalid_request_error' } });
|
||||
}
|
||||
if (!ensureModelAllowedForDownstreamKey(request, reply, requestedModel)) return;
|
||||
const downstreamPolicy = getDownstreamRoutingPolicy(request);
|
||||
|
||||
const isStream = body.stream === true;
|
||||
const excludeChannelIds: number[] = [];
|
||||
let retryCount = 0;
|
||||
|
||||
while (retryCount <= MAX_RETRIES) {
|
||||
let selected = retryCount === 0
|
||||
? tokenRouter.selectChannel(requestedModel)
|
||||
: tokenRouter.selectNextChannel(requestedModel, excludeChannelIds);
|
||||
|
||||
if (!selected && retryCount === 0) {
|
||||
await refreshModelsAndRebuildRoutes();
|
||||
selected = tokenRouter.selectChannel(requestedModel);
|
||||
}
|
||||
while (retryCount <= MAX_RETRIES) {
|
||||
let selected = retryCount === 0
|
||||
? tokenRouter.selectChannel(requestedModel, downstreamPolicy)
|
||||
: tokenRouter.selectNextChannel(requestedModel, excludeChannelIds, downstreamPolicy);
|
||||
|
||||
if (!selected && retryCount === 0) {
|
||||
await refreshModelsAndRebuildRoutes();
|
||||
selected = tokenRouter.selectChannel(requestedModel, downstreamPolicy);
|
||||
}
|
||||
|
||||
if (!selected) {
|
||||
await reportProxyAllFailed({
|
||||
@@ -51,14 +55,14 @@ export async function completionsProxyRoute(app: FastifyInstance) {
|
||||
const startTime = Date.now();
|
||||
|
||||
try {
|
||||
const upstream = await fetch(targetUrl, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Authorization': `Bearer ${selected.tokenValue}`,
|
||||
},
|
||||
body: JSON.stringify(forwardBody),
|
||||
});
|
||||
const upstream = await fetch(targetUrl, withExplicitProxyRequestInit(selected.site.proxyUrl, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Authorization': `Bearer ${selected.tokenValue}`,
|
||||
},
|
||||
body: JSON.stringify(forwardBody),
|
||||
}));
|
||||
|
||||
if (!upstream.ok) {
|
||||
const errText = await upstream.text().catch(() => 'unknown error');
|
||||
@@ -159,6 +163,7 @@ export async function completionsProxyRoute(app: FastifyInstance) {
|
||||
estimatedCost = resolvedUsage.estimatedCostFromQuota;
|
||||
}
|
||||
tokenRouter.recordSuccess(selected.channel.id, latency, estimatedCost);
|
||||
recordDownstreamCostUsage(request, estimatedCost);
|
||||
logProxy(
|
||||
selected, requestedModel, 'success', 200, latency, null, retryCount,
|
||||
resolvedUsage.promptTokens, resolvedUsage.completionTokens, resolvedUsage.totalTokens, estimatedCost,
|
||||
@@ -197,6 +202,7 @@ export async function completionsProxyRoute(app: FastifyInstance) {
|
||||
}
|
||||
|
||||
tokenRouter.recordSuccess(selected.channel.id, latency, estimatedCost);
|
||||
recordDownstreamCostUsage(request, estimatedCost);
|
||||
logProxy(
|
||||
selected, requestedModel, 'success', 200, latency, null, retryCount,
|
||||
resolvedUsage.promptTokens, resolvedUsage.completionTokens, resolvedUsage.totalTokens, estimatedCost,
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
import type { FastifyReply, FastifyRequest } from 'fastify';
|
||||
import { getProxyAuthContext } from '../../middleware/auth.js';
|
||||
import { isModelAllowedByPolicyOrAllowedRoutes, recordManagedKeyCostUsage } from '../../services/downstreamApiKeyService.js';
|
||||
import { EMPTY_DOWNSTREAM_ROUTING_POLICY, type DownstreamRoutingPolicy } from '../../services/downstreamPolicyTypes.js';
|
||||
|
||||
export function getDownstreamRoutingPolicy(request: FastifyRequest): DownstreamRoutingPolicy {
|
||||
return getProxyAuthContext(request)?.policy || EMPTY_DOWNSTREAM_ROUTING_POLICY;
|
||||
}
|
||||
|
||||
export function ensureModelAllowedForDownstreamKey(
|
||||
request: FastifyRequest,
|
||||
reply: FastifyReply,
|
||||
requestedModel: string,
|
||||
): boolean {
|
||||
const authContext = getProxyAuthContext(request);
|
||||
if (!authContext) return true;
|
||||
|
||||
if (isModelAllowedByPolicyOrAllowedRoutes(requestedModel, authContext.policy)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
reply.code(403).send({
|
||||
error: {
|
||||
message: `Model not allowed for this API key: ${requestedModel}`,
|
||||
type: 'permission_error',
|
||||
},
|
||||
});
|
||||
return false;
|
||||
}
|
||||
|
||||
export function recordDownstreamCostUsage(request: FastifyRequest, estimatedCost: number): void {
|
||||
const authContext = getProxyAuthContext(request);
|
||||
if (!authContext || authContext.keyId === null) return;
|
||||
recordManagedKeyCostUsage(authContext.keyId, estimatedCost);
|
||||
}
|
||||
@@ -9,29 +9,33 @@ import { estimateProxyCost } from '../../services/modelPricingService.js';
|
||||
import { shouldRetryProxyRequest } from '../../services/proxyRetryPolicy.js';
|
||||
import { resolveProxyUsageWithSelfLogFallback } from '../../services/proxyUsageFallbackService.js';
|
||||
import { parseProxyUsage } from '../../services/proxyUsageParser.js';
|
||||
import { ensureModelAllowedForDownstreamKey, getDownstreamRoutingPolicy, recordDownstreamCostUsage } from './downstreamPolicy.js';
|
||||
import { withExplicitProxyRequestInit } from '../../services/siteProxy.js';
|
||||
|
||||
const MAX_RETRIES = 2;
|
||||
|
||||
export async function embeddingsProxyRoute(app: FastifyInstance) {
|
||||
app.post('/v1/embeddings', async (request: FastifyRequest, reply: FastifyReply) => {
|
||||
const body = request.body as any;
|
||||
const requestedModel = body?.model;
|
||||
if (!requestedModel) {
|
||||
return reply.code(400).send({ error: { message: 'model is required', type: 'invalid_request_error' } });
|
||||
}
|
||||
|
||||
const excludeChannelIds: number[] = [];
|
||||
const requestedModel = body?.model;
|
||||
if (!requestedModel) {
|
||||
return reply.code(400).send({ error: { message: 'model is required', type: 'invalid_request_error' } });
|
||||
}
|
||||
if (!ensureModelAllowedForDownstreamKey(request, reply, requestedModel)) return;
|
||||
const downstreamPolicy = getDownstreamRoutingPolicy(request);
|
||||
|
||||
const excludeChannelIds: number[] = [];
|
||||
let retryCount = 0;
|
||||
|
||||
while (retryCount <= MAX_RETRIES) {
|
||||
let selected = retryCount === 0
|
||||
? tokenRouter.selectChannel(requestedModel)
|
||||
: tokenRouter.selectNextChannel(requestedModel, excludeChannelIds);
|
||||
|
||||
if (!selected && retryCount === 0) {
|
||||
await refreshModelsAndRebuildRoutes();
|
||||
selected = tokenRouter.selectChannel(requestedModel);
|
||||
}
|
||||
while (retryCount <= MAX_RETRIES) {
|
||||
let selected = retryCount === 0
|
||||
? tokenRouter.selectChannel(requestedModel, downstreamPolicy)
|
||||
: tokenRouter.selectNextChannel(requestedModel, excludeChannelIds, downstreamPolicy);
|
||||
|
||||
if (!selected && retryCount === 0) {
|
||||
await refreshModelsAndRebuildRoutes();
|
||||
selected = tokenRouter.selectChannel(requestedModel, downstreamPolicy);
|
||||
}
|
||||
|
||||
if (!selected) {
|
||||
await reportProxyAllFailed({
|
||||
@@ -48,14 +52,14 @@ export async function embeddingsProxyRoute(app: FastifyInstance) {
|
||||
const startTime = Date.now();
|
||||
|
||||
try {
|
||||
const upstream = await fetch(targetUrl, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Authorization': `Bearer ${selected.tokenValue}`,
|
||||
},
|
||||
body: JSON.stringify(forwardBody),
|
||||
});
|
||||
const upstream = await fetch(targetUrl, withExplicitProxyRequestInit(selected.site.proxyUrl, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Authorization': `Bearer ${selected.tokenValue}`,
|
||||
},
|
||||
body: JSON.stringify(forwardBody),
|
||||
}));
|
||||
|
||||
const text = await upstream.text();
|
||||
if (!upstream.ok) {
|
||||
@@ -115,6 +119,7 @@ export async function embeddingsProxyRoute(app: FastifyInstance) {
|
||||
}
|
||||
|
||||
tokenRouter.recordSuccess(selected.channel.id, latency, estimatedCost);
|
||||
recordDownstreamCostUsage(request, estimatedCost);
|
||||
logProxy(
|
||||
selected, requestedModel, 'success', upstream.status, latency, null, retryCount,
|
||||
resolvedUsage.promptTokens, resolvedUsage.completionTokens, resolvedUsage.totalTokens, estimatedCost,
|
||||
|
||||
@@ -7,25 +7,29 @@ import { reportProxyAllFailed, reportTokenExpired } from '../../services/alertSe
|
||||
import { isTokenExpiredError } from '../../services/alertRules.js';
|
||||
import { estimateProxyCost } from '../../services/modelPricingService.js';
|
||||
import { shouldRetryProxyRequest } from '../../services/proxyRetryPolicy.js';
|
||||
import { ensureModelAllowedForDownstreamKey, getDownstreamRoutingPolicy, recordDownstreamCostUsage } from './downstreamPolicy.js';
|
||||
import { withExplicitProxyRequestInit } from '../../services/siteProxy.js';
|
||||
|
||||
const MAX_RETRIES = 2;
|
||||
|
||||
export async function imagesProxyRoute(app: FastifyInstance) {
|
||||
app.post('/v1/images/generations', async (request: FastifyRequest, reply: FastifyReply) => {
|
||||
const body = request.body as any;
|
||||
const requestedModel = body?.model || 'gpt-image-1';
|
||||
const excludeChannelIds: number[] = [];
|
||||
let retryCount = 0;
|
||||
|
||||
while (retryCount <= MAX_RETRIES) {
|
||||
let selected = retryCount === 0
|
||||
? tokenRouter.selectChannel(requestedModel)
|
||||
: tokenRouter.selectNextChannel(requestedModel, excludeChannelIds);
|
||||
|
||||
if (!selected && retryCount === 0) {
|
||||
await refreshModelsAndRebuildRoutes();
|
||||
selected = tokenRouter.selectChannel(requestedModel);
|
||||
}
|
||||
const body = request.body as any;
|
||||
const requestedModel = body?.model || 'gpt-image-1';
|
||||
if (!ensureModelAllowedForDownstreamKey(request, reply, requestedModel)) return;
|
||||
const downstreamPolicy = getDownstreamRoutingPolicy(request);
|
||||
const excludeChannelIds: number[] = [];
|
||||
let retryCount = 0;
|
||||
|
||||
while (retryCount <= MAX_RETRIES) {
|
||||
let selected = retryCount === 0
|
||||
? tokenRouter.selectChannel(requestedModel, downstreamPolicy)
|
||||
: tokenRouter.selectNextChannel(requestedModel, excludeChannelIds, downstreamPolicy);
|
||||
|
||||
if (!selected && retryCount === 0) {
|
||||
await refreshModelsAndRebuildRoutes();
|
||||
selected = tokenRouter.selectChannel(requestedModel, downstreamPolicy);
|
||||
}
|
||||
|
||||
if (!selected) {
|
||||
await reportProxyAllFailed({
|
||||
@@ -44,14 +48,14 @@ export async function imagesProxyRoute(app: FastifyInstance) {
|
||||
const startTime = Date.now();
|
||||
|
||||
try {
|
||||
const upstream = await fetch(targetUrl, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Authorization': `Bearer ${selected.tokenValue}`,
|
||||
},
|
||||
body: JSON.stringify(forwardBody),
|
||||
});
|
||||
const upstream = await fetch(targetUrl, withExplicitProxyRequestInit(selected.site.proxyUrl, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Authorization': `Bearer ${selected.tokenValue}`,
|
||||
},
|
||||
body: JSON.stringify(forwardBody),
|
||||
}));
|
||||
|
||||
const text = await upstream.text();
|
||||
if (!upstream.ok) {
|
||||
@@ -80,17 +84,18 @@ export async function imagesProxyRoute(app: FastifyInstance) {
|
||||
try { data = JSON.parse(text); } catch { data = { data: [] }; }
|
||||
|
||||
const latency = Date.now() - startTime;
|
||||
const estimatedCost = await estimateProxyCost({
|
||||
site: selected.site,
|
||||
account: selected.account,
|
||||
modelName: selected.actualModel || requestedModel,
|
||||
const estimatedCost = await estimateProxyCost({
|
||||
site: selected.site,
|
||||
account: selected.account,
|
||||
modelName: selected.actualModel || requestedModel,
|
||||
promptTokens: 0,
|
||||
completionTokens: 0,
|
||||
totalTokens: 0,
|
||||
});
|
||||
tokenRouter.recordSuccess(selected.channel.id, latency, estimatedCost);
|
||||
logProxy(selected, requestedModel, 'success', upstream.status, latency, null, retryCount, estimatedCost);
|
||||
return reply.code(upstream.status).send(data);
|
||||
});
|
||||
tokenRouter.recordSuccess(selected.channel.id, latency, estimatedCost);
|
||||
recordDownstreamCostUsage(request, estimatedCost);
|
||||
logProxy(selected, requestedModel, 'success', upstream.status, latency, null, retryCount, estimatedCost);
|
||||
return reply.code(upstream.status).send(data);
|
||||
} catch (err: any) {
|
||||
tokenRouter.recordFailure(selected.channel.id);
|
||||
logProxy(selected, requestedModel, 'failed', 0, Date.now() - startTime, err.message, retryCount);
|
||||
|
||||
@@ -2,9 +2,13 @@ import { FastifyInstance } from 'fastify';
|
||||
import { db, schema } from '../../db/index.js';
|
||||
import { and, eq } from 'drizzle-orm';
|
||||
import { refreshModelsAndRebuildRoutes } from '../../services/modelService.js';
|
||||
import { getDownstreamRoutingPolicy } from './downstreamPolicy.js';
|
||||
import { isModelAllowedByPolicyOrAllowedRoutes } from '../../services/downstreamApiKeyService.js';
|
||||
|
||||
export async function modelsProxyRoute(app: FastifyInstance) {
|
||||
app.get('/v1/models', async (request) => {
|
||||
const downstreamPolicy = getDownstreamRoutingPolicy(request);
|
||||
|
||||
const readModels = () => {
|
||||
const rows = db.select({ modelName: schema.modelAvailability.modelName })
|
||||
.from(schema.modelAvailability)
|
||||
@@ -18,7 +22,17 @@ export async function modelsProxyRoute(app: FastifyInstance) {
|
||||
),
|
||||
)
|
||||
.all();
|
||||
return Array.from(new Set(rows.map((r) => r.modelName))).sort();
|
||||
const routeAliases = db.select({ displayName: schema.tokenRoutes.displayName })
|
||||
.from(schema.tokenRoutes)
|
||||
.where(eq(schema.tokenRoutes.enabled, true))
|
||||
.all()
|
||||
.map((row) => (row.displayName || '').trim())
|
||||
.filter((name) => name.length > 0);
|
||||
const deduped = Array.from(new Set([
|
||||
...rows.map((r) => r.modelName),
|
||||
...routeAliases,
|
||||
])).sort();
|
||||
return deduped.filter((modelName) => isModelAllowedByPolicyOrAllowedRoutes(modelName, downstreamPolicy));
|
||||
};
|
||||
|
||||
let models = readModels();
|
||||
|
||||
@@ -9,6 +9,7 @@ import { estimateProxyCost } from '../../services/modelPricingService.js';
|
||||
import { shouldRetryProxyRequest } from '../../services/proxyRetryPolicy.js';
|
||||
import { resolveProxyUsageWithSelfLogFallback } from '../../services/proxyUsageFallbackService.js';
|
||||
import { mergeProxyUsage, parseProxyUsage } from '../../services/proxyUsageParser.js';
|
||||
import { withExplicitProxyRequestInit } from '../../services/siteProxy.js';
|
||||
import {
|
||||
createStreamTransformContext,
|
||||
normalizeUpstreamFinalResponse,
|
||||
@@ -22,6 +23,7 @@ import {
|
||||
resolveUpstreamEndpointCandidates,
|
||||
type UpstreamEndpoint,
|
||||
} from './upstreamEndpoint.js';
|
||||
import { ensureModelAllowedForDownstreamKey, getDownstreamRoutingPolicy, recordDownstreamCostUsage } from './downstreamPolicy.js';
|
||||
|
||||
const MAX_RETRIES = 2;
|
||||
|
||||
@@ -613,6 +615,8 @@ export async function responsesProxyRoute(app: FastifyInstance) {
|
||||
if (!requestedModel) {
|
||||
return reply.code(400).send({ error: { message: 'model is required', type: 'invalid_request_error' } });
|
||||
}
|
||||
if (!ensureModelAllowedForDownstreamKey(request, reply, requestedModel)) return;
|
||||
const downstreamPolicy = getDownstreamRoutingPolicy(request);
|
||||
|
||||
const isStream = body.stream === true;
|
||||
const excludeChannelIds: number[] = [];
|
||||
@@ -620,12 +624,12 @@ export async function responsesProxyRoute(app: FastifyInstance) {
|
||||
|
||||
while (retryCount <= MAX_RETRIES) {
|
||||
let selected = retryCount === 0
|
||||
? tokenRouter.selectChannel(requestedModel)
|
||||
: tokenRouter.selectNextChannel(requestedModel, excludeChannelIds);
|
||||
? tokenRouter.selectChannel(requestedModel, downstreamPolicy)
|
||||
: tokenRouter.selectNextChannel(requestedModel, excludeChannelIds, downstreamPolicy);
|
||||
|
||||
if (!selected && retryCount === 0) {
|
||||
await refreshModelsAndRebuildRoutes();
|
||||
selected = tokenRouter.selectChannel(requestedModel);
|
||||
selected = tokenRouter.selectChannel(requestedModel, downstreamPolicy);
|
||||
}
|
||||
|
||||
if (!selected) {
|
||||
@@ -669,6 +673,8 @@ export async function responsesProxyRoute(app: FastifyInstance) {
|
||||
modelName,
|
||||
stream: isStream,
|
||||
tokenValue: selected.tokenValue,
|
||||
sitePlatform: selected.site.platform,
|
||||
siteUrl: selected.site.url,
|
||||
openaiBody: openAiBody,
|
||||
downstreamFormat: 'responses',
|
||||
responsesOriginalBody: body,
|
||||
@@ -676,11 +682,11 @@ export async function responsesProxyRoute(app: FastifyInstance) {
|
||||
});
|
||||
const targetUrl = `${selected.site.url}${endpointRequest.path}`;
|
||||
|
||||
const response = await fetch(targetUrl, {
|
||||
const response = await fetch(targetUrl, withExplicitProxyRequestInit(selected.site.proxyUrl, {
|
||||
method: 'POST',
|
||||
headers: endpointRequest.headers,
|
||||
body: JSON.stringify(endpointRequest.body),
|
||||
});
|
||||
}));
|
||||
|
||||
if (response.ok) {
|
||||
upstream = response;
|
||||
@@ -858,6 +864,7 @@ export async function responsesProxyRoute(app: FastifyInstance) {
|
||||
estimatedCost = resolvedUsage.estimatedCostFromQuota;
|
||||
}
|
||||
tokenRouter.recordSuccess(selected.channel.id, latency, estimatedCost);
|
||||
recordDownstreamCostUsage(request, estimatedCost);
|
||||
logProxy(
|
||||
selected, requestedModel, 'success', 200, latency, null, retryCount,
|
||||
resolvedUsage.promptTokens, resolvedUsage.completionTokens, resolvedUsage.totalTokens, estimatedCost,
|
||||
@@ -909,6 +916,7 @@ export async function responsesProxyRoute(app: FastifyInstance) {
|
||||
}
|
||||
|
||||
tokenRouter.recordSuccess(selected.channel.id, latency, estimatedCost);
|
||||
recordDownstreamCostUsage(request, estimatedCost);
|
||||
logProxy(
|
||||
selected, requestedModel, 'success', 200, latency, null, retryCount,
|
||||
resolvedUsage.promptTokens, resolvedUsage.completionTokens, resolvedUsage.totalTokens, estimatedCost,
|
||||
|
||||
@@ -26,6 +26,10 @@ function asTrimmedString(value: unknown): string {
|
||||
return typeof value === 'string' ? value.trim() : '';
|
||||
}
|
||||
|
||||
function normalizePlatformName(platform: unknown): string {
|
||||
return asTrimmedString(platform).toLowerCase();
|
||||
}
|
||||
|
||||
function headerValueToString(value: unknown): string | null {
|
||||
if (typeof value === 'string') {
|
||||
const trimmed = value.trim();
|
||||
@@ -167,6 +171,28 @@ function sanitizeMessagesBodyForAnthropic(body: Record<string, unknown>): Record
|
||||
if (hasTemperature && hasTopP) {
|
||||
delete sanitized.top_p;
|
||||
}
|
||||
|
||||
// Claude Code may send thinking.type = "adaptive". Many Anthropic-compatible
|
||||
// upstreams only accept enabled/disabled.
|
||||
const thinking = sanitized.thinking;
|
||||
if (isRecord(thinking)) {
|
||||
const rawType = asTrimmedString(thinking.type).toLowerCase();
|
||||
if (rawType === 'adaptive') {
|
||||
const budgetTokens = toFiniteNumber(thinking.budget_tokens);
|
||||
if (budgetTokens !== null && budgetTokens > 0) {
|
||||
sanitized.thinking = {
|
||||
...thinking,
|
||||
type: 'enabled',
|
||||
budget_tokens: Math.trunc(budgetTokens),
|
||||
};
|
||||
} else {
|
||||
sanitized.thinking = { type: 'disabled' };
|
||||
}
|
||||
} else if (rawType && rawType !== 'enabled' && rawType !== 'disabled') {
|
||||
sanitized.thinking = { type: 'disabled' };
|
||||
}
|
||||
}
|
||||
|
||||
return sanitized;
|
||||
}
|
||||
|
||||
@@ -337,7 +363,24 @@ function normalizeEndpointTypes(value: unknown): UpstreamEndpoint[] {
|
||||
return Array.from(normalized);
|
||||
}
|
||||
|
||||
function preferredEndpointOrder(downstreamFormat: EndpointPreference): UpstreamEndpoint[] {
|
||||
function preferredEndpointOrder(downstreamFormat: EndpointPreference, sitePlatform?: string): UpstreamEndpoint[] {
|
||||
const platform = normalizePlatformName(sitePlatform);
|
||||
|
||||
if (platform === 'gemini') {
|
||||
// Gemini upstream is routed through OpenAI-compatible chat endpoint.
|
||||
return ['chat'];
|
||||
}
|
||||
|
||||
if (platform === 'openai') {
|
||||
return downstreamFormat === 'responses'
|
||||
? ['responses', 'chat']
|
||||
: ['chat', 'responses'];
|
||||
}
|
||||
|
||||
if (platform === 'claude') {
|
||||
return ['messages'];
|
||||
}
|
||||
|
||||
if (downstreamFormat === 'responses') {
|
||||
return ['responses', 'chat', 'messages'];
|
||||
}
|
||||
@@ -352,14 +395,15 @@ export async function resolveUpstreamEndpointCandidates(
|
||||
modelName: string,
|
||||
downstreamFormat: EndpointPreference,
|
||||
): Promise<UpstreamEndpoint[]> {
|
||||
if ((context.site.platform || '').toLowerCase() === 'anyrouter') {
|
||||
const sitePlatform = normalizePlatformName(context.site.platform);
|
||||
if (sitePlatform === 'anyrouter') {
|
||||
// anyrouter deployments are effectively anthropic-protocol first.
|
||||
return downstreamFormat === 'responses'
|
||||
? ['responses', 'messages', 'chat']
|
||||
: ['messages', 'chat'];
|
||||
}
|
||||
|
||||
const preferred = preferredEndpointOrder(downstreamFormat);
|
||||
const preferred = preferredEndpointOrder(downstreamFormat, context.site.platform);
|
||||
|
||||
try {
|
||||
const catalog = await fetchModelPricingCatalog({
|
||||
@@ -415,6 +459,8 @@ export function buildUpstreamEndpointRequest(input: {
|
||||
modelName: string;
|
||||
stream: boolean;
|
||||
tokenValue: string;
|
||||
sitePlatform?: string;
|
||||
siteUrl?: string;
|
||||
openaiBody: Record<string, unknown>;
|
||||
downstreamFormat: EndpointPreference;
|
||||
claudeOriginalBody?: Record<string, unknown>;
|
||||
@@ -425,12 +471,49 @@ export function buildUpstreamEndpointRequest(input: {
|
||||
headers: Record<string, string>;
|
||||
body: Record<string, unknown>;
|
||||
} {
|
||||
const sitePlatform = normalizePlatformName(input.sitePlatform);
|
||||
const isClaudeUpstream = sitePlatform === 'claude';
|
||||
|
||||
const resolveGeminiEndpointPath = (endpoint: UpstreamEndpoint): string => {
|
||||
const normalizedSiteUrl = asTrimmedString(input.siteUrl).toLowerCase();
|
||||
const openAiCompatBase = /\/openai(?:\/|$)/.test(normalizedSiteUrl);
|
||||
if (openAiCompatBase) {
|
||||
return endpoint === 'responses'
|
||||
? '/responses'
|
||||
: '/chat/completions';
|
||||
}
|
||||
return endpoint === 'responses'
|
||||
? '/v1beta/openai/responses'
|
||||
: '/v1beta/openai/chat/completions';
|
||||
};
|
||||
|
||||
const resolveEndpointPath = (endpoint: UpstreamEndpoint): string => {
|
||||
if (sitePlatform === 'gemini') {
|
||||
return resolveGeminiEndpointPath(endpoint);
|
||||
}
|
||||
|
||||
if (sitePlatform === 'openai') {
|
||||
if (endpoint === 'responses') return '/v1/responses';
|
||||
return '/v1/chat/completions';
|
||||
}
|
||||
|
||||
if (sitePlatform === 'claude') {
|
||||
return '/v1/messages';
|
||||
}
|
||||
|
||||
if (endpoint === 'messages') return '/v1/messages';
|
||||
if (endpoint === 'responses') return '/v1/responses';
|
||||
return '/v1/chat/completions';
|
||||
};
|
||||
|
||||
const passthroughHeaders = extractSafePassthroughHeaders(input.downstreamHeaders);
|
||||
const commonHeaders = {
|
||||
const commonHeaders: Record<string, string> = {
|
||||
...passthroughHeaders,
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: `Bearer ${input.tokenValue}`,
|
||||
};
|
||||
if (!isClaudeUpstream) {
|
||||
commonHeaders.Authorization = `Bearer ${input.tokenValue}`;
|
||||
}
|
||||
|
||||
if (input.endpoint === 'messages') {
|
||||
const claudeHeaders = input.downstreamFormat === 'claude'
|
||||
@@ -460,7 +543,7 @@ export function buildUpstreamEndpointRequest(input: {
|
||||
}, input.stream);
|
||||
|
||||
return {
|
||||
path: '/v1/messages',
|
||||
path: resolveEndpointPath('messages'),
|
||||
headers,
|
||||
body: sanitizedBody,
|
||||
};
|
||||
@@ -486,7 +569,7 @@ export function buildUpstreamEndpointRequest(input: {
|
||||
}, input.stream);
|
||||
|
||||
return {
|
||||
path: '/v1/responses',
|
||||
path: resolveEndpointPath('responses'),
|
||||
headers,
|
||||
body,
|
||||
};
|
||||
@@ -494,7 +577,7 @@ export function buildUpstreamEndpointRequest(input: {
|
||||
|
||||
const headers = ensureStreamAcceptHeader(commonHeaders, input.stream);
|
||||
return {
|
||||
path: '/v1/chat/completions',
|
||||
path: resolveEndpointPath('chat'),
|
||||
headers,
|
||||
body: {
|
||||
...input.openaiBody,
|
||||
|
||||
Reference in New Issue
Block a user