Merge remote-tracking branch 'origin/main' into feat/account-level-proxy
# Conflicts: # src/server/routes/proxy/chat.ts # src/server/routes/proxy/responses.ts
This commit is contained in:
Hureru
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,159 @@
|
||||
import type { FastifyInstance, FastifyReply, FastifyRequest } from 'fastify';
|
||||
import { getProxyResourceOwner } from '../../middleware/auth.js';
|
||||
import {
|
||||
getProxyFileByPublicIdForOwner,
|
||||
getProxyFileContentByPublicIdForOwner,
|
||||
listProxyFilesByOwner,
|
||||
saveProxyFile,
|
||||
softDeleteProxyFileByPublicIdForOwner,
|
||||
} from '../../services/proxyFileStore.js';
|
||||
import { ensureMultipartBufferParser, parseMultipartFormData } from '../../routes/proxy/multipart.js';
|
||||
|
||||
function invalidRequest(reply: FastifyReply, message: string) {
|
||||
return reply.code(400).send({ error: { message, type: 'invalid_request_error' } });
|
||||
}
|
||||
|
||||
function notFound(reply: FastifyReply, message = 'file not found') {
|
||||
return reply.code(404).send({ error: { message, type: 'not_found_error' } });
|
||||
}
|
||||
|
||||
function toUnixSeconds(sqlDateTime: string | null | undefined): number {
|
||||
if (!sqlDateTime) return Math.floor(Date.now() / 1000);
|
||||
const parsed = Date.parse(sqlDateTime.replace(' ', 'T') + 'Z');
|
||||
if (!Number.isFinite(parsed)) return Math.floor(Date.now() / 1000);
|
||||
return Math.floor(parsed / 1000);
|
||||
}
|
||||
|
||||
function toFileObject(record: {
|
||||
publicId: string;
|
||||
filename: string;
|
||||
mimeType: string;
|
||||
purpose: string | null;
|
||||
byteSize: number;
|
||||
createdAt: string | null;
|
||||
}) {
|
||||
return {
|
||||
id: record.publicId,
|
||||
object: 'file',
|
||||
bytes: record.byteSize,
|
||||
created_at: toUnixSeconds(record.createdAt),
|
||||
filename: record.filename,
|
||||
purpose: record.purpose || 'assistants',
|
||||
mime_type: record.mimeType,
|
||||
};
|
||||
}
|
||||
|
||||
function inferMimeTypeFromFilename(filename: string): string {
|
||||
const normalized = filename.trim().toLowerCase();
|
||||
if (normalized.endsWith('.pdf')) return 'application/pdf';
|
||||
if (normalized.endsWith('.txt')) return 'text/plain';
|
||||
if (normalized.endsWith('.md') || normalized.endsWith('.markdown')) return 'text/markdown';
|
||||
if (normalized.endsWith('.json')) return 'application/json';
|
||||
if (normalized.endsWith('.png')) return 'image/png';
|
||||
if (normalized.endsWith('.jpg') || normalized.endsWith('.jpeg')) return 'image/jpeg';
|
||||
if (normalized.endsWith('.gif')) return 'image/gif';
|
||||
if (normalized.endsWith('.webp')) return 'image/webp';
|
||||
if (normalized.endsWith('.wav')) return 'audio/wav';
|
||||
if (normalized.endsWith('.mp3')) return 'audio/mpeg';
|
||||
return 'application/octet-stream';
|
||||
}
|
||||
|
||||
function isSupportedProxyFileMimeType(mimeType: string): boolean {
|
||||
return mimeType === 'application/pdf'
|
||||
|| mimeType === 'text/plain'
|
||||
|| mimeType === 'text/markdown'
|
||||
|| mimeType === 'application/json'
|
||||
|| mimeType.startsWith('image/')
|
||||
|| mimeType.startsWith('audio/');
|
||||
}
|
||||
|
||||
export async function filesProxyRoute(app: FastifyInstance) {
|
||||
ensureMultipartBufferParser(app);
|
||||
|
||||
app.post('/v1/files', async (request: FastifyRequest, reply: FastifyReply) => {
|
||||
const owner = getProxyResourceOwner(request);
|
||||
if (!owner) {
|
||||
return reply.code(401).send({ error: { message: 'Missing proxy auth context', type: 'authentication_error' } });
|
||||
}
|
||||
|
||||
const formData = await parseMultipartFormData(request);
|
||||
if (!formData) {
|
||||
return invalidRequest(reply, 'multipart/form-data with a file field is required');
|
||||
}
|
||||
|
||||
const fileEntry = formData.get('file');
|
||||
if (!fileEntry || typeof fileEntry !== 'object' || typeof (fileEntry as File).arrayBuffer !== 'function') {
|
||||
return invalidRequest(reply, 'file field is required');
|
||||
}
|
||||
|
||||
const filename = fileEntry.name || 'upload.bin';
|
||||
const mimeType = (fileEntry.type || inferMimeTypeFromFilename(filename)).trim().toLowerCase();
|
||||
if (!isSupportedProxyFileMimeType(mimeType)) {
|
||||
return invalidRequest(reply, `unsupported file mime type: ${mimeType || 'application/octet-stream'}`);
|
||||
}
|
||||
|
||||
const purposeValue = formData.get('purpose');
|
||||
const purpose = typeof purposeValue === 'string' && purposeValue.trim().length > 0
|
||||
? purposeValue.trim()
|
||||
: 'assistants';
|
||||
const buffer = Buffer.from(await fileEntry.arrayBuffer());
|
||||
|
||||
const saved = await saveProxyFile({
|
||||
...owner,
|
||||
filename,
|
||||
mimeType,
|
||||
purpose,
|
||||
contentBase64: buffer.toString('base64'),
|
||||
});
|
||||
return reply.send(toFileObject(saved));
|
||||
});
|
||||
|
||||
app.get('/v1/files', async (request: FastifyRequest, reply: FastifyReply) => {
|
||||
const owner = getProxyResourceOwner(request);
|
||||
if (!owner) {
|
||||
return reply.code(401).send({ error: { message: 'Missing proxy auth context', type: 'authentication_error' } });
|
||||
}
|
||||
const files = await listProxyFilesByOwner(owner);
|
||||
return reply.send({
|
||||
object: 'list',
|
||||
data: files.map((item) => toFileObject(item)),
|
||||
has_more: false,
|
||||
});
|
||||
});
|
||||
|
||||
app.get<{ Params: { fileId: string } }>('/v1/files/:fileId', async (request, reply) => {
|
||||
const owner = getProxyResourceOwner(request);
|
||||
if (!owner) {
|
||||
return reply.code(401).send({ error: { message: 'Missing proxy auth context', type: 'authentication_error' } });
|
||||
}
|
||||
const file = await getProxyFileByPublicIdForOwner(request.params.fileId, owner);
|
||||
if (!file) return notFound(reply);
|
||||
return reply.send(toFileObject(file));
|
||||
});
|
||||
|
||||
app.get<{ Params: { fileId: string } }>('/v1/files/:fileId/content', async (request, reply) => {
|
||||
const owner = getProxyResourceOwner(request);
|
||||
if (!owner) {
|
||||
return reply.code(401).send({ error: { message: 'Missing proxy auth context', type: 'authentication_error' } });
|
||||
}
|
||||
const file = await getProxyFileContentByPublicIdForOwner(request.params.fileId, owner);
|
||||
if (!file) return notFound(reply);
|
||||
reply.type(file.mimeType);
|
||||
reply.header('Content-Disposition', `inline; filename="${encodeURIComponent(file.filename)}"`);
|
||||
return reply.send(file.buffer);
|
||||
});
|
||||
|
||||
app.delete<{ Params: { fileId: string } }>('/v1/files/:fileId', async (request, reply) => {
|
||||
const owner = getProxyResourceOwner(request);
|
||||
if (!owner) {
|
||||
return reply.code(401).send({ error: { message: 'Missing proxy auth context', type: 'authentication_error' } });
|
||||
}
|
||||
const deleted = await softDeleteProxyFileByPublicIdForOwner(request.params.fileId, owner);
|
||||
if (!deleted) return notFound(reply);
|
||||
return reply.send({
|
||||
id: request.params.fileId,
|
||||
object: 'file',
|
||||
deleted: true,
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,843 @@
|
||||
import type { FastifyInstance, FastifyReply, FastifyRequest } from 'fastify';
|
||||
import { TextDecoder } from 'node:util';
|
||||
import { fetch } from 'undici';
|
||||
import { and, eq } from 'drizzle-orm';
|
||||
import { db, schema } from '../../db/index.js';
|
||||
import { formatUtcSqlDateTime } from '../../services/localTimeService.js';
|
||||
import { parseProxyUsage } from '../../services/proxyUsageParser.js';
|
||||
import { isModelAllowedByPolicyOrAllowedRoutes } from '../../services/downstreamApiKeyService.js';
|
||||
import { tokenRouter } from '../../services/tokenRouter.js';
|
||||
import { buildOauthProviderHeaders } from '../../services/oauth/service.js';
|
||||
import { getOauthInfoFromExtraConfig } from '../../services/oauth/oauthAccount.js';
|
||||
import { withSiteProxyRequestInit } from '../../services/siteProxy.js';
|
||||
import { refreshModelsAndRebuildRoutes } from '../../services/modelService.js';
|
||||
import { getDownstreamRoutingPolicy } from '../../routes/proxy/downstreamPolicy.js';
|
||||
import { executeEndpointFlow, type BuiltEndpointRequest } from '../../routes/proxy/endpointFlow.js';
|
||||
import { composeProxyLogMessage } from '../../routes/proxy/logPathMeta.js';
|
||||
import {
|
||||
buildUpstreamEndpointRequest,
|
||||
resolveUpstreamEndpointCandidates,
|
||||
} from '../../routes/proxy/upstreamEndpoint.js';
|
||||
import {
|
||||
geminiGenerateContentTransformer,
|
||||
} from '../../transformers/gemini/generate-content/index.js';
|
||||
import { createChatEndpointStrategy } from '../../transformers/shared/chatEndpointStrategy.js';
|
||||
import { normalizeUpstreamFinalResponse } from '../../transformers/shared/normalized.js';
|
||||
import {
|
||||
createGeminiCliStreamReader,
|
||||
unwrapGeminiCliPayload,
|
||||
wrapGeminiCliRequest,
|
||||
} from '../../routes/proxy/geminiCliCompat.js';
|
||||
import { dispatchRuntimeRequest } from '../../routes/proxy/runtimeExecutor.js';
|
||||
|
||||
const MAX_RETRIES = 2;
|
||||
const GEMINI_MODEL_PROBES = [
|
||||
'gemini-2.5-flash',
|
||||
'gemini-2.0-flash',
|
||||
'gemini-1.5-flash',
|
||||
'gemini-pro',
|
||||
];
|
||||
const GEMINI_CLI_STATIC_MODELS = [
|
||||
{ name: 'models/gemini-2.5-pro', displayName: 'Gemini 2.5 Pro' },
|
||||
{ name: 'models/gemini-2.5-flash', displayName: 'Gemini 2.5 Flash' },
|
||||
{ name: 'models/gemini-2.5-flash-lite', displayName: 'Gemini 2.5 Flash Lite' },
|
||||
{ name: 'models/gemini-3-pro-preview', displayName: 'Gemini 3 Pro Preview' },
|
||||
{ name: 'models/gemini-3.1-pro-preview', displayName: 'Gemini 3.1 Pro Preview' },
|
||||
{ name: 'models/gemini-3-flash-preview', displayName: 'Gemini 3 Flash Preview' },
|
||||
{ name: 'models/gemini-3.1-flash-lite-preview', displayName: 'Gemini 3.1 Flash Lite Preview' },
|
||||
];
|
||||
const EMPTY_PROXY_USAGE = {
|
||||
promptTokens: 0,
|
||||
completionTokens: 0,
|
||||
totalTokens: 0,
|
||||
};
|
||||
|
||||
function isGeminiCliPlatform(platform: unknown): boolean {
|
||||
return String(platform || '').trim().toLowerCase() === 'gemini-cli';
|
||||
}
|
||||
|
||||
function isAntigravityPlatform(platform: unknown): boolean {
|
||||
return String(platform || '').trim().toLowerCase() === 'antigravity';
|
||||
}
|
||||
|
||||
function isInternalGeminiPlatform(platform: unknown): boolean {
|
||||
return isGeminiCliPlatform(platform) || isAntigravityPlatform(platform);
|
||||
}
|
||||
|
||||
function buildGeminiCliActionPath(input: {
|
||||
isStreamAction: boolean;
|
||||
isCountTokensAction: boolean;
|
||||
}) {
|
||||
if (input.isCountTokensAction) return '/v1internal:countTokens';
|
||||
if (input.isStreamAction) return '/v1internal:streamGenerateContent?alt=sse';
|
||||
return '/v1internal:generateContent';
|
||||
}
|
||||
|
||||
function isDirectGeminiFamilyPlatform(platform: unknown): boolean {
|
||||
const normalized = String(platform || '').trim().toLowerCase();
|
||||
return normalized === 'gemini' || normalized === 'gemini-cli' || normalized === 'antigravity';
|
||||
}
|
||||
|
||||
function omitGeminiCliModelField(body: unknown): Record<string, unknown> {
|
||||
if (!body || typeof body !== 'object' || Array.isArray(body)) return {};
|
||||
const { model: _model, ...rest } = body as Record<string, unknown>;
|
||||
return rest;
|
||||
}
|
||||
|
||||
async function selectGeminiChannel(request: FastifyRequest) {
|
||||
const policy = getDownstreamRoutingPolicy(request);
|
||||
for (const candidate of GEMINI_MODEL_PROBES) {
|
||||
const selected = await tokenRouter.selectChannel(candidate, policy);
|
||||
if (selected) return selected;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
async function selectNextGeminiProbeChannel(request: FastifyRequest, excludeChannelIds: number[]) {
|
||||
const policy = getDownstreamRoutingPolicy(request);
|
||||
for (const candidate of GEMINI_MODEL_PROBES) {
|
||||
const selected = await tokenRouter.selectNextChannel(candidate, excludeChannelIds, policy);
|
||||
if (selected) return selected;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function resolveDownstreamPath(request: FastifyRequest): string {
|
||||
const rawUrl = request.raw.url || request.url || '';
|
||||
const withoutQuery = rawUrl.split('?')[0] || rawUrl;
|
||||
return withoutQuery || '/v1beta/models';
|
||||
}
|
||||
|
||||
function resolveUpstreamPath(apiVersion: string, modelActionPath: string): string {
|
||||
const normalizedVersion = apiVersion.replace(/^\/+/, '');
|
||||
const normalizedAction = modelActionPath.replace(/^\/+/, '');
|
||||
return `/${normalizedVersion}/${normalizedAction}`;
|
||||
}
|
||||
|
||||
function hasDownstreamModelRestrictions(policy: { supportedModels?: unknown; allowedRouteIds?: unknown }): boolean {
|
||||
const supportedModels = Array.isArray(policy.supportedModels) ? policy.supportedModels : [];
|
||||
const allowedRouteIds = Array.isArray(policy.allowedRouteIds) ? policy.allowedRouteIds : [];
|
||||
return supportedModels.length > 0 || allowedRouteIds.length > 0;
|
||||
}
|
||||
|
||||
function extractGeminiListedModelName(item: unknown): string {
|
||||
if (!item || typeof item !== 'object') return '';
|
||||
const rawName = typeof (item as { name?: unknown }).name === 'string'
|
||||
? (item as { name: string }).name.trim()
|
||||
: '';
|
||||
if (!rawName) return '';
|
||||
return rawName.startsWith('models/') ? rawName.slice('models/'.length) : rawName;
|
||||
}
|
||||
|
||||
async function filterGeminiListedModelsForPolicy(
|
||||
payload: unknown,
|
||||
request: FastifyRequest,
|
||||
): Promise<unknown> {
|
||||
if (!payload || typeof payload !== 'object' || !Array.isArray((payload as { models?: unknown[] }).models)) {
|
||||
return payload;
|
||||
}
|
||||
|
||||
const policy = getDownstreamRoutingPolicy(request);
|
||||
if (!hasDownstreamModelRestrictions(policy)) {
|
||||
return payload;
|
||||
}
|
||||
|
||||
const filteredModels: unknown[] = [];
|
||||
for (const item of (payload as { models: unknown[] }).models) {
|
||||
const modelName = extractGeminiListedModelName(item);
|
||||
if (!modelName) continue;
|
||||
if (!await isModelAllowedByPolicyOrAllowedRoutes(modelName, policy)) continue;
|
||||
const decision = await tokenRouter.explainSelection?.(modelName, [], policy);
|
||||
if (decision && typeof decision.selectedChannelId !== 'number') continue;
|
||||
filteredModels.push(item);
|
||||
}
|
||||
|
||||
return {
|
||||
...(payload as Record<string, unknown>),
|
||||
models: filteredModels,
|
||||
};
|
||||
}
|
||||
|
||||
async function readRouteAwareGeminiModels(request: FastifyRequest): Promise<Array<{ name: string; displayName: string }>> {
|
||||
const policy = getDownstreamRoutingPolicy(request);
|
||||
const rows = await db.select({ modelName: schema.modelAvailability.modelName })
|
||||
.from(schema.modelAvailability)
|
||||
.innerJoin(schema.accounts, eq(schema.modelAvailability.accountId, schema.accounts.id))
|
||||
.innerJoin(schema.sites, eq(schema.accounts.siteId, schema.sites.id))
|
||||
.where(and(
|
||||
eq(schema.modelAvailability.available, true),
|
||||
eq(schema.accounts.status, 'active'),
|
||||
eq(schema.sites.status, 'active'),
|
||||
))
|
||||
.all();
|
||||
const routeAliases = await db.select({ displayName: schema.tokenRoutes.displayName })
|
||||
.from(schema.tokenRoutes)
|
||||
.where(eq(schema.tokenRoutes.enabled, true))
|
||||
.all();
|
||||
const deduped = Array.from(new Set([
|
||||
...rows.map((row) => String(row.modelName || '').trim()).filter(Boolean),
|
||||
...routeAliases.map((row) => String(row.displayName || '').trim()).filter(Boolean),
|
||||
])).sort();
|
||||
|
||||
const allowed: Array<{ name: string; displayName: string }> = [];
|
||||
for (const modelName of deduped) {
|
||||
if (!await isModelAllowedByPolicyOrAllowedRoutes(modelName, policy)) continue;
|
||||
const decision = await tokenRouter.explainSelection?.(modelName, [], policy);
|
||||
if (decision && typeof decision.selectedChannelId !== 'number') continue;
|
||||
allowed.push({
|
||||
name: `models/${modelName}`,
|
||||
displayName: modelName,
|
||||
});
|
||||
}
|
||||
|
||||
return allowed;
|
||||
}
|
||||
|
||||
async function logProxy(
|
||||
selected: any,
|
||||
modelRequested: string,
|
||||
status: string,
|
||||
httpStatus: number,
|
||||
latencyMs: number,
|
||||
errorMessage: string | null,
|
||||
retryCount: number,
|
||||
downstreamPath: string,
|
||||
upstreamPath: string | null,
|
||||
promptTokens = 0,
|
||||
completionTokens = 0,
|
||||
totalTokens = 0,
|
||||
) {
|
||||
try {
|
||||
const createdAt = formatUtcSqlDateTime(new Date());
|
||||
const normalizedErrorMessage = composeProxyLogMessage({
|
||||
downstreamPath,
|
||||
upstreamPath,
|
||||
errorMessage,
|
||||
});
|
||||
await db.insert(schema.proxyLogs).values({
|
||||
routeId: selected.channel.routeId,
|
||||
channelId: selected.channel.id,
|
||||
accountId: selected.account.id,
|
||||
modelRequested,
|
||||
modelActual: selected.actualModel || modelRequested,
|
||||
status,
|
||||
httpStatus,
|
||||
latencyMs,
|
||||
promptTokens,
|
||||
completionTokens,
|
||||
totalTokens,
|
||||
estimatedCost: 0,
|
||||
billingDetails: null,
|
||||
errorMessage: normalizedErrorMessage,
|
||||
retryCount,
|
||||
createdAt,
|
||||
}).run();
|
||||
} catch (error) {
|
||||
console.warn('[proxy/gemini] failed to write proxy log', error);
|
||||
}
|
||||
}
|
||||
|
||||
export async function geminiProxyRoute(app: FastifyInstance) {
|
||||
const listModels = async (request: FastifyRequest, reply: FastifyReply) => {
|
||||
const apiVersion = geminiGenerateContentTransformer.resolveProxyApiVersion(
|
||||
request.params as { geminiApiVersion?: string } | undefined,
|
||||
);
|
||||
const excludeChannelIds: number[] = [];
|
||||
let retryCount = 0;
|
||||
let lastStatus = 503;
|
||||
let lastText = 'No available channels for Gemini models';
|
||||
let lastContentType = 'application/json';
|
||||
|
||||
while (retryCount <= MAX_RETRIES) {
|
||||
const selected = retryCount === 0
|
||||
? await selectGeminiChannel(request)
|
||||
: await selectNextGeminiProbeChannel(request, excludeChannelIds);
|
||||
if (!selected) {
|
||||
return reply.code(lastStatus).type(lastContentType).send(lastText);
|
||||
}
|
||||
|
||||
excludeChannelIds.push(selected.channel.id);
|
||||
|
||||
try {
|
||||
if (!isDirectGeminiFamilyPlatform(selected.site.platform)) {
|
||||
let models = await readRouteAwareGeminiModels(request);
|
||||
if (models.length <= 0) {
|
||||
await refreshModelsAndRebuildRoutes();
|
||||
models = await readRouteAwareGeminiModels(request);
|
||||
}
|
||||
return reply.code(200).send({ models });
|
||||
}
|
||||
|
||||
if (isGeminiCliPlatform(selected.site.platform)) {
|
||||
const filtered = await filterGeminiListedModelsForPolicy(
|
||||
{ models: GEMINI_CLI_STATIC_MODELS },
|
||||
request,
|
||||
);
|
||||
return reply.code(200).send(filtered);
|
||||
}
|
||||
|
||||
const upstream = await fetch(
|
||||
geminiGenerateContentTransformer.resolveModelsUrl(selected.site.url, apiVersion, selected.tokenValue),
|
||||
{ method: 'GET' },
|
||||
);
|
||||
const text = await upstream.text();
|
||||
if (!upstream.ok) {
|
||||
lastStatus = upstream.status;
|
||||
lastText = text;
|
||||
lastContentType = upstream.headers.get('content-type') || 'application/json';
|
||||
await tokenRouter.recordFailure?.(selected.channel.id);
|
||||
if (retryCount < MAX_RETRIES) {
|
||||
retryCount += 1;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const parsed = JSON.parse(text);
|
||||
const filtered = await filterGeminiListedModelsForPolicy(parsed, request);
|
||||
return reply.code(upstream.status).send(filtered);
|
||||
} catch {
|
||||
return reply.code(upstream.status).type(upstream.headers.get('content-type') || 'application/json').send(text);
|
||||
}
|
||||
} catch (error) {
|
||||
await tokenRouter.recordFailure?.(selected.channel.id);
|
||||
lastStatus = 502;
|
||||
lastContentType = 'application/json';
|
||||
lastText = JSON.stringify({
|
||||
error: {
|
||||
message: error instanceof Error ? error.message : 'Gemini upstream request failed',
|
||||
type: 'upstream_error',
|
||||
},
|
||||
});
|
||||
if (retryCount < MAX_RETRIES) {
|
||||
retryCount += 1;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const handleGenerateContent = async (
|
||||
request: FastifyRequest,
|
||||
reply: FastifyReply,
|
||||
options?: {
|
||||
downstreamProtocol?: 'gemini' | 'gemini-cli';
|
||||
action?: 'generateContent' | 'streamGenerateContent' | 'countTokens';
|
||||
},
|
||||
) => {
|
||||
const downstreamProtocol = options?.downstreamProtocol || 'gemini';
|
||||
const isGeminiCliDownstream = downstreamProtocol === 'gemini-cli';
|
||||
const cliRequestedModel = isGeminiCliDownstream
|
||||
? (typeof (request.body as Record<string, unknown> | null | undefined)?.model === 'string'
|
||||
? String((request.body as Record<string, unknown>).model).trim()
|
||||
: '')
|
||||
: '';
|
||||
const parsedPath = isGeminiCliDownstream
|
||||
? {
|
||||
apiVersion: 'v1beta',
|
||||
modelActionPath: `models/${cliRequestedModel}:${options?.action || 'generateContent'}`,
|
||||
isStreamAction: options?.action === 'streamGenerateContent',
|
||||
requestedModel: cliRequestedModel,
|
||||
}
|
||||
: geminiGenerateContentTransformer.parseProxyRequestPath({
|
||||
rawUrl: request.raw.url || request.url || '',
|
||||
params: request.params as { geminiApiVersion?: string } | undefined,
|
||||
});
|
||||
const { apiVersion, modelActionPath, isStreamAction, requestedModel } = parsedPath;
|
||||
const isCountTokensAction = isGeminiCliDownstream
|
||||
? options?.action === 'countTokens'
|
||||
: modelActionPath.endsWith(':countTokens');
|
||||
if (!requestedModel) {
|
||||
return reply.code(400).send({
|
||||
error: { message: 'Gemini model path is required', type: 'invalid_request_error' },
|
||||
});
|
||||
}
|
||||
|
||||
const policy = getDownstreamRoutingPolicy(request);
|
||||
const downstreamPath = resolveDownstreamPath(request);
|
||||
const excludeChannelIds: number[] = [];
|
||||
let retryCount = 0;
|
||||
let lastStatus = 503;
|
||||
let lastText = 'No available channels for this model';
|
||||
let lastContentType = 'application/json';
|
||||
|
||||
while (retryCount <= MAX_RETRIES) {
|
||||
const selected = retryCount === 0
|
||||
? await tokenRouter.selectChannel(requestedModel, policy)
|
||||
: await tokenRouter.selectNextChannel(requestedModel, excludeChannelIds, policy);
|
||||
if (!selected) {
|
||||
return reply.code(lastStatus).type(lastContentType).send(lastText);
|
||||
}
|
||||
|
||||
excludeChannelIds.push(selected.channel.id);
|
||||
|
||||
const actualModel = selected.actualModel || requestedModel;
|
||||
const normalizedBody = geminiGenerateContentTransformer.inbound.normalizeRequest(
|
||||
isGeminiCliDownstream ? omitGeminiCliModelField(request.body) : (request.body || {}),
|
||||
actualModel,
|
||||
);
|
||||
const oauth = getOauthInfoFromExtraConfig(selected.account.extraConfig);
|
||||
const isGeminiCli = isGeminiCliPlatform(selected.site.platform);
|
||||
const isInternalGemini = isInternalGeminiPlatform(selected.site.platform);
|
||||
const isDirectGeminiFamily = isDirectGeminiFamilyPlatform(selected.site.platform);
|
||||
const startTime = Date.now();
|
||||
let upstreamPath = '';
|
||||
|
||||
try {
|
||||
if (isDirectGeminiFamily) {
|
||||
if (isGeminiCli && !oauth?.projectId) {
|
||||
lastStatus = 500;
|
||||
lastContentType = 'application/json';
|
||||
lastText = JSON.stringify({
|
||||
error: {
|
||||
message: 'Gemini CLI OAuth project is missing',
|
||||
type: 'server_error',
|
||||
},
|
||||
});
|
||||
await tokenRouter.recordFailure?.(selected.channel.id);
|
||||
if (retryCount < MAX_RETRIES) {
|
||||
retryCount += 1;
|
||||
continue;
|
||||
}
|
||||
return reply.code(lastStatus).type(lastContentType).send(lastText);
|
||||
}
|
||||
|
||||
const actualModelAction = modelActionPath.replace(
|
||||
/^models\/[^:]+/,
|
||||
`models/${actualModel}`,
|
||||
);
|
||||
upstreamPath = isInternalGemini
|
||||
? buildGeminiCliActionPath({ isStreamAction, isCountTokensAction })
|
||||
: resolveUpstreamPath(apiVersion, actualModelAction);
|
||||
const query = new URLSearchParams(request.query as Record<string, string>).toString();
|
||||
const requestBody = isInternalGemini
|
||||
? (
|
||||
isCountTokensAction
|
||||
? { request: normalizedBody }
|
||||
: wrapGeminiCliRequest({
|
||||
modelName: actualModel,
|
||||
projectId: oauth?.projectId || '',
|
||||
request: normalizedBody as Record<string, unknown>,
|
||||
})
|
||||
)
|
||||
: normalizedBody;
|
||||
const requestHeaders = isInternalGemini
|
||||
? {
|
||||
'Content-Type': 'application/json',
|
||||
...(isStreamAction ? { Accept: 'text/event-stream' } : {}),
|
||||
Authorization: `Bearer ${selected.tokenValue}`,
|
||||
...buildOauthProviderHeaders({
|
||||
extraConfig: typeof selected.account.extraConfig === 'string' ? selected.account.extraConfig : null,
|
||||
downstreamHeaders: request.headers as Record<string, unknown>,
|
||||
}),
|
||||
}
|
||||
: {
|
||||
'Content-Type': 'application/json',
|
||||
};
|
||||
const targetUrl = isInternalGemini
|
||||
? `${selected.site.url}${upstreamPath}`
|
||||
: geminiGenerateContentTransformer.resolveActionUrl(
|
||||
selected.site.url,
|
||||
apiVersion,
|
||||
actualModelAction,
|
||||
selected.tokenValue,
|
||||
query,
|
||||
);
|
||||
const upstream = isInternalGemini
|
||||
? await dispatchRuntimeRequest({
|
||||
siteUrl: selected.site.url,
|
||||
targetUrl,
|
||||
request: {
|
||||
endpoint: 'chat',
|
||||
path: upstreamPath,
|
||||
headers: requestHeaders,
|
||||
body: requestBody as Record<string, unknown>,
|
||||
runtime: {
|
||||
executor: isGeminiCli ? 'gemini-cli' : 'antigravity',
|
||||
modelName: actualModel,
|
||||
stream: isStreamAction,
|
||||
oauthProjectId: oauth?.projectId || null,
|
||||
action: isCountTokensAction
|
||||
? 'countTokens'
|
||||
: (isStreamAction ? 'streamGenerateContent' : 'generateContent'),
|
||||
},
|
||||
},
|
||||
buildInit: async (requestUrl, requestForFetch) => withSiteProxyRequestInit(requestUrl, {
|
||||
method: 'POST',
|
||||
headers: requestForFetch.headers,
|
||||
body: JSON.stringify(requestForFetch.body),
|
||||
}),
|
||||
})
|
||||
: await fetch(targetUrl, {
|
||||
method: 'POST',
|
||||
headers: requestHeaders,
|
||||
body: JSON.stringify(requestBody),
|
||||
});
|
||||
const contentType = upstream.headers.get('content-type') || 'application/json';
|
||||
if (!upstream.ok) {
|
||||
lastStatus = upstream.status;
|
||||
lastContentType = contentType;
|
||||
lastText = await upstream.text();
|
||||
await tokenRouter.recordFailure?.(selected.channel.id);
|
||||
await logProxy(
|
||||
selected,
|
||||
requestedModel,
|
||||
'failed',
|
||||
lastStatus,
|
||||
Date.now() - startTime,
|
||||
lastText,
|
||||
retryCount,
|
||||
downstreamPath,
|
||||
upstreamPath,
|
||||
);
|
||||
if (retryCount < MAX_RETRIES) {
|
||||
retryCount += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
return reply.code(lastStatus).send(JSON.parse(lastText));
|
||||
} catch {
|
||||
return reply.code(lastStatus).type(lastContentType).send(lastText);
|
||||
}
|
||||
}
|
||||
|
||||
if (geminiGenerateContentTransformer.stream.isSseContentType(contentType)) {
|
||||
reply.hijack();
|
||||
reply.raw.statusCode = upstream.status;
|
||||
reply.raw.setHeader('Content-Type', contentType || 'text/event-stream');
|
||||
const upstreamReader = upstream.body?.getReader();
|
||||
const reader = isInternalGemini && !isGeminiCliDownstream && upstreamReader
|
||||
? createGeminiCliStreamReader(upstreamReader)
|
||||
: upstreamReader;
|
||||
if (!reader) {
|
||||
const latency = Date.now() - startTime;
|
||||
await tokenRouter.recordSuccess?.(selected.channel.id, latency, 0);
|
||||
await logProxy(
|
||||
selected,
|
||||
requestedModel,
|
||||
'success',
|
||||
upstream.status,
|
||||
latency,
|
||||
null,
|
||||
retryCount,
|
||||
downstreamPath,
|
||||
upstreamPath,
|
||||
);
|
||||
reply.raw.end();
|
||||
return;
|
||||
}
|
||||
const aggregateState = geminiGenerateContentTransformer.stream.createAggregateState();
|
||||
const decoder = new TextDecoder();
|
||||
let rest = '';
|
||||
try {
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
if (!value) continue;
|
||||
const chunkText = decoder.decode(value, { stream: true });
|
||||
const consumed = geminiGenerateContentTransformer.stream.consumeUpstreamSseBuffer(
|
||||
aggregateState,
|
||||
rest + chunkText,
|
||||
);
|
||||
rest = consumed.rest;
|
||||
for (const line of consumed.lines) {
|
||||
reply.raw.write(line);
|
||||
}
|
||||
}
|
||||
const tail = decoder.decode();
|
||||
if (tail) {
|
||||
const consumed = geminiGenerateContentTransformer.stream.consumeUpstreamSseBuffer(
|
||||
aggregateState,
|
||||
rest + tail,
|
||||
);
|
||||
for (const line of consumed.lines) {
|
||||
reply.raw.write(line);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
reader.releaseLock();
|
||||
reply.raw.end();
|
||||
}
|
||||
const parsedUsage = parseProxyUsage(aggregateState);
|
||||
const latency = Date.now() - startTime;
|
||||
await tokenRouter.recordSuccess?.(selected.channel.id, latency, 0);
|
||||
await logProxy(
|
||||
selected,
|
||||
requestedModel,
|
||||
'success',
|
||||
upstream.status,
|
||||
latency,
|
||||
null,
|
||||
retryCount,
|
||||
downstreamPath,
|
||||
upstreamPath,
|
||||
parsedUsage.promptTokens,
|
||||
parsedUsage.completionTokens,
|
||||
parsedUsage.totalTokens,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const text = await upstream.text();
|
||||
const aggregateState = geminiGenerateContentTransformer.stream.createAggregateState();
|
||||
let parsedUsage = EMPTY_PROXY_USAGE;
|
||||
try {
|
||||
const parsed = JSON.parse(text);
|
||||
const unwrappedPayload = isInternalGemini && !isGeminiCliDownstream
|
||||
? unwrapGeminiCliPayload(parsed)
|
||||
: parsed;
|
||||
const responsePayload = isCountTokensAction
|
||||
? unwrappedPayload
|
||||
: geminiGenerateContentTransformer.stream.serializeUpstreamJsonPayload(
|
||||
aggregateState,
|
||||
unwrappedPayload,
|
||||
isStreamAction,
|
||||
);
|
||||
parsedUsage = parseProxyUsage(aggregateState);
|
||||
const latency = Date.now() - startTime;
|
||||
await tokenRouter.recordSuccess?.(selected.channel.id, latency, 0);
|
||||
await logProxy(
|
||||
selected,
|
||||
requestedModel,
|
||||
'success',
|
||||
upstream.status,
|
||||
latency,
|
||||
null,
|
||||
retryCount,
|
||||
downstreamPath,
|
||||
upstreamPath,
|
||||
parsedUsage.promptTokens,
|
||||
parsedUsage.completionTokens,
|
||||
parsedUsage.totalTokens,
|
||||
);
|
||||
return reply.code(upstream.status).send(
|
||||
isGeminiCliDownstream && !isCountTokensAction
|
||||
? { response: responsePayload }
|
||||
: responsePayload,
|
||||
);
|
||||
} catch {
|
||||
const latency = Date.now() - startTime;
|
||||
await tokenRouter.recordSuccess?.(selected.channel.id, latency, 0);
|
||||
await logProxy(
|
||||
selected,
|
||||
requestedModel,
|
||||
'success',
|
||||
upstream.status,
|
||||
latency,
|
||||
null,
|
||||
retryCount,
|
||||
downstreamPath,
|
||||
upstreamPath,
|
||||
);
|
||||
return reply.code(upstream.status).type(contentType || 'application/json').send(text);
|
||||
}
|
||||
}
|
||||
|
||||
if (isCountTokensAction) {
|
||||
lastStatus = 501;
|
||||
lastContentType = 'application/json';
|
||||
lastText = JSON.stringify({
|
||||
error: {
|
||||
message: 'Gemini countTokens compatibility is not implemented for this upstream',
|
||||
type: 'invalid_request_error',
|
||||
},
|
||||
});
|
||||
return reply.code(lastStatus).type(lastContentType).send(lastText);
|
||||
}
|
||||
|
||||
const openAiBody = geminiGenerateContentTransformer.compatibility.buildOpenAiBodyFromGeminiRequest({
|
||||
body: normalizedBody as Record<string, unknown>,
|
||||
modelName: actualModel,
|
||||
stream: isStreamAction,
|
||||
});
|
||||
const endpointCandidates = await resolveUpstreamEndpointCandidates(
|
||||
{
|
||||
site: selected.site,
|
||||
account: selected.account,
|
||||
},
|
||||
actualModel,
|
||||
'openai',
|
||||
requestedModel,
|
||||
);
|
||||
const buildEndpointRequest = (
|
||||
endpoint: 'chat' | 'messages' | 'responses',
|
||||
requestOptions: { forceNormalizeClaudeBody?: boolean } = {},
|
||||
) => {
|
||||
const endpointRequest = buildUpstreamEndpointRequest({
|
||||
endpoint,
|
||||
modelName: actualModel,
|
||||
stream: isStreamAction,
|
||||
tokenValue: selected.tokenValue,
|
||||
oauthProvider: oauth?.provider,
|
||||
oauthProjectId: oauth?.projectId,
|
||||
sitePlatform: selected.site.platform,
|
||||
siteUrl: selected.site.url,
|
||||
openaiBody: openAiBody,
|
||||
downstreamFormat: 'openai',
|
||||
forceNormalizeClaudeBody: requestOptions.forceNormalizeClaudeBody,
|
||||
downstreamHeaders: request.headers as Record<string, unknown>,
|
||||
providerHeaders: buildOauthProviderHeaders({
|
||||
extraConfig: typeof selected.account.extraConfig === 'string' ? selected.account.extraConfig : null,
|
||||
downstreamHeaders: request.headers as Record<string, unknown>,
|
||||
}),
|
||||
});
|
||||
return {
|
||||
endpoint,
|
||||
path: endpointRequest.path,
|
||||
headers: endpointRequest.headers,
|
||||
body: endpointRequest.body as Record<string, unknown>,
|
||||
runtime: endpointRequest.runtime,
|
||||
};
|
||||
};
|
||||
const dispatchRequest = (compatibilityRequest: BuiltEndpointRequest, targetUrl?: string) => (
|
||||
dispatchRuntimeRequest({
|
||||
siteUrl: selected.site.url,
|
||||
targetUrl,
|
||||
request: compatibilityRequest,
|
||||
buildInit: async (requestUrl, requestForFetch) => withSiteProxyRequestInit(requestUrl, {
|
||||
method: 'POST',
|
||||
headers: requestForFetch.headers,
|
||||
body: JSON.stringify(requestForFetch.body),
|
||||
}),
|
||||
})
|
||||
);
|
||||
const endpointStrategy = createChatEndpointStrategy({
|
||||
downstreamFormat: 'openai',
|
||||
endpointCandidates,
|
||||
modelName: actualModel,
|
||||
requestedModelHint: requestedModel,
|
||||
sitePlatform: selected.site.platform,
|
||||
isStream: isStreamAction,
|
||||
buildRequest: ({ endpoint, forceNormalizeClaudeBody }) => buildEndpointRequest(
|
||||
endpoint,
|
||||
{ forceNormalizeClaudeBody },
|
||||
),
|
||||
dispatchRequest,
|
||||
});
|
||||
const endpointResult = await executeEndpointFlow({
|
||||
siteUrl: selected.site.url,
|
||||
endpointCandidates,
|
||||
buildRequest: (endpoint) => buildEndpointRequest(endpoint),
|
||||
dispatchRequest,
|
||||
tryRecover: endpointStrategy.tryRecover,
|
||||
shouldDowngrade: endpointStrategy.shouldDowngrade,
|
||||
});
|
||||
if (!endpointResult.ok) {
|
||||
lastStatus = endpointResult.status;
|
||||
lastContentType = 'application/json';
|
||||
lastText = JSON.stringify({
|
||||
error: {
|
||||
message: endpointResult.errText,
|
||||
type: 'upstream_error',
|
||||
},
|
||||
});
|
||||
await tokenRouter.recordFailure?.(selected.channel.id);
|
||||
await logProxy(
|
||||
selected,
|
||||
requestedModel,
|
||||
'failed',
|
||||
lastStatus,
|
||||
Date.now() - startTime,
|
||||
endpointResult.errText,
|
||||
retryCount,
|
||||
downstreamPath,
|
||||
null,
|
||||
);
|
||||
if (retryCount < MAX_RETRIES) {
|
||||
retryCount += 1;
|
||||
continue;
|
||||
}
|
||||
return reply.code(lastStatus).type(lastContentType).send(lastText);
|
||||
}
|
||||
|
||||
upstreamPath = endpointResult.upstreamPath;
|
||||
const upstream = endpointResult.upstream;
|
||||
const rawText = await upstream.text();
|
||||
let upstreamData: unknown = rawText;
|
||||
try {
|
||||
upstreamData = JSON.parse(rawText);
|
||||
} catch {}
|
||||
const parsedUsage = parseProxyUsage(upstreamData);
|
||||
const normalizedFinal = normalizeUpstreamFinalResponse(upstreamData, actualModel, rawText);
|
||||
const geminiResponse = geminiGenerateContentTransformer.compatibility.serializeNormalizedFinalToGemini({
|
||||
normalized: normalizedFinal,
|
||||
usage: {
|
||||
promptTokens: parsedUsage.promptTokens,
|
||||
completionTokens: parsedUsage.completionTokens,
|
||||
totalTokens: parsedUsage.totalTokens,
|
||||
},
|
||||
});
|
||||
const latency = Date.now() - startTime;
|
||||
await tokenRouter.recordSuccess?.(selected.channel.id, latency, 0);
|
||||
await logProxy(
|
||||
selected,
|
||||
requestedModel,
|
||||
'success',
|
||||
upstream.status,
|
||||
latency,
|
||||
null,
|
||||
retryCount,
|
||||
downstreamPath,
|
||||
upstreamPath,
|
||||
parsedUsage.promptTokens,
|
||||
parsedUsage.completionTokens,
|
||||
parsedUsage.totalTokens,
|
||||
);
|
||||
return reply.code(upstream.status).send(
|
||||
isGeminiCliDownstream
|
||||
? { response: geminiResponse }
|
||||
: geminiResponse,
|
||||
);
|
||||
} catch (error) {
|
||||
lastStatus = 502;
|
||||
lastContentType = 'application/json';
|
||||
lastText = JSON.stringify({
|
||||
error: {
|
||||
message: error instanceof Error ? error.message : 'Gemini upstream request failed',
|
||||
type: 'upstream_error',
|
||||
},
|
||||
});
|
||||
await tokenRouter.recordFailure?.(selected.channel.id);
|
||||
await logProxy(
|
||||
selected,
|
||||
requestedModel,
|
||||
'failed',
|
||||
0,
|
||||
Date.now() - startTime,
|
||||
error instanceof Error ? error.message : 'Gemini upstream request failed',
|
||||
retryCount,
|
||||
downstreamPath,
|
||||
upstreamPath || null,
|
||||
);
|
||||
if (retryCount < MAX_RETRIES) {
|
||||
retryCount += 1;
|
||||
continue;
|
||||
}
|
||||
return reply.code(lastStatus).type(lastContentType).send(lastText);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const generateContent = async (request: FastifyRequest, reply: FastifyReply) => handleGenerateContent(request, reply);
|
||||
const geminiCliGenerateContent = async (request: FastifyRequest, reply: FastifyReply) => handleGenerateContent(request, reply, {
|
||||
downstreamProtocol: 'gemini-cli',
|
||||
action: 'generateContent',
|
||||
});
|
||||
const geminiCliStreamGenerateContent = async (request: FastifyRequest, reply: FastifyReply) => handleGenerateContent(request, reply, {
|
||||
downstreamProtocol: 'gemini-cli',
|
||||
action: 'streamGenerateContent',
|
||||
});
|
||||
const geminiCliCountTokens = async (request: FastifyRequest, reply: FastifyReply) => handleGenerateContent(request, reply, {
|
||||
downstreamProtocol: 'gemini-cli',
|
||||
action: 'countTokens',
|
||||
});
|
||||
|
||||
app.get('/v1beta/models', listModels);
|
||||
app.get('/gemini/:geminiApiVersion/models', listModels);
|
||||
app.post('/v1beta/models/*', generateContent);
|
||||
app.post('/gemini/:geminiApiVersion/models/*', generateContent);
|
||||
app.post('/v1internal::generateContent', geminiCliGenerateContent);
|
||||
app.post('/v1internal::streamGenerateContent', geminiCliStreamGenerateContent);
|
||||
app.post('/v1internal::countTokens', geminiCliCountTokens);
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
import type { ProxyResourceOwner } from '../../middleware/auth.js';
|
||||
import { getProxyFileByPublicIdForOwner } from '../../services/proxyFileStore.js';
|
||||
import {
|
||||
inferInputFileMimeType,
|
||||
normalizeInputFileBlock,
|
||||
toOpenAiChatFileBlock,
|
||||
toResponsesInputFileBlock,
|
||||
type NormalizedInputFile,
|
||||
} from '../../transformers/shared/inputFile.js';
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return !!value && typeof value === 'object';
|
||||
}
|
||||
|
||||
function isSupportedFileMimeType(mimeType: string): boolean {
|
||||
return mimeType === 'application/pdf'
|
||||
|| mimeType === 'text/plain'
|
||||
|| mimeType === 'text/markdown'
|
||||
|| mimeType === 'application/json'
|
||||
|| mimeType.startsWith('image/')
|
||||
|| mimeType.startsWith('audio/');
|
||||
}
|
||||
|
||||
export class ProxyInputFileResolutionError extends Error {
|
||||
statusCode: number;
|
||||
payload: unknown;
|
||||
|
||||
constructor(statusCode: number, payload: unknown) {
|
||||
super(typeof payload === 'object' && payload && 'error' in (payload as Record<string, unknown>)
|
||||
? String(((payload as { error?: { message?: unknown } }).error?.message) || 'input file resolution failed')
|
||||
: 'input file resolution failed');
|
||||
this.name = 'ProxyInputFileResolutionError';
|
||||
this.statusCode = statusCode;
|
||||
this.payload = payload;
|
||||
}
|
||||
}
|
||||
|
||||
function buildInvalidRequest(message: string): ProxyInputFileResolutionError {
|
||||
return new ProxyInputFileResolutionError(400, {
|
||||
error: {
|
||||
message,
|
||||
type: 'invalid_request_error',
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function buildNotFound(message: string): ProxyInputFileResolutionError {
|
||||
return new ProxyInputFileResolutionError(404, {
|
||||
error: {
|
||||
message,
|
||||
type: 'not_found_error',
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async function resolveInputFileBlock(
|
||||
item: Record<string, unknown>,
|
||||
owner: ProxyResourceOwner,
|
||||
): Promise<Record<string, unknown>> {
|
||||
const normalized = normalizeInputFileBlock(item);
|
||||
if (!normalized) return item;
|
||||
|
||||
let resolved: NormalizedInputFile = { ...normalized };
|
||||
|
||||
if (!resolved.fileData && resolved.fileId?.startsWith('file-metapi-')) {
|
||||
const stored = await getProxyFileByPublicIdForOwner(resolved.fileId, owner);
|
||||
if (!stored) {
|
||||
throw buildNotFound(`file not found: ${resolved.fileId}`);
|
||||
}
|
||||
resolved = {
|
||||
filename: stored.filename,
|
||||
mimeType: stored.mimeType,
|
||||
fileData: stored.contentBase64,
|
||||
};
|
||||
}
|
||||
|
||||
const inferredMimeType = inferInputFileMimeType(resolved);
|
||||
if (resolved.fileData && (!inferredMimeType || !isSupportedFileMimeType(inferredMimeType))) {
|
||||
throw buildInvalidRequest(`unsupported file mime type: ${inferredMimeType || 'application/octet-stream'}`);
|
||||
}
|
||||
if (inferredMimeType) {
|
||||
resolved.mimeType = inferredMimeType;
|
||||
}
|
||||
|
||||
const type = typeof item.type === 'string' ? item.type : '';
|
||||
if (type === 'input_file') {
|
||||
return toResponsesInputFileBlock(resolved);
|
||||
}
|
||||
if (type === 'file') {
|
||||
return toOpenAiChatFileBlock(resolved);
|
||||
}
|
||||
return item;
|
||||
}
|
||||
|
||||
export async function inlineLocalInputFileReferences(
|
||||
value: unknown,
|
||||
owner: ProxyResourceOwner,
|
||||
): Promise<unknown> {
|
||||
if (Array.isArray(value)) {
|
||||
return Promise.all(value.map((item) => inlineLocalInputFileReferences(item, owner)));
|
||||
}
|
||||
|
||||
if (!isRecord(value)) return value;
|
||||
|
||||
const type = typeof value.type === 'string' ? value.type : '';
|
||||
if (type === 'input_file' || type === 'file') {
|
||||
return resolveInputFileBlock(value, owner);
|
||||
}
|
||||
|
||||
const entries = await Promise.all(
|
||||
Object.entries(value).map(async ([key, entryValue]) => [key, await inlineLocalInputFileReferences(entryValue, owner)] as const),
|
||||
);
|
||||
return Object.fromEntries(entries);
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
import { describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import { listModelsSurface } from './modelsSurface.js';
|
||||
|
||||
describe('listModelsSurface', () => {
|
||||
it('returns OpenAI list shape and hides models without a resolvable channel', async () => {
|
||||
const result = await listModelsSurface({
|
||||
downstreamPolicy: { type: 'all' },
|
||||
responseFormat: 'openai',
|
||||
tokenRouter: {
|
||||
getAvailableModels: vi.fn().mockResolvedValue(['routable-model', 'orphan-model']),
|
||||
explainSelection: vi.fn()
|
||||
.mockResolvedValueOnce({ selectedChannelId: null })
|
||||
.mockResolvedValueOnce({ selectedChannelId: 11 }),
|
||||
},
|
||||
refreshModelsAndRebuildRoutes: vi.fn(),
|
||||
isModelAllowed: vi.fn().mockResolvedValue(true),
|
||||
now: () => new Date('2026-03-19T00:00:00.000Z'),
|
||||
});
|
||||
|
||||
expect(result).toEqual({
|
||||
object: 'list',
|
||||
data: [
|
||||
{
|
||||
id: 'routable-model',
|
||||
object: 'model',
|
||||
created: 1773878400,
|
||||
owned_by: 'metapi',
|
||||
},
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
it('returns Claude list shape when requested', async () => {
|
||||
const result = await listModelsSurface({
|
||||
downstreamPolicy: { type: 'all' },
|
||||
responseFormat: 'claude',
|
||||
tokenRouter: {
|
||||
getAvailableModels: vi.fn().mockResolvedValue(['claude-opus-4-6']),
|
||||
explainSelection: vi.fn().mockResolvedValue({ selectedChannelId: 22 }),
|
||||
},
|
||||
refreshModelsAndRebuildRoutes: vi.fn(),
|
||||
isModelAllowed: vi.fn().mockResolvedValue(true),
|
||||
now: () => new Date('2026-03-19T00:00:00.000Z'),
|
||||
});
|
||||
|
||||
expect(result).toEqual({
|
||||
data: [
|
||||
{
|
||||
id: 'claude-opus-4-6',
|
||||
type: 'model',
|
||||
display_name: 'claude-opus-4-6',
|
||||
created_at: '2026-03-19T00:00:00.000Z',
|
||||
},
|
||||
],
|
||||
first_id: 'claude-opus-4-6',
|
||||
last_id: 'claude-opus-4-6',
|
||||
has_more: false,
|
||||
});
|
||||
});
|
||||
|
||||
it('applies downstream policy filtering before selection checks and refreshes once when the first read is empty', async () => {
|
||||
const getAvailableModels = vi.fn()
|
||||
.mockResolvedValueOnce(['blocked-model'])
|
||||
.mockResolvedValueOnce(['allowed-model']);
|
||||
const refreshModelsAndRebuildRoutes = vi.fn().mockResolvedValue(undefined);
|
||||
const isModelAllowed = vi.fn()
|
||||
.mockResolvedValueOnce(false)
|
||||
.mockResolvedValueOnce(true);
|
||||
const explainSelection = vi.fn().mockResolvedValue({ selectedChannelId: 33 });
|
||||
|
||||
const result = await listModelsSurface({
|
||||
downstreamPolicy: { type: 'whitelist' },
|
||||
responseFormat: 'openai',
|
||||
tokenRouter: {
|
||||
getAvailableModels,
|
||||
explainSelection,
|
||||
},
|
||||
refreshModelsAndRebuildRoutes,
|
||||
isModelAllowed,
|
||||
now: () => new Date('2026-03-19T00:00:00.000Z'),
|
||||
});
|
||||
|
||||
expect(refreshModelsAndRebuildRoutes).toHaveBeenCalledTimes(1);
|
||||
expect(result).toEqual({
|
||||
object: 'list',
|
||||
data: [
|
||||
{
|
||||
id: 'allowed-model',
|
||||
object: 'model',
|
||||
created: 1773878400,
|
||||
owned_by: 'metapi',
|
||||
},
|
||||
],
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,70 @@
|
||||
function isSearchPseudoModel(modelName: string): boolean {
|
||||
const normalized = (modelName || '').trim().toLowerCase();
|
||||
if (!normalized) return false;
|
||||
return normalized === '__search' || /^__.+_search$/.test(normalized);
|
||||
}
|
||||
|
||||
type ModelsSurfaceInput = {
|
||||
downstreamPolicy: unknown;
|
||||
responseFormat: 'openai' | 'claude';
|
||||
tokenRouter: {
|
||||
getAvailableModels(): Promise<string[]>;
|
||||
explainSelection(modelName: string, excludeChannelIds: number[], downstreamPolicy: unknown): Promise<{
|
||||
selectedChannelId?: number | null;
|
||||
}>;
|
||||
};
|
||||
refreshModelsAndRebuildRoutes(): Promise<unknown>;
|
||||
isModelAllowed(modelName: string, downstreamPolicy: unknown): Promise<boolean>;
|
||||
now?: () => Date;
|
||||
};
|
||||
|
||||
async function readVisibleModels(input: ModelsSurfaceInput): Promise<string[]> {
|
||||
const deduped = Array.from(new Set(await input.tokenRouter.getAvailableModels()))
|
||||
.filter((modelName) => !isSearchPseudoModel(modelName))
|
||||
.sort();
|
||||
const allowed: string[] = [];
|
||||
for (const modelName of deduped) {
|
||||
if (!await input.isModelAllowed(modelName, input.downstreamPolicy)) {
|
||||
continue;
|
||||
}
|
||||
const decision = await input.tokenRouter.explainSelection(modelName, [], input.downstreamPolicy);
|
||||
if (typeof decision.selectedChannelId === 'number') {
|
||||
allowed.push(modelName);
|
||||
}
|
||||
}
|
||||
return allowed;
|
||||
}
|
||||
|
||||
export async function listModelsSurface(input: ModelsSurfaceInput) {
|
||||
let models = await readVisibleModels(input);
|
||||
if (models.length === 0) {
|
||||
await input.refreshModelsAndRebuildRoutes();
|
||||
models = await readVisibleModels(input);
|
||||
}
|
||||
|
||||
const now = input.now?.() ?? new Date();
|
||||
if (input.responseFormat === 'claude') {
|
||||
const data = models.map((id) => ({
|
||||
id,
|
||||
type: 'model' as const,
|
||||
display_name: id,
|
||||
created_at: now.toISOString(),
|
||||
}));
|
||||
return {
|
||||
data,
|
||||
first_id: data[0]?.id || null,
|
||||
last_id: data[data.length - 1]?.id || null,
|
||||
has_more: false,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
object: 'list' as const,
|
||||
data: models.map((id) => ({
|
||||
id,
|
||||
object: 'model' as const,
|
||||
created: Math.floor(now.getTime() / 1000),
|
||||
owned_by: 'metapi',
|
||||
})),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,870 @@
|
||||
import { TextDecoder } from 'node:util';
|
||||
import type { FastifyReply, FastifyRequest } from 'fastify';
|
||||
import { db, hasProxyLogDownstreamApiKeyIdColumn, schema } from '../../db/index.js';
|
||||
import { tokenRouter } from '../../services/tokenRouter.js';
|
||||
import { refreshModelsAndRebuildRoutes } from '../../services/modelService.js';
|
||||
import { reportProxyAllFailed, reportTokenExpired } from '../../services/alertService.js';
|
||||
import { isTokenExpiredError } from '../../services/alertRules.js';
|
||||
import { shouldRetryProxyRequest } from '../../services/proxyRetryPolicy.js';
|
||||
import { resolveProxyUsageWithSelfLogFallback } from '../../services/proxyUsageFallbackService.js';
|
||||
import { mergeProxyUsage, parseProxyUsage } from '../../services/proxyUsageParser.js';
|
||||
import { resolveChannelProxyUrl, withSiteRecordProxyRequestInit } from '../../services/siteProxy.js';
|
||||
import { openAiResponsesTransformer } from '../../transformers/openai/responses/index.js';
|
||||
import {
|
||||
buildUpstreamEndpointRequest,
|
||||
resolveUpstreamEndpointCandidates,
|
||||
} from '../../routes/proxy/upstreamEndpoint.js';
|
||||
import { ensureModelAllowedForDownstreamKey, getDownstreamRoutingPolicy, recordDownstreamCostUsage } from '../../routes/proxy/downstreamPolicy.js';
|
||||
import { composeProxyLogMessage } from '../../routes/proxy/logPathMeta.js';
|
||||
import { executeEndpointFlow, type BuiltEndpointRequest } from '../../routes/proxy/endpointFlow.js';
|
||||
import { detectProxyFailure } from '../../routes/proxy/proxyFailureJudge.js';
|
||||
import { formatUtcSqlDateTime } from '../../services/localTimeService.js';
|
||||
import { resolveProxyLogBilling } from '../../routes/proxy/proxyBilling.js';
|
||||
import { getProxyAuthContext, getProxyResourceOwner } from '../../middleware/auth.js';
|
||||
import { dispatchRuntimeRequest } from '../../routes/proxy/runtimeExecutor.js';
|
||||
import { normalizeInputFileBlock } from '../../transformers/shared/inputFile.js';
|
||||
import {
|
||||
ProxyInputFileResolutionError,
|
||||
hasNonImageFileInputInOpenAiBody,
|
||||
resolveResponsesBodyInputFiles,
|
||||
} from '../../services/proxyInputFileResolver.js';
|
||||
import {
|
||||
buildOauthProviderHeaders,
|
||||
} from '../../services/oauth/service.js';
|
||||
import { getOauthInfoFromExtraConfig } from '../../services/oauth/oauthAccount.js';
|
||||
import { recordOauthQuotaResetHint } from '../../services/oauth/quota.js';
|
||||
import { refreshOauthAccessTokenSingleflight } from '../../services/oauth/refreshSingleflight.js';
|
||||
import { collectResponsesFinalPayloadFromSse } from '../../routes/proxy/responsesSseFinal.js';
|
||||
import {
|
||||
createGeminiCliStreamReader,
|
||||
unwrapGeminiCliPayload,
|
||||
} from '../../routes/proxy/geminiCliCompat.js';
|
||||
import { isCodexResponsesSurface } from '../cliProfiles/codexProfile.js';
|
||||
import { detectCliProfile } from '../cliProfiles/registry.js';
|
||||
import type { CliProfileId } from '../cliProfiles/types.js';
|
||||
|
||||
const MAX_RETRIES = 2;
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return !!value && typeof value === 'object';
|
||||
}
|
||||
|
||||
function normalizeIncludeList(value: unknown): string[] {
|
||||
if (typeof value === 'string') {
|
||||
const trimmed = value.trim();
|
||||
return trimmed ? [trimmed] : [];
|
||||
}
|
||||
if (!Array.isArray(value)) return [];
|
||||
return value
|
||||
.map((item) => (typeof item === 'string' ? item.trim() : ''))
|
||||
.filter((item) => item.length > 0);
|
||||
}
|
||||
|
||||
function hasExplicitInclude(body: Record<string, unknown>): boolean {
|
||||
return Object.prototype.hasOwnProperty.call(body, 'include');
|
||||
}
|
||||
|
||||
function hasResponsesReasoningRequest(value: unknown): boolean {
|
||||
if (!isRecord(value)) return false;
|
||||
const relevantKeys = ['effort', 'budget_tokens', 'budgetTokens', 'max_tokens', 'maxTokens', 'summary'];
|
||||
return relevantKeys.some((key) => {
|
||||
const entry = value[key];
|
||||
if (typeof entry === 'string') return entry.trim().length > 0;
|
||||
return entry !== undefined && entry !== null;
|
||||
});
|
||||
}
|
||||
|
||||
function carriesResponsesReasoningContinuity(value: unknown): boolean {
|
||||
if (Array.isArray(value)) {
|
||||
return value.some((item) => carriesResponsesReasoningContinuity(item));
|
||||
}
|
||||
if (!isRecord(value)) return false;
|
||||
|
||||
const type = typeof value.type === 'string' ? value.type.trim().toLowerCase() : '';
|
||||
if (type === 'reasoning') {
|
||||
if (typeof value.encrypted_content === 'string' && value.encrypted_content.trim()) {
|
||||
return true;
|
||||
}
|
||||
if (Array.isArray(value.summary) && value.summary.length > 0) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
if (typeof value.reasoning_signature === 'string' && value.reasoning_signature.trim()) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return carriesResponsesReasoningContinuity(value.input)
|
||||
|| carriesResponsesReasoningContinuity(value.content);
|
||||
}
|
||||
|
||||
function wantsNativeResponsesReasoning(body: unknown): boolean {
|
||||
if (!isRecord(body)) return false;
|
||||
const include = normalizeIncludeList(body.include);
|
||||
if (include.some((item) => item.toLowerCase() === 'reasoning.encrypted_content')) {
|
||||
return true;
|
||||
}
|
||||
if (carriesResponsesReasoningContinuity(body.input)) {
|
||||
return true;
|
||||
}
|
||||
if (hasExplicitInclude(body)) {
|
||||
return false;
|
||||
}
|
||||
return hasResponsesReasoningRequest(body.reasoning);
|
||||
}
|
||||
|
||||
function carriesResponsesFileUrlInput(value: unknown): boolean {
|
||||
if (Array.isArray(value)) {
|
||||
return value.some((item) => carriesResponsesFileUrlInput(item));
|
||||
}
|
||||
if (!isRecord(value)) return false;
|
||||
|
||||
const normalizedFile = normalizeInputFileBlock(value);
|
||||
if (normalizedFile?.fileUrl) return true;
|
||||
|
||||
return Object.values(value).some((entry) => carriesResponsesFileUrlInput(entry));
|
||||
}
|
||||
|
||||
type UsageSummary = ReturnType<typeof parseProxyUsage>;
|
||||
|
||||
type DownstreamClientContext = {
|
||||
clientKind: CliProfileId;
|
||||
sessionId?: string;
|
||||
traceHint?: string;
|
||||
};
|
||||
|
||||
function detectDownstreamClientContext(input: {
|
||||
downstreamPath: string;
|
||||
headers?: Record<string, unknown>;
|
||||
body?: unknown;
|
||||
}): DownstreamClientContext {
|
||||
const detected = detectCliProfile(input);
|
||||
return {
|
||||
clientKind: detected.id,
|
||||
...(detected.sessionId ? { sessionId: detected.sessionId } : {}),
|
||||
...(detected.traceHint ? { traceHint: detected.traceHint } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
function deriveCodexExplicitSessionId(body: Record<string, unknown>): string | null {
|
||||
const promptCacheKey = typeof body.prompt_cache_key === 'string'
|
||||
? body.prompt_cache_key.trim()
|
||||
: '';
|
||||
return promptCacheKey || null;
|
||||
}
|
||||
|
||||
export async function handleOpenAiResponsesSurfaceRequest(
|
||||
request: FastifyRequest,
|
||||
reply: FastifyReply,
|
||||
downstreamPath: '/v1/responses' | '/v1/responses/compact',
|
||||
) {
|
||||
const body = request.body as Record<string, unknown>;
|
||||
const clientContext = detectDownstreamClientContext({
|
||||
downstreamPath,
|
||||
headers: request.headers as Record<string, unknown>,
|
||||
body,
|
||||
});
|
||||
const defaultEncryptedReasoningInclude = isCodexResponsesSurface(
|
||||
request.headers as Record<string, unknown>,
|
||||
);
|
||||
const parsedRequestEnvelope = openAiResponsesTransformer.transformRequest(body, {
|
||||
defaultEncryptedReasoningInclude,
|
||||
});
|
||||
if (parsedRequestEnvelope.error) {
|
||||
return reply.code(parsedRequestEnvelope.error.statusCode).send(parsedRequestEnvelope.error.payload);
|
||||
}
|
||||
const requestEnvelope = parsedRequestEnvelope.value!;
|
||||
const requestedModel = requestEnvelope.model;
|
||||
const isStream = requestEnvelope.stream;
|
||||
const isCompactRequest = downstreamPath === '/v1/responses/compact';
|
||||
if (isCompactRequest && isStream) {
|
||||
return reply.code(400).send({
|
||||
error: {
|
||||
message: 'stream is not supported on /v1/responses/compact',
|
||||
type: 'invalid_request_error',
|
||||
},
|
||||
});
|
||||
}
|
||||
if (!await ensureModelAllowedForDownstreamKey(request, reply, requestedModel)) return;
|
||||
const downstreamPolicy = getDownstreamRoutingPolicy(request);
|
||||
const downstreamApiKeyId = getProxyAuthContext(request)?.keyId ?? null;
|
||||
const logDownstreamApiKeyId = downstreamApiKeyId !== null
|
||||
&& await hasProxyLogDownstreamApiKeyIdColumn();
|
||||
const excludeChannelIds: number[] = [];
|
||||
let retryCount = 0;
|
||||
|
||||
while (retryCount <= MAX_RETRIES) {
|
||||
let selected = retryCount === 0
|
||||
? await tokenRouter.selectChannel(requestedModel, downstreamPolicy)
|
||||
: await tokenRouter.selectNextChannel(requestedModel, excludeChannelIds, downstreamPolicy);
|
||||
|
||||
if (!selected && retryCount === 0) {
|
||||
await refreshModelsAndRebuildRoutes();
|
||||
selected = await tokenRouter.selectChannel(requestedModel, downstreamPolicy);
|
||||
}
|
||||
|
||||
if (!selected) {
|
||||
await reportProxyAllFailed({
|
||||
model: requestedModel,
|
||||
reason: 'No available channels after retries',
|
||||
});
|
||||
return reply.code(503).send({
|
||||
error: { message: 'No available channels for this model', type: 'server_error' },
|
||||
});
|
||||
}
|
||||
|
||||
excludeChannelIds.push(selected.channel.id);
|
||||
|
||||
const modelName = selected.actualModel || requestedModel;
|
||||
const oauth = getOauthInfoFromExtraConfig(selected.account.extraConfig);
|
||||
const isCodexSite = String(selected.site.platform || '').trim().toLowerCase() === 'codex';
|
||||
const owner = getProxyResourceOwner(request);
|
||||
let normalizedResponsesBody: Record<string, unknown> = {
|
||||
...requestEnvelope.parsed.normalizedBody,
|
||||
model: modelName,
|
||||
stream: isStream,
|
||||
};
|
||||
if (body.generate === false) {
|
||||
normalizedResponsesBody.generate = false;
|
||||
}
|
||||
if (owner) {
|
||||
try {
|
||||
normalizedResponsesBody = await resolveResponsesBodyInputFiles(normalizedResponsesBody, owner);
|
||||
} catch (error) {
|
||||
if (error instanceof ProxyInputFileResolutionError) {
|
||||
return reply.code(error.statusCode).send(error.payload);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
const openAiBody = openAiResponsesTransformer.inbound.toOpenAiBody(
|
||||
normalizedResponsesBody,
|
||||
modelName,
|
||||
isStream,
|
||||
{ defaultEncryptedReasoningInclude },
|
||||
);
|
||||
const hasNonImageFileInput = hasNonImageFileInputInOpenAiBody(openAiBody);
|
||||
const prefersNativeResponsesReasoning = wantsNativeResponsesReasoning(normalizedResponsesBody);
|
||||
const requiresNativeResponsesFileUrl = carriesResponsesFileUrlInput(normalizedResponsesBody.input);
|
||||
if (requiresNativeResponsesFileUrl && String(selected.site.platform || '').trim().toLowerCase() === 'claude') {
|
||||
return reply.code(400).send({
|
||||
error: {
|
||||
message: 'Responses input_file.file_url requires an upstream /v1/responses endpoint; current site only supports /v1/messages.',
|
||||
type: 'invalid_request_error',
|
||||
},
|
||||
});
|
||||
}
|
||||
const endpointCandidates = await resolveUpstreamEndpointCandidates(
|
||||
{
|
||||
site: selected.site,
|
||||
account: selected.account,
|
||||
},
|
||||
modelName,
|
||||
'responses',
|
||||
requestedModel,
|
||||
{
|
||||
hasNonImageFileInput,
|
||||
wantsNativeResponsesReasoning: prefersNativeResponsesReasoning,
|
||||
},
|
||||
);
|
||||
if (requiresNativeResponsesFileUrl) {
|
||||
endpointCandidates.splice(0, endpointCandidates.length, 'responses');
|
||||
}
|
||||
if (endpointCandidates.length === 0) {
|
||||
endpointCandidates.push('responses', 'chat', 'messages');
|
||||
}
|
||||
const buildProviderHeaders = () => (
|
||||
buildOauthProviderHeaders({
|
||||
extraConfig: typeof selected.account.extraConfig === 'string' ? selected.account.extraConfig : null,
|
||||
downstreamHeaders: request.headers as Record<string, unknown>,
|
||||
})
|
||||
);
|
||||
const buildEndpointRequest = (endpoint: 'chat' | 'messages' | 'responses') => {
|
||||
const upstreamStream = isStream || (isCodexSite && endpoint === 'responses');
|
||||
const codexExplicitSessionId = deriveCodexExplicitSessionId(normalizedResponsesBody);
|
||||
const endpointRequest = buildUpstreamEndpointRequest({
|
||||
endpoint,
|
||||
modelName,
|
||||
stream: upstreamStream,
|
||||
tokenValue: selected.tokenValue,
|
||||
oauthProvider: oauth?.provider,
|
||||
oauthProjectId: oauth?.projectId,
|
||||
sitePlatform: selected.site.platform,
|
||||
siteUrl: selected.site.url,
|
||||
openaiBody: openAiBody,
|
||||
downstreamFormat: 'responses',
|
||||
responsesOriginalBody: normalizedResponsesBody,
|
||||
downstreamHeaders: request.headers as Record<string, unknown>,
|
||||
providerHeaders: buildProviderHeaders(),
|
||||
codexExplicitSessionId,
|
||||
});
|
||||
const upstreamPath = (
|
||||
isCompactRequest && endpoint === 'responses'
|
||||
? `${endpointRequest.path}/compact`
|
||||
: endpointRequest.path
|
||||
);
|
||||
return {
|
||||
endpoint,
|
||||
path: upstreamPath,
|
||||
headers: endpointRequest.headers,
|
||||
body: endpointRequest.body as Record<string, unknown>,
|
||||
runtime: endpointRequest.runtime,
|
||||
};
|
||||
};
|
||||
const dispatchRequest = (compatibilityRequest: BuiltEndpointRequest, targetUrl?: string) => (
|
||||
dispatchRuntimeRequest({
|
||||
siteUrl: selected.site.url,
|
||||
targetUrl,
|
||||
request: compatibilityRequest,
|
||||
buildInit: (_requestUrl, requestForFetch) => withSiteRecordProxyRequestInit(selected.site, {
|
||||
method: 'POST',
|
||||
headers: requestForFetch.headers,
|
||||
body: JSON.stringify(requestForFetch.body),
|
||||
}),
|
||||
})
|
||||
);
|
||||
const endpointStrategy = openAiResponsesTransformer.compatibility.createEndpointStrategy({
|
||||
isStream: isStream || isCodexSite,
|
||||
requiresNativeResponsesFileUrl,
|
||||
dispatchRequest,
|
||||
});
|
||||
const tryRecover = async (ctx: Parameters<NonNullable<typeof endpointStrategy.tryRecover>>[0]) => {
|
||||
if (ctx.response.status === 401 && oauth) {
|
||||
try {
|
||||
const refreshed = await refreshOauthAccessTokenSingleflight(selected.account.id);
|
||||
selected.tokenValue = refreshed.accessToken;
|
||||
selected.account = {
|
||||
...selected.account,
|
||||
accessToken: refreshed.accessToken,
|
||||
extraConfig: refreshed.extraConfig ?? selected.account.extraConfig,
|
||||
};
|
||||
const refreshedRequest = buildEndpointRequest(ctx.request.endpoint);
|
||||
const refreshedTargetUrl = `${selected.site.url}${refreshedRequest.path}`;
|
||||
const refreshedResponse = await dispatchRequest(refreshedRequest, refreshedTargetUrl);
|
||||
if (refreshedResponse.ok) {
|
||||
return {
|
||||
upstream: refreshedResponse,
|
||||
upstreamPath: refreshedRequest.path,
|
||||
};
|
||||
}
|
||||
ctx.request = refreshedRequest;
|
||||
ctx.response = refreshedResponse;
|
||||
ctx.rawErrText = await refreshedResponse.text().catch(() => 'unknown error');
|
||||
} catch {
|
||||
return endpointStrategy.tryRecover(ctx);
|
||||
}
|
||||
}
|
||||
return endpointStrategy.tryRecover(ctx);
|
||||
};
|
||||
|
||||
const startTime = Date.now();
|
||||
|
||||
try {
|
||||
const endpointResult = await executeEndpointFlow({
|
||||
siteUrl: selected.site.url,
|
||||
proxyUrl: resolveChannelProxyUrl(selected.site, selected.account.extraConfig),
|
||||
endpointCandidates,
|
||||
buildRequest: (endpoint) => buildEndpointRequest(endpoint),
|
||||
dispatchRequest,
|
||||
tryRecover,
|
||||
shouldDowngrade: endpointStrategy.shouldDowngrade,
|
||||
onDowngrade: (ctx) => {
|
||||
logProxy(
|
||||
selected,
|
||||
requestedModel,
|
||||
'failed',
|
||||
ctx.response.status,
|
||||
Date.now() - startTime,
|
||||
ctx.errText,
|
||||
retryCount,
|
||||
downstreamPath,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
null,
|
||||
null,
|
||||
clientContext,
|
||||
logDownstreamApiKeyId ? downstreamApiKeyId : null,
|
||||
);
|
||||
},
|
||||
});
|
||||
|
||||
if (!endpointResult.ok) {
|
||||
const status = endpointResult.status || 502;
|
||||
const errText = endpointResult.errText || 'unknown error';
|
||||
const rawErrText = endpointResult.rawErrText || errText;
|
||||
tokenRouter.recordFailure(selected.channel.id);
|
||||
logProxy(
|
||||
selected,
|
||||
requestedModel,
|
||||
'failed',
|
||||
status,
|
||||
Date.now() - startTime,
|
||||
errText,
|
||||
retryCount,
|
||||
downstreamPath,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
null,
|
||||
null,
|
||||
clientContext,
|
||||
logDownstreamApiKeyId ? downstreamApiKeyId : null,
|
||||
);
|
||||
await recordOauthQuotaResetHint({
|
||||
accountId: selected.account.id,
|
||||
statusCode: status,
|
||||
errorText: rawErrText,
|
||||
});
|
||||
|
||||
if (isTokenExpiredError({ status, message: errText })) {
|
||||
await reportTokenExpired({
|
||||
accountId: selected.account.id,
|
||||
username: selected.account.username,
|
||||
siteName: selected.site.name,
|
||||
detail: `HTTP ${status}`,
|
||||
});
|
||||
}
|
||||
|
||||
if (shouldRetryProxyRequest(status, errText) && retryCount < MAX_RETRIES) {
|
||||
retryCount += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
await reportProxyAllFailed({
|
||||
model: requestedModel,
|
||||
reason: `upstream returned HTTP ${status}`,
|
||||
});
|
||||
return reply.code(status).send({ error: { message: errText, type: 'upstream_error' } });
|
||||
}
|
||||
|
||||
const upstream = endpointResult.upstream;
|
||||
const successfulUpstreamPath = endpointResult.upstreamPath;
|
||||
const finalizeStreamSuccess = async (parsedUsage: UsageSummary, latency: number) => {
|
||||
let usageForLog = {
|
||||
promptTokens: parsedUsage.promptTokens,
|
||||
completionTokens: parsedUsage.completionTokens,
|
||||
totalTokens: parsedUsage.totalTokens,
|
||||
};
|
||||
let estimatedCost = 0;
|
||||
let billingDetails: unknown = null;
|
||||
|
||||
try {
|
||||
const resolvedUsage = await resolveProxyUsageWithSelfLogFallback({
|
||||
site: selected.site,
|
||||
account: selected.account,
|
||||
tokenValue: selected.tokenValue,
|
||||
tokenName: selected.tokenName,
|
||||
modelName: selected.actualModel || requestedModel,
|
||||
requestStartedAtMs: startTime,
|
||||
requestEndedAtMs: startTime + latency,
|
||||
localLatencyMs: latency,
|
||||
usage: {
|
||||
promptTokens: parsedUsage.promptTokens,
|
||||
completionTokens: parsedUsage.completionTokens,
|
||||
totalTokens: parsedUsage.totalTokens,
|
||||
},
|
||||
});
|
||||
usageForLog = {
|
||||
promptTokens: resolvedUsage.promptTokens,
|
||||
completionTokens: resolvedUsage.completionTokens,
|
||||
totalTokens: resolvedUsage.totalTokens,
|
||||
};
|
||||
const billing = await resolveProxyLogBilling({
|
||||
site: selected.site,
|
||||
account: selected.account,
|
||||
modelName: selected.actualModel || requestedModel,
|
||||
parsedUsage,
|
||||
resolvedUsage,
|
||||
});
|
||||
estimatedCost = billing.estimatedCost;
|
||||
billingDetails = billing.billingDetails;
|
||||
} catch (error) {
|
||||
console.error('[responses] post-stream bookkeeping failed:', error);
|
||||
}
|
||||
|
||||
try {
|
||||
tokenRouter.recordSuccess(selected.channel.id, latency, estimatedCost);
|
||||
recordDownstreamCostUsage(request, estimatedCost);
|
||||
logProxy(
|
||||
selected, requestedModel, 'success', 200, latency, null, retryCount, downstreamPath,
|
||||
usageForLog.promptTokens, usageForLog.completionTokens, usageForLog.totalTokens, estimatedCost, billingDetails,
|
||||
successfulUpstreamPath,
|
||||
clientContext,
|
||||
logDownstreamApiKeyId ? downstreamApiKeyId : null,
|
||||
);
|
||||
} catch (error) {
|
||||
console.error('[responses] post-stream success logging failed:', error);
|
||||
}
|
||||
};
|
||||
|
||||
if (isStream) {
|
||||
const upstreamContentType = (upstream.headers.get('content-type') || '').toLowerCase();
|
||||
const startSseResponse = () => {
|
||||
reply.hijack();
|
||||
reply.raw.statusCode = 200;
|
||||
reply.raw.setHeader('Content-Type', 'text/event-stream; charset=utf-8');
|
||||
reply.raw.setHeader('Cache-Control', 'no-cache, no-transform');
|
||||
reply.raw.setHeader('Connection', 'keep-alive');
|
||||
reply.raw.setHeader('X-Accel-Buffering', 'no');
|
||||
};
|
||||
|
||||
let parsedUsage: UsageSummary = {
|
||||
promptTokens: 0,
|
||||
completionTokens: 0,
|
||||
totalTokens: 0,
|
||||
cacheReadTokens: 0,
|
||||
cacheCreationTokens: 0,
|
||||
promptTokensIncludeCache: null,
|
||||
};
|
||||
const writeLines = (lines: string[]) => {
|
||||
for (const line of lines) reply.raw.write(line);
|
||||
};
|
||||
const streamSession = openAiResponsesTransformer.proxyStream.createSession({
|
||||
modelName,
|
||||
successfulUpstreamPath,
|
||||
strictTerminalEvents: Object.entries(request.headers as Record<string, unknown>)
|
||||
.some(([rawKey, rawValue]) => rawKey.trim().toLowerCase() === 'x-metapi-responses-websocket-transport'
|
||||
&& String(rawValue).trim() === '1'),
|
||||
getUsage: () => parsedUsage,
|
||||
onParsedPayload: (payload) => {
|
||||
if (payload && typeof payload === 'object') {
|
||||
parsedUsage = mergeProxyUsage(parsedUsage, parseProxyUsage(payload));
|
||||
}
|
||||
},
|
||||
writeLines,
|
||||
writeRaw: (chunk) => {
|
||||
reply.raw.write(chunk);
|
||||
},
|
||||
});
|
||||
if (!upstreamContentType.includes('text/event-stream')) {
|
||||
const rawText = await upstream.text();
|
||||
let upstreamData: unknown = rawText;
|
||||
try {
|
||||
upstreamData = JSON.parse(rawText);
|
||||
} catch {
|
||||
upstreamData = rawText;
|
||||
}
|
||||
if (String(selected.site.platform || '').trim().toLowerCase() === 'gemini-cli') {
|
||||
upstreamData = unwrapGeminiCliPayload(upstreamData);
|
||||
}
|
||||
|
||||
parsedUsage = parseProxyUsage(upstreamData);
|
||||
const latency = Date.now() - startTime;
|
||||
const failure = detectProxyFailure({ rawText, usage: parsedUsage });
|
||||
if (failure) {
|
||||
tokenRouter.recordFailure(selected.channel.id);
|
||||
logProxy(
|
||||
selected,
|
||||
requestedModel,
|
||||
'failed',
|
||||
failure.status,
|
||||
latency,
|
||||
failure.reason,
|
||||
retryCount,
|
||||
downstreamPath,
|
||||
parsedUsage.promptTokens,
|
||||
parsedUsage.completionTokens,
|
||||
parsedUsage.totalTokens,
|
||||
0,
|
||||
null,
|
||||
successfulUpstreamPath,
|
||||
clientContext,
|
||||
logDownstreamApiKeyId ? downstreamApiKeyId : null,
|
||||
);
|
||||
|
||||
if (shouldRetryProxyRequest(failure.status, failure.reason) && retryCount < MAX_RETRIES) {
|
||||
retryCount += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
await reportProxyAllFailed({
|
||||
model: requestedModel,
|
||||
reason: failure.reason,
|
||||
});
|
||||
return reply.code(failure.status).send({ error: { message: failure.reason, type: 'upstream_error' } });
|
||||
}
|
||||
|
||||
startSseResponse();
|
||||
const streamResult = streamSession.consumeUpstreamFinalPayload(upstreamData, rawText, reply.raw);
|
||||
if (streamResult.status === 'failed') {
|
||||
tokenRouter.recordFailure(selected.channel.id);
|
||||
logProxy(
|
||||
selected,
|
||||
requestedModel,
|
||||
'failed',
|
||||
200,
|
||||
latency,
|
||||
streamResult.errorMessage,
|
||||
retryCount,
|
||||
downstreamPath,
|
||||
parsedUsage.promptTokens,
|
||||
parsedUsage.completionTokens,
|
||||
parsedUsage.totalTokens,
|
||||
0,
|
||||
null,
|
||||
successfulUpstreamPath,
|
||||
clientContext,
|
||||
logDownstreamApiKeyId ? downstreamApiKeyId : null,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
await finalizeStreamSuccess(parsedUsage, latency);
|
||||
return;
|
||||
}
|
||||
|
||||
startSseResponse();
|
||||
|
||||
const upstreamReader = upstream.body?.getReader();
|
||||
const baseReader = String(selected.site.platform || '').trim().toLowerCase() === 'gemini-cli' && upstreamReader
|
||||
? createGeminiCliStreamReader(upstreamReader)
|
||||
: upstreamReader;
|
||||
let rawText = '';
|
||||
const decoder = new TextDecoder();
|
||||
const reader = baseReader
|
||||
? {
|
||||
async read() {
|
||||
const result = await baseReader.read();
|
||||
if (result.value) {
|
||||
rawText += decoder.decode(result.value, { stream: true });
|
||||
}
|
||||
return result;
|
||||
},
|
||||
async cancel(reason?: unknown) {
|
||||
return baseReader.cancel(reason);
|
||||
},
|
||||
releaseLock() {
|
||||
return baseReader.releaseLock();
|
||||
},
|
||||
}
|
||||
: baseReader;
|
||||
const streamResult = await streamSession.run(reader, reply.raw);
|
||||
rawText += decoder.decode();
|
||||
|
||||
const latency = Date.now() - startTime;
|
||||
if (streamResult.status === 'failed') {
|
||||
tokenRouter.recordFailure(selected.channel.id);
|
||||
logProxy(
|
||||
selected,
|
||||
requestedModel,
|
||||
'failed',
|
||||
200,
|
||||
latency,
|
||||
streamResult.errorMessage,
|
||||
retryCount,
|
||||
downstreamPath,
|
||||
parsedUsage.promptTokens,
|
||||
parsedUsage.completionTokens,
|
||||
parsedUsage.totalTokens,
|
||||
0,
|
||||
null,
|
||||
successfulUpstreamPath,
|
||||
clientContext,
|
||||
logDownstreamApiKeyId ? downstreamApiKeyId : null,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// Once SSE has been hijacked and bytes may already be on the wire, we
|
||||
// must not attempt to convert stream failures into a fresh HTTP error
|
||||
// response or retry on another channel. Responses stream failures are
|
||||
// handled in-band by the proxy stream session.
|
||||
|
||||
await finalizeStreamSuccess(parsedUsage, latency);
|
||||
return;
|
||||
}
|
||||
|
||||
const upstreamContentType = (upstream.headers.get('content-type') || '').toLowerCase();
|
||||
let rawText = '';
|
||||
let upstreamData: unknown;
|
||||
if (
|
||||
upstreamContentType.includes('text/event-stream')
|
||||
&& (
|
||||
successfulUpstreamPath.endsWith('/responses')
|
||||
|| successfulUpstreamPath.endsWith('/responses/compact')
|
||||
)
|
||||
) {
|
||||
const collected = await collectResponsesFinalPayloadFromSse(upstream, modelName);
|
||||
rawText = collected.rawText;
|
||||
upstreamData = collected.payload;
|
||||
} else {
|
||||
rawText = await upstream.text();
|
||||
upstreamData = rawText;
|
||||
try {
|
||||
upstreamData = JSON.parse(rawText);
|
||||
} catch {
|
||||
upstreamData = rawText;
|
||||
}
|
||||
}
|
||||
if (String(selected.site.platform || '').trim().toLowerCase() === 'gemini-cli') {
|
||||
upstreamData = unwrapGeminiCliPayload(upstreamData);
|
||||
}
|
||||
const latency = Date.now() - startTime;
|
||||
const parsedUsage = parseProxyUsage(upstreamData);
|
||||
const failure = detectProxyFailure({ rawText, usage: parsedUsage });
|
||||
if (failure) {
|
||||
tokenRouter.recordFailure(selected.channel.id);
|
||||
logProxy(
|
||||
selected,
|
||||
requestedModel,
|
||||
'failed',
|
||||
failure.status,
|
||||
latency,
|
||||
failure.reason,
|
||||
retryCount,
|
||||
downstreamPath,
|
||||
parsedUsage.promptTokens,
|
||||
parsedUsage.completionTokens,
|
||||
parsedUsage.totalTokens,
|
||||
0,
|
||||
null,
|
||||
successfulUpstreamPath,
|
||||
clientContext,
|
||||
logDownstreamApiKeyId ? downstreamApiKeyId : null,
|
||||
);
|
||||
if (shouldRetryProxyRequest(failure.status, failure.reason) && retryCount < MAX_RETRIES) {
|
||||
retryCount += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
await reportProxyAllFailed({
|
||||
model: requestedModel,
|
||||
reason: failure.reason,
|
||||
});
|
||||
return reply.code(failure.status).send({ error: { message: failure.reason, type: 'upstream_error' } });
|
||||
}
|
||||
const normalized = openAiResponsesTransformer.transformFinalResponse(
|
||||
upstreamData,
|
||||
modelName,
|
||||
rawText,
|
||||
);
|
||||
const downstreamData = openAiResponsesTransformer.outbound.serializeFinal({
|
||||
upstreamPayload: upstreamData,
|
||||
normalized,
|
||||
usage: parsedUsage,
|
||||
serializationMode: isCompactRequest ? 'compact' : 'response',
|
||||
});
|
||||
const resolvedUsage = await resolveProxyUsageWithSelfLogFallback({
|
||||
site: selected.site,
|
||||
account: selected.account,
|
||||
tokenValue: selected.tokenValue,
|
||||
tokenName: selected.tokenName,
|
||||
modelName: selected.actualModel || requestedModel,
|
||||
requestStartedAtMs: startTime,
|
||||
requestEndedAtMs: startTime + latency,
|
||||
localLatencyMs: latency,
|
||||
usage: {
|
||||
promptTokens: parsedUsage.promptTokens,
|
||||
completionTokens: parsedUsage.completionTokens,
|
||||
totalTokens: parsedUsage.totalTokens,
|
||||
},
|
||||
});
|
||||
const { estimatedCost, billingDetails } = await resolveProxyLogBilling({
|
||||
site: selected.site,
|
||||
account: selected.account,
|
||||
modelName: selected.actualModel || requestedModel,
|
||||
parsedUsage,
|
||||
resolvedUsage,
|
||||
});
|
||||
|
||||
tokenRouter.recordSuccess(selected.channel.id, latency, estimatedCost);
|
||||
recordDownstreamCostUsage(request, estimatedCost);
|
||||
logProxy(
|
||||
selected, requestedModel, 'success', 200, latency, null, retryCount, downstreamPath,
|
||||
resolvedUsage.promptTokens, resolvedUsage.completionTokens, resolvedUsage.totalTokens, estimatedCost, billingDetails,
|
||||
successfulUpstreamPath,
|
||||
clientContext,
|
||||
logDownstreamApiKeyId ? downstreamApiKeyId : null,
|
||||
);
|
||||
return reply.send(downstreamData);
|
||||
} catch (err: any) {
|
||||
tokenRouter.recordFailure(selected.channel.id);
|
||||
logProxy(
|
||||
selected,
|
||||
requestedModel,
|
||||
'failed',
|
||||
0,
|
||||
Date.now() - startTime,
|
||||
err.message,
|
||||
retryCount,
|
||||
downstreamPath,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
null,
|
||||
null,
|
||||
clientContext,
|
||||
logDownstreamApiKeyId ? downstreamApiKeyId : null,
|
||||
);
|
||||
if (retryCount < MAX_RETRIES) {
|
||||
retryCount += 1;
|
||||
continue;
|
||||
}
|
||||
await reportProxyAllFailed({
|
||||
model: requestedModel,
|
||||
reason: err.message || 'network failure',
|
||||
});
|
||||
return reply.code(502).send({
|
||||
error: { message: `Upstream error: ${err.message}`, type: 'upstream_error' },
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function logProxy(
|
||||
selected: any,
|
||||
modelRequested: string,
|
||||
status: string,
|
||||
httpStatus: number,
|
||||
latencyMs: number,
|
||||
errorMessage: string | null,
|
||||
retryCount: number,
|
||||
downstreamPath: string,
|
||||
promptTokens = 0,
|
||||
completionTokens = 0,
|
||||
totalTokens = 0,
|
||||
estimatedCost = 0,
|
||||
billingDetails: unknown = null,
|
||||
upstreamPath: string | null = null,
|
||||
clientContext: DownstreamClientContext | null = null,
|
||||
downstreamApiKeyId: number | null = null,
|
||||
) {
|
||||
try {
|
||||
const createdAt = formatUtcSqlDateTime(new Date());
|
||||
const normalizedErrorMessage = composeProxyLogMessage({
|
||||
clientKind: clientContext?.clientKind && clientContext.clientKind !== 'generic'
|
||||
? clientContext.clientKind
|
||||
: null,
|
||||
sessionId: clientContext?.sessionId || null,
|
||||
traceHint: clientContext?.traceHint || null,
|
||||
downstreamPath,
|
||||
upstreamPath,
|
||||
errorMessage,
|
||||
});
|
||||
await db.insert(schema.proxyLogs).values({
|
||||
routeId: selected.channel.routeId,
|
||||
channelId: selected.channel.id,
|
||||
accountId: selected.account.id,
|
||||
...(downstreamApiKeyId !== null ? { downstreamApiKeyId } : {}),
|
||||
modelRequested,
|
||||
modelActual: selected.actualModel,
|
||||
status,
|
||||
httpStatus,
|
||||
latencyMs,
|
||||
promptTokens,
|
||||
completionTokens,
|
||||
totalTokens,
|
||||
estimatedCost,
|
||||
billingDetails: billingDetails ? JSON.stringify(billingDetails) : null,
|
||||
errorMessage: normalizedErrorMessage,
|
||||
retryCount,
|
||||
createdAt,
|
||||
}).run();
|
||||
} catch (error) {
|
||||
console.warn('[proxy/responses] failed to write proxy log', error);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user