LaPaoPao/BoosAPI
1
0
Fork 0
forked from boos/BoosAPI
Code Pull Requests 1 Activity

feat: add proxy first-byte timeout and log badges (#383)

Browse Source 
* add proxy first-byte timeout and log badges

* fix first-byte timeout review feedback
This commit is contained in:
Cita
2026-04-02 10:00:11 +08:00
committed by GitHub
parent bcd758edac
commit 180d17adfc
43 changed files with 4049 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/server/proxy-core/surfaces/chatSurface.ts
+11
View File
@@ -45,6 +45,7 @@ import {
unwrapGeminiCliPayload,
} from '../../routes/proxy/geminiCliCompat.js';
import { summarizeConversationFileInputsInOpenAiBody } from '../capabilities/conversationFileCapabilities.js';
import { getObservedResponseMeta } from '../firstByteTimeout.js';
import { getRuntimeResponseReader, readRuntimeResponseText } from '../executors/types.js';
import { detectDownstreamClientContext } from '../../routes/proxy/downstreamClientContext.js';
import { getProxyMaxChannelRetries } from '../../services/proxyChannelRetry.js';
@@ -394,6 +395,7 @@ export async function handleChatSurfaceRequest(
return executeEndpointFlow({
siteUrl: siteApiBaseUrl,
disableCrossProtocolFallback: config.disableCrossProtocolFallback,
firstByteTimeoutMs: Math.max(0, Math.trunc((config.proxyFirstByteTimeoutSec || 0) * 1000)),
endpointCandidates,
buildRequest: (endpoint) => buildEndpointRequest(endpoint),
dispatchRequest,
@@ -525,6 +527,7 @@ export async function handleChatSurfaceRequest(
const upstream = endpointResult.upstream;
const successfulUpstreamPath = endpointResult.upstreamPath;
const firstByteLatencyMs = getObservedResponseMeta(upstream)?.firstByteLatencyMs ?? null;
if (isStream) {
const upstreamContentType = (upstream.headers.get('content-type') || '').toLowerCase();
@@ -557,6 +560,8 @@ export async function handleChatSurfaceRequest(
parsedUsage,
upstreamUsagePresent,
requestStartedAtMs: startTime,
isStream: true,
firstByteLatencyMs,
latencyMs,
retryCount,
upstreamPath: successfulUpstreamPath,
@@ -918,6 +923,8 @@ export async function handleChatSurfaceRequest(
parsedUsage,
upstreamUsagePresent,
requestStartedAtMs: startTime,
isStream: false,
firstByteLatencyMs,
latencyMs: latency,
retryCount,
upstreamPath: successfulUpstreamPath,
@@ -961,6 +968,7 @@ export async function handleChatSurfaceRequest(
status: endpointFailureStatus || 502,
errText: err.message || 'unknown error',
rawErrText: err.rawErrText || err.message || 'unknown error',
isStream,
latencyMs: Date.now() - startTime,
retryCount,
});
@@ -985,6 +993,7 @@ export async function handleChatSurfaceRequest(
requestedModel,
modelName,
errorMessage: err?.message || 'network failure',
isStream,
latencyMs: Date.now() - startTime,
retryCount,
});
@@ -1387,6 +1396,7 @@ export async function handleClaudeCountTokensSurfaceRequest(
status: endpointFailureStatus || 502,
errText: error.message || 'unknown error',
rawErrText: error.rawErrText || error.message || 'unknown error',
isStream: false,
latencyMs: Date.now() - startTime,
retryCount,
});
@@ -1407,6 +1417,7 @@ export async function handleClaudeCountTokensSurfaceRequest(
requestedModel,
modelName,
errorMessage: error?.message || 'network failure',
isStream: false,
latencyMs: Date.now() - startTime,
retryCount,
});
src/server/proxy-core/surfaces/geminiSurface.ts
+61 -4
View File
@@ -40,6 +40,7 @@ import { detectDownstreamClientContext, type DownstreamClientContext } from '../
import { insertProxyLog } from '../../services/proxyLogStore.js';
import { summarizeConversationFileInputsInOpenAiBody } from '../capabilities/conversationFileCapabilities.js';
import { getRuntimeResponseReader, readRuntimeResponseText } from '../executors/types.js';
import { fetchWithObservedFirstByte, getObservedResponseMeta } from '../firstByteTimeout.js';
import { getProxyMaxChannelRetries } from '../../services/proxyChannelRetry.js';
import { shouldAbortSameSiteEndpointFallback } from '../../services/proxyRetryPolicy.js';
import {
@@ -253,6 +254,8 @@ async function logProxy(
promptTokens = 0,
completionTokens = 0,
totalTokens = 0,
isStream = false,
firstByteLatencyMs: number | null = null,
) {
try {
const createdAt = formatUtcSqlDateTime(new Date());
@@ -274,6 +277,8 @@ async function logProxy(
modelActual: selected.actualModel || modelRequested,
status,
httpStatus,
isStream,
firstByteLatencyMs,
latencyMs,
promptTokens,
completionTokens,
@@ -611,6 +616,7 @@ export async function geminiProxyRoute(app: FastifyInstance) {
const isInternalGemini = isInternalGeminiPlatform(selected.site.platform);
const isDirectGeminiFamily = isDirectGeminiFamilyPlatform(selected.site.platform);
const startTime = Date.now();
const firstByteTimeoutMs = Math.max(0, Math.trunc((config.proxyFirstByteTimeoutSec || 0) * 1000));
let upstreamPath = '';
try {
@@ -698,11 +704,12 @@ export async function geminiProxyRoute(app: FastifyInstance) {
requestHeaders,
targetUrl,
runtimeExecutor,
dispatch: () => (
dispatch: (signal?: AbortSignal) => (
isInternalGemini
? dispatchRuntimeRequest({
siteUrl: selected.site.url,
targetUrl,
signal,
request: {
endpoint: 'chat',
path: upstreamPath,
@@ -728,13 +735,22 @@ export async function geminiProxyRoute(app: FastifyInstance) {
method: 'POST',
headers: requestHeaders,
body: JSON.stringify(requestBody),
signal,
})
),
};
};
let directDispatchState = buildDirectDispatchState();
let upstream = await directDispatchState.dispatch();
const dispatchWithObservedFirstByte = async () => fetchWithObservedFirstByte(
(signal) => directDispatchState.dispatch(signal),
{
firstByteTimeoutMs,
startedAtMs: Date.now(),
},
);
let upstream = await dispatchWithObservedFirstByte();
let firstByteLatencyMs = getObservedResponseMeta(upstream)?.firstByteLatencyMs ?? null;
let contentType = upstream.headers.get('content-type') || 'application/json';
let recoverApplied = false;
if (upstream.status === 401 && oauth) {
@@ -748,7 +764,8 @@ export async function geminiProxyRoute(app: FastifyInstance) {
};
oauth = getOauthInfoFromAccount(selected.account);
directDispatchState = buildDirectDispatchState();
upstream = await directDispatchState.dispatch();
upstream = await dispatchWithObservedFirstByte();
firstByteLatencyMs = getObservedResponseMeta(upstream)?.firstByteLatencyMs ?? null;
contentType = upstream.headers.get('content-type') || 'application/json';
recoverApplied = true;
} catch {
@@ -791,6 +808,11 @@ export async function geminiProxyRoute(app: FastifyInstance) {
downstreamPath,
upstreamPath,
clientContext,
0,
0,
0,
isStreamAction,
firstByteLatencyMs,
);
if (canRetryChannelSelection(retryCount, forcedChannelId)) {
retryCount += 1;
@@ -829,6 +851,11 @@ export async function geminiProxyRoute(app: FastifyInstance) {
downstreamPath,
upstreamPath,
clientContext,
0,
0,
0,
isStreamAction,
firstByteLatencyMs,
);
await safeInsertSurfaceProxyDebugAttempt(debugTrace, {
attemptIndex: retryCount,
@@ -913,6 +940,8 @@ export async function geminiProxyRoute(app: FastifyInstance) {
parsedUsage.promptTokens,
parsedUsage.completionTokens,
parsedUsage.totalTokens,
isStreamAction,
firstByteLatencyMs,
);
await safeInsertSurfaceProxyDebugAttempt(debugTrace, {
attemptIndex: retryCount,
@@ -965,6 +994,8 @@ export async function geminiProxyRoute(app: FastifyInstance) {
parsedUsage.promptTokens,
parsedUsage.completionTokens,
parsedUsage.totalTokens,
isStreamAction,
firstByteLatencyMs,
);
await safeInsertSurfaceProxyDebugAttempt(debugTrace, {
attemptIndex: retryCount,
@@ -1029,6 +1060,8 @@ export async function geminiProxyRoute(app: FastifyInstance) {
parsedUsage.promptTokens,
parsedUsage.completionTokens,
parsedUsage.totalTokens,
isStreamAction,
firstByteLatencyMs,
);
await safeInsertSurfaceProxyDebugAttempt(debugTrace, {
attemptIndex: retryCount,
@@ -1074,6 +1107,11 @@ export async function geminiProxyRoute(app: FastifyInstance) {
downstreamPath,
upstreamPath,
clientContext,
0,
0,
0,
isStreamAction,
firstByteLatencyMs,
);
await safeInsertSurfaceProxyDebugAttempt(debugTrace, {
attemptIndex: retryCount,
@@ -1192,10 +1230,15 @@ export async function geminiProxyRoute(app: FastifyInstance) {
};
};
const channelProxyUrl = resolveChannelProxyUrl(selected.site, selected.account.extraConfig);
const dispatchRequest = (compatibilityRequest: BuiltEndpointRequest, targetUrl?: string) => (
const dispatchRequest = (
compatibilityRequest: BuiltEndpointRequest,
targetUrl?: string,
signal?: AbortSignal,
) => (
dispatchRuntimeRequest({
siteUrl: selected.site.url,
targetUrl,
signal,
request: compatibilityRequest,
buildInit: async (_requestUrl, requestForFetch) => withSiteRecordProxyRequestInit(selected.site, {
method: 'POST',
@@ -1221,6 +1264,7 @@ export async function geminiProxyRoute(app: FastifyInstance) {
const endpointResult = await executeEndpointFlow({
siteUrl: selected.site.url,
disableCrossProtocolFallback: config.disableCrossProtocolFallback,
firstByteTimeoutMs,
endpointCandidates,
buildRequest: (endpoint) => buildEndpointRequest(endpoint),
dispatchRequest,
@@ -1311,6 +1355,11 @@ export async function geminiProxyRoute(app: FastifyInstance) {
downstreamPath,
null,
clientContext,
0,
0,
0,
isStreamAction,
null,
);
if (canRetryChannelSelection(retryCount, forcedChannelId)) {
retryCount += 1;
@@ -1324,6 +1373,7 @@ export async function geminiProxyRoute(app: FastifyInstance) {
upstreamPath = endpointResult.upstreamPath;
const upstream = endpointResult.upstream;
const firstByteLatencyMs = getObservedResponseMeta(upstream)?.firstByteLatencyMs ?? null;
const rawText = await readRuntimeResponseText(upstream);
let upstreamData: unknown = rawText;
try {
@@ -1355,6 +1405,8 @@ export async function geminiProxyRoute(app: FastifyInstance) {
parsedUsage.promptTokens,
parsedUsage.completionTokens,
parsedUsage.totalTokens,
isStreamAction,
firstByteLatencyMs,
);
const downstreamPayload = isGeminiCliDownstream
? { response: geminiResponse }
@@ -1397,6 +1449,11 @@ export async function geminiProxyRoute(app: FastifyInstance) {
downstreamPath,
upstreamPath || null,
clientContext,
0,
0,
0,
isStreamAction,
null,
);
if (canRetryChannelSelection(retryCount, forcedChannelId)) {
retryCount += 1;
src/server/proxy-core/surfaces/openAiResponsesSurface.ts
+9
View File
@@ -44,6 +44,7 @@ import {
unwrapGeminiCliPayload,
} from '../../routes/proxy/geminiCliCompat.js';
import { isCodexResponsesSurface } from '../cliProfiles/codexProfile.js';
import { getObservedResponseMeta } from '../firstByteTimeout.js';
import { getRuntimeResponseReader, readRuntimeResponseText } from '../executors/types.js';
import { runCodexHttpSessionTask } from '../runtime/codexHttpSessionQueue.js';
import {
@@ -580,6 +581,7 @@ export async function handleOpenAiResponsesSurfaceRequest(
return executeEndpointFlow({
siteUrl: siteApiBaseUrl,
disableCrossProtocolFallback: config.disableCrossProtocolFallback,
firstByteTimeoutMs: Math.max(0, Math.trunc((config.proxyFirstByteTimeoutSec || 0) * 1000)),
endpointCandidates,
buildRequest: (endpoint) => buildEndpointRequest(endpoint),
dispatchRequest,
@@ -712,6 +714,7 @@ export async function handleOpenAiResponsesSurfaceRequest(
const upstream = endpointResult.upstream;
const successfulUpstreamPath = endpointResult.upstreamPath;
const firstByteLatencyMs = getObservedResponseMeta(upstream)?.firstByteLatencyMs ?? null;
const finalizeStreamSuccess = async (
parsedUsage: UsageSummary,
latency: number,
@@ -726,6 +729,8 @@ export async function handleOpenAiResponsesSurfaceRequest(
parsedUsage,
upstreamUsagePresent,
requestStartedAtMs: startTime,
isStream: true,
firstByteLatencyMs,
latencyMs: latency,
retryCount,
upstreamPath: successfulUpstreamPath,
@@ -1174,6 +1179,8 @@ export async function handleOpenAiResponsesSurfaceRequest(
parsedUsage,
upstreamUsagePresent,
requestStartedAtMs: startTime,
isStream: false,
firstByteLatencyMs,
latencyMs: latency,
retryCount,
upstreamPath: successfulUpstreamPath,
@@ -1219,6 +1226,7 @@ export async function handleOpenAiResponsesSurfaceRequest(
status: endpointFailureStatus || 502,
errText: err?.message || 'unknown error',
rawErrText: err?.rawErrText || err?.message || 'unknown error',
isStream,
latencyMs: Date.now() - startTime,
retryCount,
});
@@ -1243,6 +1251,7 @@ export async function handleOpenAiResponsesSurfaceRequest(
requestedModel,
modelName,
errorMessage: err?.message || 'network failure',
isStream,
latencyMs: Date.now() - startTime,
retryCount,
});
src/server/proxy-core/surfaces/sharedSurface.ts
+33 -1
View File
@@ -191,6 +191,8 @@ export async function writeSurfaceProxyLog(input: {
modelRequested: string;
status: string;
httpStatus: number;
isStream?: boolean | null;
firstByteLatencyMs?: number | null;
latencyMs: number;
errorMessage: string | null;
retryCount: number;
@@ -227,6 +229,8 @@ export async function writeSurfaceProxyLog(input: {
modelActual: input.selected.actualModel ?? null,
status: input.status,
httpStatus: input.httpStatus,
isStream: input.isStream ?? null,
firstByteLatencyMs: input.firstByteLatencyMs ?? null,
latencyMs: input.latencyMs,
promptTokens: input.promptTokens ?? null,
completionTokens: input.completionTokens ?? null,
@@ -255,10 +259,12 @@ export function createSurfaceDispatchRequest(input: {
return (
request: BuiltEndpointRequest,
targetUrl?: string,
) => (
signal?: AbortSignal,
) => (
dispatchRuntimeRequest({
siteUrl: input.siteUrl ?? input.site.url,
targetUrl,
signal,
request,
buildInit: (_requestUrl, requestForFetch) => withSiteRecordProxyRequestInit(input.site, {
method: 'POST',
@@ -326,6 +332,8 @@ export async function recordSurfaceSuccess(input: {
parsedUsage: SurfaceUsageSummary;
upstreamUsagePresent?: boolean;
requestStartedAtMs: number;
isStream?: boolean | null;
firstByteLatencyMs?: number | null;
latencyMs: number;
retryCount: number;
upstreamPath?: string | null;
@@ -334,6 +342,8 @@ export async function recordSurfaceSuccess(input: {
modelRequested: string;
status: string;
httpStatus: number;
isStream?: boolean | null;
firstByteLatencyMs?: number | null;
latencyMs: number;
errorMessage: string | null;
retryCount: number;
@@ -427,6 +437,8 @@ export async function recordSurfaceSuccess(input: {
modelRequested: input.requestedModel,
status: 'success',
httpStatus: 200,
isStream: input.isStream ?? null,
firstByteLatencyMs: input.firstByteLatencyMs ?? null,
latencyMs: input.latencyMs,
errorMessage: null,
retryCount: input.retryCount,
@@ -458,6 +470,8 @@ export function createSurfaceFailureToolkit(input: {
modelRequested: string;
status: string;
httpStatus: number;
isStream?: boolean | null;
firstByteLatencyMs?: number | null;
latencyMs: number;
errorMessage: string | null;
retryCount: number;
@@ -474,6 +488,8 @@ export function createSurfaceFailureToolkit(input: {
modelRequested: args.modelRequested,
status: args.status,
httpStatus: args.httpStatus,
isStream: args.isStream ?? null,
firstByteLatencyMs: args.firstByteLatencyMs ?? null,
latencyMs: args.latencyMs,
errorMessage: args.errorMessage,
retryCount: args.retryCount,
@@ -510,6 +526,8 @@ export function createSurfaceFailureToolkit(input: {
status: number;
errText: string;
rawErrText?: string | null;
isStream?: boolean | null;
firstByteLatencyMs?: number | null;
latencyMs: number;
retryCount: number;
}): Promise<SurfaceFailureOutcome> {
@@ -524,6 +542,8 @@ export function createSurfaceFailureToolkit(input: {
modelRequested: args.requestedModel,
status: 'failed',
httpStatus: args.status,
isStream: args.isStream ?? null,
firstByteLatencyMs: args.firstByteLatencyMs ?? null,
latencyMs: args.latencyMs,
errorMessage: args.errText,
retryCount: args.retryCount,
@@ -570,6 +590,8 @@ export function createSurfaceFailureToolkit(input: {
requestedModel: string;
modelName: string;
failure: { status: number; reason: string };
isStream?: boolean | null;
firstByteLatencyMs?: number | null;
latencyMs: number;
retryCount: number;
promptTokens?: number | null;
@@ -587,6 +609,8 @@ export function createSurfaceFailureToolkit(input: {
modelRequested: args.requestedModel,
status: 'failed',
httpStatus: args.failure.status,
isStream: args.isStream ?? null,
firstByteLatencyMs: args.firstByteLatencyMs ?? null,
latencyMs: args.latencyMs,
errorMessage: args.failure.reason,
retryCount: args.retryCount,
@@ -623,6 +647,8 @@ export function createSurfaceFailureToolkit(input: {
requestedModel: string;
modelName: string;
errorMessage: string;
isStream?: boolean | null;
firstByteLatencyMs?: number | null;
latencyMs: number;
retryCount: number;
}): Promise<SurfaceFailureOutcome> {
@@ -635,6 +661,8 @@ export function createSurfaceFailureToolkit(input: {
modelRequested: args.requestedModel,
status: 'failed',
httpStatus: 0,
isStream: args.isStream ?? null,
firstByteLatencyMs: args.firstByteLatencyMs ?? null,
latencyMs: args.latencyMs,
errorMessage: args.errorMessage,
retryCount: args.retryCount,
@@ -665,6 +693,8 @@ export function createSurfaceFailureToolkit(input: {
requestedModel: string;
modelName: string;
errorMessage: string | null;
isStream?: boolean | null;
firstByteLatencyMs?: number | null;
latencyMs: number;
retryCount: number;
promptTokens?: number | null;
@@ -692,6 +722,8 @@ export function createSurfaceFailureToolkit(input: {
modelRequested: args.requestedModel,
status: 'failed',
httpStatus: args.httpStatus ?? 200,
isStream: args.isStream ?? null,
firstByteLatencyMs: args.firstByteLatencyMs ?? null,
latencyMs: args.latencyMs,
errorMessage,
retryCount: args.retryCount,