Nvidia/BossAI
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add proxy first-byte timeout and log badges (#383)

Browse Source 
* add proxy first-byte timeout and log badges

* fix first-byte timeout review feedback
This commit is contained in:
Cita
2026-04-02 10:00:11 +08:00
committed by GitHub
parent bcd758edac
commit 180d17adfc
43 changed files with 4049 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/server/proxy-core/executors/types.ts
+8 -1
View File
@@ -35,6 +35,7 @@ export type RuntimeDispatchInput = {
siteUrl: string;
request: ProxyRuntimeRequest;
targetUrl?: string;
signal?: AbortSignal;
buildInit: (requestUrl: string, request: ProxyRuntimeRequest) => Promise<UndiciRequestInit> | UndiciRequestInit;
};
@@ -72,7 +73,13 @@ export async function performFetch(
requestUrl = input.targetUrl || buildUpstreamUrl(input.siteUrl, request.path),
): Promise<RuntimeResponse> {
const init = await input.buildInit(requestUrl, request);
return fetch(requestUrl, init);
const combinedSignal = input.signal && init.signal
? AbortSignal.any([input.signal, init.signal as AbortSignal])
: (input.signal ?? init.signal);
return fetch(requestUrl, {
...init,
signal: combinedSignal,
});
}
function hasZstdContentEncoding(contentEncoding: string | null): boolean {
src/server/proxy-core/firstByteTimeout.test.ts
+62
View File
@@ -0,0 +1,62 @@
import { describe, expect, it } from 'vitest';
import {
fetchWithObservedFirstByte,
getObservedResponseMeta,
isObservedFirstByteTimeoutResponse,
} from './firstByteTimeout.js';
function buildDelayedResponse(bodyText: string, delayMs: number, status = 200): Response {
const encoder = new TextEncoder();
const body = new ReadableStream<Uint8Array>({
start(controller) {
setTimeout(() => {
controller.enqueue(encoder.encode(bodyText));
controller.close();
}, delayMs);
},
});
return new Response(body, {
status,
headers: { 'content-type': 'text/plain; charset=utf-8' },
});
}
describe('fetchWithObservedFirstByte', () => {
it('replays the first chunk and records first-byte latency when upstream responds in time', async () => {
const response = await fetchWithObservedFirstByte(
async () => buildDelayedResponse('hello world', 5),
{
firstByteTimeoutMs: 100,
startedAtMs: Date.now(),
},
);
expect(response.status).toBe(200);
expect(await response.text()).toBe('hello world');
const meta = getObservedResponseMeta(response);
expect(meta?.timedOutBeforeFirstByte).toBe(false);
expect(meta?.firstByteLatencyMs).not.toBeNull();
expect((meta?.firstByteLatencyMs || 0)).toBeGreaterThanOrEqual(0);
expect((meta?.firstByteLatencyMs || 0)).toBeLessThan(100);
});
it('returns a synthetic timeout response when upstream sends no first byte before the deadline', async () => {
const response = await fetchWithObservedFirstByte(
async () => buildDelayedResponse('too late', 80),
{
firstByteTimeoutMs: 10,
startedAtMs: Date.now(),
},
);
expect(response.status).toBe(408);
expect(await response.text()).toContain('first byte timeout');
expect(isObservedFirstByteTimeoutResponse(response)).toBe(true);
const meta = getObservedResponseMeta(response);
expect(meta?.timedOutBeforeFirstByte).toBe(true);
expect(meta?.firstByteLatencyMs).toBeNull();
});
});
src/server/proxy-core/firstByteTimeout.ts
+179
View File
@@ -0,0 +1,179 @@
import { Headers, Response } from 'undici';
type ObservedResponseMeta = {
firstByteLatencyMs: number | null;
timedOutBeforeFirstByte: boolean;
};
const observedResponseMeta = new WeakMap<Response, ObservedResponseMeta>();
function setObservedResponseMeta<T extends Response>(response: T, meta: ObservedResponseMeta): T {
observedResponseMeta.set(response, meta);
return response;
}
function clearTimer(timer: ReturnType<typeof setTimeout> | null) {
if (!timer) return;
clearTimeout(timer);
}
function buildFirstByteTimeoutMessage(timeoutMs: number): string {
const seconds = Math.max(1, Math.round(timeoutMs / 1000));
return `first byte timeout (${seconds}s)`;
}
function buildObservedTimeoutResponse(timeoutMs: number): Response {
return setObservedResponseMeta(new Response(buildFirstByteTimeoutMessage(timeoutMs), {
status: 408,
headers: { 'content-type': 'text/plain; charset=utf-8' },
}), {
firstByteLatencyMs: null,
timedOutBeforeFirstByte: true,
});
}
async function cancelReaderQuietly(reader: ReadableStreamDefaultReader<Uint8Array> | null) {
if (!reader) return;
try {
await reader.cancel();
} catch {
// Ignore cancellation errors from already-closed streams.
}
try {
reader.releaseLock();
} catch {
// Ignore release errors from already-released readers.
}
}
function buildReplayResponse<T extends Response>(
response: T,
reader: ReadableStreamDefaultReader<Uint8Array>,
firstChunk: ReadableStreamReadResult<Uint8Array>,
): T {
let firstChunkDelivered = false;
let readerReleased = false;
const releaseReader = () => {
if (readerReleased) return;
readerReleased = true;
try {
reader.releaseLock();
} catch {
// Ignore release failures from cancelled/closed readers.
}
};
const stream = new ReadableStream<Uint8Array>({
async pull(controller) {
if (!firstChunkDelivered) {
firstChunkDelivered = true;
if (firstChunk.done) {
controller.close();
releaseReader();
return;
}
controller.enqueue(firstChunk.value);
return;
}
try {
const next = await reader.read();
if (next.done) {
controller.close();
releaseReader();
return;
}
controller.enqueue(next.value);
} catch (error) {
controller.error(error);
releaseReader();
}
},
async cancel(reason) {
try {
await reader.cancel(reason);
} catch {
// Ignore cancellation failures from already-closed streams.
}
releaseReader();
},
});
return new Response(stream, {
status: response.status,
statusText: response.statusText,
headers: new Headers(response.headers),
}) as T;
}
export async function fetchWithObservedFirstByte<T extends Response>(
dispatch: (signal?: AbortSignal) => Promise<T>,
options: {
firstByteTimeoutMs?: number;
startedAtMs?: number;
} = {},
): Promise<T> {
const timeoutMs = Math.max(0, Math.trunc(options.firstByteTimeoutMs ?? 0));
const startedAtMs = options.startedAtMs ?? Date.now();
const controller = timeoutMs > 0 ? new AbortController() : null;
let timer: ReturnType<typeof setTimeout> | null = null;
let timedOutBeforeFirstByte = false;
const timeoutSentinel = Symbol('first-byte-timeout');
const timeoutPromise = timeoutMs > 0
? new Promise<typeof timeoutSentinel>((resolve) => {
timer = setTimeout(() => {
timedOutBeforeFirstByte = true;
controller?.abort(new Error(buildFirstByteTimeoutMessage(timeoutMs)));
resolve(timeoutSentinel);
}, timeoutMs);
})
: null;
try {
const dispatched = timeoutPromise
? await Promise.race([dispatch(controller?.signal), timeoutPromise])
: await dispatch(controller?.signal);
if (dispatched === timeoutSentinel) {
return buildObservedTimeoutResponse(timeoutMs) as T;
}
const response = dispatched as T;
if (!response.body) {
clearTimer(timer);
return setObservedResponseMeta(response, {
firstByteLatencyMs: Math.max(0, Date.now() - startedAtMs),
timedOutBeforeFirstByte: false,
});
}
const reader = response.body.getReader() as ReadableStreamDefaultReader<Uint8Array>;
const firstChunk = timeoutPromise
? await Promise.race([reader.read(), timeoutPromise])
: await reader.read();
if (firstChunk === timeoutSentinel) {
await cancelReaderQuietly(reader);
return buildObservedTimeoutResponse(timeoutMs) as T;
}
clearTimer(timer);
return setObservedResponseMeta(buildReplayResponse(response, reader, firstChunk), {
firstByteLatencyMs: Math.max(0, Date.now() - startedAtMs),
timedOutBeforeFirstByte: false,
});
} catch (error) {
clearTimer(timer);
if (timedOutBeforeFirstByte && timeoutMs > 0) {
return buildObservedTimeoutResponse(timeoutMs) as T;
}
throw error;
} finally {
clearTimer(timer);
}
}
export function getObservedResponseMeta(response: Response | null | undefined): ObservedResponseMeta | null {
if (!response) return null;
return observedResponseMeta.get(response) ?? null;
}
export function isObservedFirstByteTimeoutResponse(response: Response | null | undefined): boolean {
return getObservedResponseMeta(response)?.timedOutBeforeFirstByte === true;
}
src/server/proxy-core/orchestration/endpointFlow.ts
+37 -8
View File
@@ -1,5 +1,6 @@
import { fetch } from 'undici';
import { readRuntimeResponseText } from '../executors/types.js';
import { fetchWithObservedFirstByte, isObservedFirstByteTimeoutResponse } from '../firstByteTimeout.js';
import { withSiteProxyRequestInit } from '../../services/siteProxy.js';
import {
buildUpstreamUrl,
@@ -69,7 +70,9 @@ export type ExecuteEndpointFlowInput = {
dispatchRequest?: (
request: BuiltEndpointRequest,
targetUrl: string,
signal?: AbortSignal,
) => Promise<Awaited<ReturnType<typeof fetch>>>;
firstByteTimeoutMs?: number;
tryRecover?: (ctx: EndpointAttemptContext) => Promise<EndpointRecoverResult>;
shouldDowngrade?: (ctx: EndpointAttemptContext) => boolean;
shouldAbortRemainingEndpoints?: (ctx: EndpointAttemptContext & { errText: string }) => boolean;
@@ -117,13 +120,23 @@ export async function executeEndpointFlow(input: ExecuteEndpointFlowInput): Prom
? buildUpstreamUrl(input.proxyUrl, request.path)
: defaultTarget;
let response = input.dispatchRequest
? await input.dispatchRequest(request, targetUrl)
: await fetch(targetUrl, await withSiteProxyRequestInit(targetUrl, {
method: 'POST',
headers: request.headers,
body: JSON.stringify(request.body),
}));
const attemptStartedAtMs = Date.now();
let response = await fetchWithObservedFirstByte(
async (signal) => (
input.dispatchRequest
? await input.dispatchRequest(request, targetUrl, signal)
: await fetch(targetUrl, await withSiteProxyRequestInit(targetUrl, {
method: 'POST',
headers: request.headers,
body: JSON.stringify(request.body),
signal,
}))
),
{
firstByteTimeoutMs: input.firstByteTimeoutMs,
startedAtMs: attemptStartedAtMs,
},
);
if (response.ok) {
await runEndpointFlowHook(input.onAttemptSuccess, {
@@ -151,6 +164,23 @@ export async function executeEndpointFlow(input: ExecuteEndpointFlowInput): Prom
rawErrText,
recoverApplied: false,
};
const isLastEndpoint = endpointIndex >= endpointCount - 1;
if (isObservedFirstByteTimeoutResponse(response) && !isLastEndpoint) {
const errText = rawErrText.trim() || 'first byte timeout';
const timeoutContext = {
...baseContext,
errText,
};
await runEndpointFlowHook(input.onAttemptFailure, timeoutContext, 'onAttemptFailure');
finalStatus = response.status || 408;
finalErrText = errText;
finalRawErrText = rawErrText;
if (input.disableCrossProtocolFallback) {
break;
}
continue;
}
if (input.tryRecover) {
const recovered = await input.tryRecover(baseContext);
@@ -192,7 +222,6 @@ export async function executeEndpointFlow(input: ExecuteEndpointFlowInput): Prom
errText,
}, 'onAttemptFailure');
const isLastEndpoint = endpointIndex >= endpointCount - 1;
if (input.disableCrossProtocolFallback && !isLastEndpoint) {
finalStatus = response.status;
finalErrText = errText;
src/server/proxy-core/surfaces/chatSurface.ts
+11
View File
@@ -45,6 +45,7 @@ import {
unwrapGeminiCliPayload,
} from '../../routes/proxy/geminiCliCompat.js';
import { summarizeConversationFileInputsInOpenAiBody } from '../capabilities/conversationFileCapabilities.js';
import { getObservedResponseMeta } from '../firstByteTimeout.js';
import { getRuntimeResponseReader, readRuntimeResponseText } from '../executors/types.js';
import { detectDownstreamClientContext } from '../../routes/proxy/downstreamClientContext.js';
import { getProxyMaxChannelRetries } from '../../services/proxyChannelRetry.js';
@@ -394,6 +395,7 @@ export async function handleChatSurfaceRequest(
return executeEndpointFlow({
siteUrl: siteApiBaseUrl,
disableCrossProtocolFallback: config.disableCrossProtocolFallback,
firstByteTimeoutMs: Math.max(0, Math.trunc((config.proxyFirstByteTimeoutSec || 0) * 1000)),
endpointCandidates,
buildRequest: (endpoint) => buildEndpointRequest(endpoint),
dispatchRequest,
@@ -525,6 +527,7 @@ export async function handleChatSurfaceRequest(
const upstream = endpointResult.upstream;
const successfulUpstreamPath = endpointResult.upstreamPath;
const firstByteLatencyMs = getObservedResponseMeta(upstream)?.firstByteLatencyMs ?? null;
if (isStream) {
const upstreamContentType = (upstream.headers.get('content-type') || '').toLowerCase();
@@ -557,6 +560,8 @@ export async function handleChatSurfaceRequest(
parsedUsage,
upstreamUsagePresent,
requestStartedAtMs: startTime,
isStream: true,
firstByteLatencyMs,
latencyMs,
retryCount,
upstreamPath: successfulUpstreamPath,
@@ -918,6 +923,8 @@ export async function handleChatSurfaceRequest(
parsedUsage,
upstreamUsagePresent,
requestStartedAtMs: startTime,
isStream: false,
firstByteLatencyMs,
latencyMs: latency,
retryCount,
upstreamPath: successfulUpstreamPath,
@@ -961,6 +968,7 @@ export async function handleChatSurfaceRequest(
status: endpointFailureStatus || 502,
errText: err.message || 'unknown error',
rawErrText: err.rawErrText || err.message || 'unknown error',
isStream,
latencyMs: Date.now() - startTime,
retryCount,
});
@@ -985,6 +993,7 @@ export async function handleChatSurfaceRequest(
requestedModel,
modelName,
errorMessage: err?.message || 'network failure',
isStream,
latencyMs: Date.now() - startTime,
retryCount,
});
@@ -1387,6 +1396,7 @@ export async function handleClaudeCountTokensSurfaceRequest(
status: endpointFailureStatus || 502,
errText: error.message || 'unknown error',
rawErrText: error.rawErrText || error.message || 'unknown error',
isStream: false,
latencyMs: Date.now() - startTime,
retryCount,
});
@@ -1407,6 +1417,7 @@ export async function handleClaudeCountTokensSurfaceRequest(
requestedModel,
modelName,
errorMessage: error?.message || 'network failure',
isStream: false,
latencyMs: Date.now() - startTime,
retryCount,
});
src/server/proxy-core/surfaces/geminiSurface.ts
+61 -4
View File
@@ -40,6 +40,7 @@ import { detectDownstreamClientContext, type DownstreamClientContext } from '../
import { insertProxyLog } from '../../services/proxyLogStore.js';
import { summarizeConversationFileInputsInOpenAiBody } from '../capabilities/conversationFileCapabilities.js';
import { getRuntimeResponseReader, readRuntimeResponseText } from '../executors/types.js';
import { fetchWithObservedFirstByte, getObservedResponseMeta } from '../firstByteTimeout.js';
import { getProxyMaxChannelRetries } from '../../services/proxyChannelRetry.js';
import { shouldAbortSameSiteEndpointFallback } from '../../services/proxyRetryPolicy.js';
import {
@@ -253,6 +254,8 @@ async function logProxy(
promptTokens = 0,
completionTokens = 0,
totalTokens = 0,
isStream = false,
firstByteLatencyMs: number | null = null,
) {
try {
const createdAt = formatUtcSqlDateTime(new Date());
@@ -274,6 +277,8 @@ async function logProxy(
modelActual: selected.actualModel || modelRequested,
status,
httpStatus,
isStream,
firstByteLatencyMs,
latencyMs,
promptTokens,
completionTokens,
@@ -611,6 +616,7 @@ export async function geminiProxyRoute(app: FastifyInstance) {
const isInternalGemini = isInternalGeminiPlatform(selected.site.platform);
const isDirectGeminiFamily = isDirectGeminiFamilyPlatform(selected.site.platform);
const startTime = Date.now();
const firstByteTimeoutMs = Math.max(0, Math.trunc((config.proxyFirstByteTimeoutSec || 0) * 1000));
let upstreamPath = '';
try {
@@ -698,11 +704,12 @@ export async function geminiProxyRoute(app: FastifyInstance) {
requestHeaders,
targetUrl,
runtimeExecutor,
dispatch: () => (
dispatch: (signal?: AbortSignal) => (
isInternalGemini
? dispatchRuntimeRequest({
siteUrl: selected.site.url,
targetUrl,
signal,
request: {
endpoint: 'chat',
path: upstreamPath,
@@ -728,13 +735,22 @@ export async function geminiProxyRoute(app: FastifyInstance) {
method: 'POST',
headers: requestHeaders,
body: JSON.stringify(requestBody),
signal,
})
),
};
};
let directDispatchState = buildDirectDispatchState();
let upstream = await directDispatchState.dispatch();
const dispatchWithObservedFirstByte = async () => fetchWithObservedFirstByte(
(signal) => directDispatchState.dispatch(signal),
{
firstByteTimeoutMs,
startedAtMs: Date.now(),
},
);
let upstream = await dispatchWithObservedFirstByte();
let firstByteLatencyMs = getObservedResponseMeta(upstream)?.firstByteLatencyMs ?? null;
let contentType = upstream.headers.get('content-type') || 'application/json';
let recoverApplied = false;
if (upstream.status === 401 && oauth) {
@@ -748,7 +764,8 @@ export async function geminiProxyRoute(app: FastifyInstance) {
};
oauth = getOauthInfoFromAccount(selected.account);
directDispatchState = buildDirectDispatchState();
upstream = await directDispatchState.dispatch();
upstream = await dispatchWithObservedFirstByte();
firstByteLatencyMs = getObservedResponseMeta(upstream)?.firstByteLatencyMs ?? null;
contentType = upstream.headers.get('content-type') || 'application/json';
recoverApplied = true;
} catch {
@@ -791,6 +808,11 @@ export async function geminiProxyRoute(app: FastifyInstance) {
downstreamPath,
upstreamPath,
clientContext,
0,
0,
0,
isStreamAction,
firstByteLatencyMs,
);
if (canRetryChannelSelection(retryCount, forcedChannelId)) {
retryCount += 1;
@@ -829,6 +851,11 @@ export async function geminiProxyRoute(app: FastifyInstance) {
downstreamPath,
upstreamPath,
clientContext,
0,
0,
0,
isStreamAction,
firstByteLatencyMs,
);
await safeInsertSurfaceProxyDebugAttempt(debugTrace, {
attemptIndex: retryCount,
@@ -913,6 +940,8 @@ export async function geminiProxyRoute(app: FastifyInstance) {
parsedUsage.promptTokens,
parsedUsage.completionTokens,
parsedUsage.totalTokens,
isStreamAction,
firstByteLatencyMs,
);
await safeInsertSurfaceProxyDebugAttempt(debugTrace, {
attemptIndex: retryCount,
@@ -965,6 +994,8 @@ export async function geminiProxyRoute(app: FastifyInstance) {
parsedUsage.promptTokens,
parsedUsage.completionTokens,
parsedUsage.totalTokens,
isStreamAction,
firstByteLatencyMs,
);
await safeInsertSurfaceProxyDebugAttempt(debugTrace, {
attemptIndex: retryCount,
@@ -1029,6 +1060,8 @@ export async function geminiProxyRoute(app: FastifyInstance) {
parsedUsage.promptTokens,
parsedUsage.completionTokens,
parsedUsage.totalTokens,
isStreamAction,
firstByteLatencyMs,
);
await safeInsertSurfaceProxyDebugAttempt(debugTrace, {
attemptIndex: retryCount,
@@ -1074,6 +1107,11 @@ export async function geminiProxyRoute(app: FastifyInstance) {
downstreamPath,
upstreamPath,
clientContext,
0,
0,
0,
isStreamAction,
firstByteLatencyMs,
);
await safeInsertSurfaceProxyDebugAttempt(debugTrace, {
attemptIndex: retryCount,
@@ -1192,10 +1230,15 @@ export async function geminiProxyRoute(app: FastifyInstance) {
};
};
const channelProxyUrl = resolveChannelProxyUrl(selected.site, selected.account.extraConfig);
const dispatchRequest = (compatibilityRequest: BuiltEndpointRequest, targetUrl?: string) => (
const dispatchRequest = (
compatibilityRequest: BuiltEndpointRequest,
targetUrl?: string,
signal?: AbortSignal,
) => (
dispatchRuntimeRequest({
siteUrl: selected.site.url,
targetUrl,
signal,
request: compatibilityRequest,
buildInit: async (_requestUrl, requestForFetch) => withSiteRecordProxyRequestInit(selected.site, {
method: 'POST',
@@ -1221,6 +1264,7 @@ export async function geminiProxyRoute(app: FastifyInstance) {
const endpointResult = await executeEndpointFlow({
siteUrl: selected.site.url,
disableCrossProtocolFallback: config.disableCrossProtocolFallback,
firstByteTimeoutMs,
endpointCandidates,
buildRequest: (endpoint) => buildEndpointRequest(endpoint),
dispatchRequest,
@@ -1311,6 +1355,11 @@ export async function geminiProxyRoute(app: FastifyInstance) {
downstreamPath,
null,
clientContext,
0,
0,
0,
isStreamAction,
null,
);
if (canRetryChannelSelection(retryCount, forcedChannelId)) {
retryCount += 1;
@@ -1324,6 +1373,7 @@ export async function geminiProxyRoute(app: FastifyInstance) {
upstreamPath = endpointResult.upstreamPath;
const upstream = endpointResult.upstream;
const firstByteLatencyMs = getObservedResponseMeta(upstream)?.firstByteLatencyMs ?? null;
const rawText = await readRuntimeResponseText(upstream);
let upstreamData: unknown = rawText;
try {
@@ -1355,6 +1405,8 @@ export async function geminiProxyRoute(app: FastifyInstance) {
parsedUsage.promptTokens,
parsedUsage.completionTokens,
parsedUsage.totalTokens,
isStreamAction,
firstByteLatencyMs,
);
const downstreamPayload = isGeminiCliDownstream
? { response: geminiResponse }
@@ -1397,6 +1449,11 @@ export async function geminiProxyRoute(app: FastifyInstance) {
downstreamPath,
upstreamPath || null,
clientContext,
0,
0,
0,
isStreamAction,
null,
);
if (canRetryChannelSelection(retryCount, forcedChannelId)) {
retryCount += 1;
src/server/proxy-core/surfaces/openAiResponsesSurface.ts
+9
View File
@@ -44,6 +44,7 @@ import {
unwrapGeminiCliPayload,
} from '../../routes/proxy/geminiCliCompat.js';
import { isCodexResponsesSurface } from '../cliProfiles/codexProfile.js';
import { getObservedResponseMeta } from '../firstByteTimeout.js';
import { getRuntimeResponseReader, readRuntimeResponseText } from '../executors/types.js';
import { runCodexHttpSessionTask } from '../runtime/codexHttpSessionQueue.js';
import {
@@ -580,6 +581,7 @@ export async function handleOpenAiResponsesSurfaceRequest(
return executeEndpointFlow({
siteUrl: siteApiBaseUrl,
disableCrossProtocolFallback: config.disableCrossProtocolFallback,
firstByteTimeoutMs: Math.max(0, Math.trunc((config.proxyFirstByteTimeoutSec || 0) * 1000)),
endpointCandidates,
buildRequest: (endpoint) => buildEndpointRequest(endpoint),
dispatchRequest,
@@ -712,6 +714,7 @@ export async function handleOpenAiResponsesSurfaceRequest(
const upstream = endpointResult.upstream;
const successfulUpstreamPath = endpointResult.upstreamPath;
const firstByteLatencyMs = getObservedResponseMeta(upstream)?.firstByteLatencyMs ?? null;
const finalizeStreamSuccess = async (
parsedUsage: UsageSummary,
latency: number,
@@ -726,6 +729,8 @@ export async function handleOpenAiResponsesSurfaceRequest(
parsedUsage,
upstreamUsagePresent,
requestStartedAtMs: startTime,
isStream: true,
firstByteLatencyMs,
latencyMs: latency,
retryCount,
upstreamPath: successfulUpstreamPath,
@@ -1174,6 +1179,8 @@ export async function handleOpenAiResponsesSurfaceRequest(
parsedUsage,
upstreamUsagePresent,
requestStartedAtMs: startTime,
isStream: false,
firstByteLatencyMs,
latencyMs: latency,
retryCount,
upstreamPath: successfulUpstreamPath,
@@ -1219,6 +1226,7 @@ export async function handleOpenAiResponsesSurfaceRequest(
status: endpointFailureStatus || 502,
errText: err?.message || 'unknown error',
rawErrText: err?.rawErrText || err?.message || 'unknown error',
isStream,
latencyMs: Date.now() - startTime,
retryCount,
});
@@ -1243,6 +1251,7 @@ export async function handleOpenAiResponsesSurfaceRequest(
requestedModel,
modelName,
errorMessage: err?.message || 'network failure',
isStream,
latencyMs: Date.now() - startTime,
retryCount,
});
src/server/proxy-core/surfaces/sharedSurface.ts
+33 -1
View File
@@ -191,6 +191,8 @@ export async function writeSurfaceProxyLog(input: {
modelRequested: string;
status: string;
httpStatus: number;
isStream?: boolean | null;
firstByteLatencyMs?: number | null;
latencyMs: number;
errorMessage: string | null;
retryCount: number;
@@ -227,6 +229,8 @@ export async function writeSurfaceProxyLog(input: {
modelActual: input.selected.actualModel ?? null,
status: input.status,
httpStatus: input.httpStatus,
isStream: input.isStream ?? null,
firstByteLatencyMs: input.firstByteLatencyMs ?? null,
latencyMs: input.latencyMs,
promptTokens: input.promptTokens ?? null,
completionTokens: input.completionTokens ?? null,
@@ -255,10 +259,12 @@ export function createSurfaceDispatchRequest(input: {
return (
request: BuiltEndpointRequest,
targetUrl?: string,
) => (
signal?: AbortSignal,
) => (
dispatchRuntimeRequest({
siteUrl: input.siteUrl ?? input.site.url,
targetUrl,
signal,
request,
buildInit: (_requestUrl, requestForFetch) => withSiteRecordProxyRequestInit(input.site, {
method: 'POST',
@@ -326,6 +332,8 @@ export async function recordSurfaceSuccess(input: {
parsedUsage: SurfaceUsageSummary;
upstreamUsagePresent?: boolean;
requestStartedAtMs: number;
isStream?: boolean | null;
firstByteLatencyMs?: number | null;
latencyMs: number;
retryCount: number;
upstreamPath?: string | null;
@@ -334,6 +342,8 @@ export async function recordSurfaceSuccess(input: {
modelRequested: string;
status: string;
httpStatus: number;
isStream?: boolean | null;
firstByteLatencyMs?: number | null;
latencyMs: number;
errorMessage: string | null;
retryCount: number;
@@ -427,6 +437,8 @@ export async function recordSurfaceSuccess(input: {
modelRequested: input.requestedModel,
status: 'success',
httpStatus: 200,
isStream: input.isStream ?? null,
firstByteLatencyMs: input.firstByteLatencyMs ?? null,
latencyMs: input.latencyMs,
errorMessage: null,
retryCount: input.retryCount,
@@ -458,6 +470,8 @@ export function createSurfaceFailureToolkit(input: {
modelRequested: string;
status: string;
httpStatus: number;
isStream?: boolean | null;
firstByteLatencyMs?: number | null;
latencyMs: number;
errorMessage: string | null;
retryCount: number;
@@ -474,6 +488,8 @@ export function createSurfaceFailureToolkit(input: {
modelRequested: args.modelRequested,
status: args.status,
httpStatus: args.httpStatus,
isStream: args.isStream ?? null,
firstByteLatencyMs: args.firstByteLatencyMs ?? null,
latencyMs: args.latencyMs,
errorMessage: args.errorMessage,
retryCount: args.retryCount,
@@ -510,6 +526,8 @@ export function createSurfaceFailureToolkit(input: {
status: number;
errText: string;
rawErrText?: string | null;
isStream?: boolean | null;
firstByteLatencyMs?: number | null;
latencyMs: number;
retryCount: number;
}): Promise<SurfaceFailureOutcome> {
@@ -524,6 +542,8 @@ export function createSurfaceFailureToolkit(input: {
modelRequested: args.requestedModel,
status: 'failed',
httpStatus: args.status,
isStream: args.isStream ?? null,
firstByteLatencyMs: args.firstByteLatencyMs ?? null,
latencyMs: args.latencyMs,
errorMessage: args.errText,
retryCount: args.retryCount,
@@ -570,6 +590,8 @@ export function createSurfaceFailureToolkit(input: {
requestedModel: string;
modelName: string;
failure: { status: number; reason: string };
isStream?: boolean | null;
firstByteLatencyMs?: number | null;
latencyMs: number;
retryCount: number;
promptTokens?: number | null;
@@ -587,6 +609,8 @@ export function createSurfaceFailureToolkit(input: {
modelRequested: args.requestedModel,
status: 'failed',
httpStatus: args.failure.status,
isStream: args.isStream ?? null,
firstByteLatencyMs: args.firstByteLatencyMs ?? null,
latencyMs: args.latencyMs,
errorMessage: args.failure.reason,
retryCount: args.retryCount,
@@ -623,6 +647,8 @@ export function createSurfaceFailureToolkit(input: {
requestedModel: string;
modelName: string;
errorMessage: string;
isStream?: boolean | null;
firstByteLatencyMs?: number | null;
latencyMs: number;
retryCount: number;
}): Promise<SurfaceFailureOutcome> {
@@ -635,6 +661,8 @@ export function createSurfaceFailureToolkit(input: {
modelRequested: args.requestedModel,
status: 'failed',
httpStatus: 0,
isStream: args.isStream ?? null,
firstByteLatencyMs: args.firstByteLatencyMs ?? null,
latencyMs: args.latencyMs,
errorMessage: args.errorMessage,
retryCount: args.retryCount,
@@ -665,6 +693,8 @@ export function createSurfaceFailureToolkit(input: {
requestedModel: string;
modelName: string;
errorMessage: string | null;
isStream?: boolean | null;
firstByteLatencyMs?: number | null;
latencyMs: number;
retryCount: number;
promptTokens?: number | null;
@@ -692,6 +722,8 @@ export function createSurfaceFailureToolkit(input: {
modelRequested: args.requestedModel,
status: 'failed',
httpStatus: args.httpStatus ?? 200,
isStream: args.isStream ?? null,
firstByteLatencyMs: args.firstByteLatencyMs ?? null,
latencyMs: args.latencyMs,
errorMessage,
retryCount: args.retryCount,