boos/BoosAPI
2
0
Fork 2
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix route grouping and codex websocket regressions

Browse Source
This commit is contained in:
cita
2026-03-18 20:38:01 +08:00
parent 332007d4fb
commit f769f43909
38 changed files with 2453 additions and 636 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/server/routes/proxy/chat.stream.test.ts
+17 -2
View File
@@ -1384,6 +1384,15 @@ describe('chat proxy stream behavior', () => {
status: 400,
headers: { 'content-type': 'application/json' },
}))
.mockResolvedValueOnce(new Response(JSON.stringify({
error: {
message: 'request validation failed',
type: 'invalid_request_error',
},
}), {
status: 400,
headers: { 'content-type': 'application/json' },
}))
.mockResolvedValueOnce(new Response(JSON.stringify({
id: 'resp_retry_minimal_headers',
object: 'response',
@@ -1413,21 +1422,27 @@ describe('chat proxy stream behavior', () => {
const body = response.json();
expect(body.output_text).toContain('ok after minimal headers retry');
expect(fetchMock).toHaveBeenCalledTimes(3);
expect(fetchMock).toHaveBeenCalledTimes(4);
const [, firstOptions] = fetchMock.mock.calls[0] as [string, any];
const [, secondOptions] = fetchMock.mock.calls[1] as [string, any];
const [, thirdOptions] = fetchMock.mock.calls[2] as [string, any];
const [, fourthOptions] = fetchMock.mock.calls[3] as [string, any];
expect(firstOptions.headers['openai-beta']).toBe('responses-2025-03-11');
expect(secondOptions.headers['openai-beta']).toBe('responses-2025-03-11');
expect(thirdOptions.headers['openai-beta']).toBeUndefined();
expect(thirdOptions.headers['openai-beta']).toBe('responses-2025-03-11');
expect(fourthOptions.headers['openai-beta']).toBeUndefined();
const firstBody = JSON.parse(firstOptions.body);
const secondBody = JSON.parse(secondOptions.body);
const thirdBody = JSON.parse(thirdOptions.body);
const fourthBody = JSON.parse(fourthOptions.body);
expect(firstBody.user).toBe('user-123');
expect(secondBody.user).toBeUndefined();
expect(thirdBody.user).toBeUndefined();
expect(secondBody.include).toEqual(['reasoning.encrypted_content']);
expect(thirdBody.include).toBeUndefined();
expect(fourthBody.user).toBeUndefined();
});
it('returns concise Cloudflare host error on /v1/responses 502 html failures', async () => {
src/server/routes/proxy/models.test.ts
+105
View File
@@ -7,11 +7,13 @@ import { join } from 'node:path';
type DbModule = typeof import('../../db/index.js');
type ProxyRouterModule = typeof import('./router.js');
type TokenRouterModule = typeof import('../../services/tokenRouter.js');
type TokensRoutesModule = typeof import('../api/tokens.js');
describe('/v1/models route', () => {
let db: DbModule['db'];
let schema: DbModule['schema'];
let proxyRoutes: ProxyRouterModule['proxyRoutes'];
let tokensRoutes: TokensRoutesModule['tokensRoutes'];
let invalidateTokenRouterCache: TokenRouterModule['invalidateTokenRouterCache'];
let app: FastifyInstance;
let dataDir = '';
@@ -24,13 +26,16 @@ describe('/v1/models route', () => {
const dbModule = await import('../../db/index.js');
const proxyRouterModule = await import('./router.js');
const tokenRouterModule = await import('../../services/tokenRouter.js');
const tokensRoutesModule = await import('../api/tokens.js');
db = dbModule.db;
schema = dbModule.schema;
proxyRoutes = proxyRouterModule.proxyRoutes;
tokensRoutes = tokensRoutesModule.tokensRoutes;
invalidateTokenRouterCache = tokenRouterModule.invalidateTokenRouterCache;
app = Fastify();
await app.register(tokensRoutes);
await app.register(proxyRoutes);
});
@@ -285,6 +290,106 @@ describe('/v1/models route', () => {
expect(ids).not.toContain('claude-sonnet-4-5');
});
it('returns only explicit-group public name while hiding source exact routes', async () => {
const site = await db.insert(schema.sites).values({
name: 'explicit-group-site',
url: 'https://explicit-group.example.com',
platform: 'openai',
status: 'active',
}).returning().get();
const account = await db.insert(schema.accounts).values({
siteId: site.id,
accessToken: 'explicit-group-access-token',
status: 'active',
}).returning().get();
const token = await db.insert(schema.accountTokens).values({
accountId: account.id,
name: 'default',
token: 'explicit-group-api-token',
enabled: true,
isDefault: true,
}).returning().get();
await db.insert(schema.modelAvailability).values([
{
accountId: account.id,
modelName: 'claude-opus-4-5',
available: true,
},
{
accountId: account.id,
modelName: 'claude-sonnet-4-5',
available: true,
},
]).run();
const sourceRouteA = await db.insert(schema.tokenRoutes).values({
modelPattern: 'claude-opus-4-5',
enabled: true,
}).returning().get();
const sourceRouteB = await db.insert(schema.tokenRoutes).values({
modelPattern: 'claude-sonnet-4-5',
enabled: true,
}).returning().get();
await db.insert(schema.routeChannels).values([
{
routeId: sourceRouteA.id,
accountId: account.id,
tokenId: token.id,
sourceModel: 'claude-opus-4-5',
enabled: true,
},
{
routeId: sourceRouteB.id,
accountId: account.id,
tokenId: token.id,
sourceModel: 'claude-sonnet-4-5',
enabled: true,
},
]).run();
const groupResponse = await app.inject({
method: 'POST',
url: '/api/routes',
payload: {
routeMode: 'explicit_group',
displayName: 'claude-opus-4-6',
sourceRouteIds: [sourceRouteA.id, sourceRouteB.id],
},
});
expect(groupResponse.statusCode).toBe(200);
const groupId = (groupResponse.json() as { id: number }).id;
await db.insert(schema.downstreamApiKeys).values({
name: 'managed-explicit-group-key',
key: 'sk-managed-explicit-group',
enabled: true,
allowedRouteIds: JSON.stringify([groupId]),
}).run();
const response = await app.inject({
method: 'GET',
url: '/v1/models',
headers: {
authorization: 'Bearer sk-managed-explicit-group',
},
});
expect(response.statusCode).toBe(200);
const body = response.json() as {
object: 'list';
data: Array<{ id: string }>;
};
const ids = body.data.map((item) => item.id);
expect(ids).toContain('claude-opus-4-6');
expect(ids).not.toContain('claude-opus-4-5');
expect(ids).not.toContain('claude-sonnet-4-5');
});
it('filters search pseudo models out of /v1/models', async () => {
const site = await db.insert(schema.sites).values({
name: 'search-site',
src/server/routes/proxy/models.ts
+1 -24
View File
@@ -1,6 +1,4 @@
import { FastifyInstance } from 'fastify';
import { db, schema } from '../../db/index.js';
import { and, eq } from 'drizzle-orm';
import { refreshModelsAndRebuildRoutes } from '../../services/modelService.js';
import { getDownstreamRoutingPolicy } from './downstreamPolicy.js';
import { tokenRouter } from '../../services/tokenRouter.js';
@@ -17,28 +15,7 @@ export async function modelsProxyRoute(app: FastifyInstance) {
const downstreamPolicy = getDownstreamRoutingPolicy(request);
const readModels = async () => {
const rows = await db.select({ modelName: schema.modelAvailability.modelName })
.from(schema.modelAvailability)
.innerJoin(schema.accounts, eq(schema.modelAvailability.accountId, schema.accounts.id))
.innerJoin(schema.sites, eq(schema.accounts.siteId, schema.sites.id))
.where(
and(
eq(schema.modelAvailability.available, true),
eq(schema.accounts.status, 'active'),
eq(schema.sites.status, 'active'),
),
)
.all();
const routeAliases = (await db.select({ displayName: schema.tokenRoutes.displayName })
.from(schema.tokenRoutes)
.where(eq(schema.tokenRoutes.enabled, true))
.all())
.map((row) => (row.displayName || '').trim())
.filter((name) => name.length > 0);
const deduped = Array.from(new Set([
...rows.map((r) => r.modelName),
...routeAliases,
]))
const deduped = Array.from(new Set(await tokenRouter.getAvailableModels()))
.filter((modelName) => !isSearchPseudoModel(modelName))
.sort();
const allowed: string[] = [];
src/server/routes/proxy/responses.ts
+6
View File
@@ -204,6 +204,9 @@ export async function responsesProxyRoute(app: FastifyInstance) {
model: modelName,
stream: isStream,
};
if (body.generate === false) {
normalizedResponsesBody.generate = false;
}
if (owner) {
try {
normalizedResponsesBody = await resolveResponsesBodyInputFiles(normalizedResponsesBody, owner);
@@ -446,6 +449,9 @@ export async function responsesProxyRoute(app: FastifyInstance) {
const streamSession = openAiResponsesTransformer.proxyStream.createSession({
modelName,
successfulUpstreamPath,
strictTerminalEvents: Object.entries(request.headers as Record<string, unknown>)
.some(([rawKey, rawValue]) => rawKey.trim().toLowerCase() === 'x-metapi-responses-websocket-transport'
&& String(rawValue).trim() === '1'),
getUsage: () => parsedUsage,
onParsedPayload: (payload) => {
if (payload && typeof payload === 'object') {
src/server/routes/proxy/responses.websocket.test.ts
+367 -22
View File
@@ -6,6 +6,7 @@ import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vites
const fetchMock = vi.fn();
const selectChannelMock = vi.fn();
const selectNextChannelMock = vi.fn();
const previewSelectedChannelMock = vi.fn();
const recordSuccessMock = vi.fn();
const recordFailureMock = vi.fn();
const refreshModelsAndRebuildRoutesMock = vi.fn();
@@ -30,6 +31,7 @@ vi.mock('../../services/tokenRouter.js', () => ({
tokenRouter: {
selectChannel: (...args: unknown[]) => selectChannelMock(...args),
selectNextChannel: (...args: unknown[]) => selectNextChannelMock(...args),
previewSelectedChannel: (...args: unknown[]) => previewSelectedChannelMock(...args),
recordSuccess: (...args: unknown[]) => recordSuccessMock(...args),
recordFailure: (...args: unknown[]) => recordFailureMock(...args),
},
@@ -91,6 +93,44 @@ function createSseResponse(chunks: string[], status = 200) {
});
}
function createSelectedChannel(options?: {
siteName?: string;
siteUrl?: string;
sitePlatform?: string;
username?: string;
extraConfig?: string;
tokenValue?: string;
actualModel?: string;
}) {
const sitePlatform = options?.sitePlatform ?? 'codex';
const isCodex = sitePlatform === 'codex';
return {
channel: { id: 11, routeId: 22 },
site: {
name: options?.siteName ?? (isCodex ? 'codex-site' : 'openai-site'),
url: options?.siteUrl ?? (isCodex ? 'https://chatgpt.com/backend-api/codex' : 'https://api.openai.com'),
platform: sitePlatform,
},
account: {
id: 33,
username: options?.username ?? (isCodex ? 'codex-user@example.com' : 'openai-user@example.com'),
extraConfig: options?.extraConfig ?? (isCodex
? JSON.stringify({
credentialMode: 'session',
oauth: {
provider: 'codex',
accountId: 'chatgpt-account-123',
email: 'codex-user@example.com',
},
})
: '{}'),
},
tokenName: 'default',
tokenValue: options?.tokenValue ?? (isCodex ? 'oauth-access-token' : 'sk-openai-token'),
actualModel: options?.actualModel ?? (isCodex ? 'gpt-5.4' : 'gpt-4.1'),
};
}
function waitForSocketOpen(socket: WebSocket) {
return new Promise<void>((resolve, reject) => {
socket.once('open', () => resolve());
@@ -98,19 +138,37 @@ function waitForSocketOpen(socket: WebSocket) {
});
}
function waitForSocketMessages(socket: WebSocket, count: number) {
function waitForSocketUpgrade(socket: WebSocket) {
return new Promise<{ headers: Record<string, string | string[] | undefined> }>((resolve, reject) => {
socket.once('upgrade', (response) => resolve({ headers: response.headers as Record<string, string | string[] | undefined> }));
socket.once('error', reject);
});
}
function waitForSocketMessages(socket: WebSocket, count: number, timeoutMs = 1000) {
return new Promise<any[]>((resolve, reject) => {
const messages: any[] = [];
const timeout = setTimeout(() => {
socket.off('message', onMessage);
socket.off('error', onError);
reject(new Error(`Timed out waiting for ${count} websocket messages`));
}, timeoutMs);
const onMessage = (payload: WebSocket.RawData) => {
messages.push(JSON.parse(String(payload)));
if (messages.length >= count) {
clearTimeout(timeout);
socket.off('message', onMessage);
socket.off('error', reject);
socket.off('error', onError);
resolve(messages);
}
};
const onError = (error: Error) => {
clearTimeout(timeout);
socket.off('message', onMessage);
reject(error);
};
socket.on('message', onMessage);
socket.once('error', reject);
socket.once('error', onError);
});
}
@@ -131,6 +189,7 @@ describe('responses websocket transport', () => {
fetchMock.mockReset();
selectChannelMock.mockReset();
selectNextChannelMock.mockReset();
previewSelectedChannelMock.mockReset();
recordSuccessMock.mockReset();
recordFailureMock.mockReset();
refreshModelsAndRebuildRoutesMock.mockReset();
@@ -139,26 +198,10 @@ describe('responses websocket transport', () => {
resolveProxyUsageWithSelfLogFallbackMock.mockClear();
dbInsertMock.mockClear();
selectChannelMock.mockReturnValue({
channel: { id: 11, routeId: 22 },
site: { name: 'codex-site', url: 'https://chatgpt.com/backend-api/codex', platform: 'codex' },
account: {
id: 33,
username: 'codex-user@example.com',
extraConfig: JSON.stringify({
credentialMode: 'session',
oauth: {
provider: 'codex',
accountId: 'chatgpt-account-123',
email: 'codex-user@example.com',
},
}),
},
tokenName: 'default',
tokenValue: 'oauth-access-token',
actualModel: 'gpt-5.4',
});
const selectedChannel = createSelectedChannel();
selectChannelMock.mockReturnValue(selectedChannel);
selectNextChannelMock.mockReturnValue(null);
previewSelectedChannelMock.mockResolvedValue(selectedChannel);
});
afterAll(async () => {
@@ -206,4 +249,306 @@ describe('responses websocket transport', () => {
expect(messages[3]?.response?.output?.[0]?.content?.[0]?.text).toBe('pong');
expect(fetchMock).toHaveBeenCalledTimes(1);
});
it('echoes x-codex-turn-state on websocket upgrade responses', async () => {
const socket = new WebSocket(`${baseUrl}/v1/responses`, {
headers: {
'x-codex-turn-state': 'turn-state-123',
},
});
const [upgrade] = await Promise.all([
waitForSocketUpgrade(socket),
waitForSocketOpen(socket),
]);
socket.close();
expect(upgrade.headers['x-codex-turn-state']).toBe('turn-state-123');
});
it('merges follow-up response.create payloads when the selected upstream does not support incremental mode', async () => {
const selectedChannel = createSelectedChannel({
sitePlatform: 'openai',
actualModel: 'gpt-4.1',
});
selectChannelMock.mockReturnValue(selectedChannel);
previewSelectedChannelMock.mockResolvedValue(selectedChannel);
fetchMock
.mockResolvedValueOnce(createSseResponse([
'event: response.output_item.done\n',
'data: {"type":"response.output_item.done","output_index":0,"item":{"id":"fc_1","type":"function_call","call_id":"call_1"}}\n\n',
'event: response.output_item.done\n',
'data: {"type":"response.output_item.done","output_index":1,"item":{"id":"msg_1","type":"message","role":"assistant","status":"completed","content":[{"type":"output_text","text":"call tool"}]}}\n\n',
'event: response.completed\n',
'data: {"type":"response.completed","response":{"id":"resp_ws_1","model":"gpt-4.1","status":"completed","output":[{"id":"fc_1","type":"function_call","call_id":"call_1"},{"id":"msg_1","type":"message","role":"assistant","status":"completed","content":[{"type":"output_text","text":"call tool"}]}],"usage":{"input_tokens":3,"output_tokens":1,"total_tokens":4}}}\n\n',
'data: [DONE]\n\n',
]))
.mockResolvedValueOnce(createSseResponse([
'event: response.completed\n',
'data: {"type":"response.completed","response":{"id":"resp_ws_2","model":"gpt-4.1","status":"completed","output":[{"id":"msg_2","type":"message","role":"assistant","status":"completed","content":[{"type":"output_text","text":"done"}]}],"usage":{"input_tokens":5,"output_tokens":1,"total_tokens":6}}}\n\n',
'data: [DONE]\n\n',
]));
const socket = new WebSocket(`${baseUrl}/v1/responses`);
await waitForSocketOpen(socket);
const firstResponsePromise = waitForSocketMessages(socket, 1);
socket.send(JSON.stringify({
type: 'response.create',
model: 'gpt-4.1',
instructions: 'be helpful',
input: [
{
id: 'msg_user_1',
type: 'message',
role: 'user',
content: [{ type: 'input_text', text: 'call the tool' }],
},
],
}));
await firstResponsePromise;
const secondResponsePromise = waitForSocketMessages(socket, 1);
socket.send(JSON.stringify({
type: 'response.create',
previous_response_id: 'resp_ws_1',
input: [
{
id: 'tool_out_1',
type: 'function_call_output',
call_id: 'call_1',
output: 'tool result',
},
],
}));
await secondResponsePromise;
socket.close();
expect(fetchMock).toHaveBeenCalledTimes(2);
const [, firstOptions] = fetchMock.mock.calls[0] as [string, RequestInit];
const [, secondOptions] = fetchMock.mock.calls[1] as [string, RequestInit];
const firstBody = JSON.parse(String(firstOptions.body));
const secondBody = JSON.parse(String(secondOptions.body));
expect(firstBody.input).toHaveLength(1);
expect(secondBody.previous_response_id).toBeUndefined();
expect(secondBody.model).toBe('gpt-4.1');
expect(secondBody.instructions).toBe('be helpful');
expect(secondBody.input).toEqual([
{
id: 'msg_user_1',
type: 'message',
role: 'user',
content: [{ type: 'input_text', text: 'call the tool' }],
},
{
id: 'fc_1',
type: 'function_call',
call_id: 'call_1',
status: 'completed',
},
{
id: 'msg_1',
type: 'message',
role: 'assistant',
status: 'completed',
content: [{ type: 'output_text', text: 'call tool' }],
},
{
id: 'tool_out_1',
type: 'function_call_output',
call_id: 'call_1',
output: 'tool result',
},
]);
});
it('preserves incremental response.create payloads with previous_response_id for websocket-capable upstreams', async () => {
fetchMock
.mockResolvedValueOnce(createSseResponse([
'event: response.output_item.done\n',
'data: {"type":"response.output_item.done","output_index":0,"item":{"id":"fc_1","type":"function_call","call_id":"call_1"}}\n\n',
'event: response.output_item.done\n',
'data: {"type":"response.output_item.done","output_index":1,"item":{"id":"msg_1","type":"message","role":"assistant","status":"completed","content":[{"type":"output_text","text":"call tool"}]}}\n\n',
'event: response.completed\n',
'data: {"type":"response.completed","response":{"id":"resp_ws_1","model":"gpt-5.4","status":"completed","output":[{"id":"fc_1","type":"function_call","call_id":"call_1"},{"id":"msg_1","type":"message","role":"assistant","status":"completed","content":[{"type":"output_text","text":"call tool"}]}],"usage":{"input_tokens":3,"output_tokens":1,"total_tokens":4}}}\n\n',
'data: [DONE]\n\n',
]))
.mockResolvedValueOnce(createSseResponse([
'event: response.completed\n',
'data: {"type":"response.completed","response":{"id":"resp_ws_2","model":"gpt-5.4","status":"completed","output":[{"id":"msg_2","type":"message","role":"assistant","status":"completed","content":[{"type":"output_text","text":"done"}]}],"usage":{"input_tokens":5,"output_tokens":1,"total_tokens":6}}}\n\n',
'data: [DONE]\n\n',
]));
const socket = new WebSocket(`${baseUrl}/v1/responses`);
await waitForSocketOpen(socket);
const firstResponsePromise = waitForSocketMessages(socket, 1);
socket.send(JSON.stringify({
type: 'response.create',
model: 'gpt-5.4',
instructions: 'be helpful',
input: [
{
id: 'msg_user_1',
type: 'message',
role: 'user',
content: [{ type: 'input_text', text: 'call the tool' }],
},
],
}));
await firstResponsePromise;
const secondResponsePromise = waitForSocketMessages(socket, 1);
socket.send(JSON.stringify({
type: 'response.create',
previous_response_id: 'resp_ws_1',
input: [
{
id: 'tool_out_1',
type: 'function_call_output',
call_id: 'call_1',
output: 'tool result',
},
],
}));
await secondResponsePromise;
socket.close();
expect(fetchMock).toHaveBeenCalledTimes(2);
const [, secondOptions] = fetchMock.mock.calls[1] as [string, RequestInit];
const secondBody = JSON.parse(String(secondOptions.body));
expect(secondBody.previous_response_id).toBe('resp_ws_1');
expect(secondBody.model).toBe('gpt-5.4');
expect(secondBody.instructions).toBe('be helpful');
expect(secondBody.input).toEqual([
{
id: 'tool_out_1',
type: 'function_call_output',
call_id: 'call_1',
output: 'tool result',
},
]);
});
it('handles generate=false locally only for non-websocket-capable upstreams', async () => {
const selectedChannel = createSelectedChannel({
sitePlatform: 'openai',
actualModel: 'gpt-4.1',
});
selectChannelMock.mockReturnValue(selectedChannel);
previewSelectedChannelMock.mockResolvedValue(selectedChannel);
fetchMock.mockResolvedValueOnce(createSseResponse([
'event: response.completed\n',
'data: {"type":"response.completed","response":{"id":"resp_ws_after_prewarm","model":"gpt-4.1","status":"completed","output":[{"id":"msg_2","type":"message","role":"assistant","status":"completed","content":[{"type":"output_text","text":"done"}]}],"usage":{"input_tokens":1,"output_tokens":1,"total_tokens":2}}}\n\n',
'data: [DONE]\n\n',
]));
const socket = new WebSocket(`${baseUrl}/v1/responses`);
await waitForSocketOpen(socket);
const prewarmMessagesPromise = waitForSocketMessages(socket, 2);
socket.send(JSON.stringify({
type: 'response.create',
model: 'gpt-4.1',
generate: false,
}));
const prewarmMessages = await prewarmMessagesPromise;
expect(prewarmMessages.map((message) => message.type)).toEqual(['response.created', 'response.completed']);
expect(fetchMock).toHaveBeenCalledTimes(0);
const secondResponsePromise = waitForSocketMessages(socket, 1);
socket.send(JSON.stringify({
type: 'response.create',
previous_response_id: prewarmMessages[0]?.response?.id,
input: [
{
id: 'msg_followup_1',
type: 'message',
role: 'user',
content: [{ type: 'input_text', text: 'continue' }],
},
],
}));
await secondResponsePromise;
socket.close();
expect(fetchMock).toHaveBeenCalledTimes(1);
const [, options] = fetchMock.mock.calls[0] as [string, RequestInit];
const forwardedBody = JSON.parse(String(options.body));
expect(forwardedBody.generate).toBeUndefined();
expect(forwardedBody.previous_response_id).toBeUndefined();
expect(forwardedBody.model).toBe('gpt-4.1');
expect(forwardedBody.input).toEqual([
{
id: 'msg_followup_1',
type: 'message',
role: 'user',
content: [{ type: 'input_text', text: 'continue' }],
},
]);
});
it('forwards generate=false upstream for websocket-capable upstreams instead of synthesizing prewarm events', async () => {
fetchMock.mockResolvedValueOnce(createSseResponse([
'event: response.completed\n',
'data: {"type":"response.completed","response":{"id":"resp_capable_generate_false","model":"gpt-5.4","status":"completed","output":[],"usage":{"input_tokens":0,"output_tokens":0,"total_tokens":0}}}\n\n',
'data: [DONE]\n\n',
]));
const socket = new WebSocket(`${baseUrl}/v1/responses`);
await waitForSocketOpen(socket);
const messagesPromise = waitForSocketMessages(socket, 1);
socket.send(JSON.stringify({
type: 'response.create',
model: 'gpt-5.4',
generate: false,
}));
const messages = await messagesPromise;
socket.close();
expect(messages.map((message) => message.type)).toEqual(['response.completed']);
expect(fetchMock).toHaveBeenCalledTimes(1);
const [, options] = fetchMock.mock.calls[0] as [string, RequestInit];
const forwardedBody = JSON.parse(String(options.body));
expect(forwardedBody.generate).toBe(false);
});
it('emits websocket error when the upstream stream closes before a terminal responses event', async () => {
fetchMock.mockResolvedValue(createSseResponse([
'event: response.created\n',
'data: {"type":"response.created","response":{"id":"resp_incomplete","model":"gpt-5.4","created_at":1706000000,"status":"in_progress","output":[]}}\n\n',
'event: response.output_text.delta\n',
'data: {"type":"response.output_text.delta","output_index":0,"item_id":"msg_ws","delta":"partial"}\n\n',
]));
const socket = new WebSocket(`${baseUrl}/v1/responses`);
await waitForSocketOpen(socket);
const messagesPromise = waitForSocketMessages(socket, 3, 400);
socket.send(JSON.stringify({
type: 'response.create',
model: 'gpt-5.4',
input: [
{
type: 'message',
role: 'user',
content: [{ type: 'input_text', text: 'hello websocket' }],
},
],
}));
const messages = await messagesPromise;
socket.close();
expect(messages.map((message) => message.type)).toEqual([
'response.created',
'response.output_text.delta',
'error',
]);
expect(messages[2]?.status).toBe(408);
expect(messages[2]?.error?.message).toContain('stream closed before response.completed');
});
});
src/server/routes/proxy/responsesWebsocket.ts
+93 -41
View File
@@ -2,9 +2,13 @@ import { randomUUID } from 'node:crypto';
import type { FastifyInstance } from 'fastify';
import type { IncomingMessage } from 'node:http';
import { WebSocketServer, type RawData, type WebSocket } from 'ws';
import { tokenRouter } from '../../services/tokenRouter.js';
import { openAiResponsesTransformer } from '../../transformers/openai/responses/index.js';
const installedApps = new WeakSet<FastifyInstance>();
const WS_TURN_STATE_HEADER = 'x-codex-turn-state';
const RESPONSES_WEBSOCKET_MODE_HEADER = 'x-metapi-responses-websocket-mode';
const RESPONSES_WEBSOCKET_TRANSPORT_HEADER = 'x-metapi-responses-websocket-transport';
type NormalizedResponsesWebsocketRequest =
| {
@@ -26,6 +30,18 @@ function asTrimmedString(value: unknown): string {
return typeof value === 'string' ? value.trim() : '';
}
function headerValueToTrimmedString(value: unknown): string {
if (typeof value === 'string') return value.trim();
if (Array.isArray(value)) {
for (const item of value) {
if (typeof item !== 'string') continue;
const trimmed = item.trim();
if (trimmed) return trimmed;
}
}
return '';
}
function parseJsonObject(raw: RawData): Record<string, unknown> | null {
try {
const parsed = JSON.parse(String(raw));
@@ -44,19 +60,11 @@ function toResponseInputArray(value: unknown): unknown[] {
}
function normalizeResponsesWebsocketRequest(
raw: RawData,
parsed: Record<string, unknown>,
lastRequest: Record<string, unknown> | null,
lastResponseOutput: unknown[],
supportsIncrementalInput: boolean,
): NormalizedResponsesWebsocketRequest {
const parsed = parseJsonObject(raw);
if (!parsed) {
return {
ok: false,
status: 400,
message: 'Invalid websocket JSON payload',
};
}
const requestType = asTrimmedString(parsed.type);
if (requestType !== 'response.create' && requestType !== 'response.append') {
return {
@@ -66,27 +74,6 @@ function normalizeResponsesWebsocketRequest(
};
}
if (requestType === 'response.create' && parsed.generate === false) {
const next = cloneJsonObject(parsed);
delete next.type;
delete next.generate;
next.stream = true;
if (!Array.isArray(next.input)) next.input = [];
const modelName = asTrimmedString(next.model);
if (!modelName) {
return {
ok: false,
status: 400,
message: 'missing model in response.create request',
};
}
return {
ok: true,
request: next,
nextRequestSnapshot: cloneJsonObject(next),
};
}
if (!lastRequest) {
if (requestType !== 'response.create') {
return {
@@ -97,6 +84,9 @@ function normalizeResponsesWebsocketRequest(
}
const next = cloneJsonObject(parsed);
delete next.type;
if (!supportsIncrementalInput && parsed.generate === false) {
delete next.generate;
}
next.stream = true;
if (!Array.isArray(next.input)) next.input = [];
const modelName = asTrimmedString(next.model);
@@ -122,15 +112,8 @@ function normalizeResponsesWebsocketRequest(
};
}
const mergedInput = [
...toResponseInputArray(lastRequest.input),
...cloneJsonObject(lastResponseOutput),
...cloneJsonObject(parsed.input),
];
const next = cloneJsonObject(parsed);
delete next.type;
delete next.previous_response_id;
next.input = mergedInput;
next.stream = true;
if (!('model' in next) && typeof lastRequest.model === 'string') {
next.model = lastRequest.model;
@@ -139,6 +122,22 @@ function normalizeResponsesWebsocketRequest(
next.instructions = cloneJsonObject(lastRequest.instructions);
}
if (supportsIncrementalInput && requestType === 'response.create' && asTrimmedString(parsed.previous_response_id)) {
return {
ok: true,
request: next,
nextRequestSnapshot: cloneJsonObject(next),
};
}
const mergedInput = [
...toResponseInputArray(lastRequest.input),
...cloneJsonObject(lastResponseOutput),
...cloneJsonObject(parsed.input),
];
delete next.previous_response_id;
next.input = mergedInput;
return {
ok: true,
request: next,
@@ -146,6 +145,16 @@ function normalizeResponsesWebsocketRequest(
};
}
function shouldHandleResponsesWebsocketPrewarmLocally(
parsed: Record<string, unknown>,
lastRequest: Record<string, unknown> | null,
supportsIncrementalInput: boolean,
): boolean {
if (supportsIncrementalInput || lastRequest) return false;
if (asTrimmedString(parsed.type) !== 'response.create') return false;
return parsed.generate === false;
}
function writeResponsesWebsocketError(
socket: WebSocket,
status: number,
@@ -244,6 +253,21 @@ function buildInjectHeaders(request: IncomingMessage): Record<string, string | s
return headers;
}
async function supportsResponsesWebsocketIncrementalInput(
parsed: Record<string, unknown>,
lastRequest: Record<string, unknown> | null,
): Promise<boolean> {
const requestModel = asTrimmedString(parsed.model) || asTrimmedString(lastRequest?.model);
if (!requestModel) return false;
try {
const selected = await tokenRouter.previewSelectedChannel(requestModel);
return asTrimmedString(selected?.site?.platform).toLowerCase() === 'codex';
} catch {
return false;
}
}
async function handleResponsesWebsocketConnection(
app: FastifyInstance,
socket: WebSocket,
@@ -259,14 +283,25 @@ async function handleResponsesWebsocketConnection(
return;
}
const normalized = normalizeResponsesWebsocketRequest(raw, lastRequest, lastResponseOutput);
const supportsIncrementalInput = await supportsResponsesWebsocketIncrementalInput(parsed, lastRequest);
const shouldHandleLocalPrewarm = shouldHandleResponsesWebsocketPrewarmLocally(
parsed,
lastRequest,
supportsIncrementalInput,
);
const normalized = normalizeResponsesWebsocketRequest(
parsed,
lastRequest,
lastResponseOutput,
supportsIncrementalInput,
);
if (!normalized.ok) {
writeResponsesWebsocketError(socket, normalized.status, normalized.message);
return;
}
lastRequest = normalized.nextRequestSnapshot;
if (parsed.generate === false && asTrimmedString(parsed.type) === 'response.create') {
if (shouldHandleLocalPrewarm) {
lastResponseOutput = [];
for (const payload of synthesizePrewarmResponsePayloads(normalized.request)) {
socket.send(JSON.stringify(payload));
@@ -277,7 +312,11 @@ async function handleResponsesWebsocketConnection(
const response = await app.inject({
method: 'POST',
url: '/v1/responses',
headers: buildInjectHeaders(request),
headers: {
...buildInjectHeaders(request),
[RESPONSES_WEBSOCKET_TRANSPORT_HEADER]: '1',
...(supportsIncrementalInput ? { [RESPONSES_WEBSOCKET_MODE_HEADER]: 'incremental' } : {}),
},
payload: normalized.request,
});
@@ -313,17 +352,25 @@ async function handleResponsesWebsocketConnection(
const pulled = openAiResponsesTransformer.pullSseEvents(response.body);
const forwardedPayloads: unknown[] = [];
let sawTerminalPayload = false;
for (const event of pulled.events) {
if (event.data === '[DONE]') continue;
try {
const payload = JSON.parse(event.data);
forwardedPayloads.push(payload);
const type = isRecord(payload) ? asTrimmedString(payload.type) : '';
if (type === 'response.completed' || type === 'response.failed') {
sawTerminalPayload = true;
}
socket.send(JSON.stringify(payload));
} catch {
// Ignore malformed SSE frames; the HTTP route already normalizes them.
}
}
lastResponseOutput = collectResponsesOutput(forwardedPayloads);
if (!sawTerminalPayload) {
writeResponsesWebsocketError(socket, 408, 'stream closed before response.completed');
}
});
}
@@ -332,6 +379,11 @@ export function ensureResponsesWebsocketTransport(app: FastifyInstance) {
installedApps.add(app);
const websocketServer = new WebSocketServer({ noServer: true });
websocketServer.on('headers', (headers, request) => {
const turnState = headerValueToTrimmedString(request.headers[WS_TURN_STATE_HEADER]);
if (!turnState) return;
headers.push(`${WS_TURN_STATE_HEADER}: ${turnState}`);
});
app.server.on('upgrade', (request, socket, head) => {
const url = new URL(request.url || '/', 'http://localhost');
src/server/routes/proxy/upstreamEndpoint.ts
+14 -2
View File
@@ -470,6 +470,9 @@ function convertCodexSystemRoleToDeveloper(input: unknown): unknown {
function applyCodexResponsesCompatibility(
body: Record<string, unknown>,
sitePlatform: string,
options?: {
preservePreviousResponseId?: boolean;
},
): Record<string, unknown> {
if (sitePlatform !== 'codex') return body;
@@ -494,12 +497,14 @@ function applyCodexResponsesCompatibility(
'truncation',
'user',
'context_management',
'previous_response_id',
'prompt_cache_retention',
'safety_identifier',
]) {
delete next[key];
}
if (!options?.preservePreviousResponseId) {
delete next.previous_response_id;
}
if (asTrimmedString(next.service_tier).toLowerCase() !== 'priority') {
delete next.service_tier;
@@ -974,6 +979,8 @@ export function buildUpstreamEndpointRequest(input: {
}
if (input.endpoint === 'responses') {
const websocketMode = Object.entries(input.downstreamHeaders || {}).find(([rawKey]) => rawKey.trim().toLowerCase() === 'x-metapi-responses-websocket-mode');
const preserveWebsocketIncrementalMode = asTrimmedString(websocketMode?.[1]).toLowerCase() === 'incremental';
const responsesHeaders = input.downstreamFormat === 'responses'
? extractResponsesPassthroughHeaders(input.downstreamHeaders)
: {};
@@ -986,11 +993,16 @@ export function buildUpstreamEndpointRequest(input: {
}
: convertOpenAiBodyToResponsesBodyViaTransformer(openaiBody, input.modelName, input.stream)
);
const sanitizedResponsesBody = sanitizeResponsesBodyForProxyViaTransformer(rawBody, input.modelName, input.stream);
if (preserveWebsocketIncrementalMode && rawBody.generate === false) {
sanitizedResponsesBody.generate = false;
}
const body = ensureCodexResponsesStoreFalse(
ensureCodexResponsesInstructions(
applyCodexResponsesCompatibility(
sanitizeResponsesBodyForProxyViaTransformer(rawBody, input.modelName, input.stream),
sanitizedResponsesBody,
sitePlatform,
{ preservePreviousResponseId: preserveWebsocketIncrementalMode },
),
sitePlatform,
),