feat(server): add proactive channel probes and load-aware stable routing / 服务端新增主动探活与负载感知稳定路由 (#330)

* wip: preserve local changes before upstream merge

* Improve proxy routing stability and add stable-first strategy

* Sync explicit-group routing strategy to eligible source routes

* Fix proxy retry and runtime health review issues

* fix: make proxy channel retry attempts configurable

* fix: harden proxy retry recovery and bookkeeping

* fix: address pr 257 review comments

* feat(proxy): add sticky session channel coordination

* feat(server): add proactive channel probes and load-aware stable routing

* test(server): stabilize session load routing selection assertion

* fix(proxy): harden codex forwarding stability

* fix(router): rotate stable-first channels by healthy order

* fix(server): address probe review feedback

* fix(server): address probe review follow-ups

* fix(ci): address CodeQL and core test failures

* feat(server): refine stable-first routing recovery and observation

* Finish PR 330 settings and review fixes

* Fix chat site endpoint CI mock

* Address latest PR 330 review comments

---------

Co-authored-by: Cita <juricek.chen@gmail.com>

src/server/proxy-core/runtime/codexSessionResponseStore.test.ts

@@ -0,0 +1,48 @@
+import { describe, expect, it } from 'vitest';
+import {
+  buildCodexSessionResponseStoreKey,
+  getCodexSessionResponseId,
+  resetCodexSessionResponseStore,
+  setCodexSessionResponseId,
+} from './codexSessionResponseStore.js';
+
+describe('codexSessionResponseStore', () => {
+  it('evicts the oldest session id when the store exceeds the cap', () => {
+    resetCodexSessionResponseStore();
+
+    for (let index = 0; index <= 10_000; index += 1) {
+      setCodexSessionResponseId(`session-${index}`, `resp-${index}`);
+    }
+
+    expect(getCodexSessionResponseId('session-0')).toBeNull();
+    expect(getCodexSessionResponseId('session-1')).toBe('resp-1');
+    expect(getCodexSessionResponseId('session-10000')).toBe('resp-10000');
+
+    resetCodexSessionResponseStore();
+  });
+
+  it('namespaces identical downstream session ids by channel scope', () => {
+    resetCodexSessionResponseStore();
+
+    const keyA = buildCodexSessionResponseStoreKey({
+      sessionId: 'session-1',
+      siteId: 10,
+      accountId: 20,
+      channelId: 30,
+    });
+    const keyB = buildCodexSessionResponseStoreKey({
+      sessionId: 'session-1',
+      siteId: 10,
+      accountId: 21,
+      channelId: 31,
+    });
+
+    setCodexSessionResponseId(keyA, 'resp-a');
+    setCodexSessionResponseId(keyB, 'resp-b');
+
+    expect(getCodexSessionResponseId(keyA)).toBe('resp-a');
+    expect(getCodexSessionResponseId(keyB)).toBe('resp-b');
+
+    resetCodexSessionResponseStore();
+  });
+});

src/server/proxy-core/runtime/codexSessionResponseStore.ts

@@ -0,0 +1,55 @@
+const MAX_CODEX_SESSION_RESPONSE_IDS = 10_000;
+
+const codexSessionResponseIds = new Map<string, string>();
+
+function normalizeSessionId(sessionId: string): string {
+  return sessionId.trim();
+}
+
+export function buildCodexSessionResponseStoreKey(input: {
+  sessionId: string;
+  siteId?: number | null;
+  accountId?: number | null;
+  channelId?: number | null;
+}): string {
+  const normalizedSessionId = normalizeSessionId(input.sessionId);
+  if (!normalizedSessionId) return '';
+  const parts = [
+    Number.isFinite(input.siteId as number) && Number(input.siteId) > 0 ? `site:${Math.trunc(Number(input.siteId))}` : '',
+    Number.isFinite(input.accountId as number) && Number(input.accountId) > 0 ? `account:${Math.trunc(Number(input.accountId))}` : '',
+    Number.isFinite(input.channelId as number) && Number(input.channelId) > 0 ? `channel:${Math.trunc(Number(input.channelId))}` : '',
+    `session:${normalizedSessionId}`,
+  ].filter(Boolean);
+  return parts.join('|');
+}
+
+export function getCodexSessionResponseId(sessionId: string): string | null {
+  const normalized = normalizeSessionId(sessionId);
+  if (!normalized) return null;
+  return codexSessionResponseIds.get(normalized) ?? null;
+}
+
+export function setCodexSessionResponseId(sessionId: string, responseId: string): void {
+  const normalizedSessionId = normalizeSessionId(sessionId);
+  const normalizedResponseId = responseId.trim();
+  if (!normalizedSessionId || !normalizedResponseId) return;
+  if (codexSessionResponseIds.has(normalizedSessionId)) {
+    codexSessionResponseIds.delete(normalizedSessionId);
+  }
+  codexSessionResponseIds.set(normalizedSessionId, normalizedResponseId);
+  while (codexSessionResponseIds.size > MAX_CODEX_SESSION_RESPONSE_IDS) {
+    const oldestKey = codexSessionResponseIds.keys().next().value;
+    if (!oldestKey) break;
+    codexSessionResponseIds.delete(oldestKey);
+  }
+}
+
+export function clearCodexSessionResponseId(sessionId: string): void {
+  const normalized = normalizeSessionId(sessionId);
+  if (!normalized) return;
+  codexSessionResponseIds.delete(normalized);
+}
+
+export function resetCodexSessionResponseStore(): void {
+  codexSessionResponseIds.clear();
+}

src/server/proxy-core/runtime/codexWebsocketRuntime.test.ts

@@ -1,6 +1,7 @@
 import { AddressInfo } from 'node:net';
 import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest';
 import { WebSocketServer } from 'ws';
+import { resetCodexSessionResponseStore } from './codexSessionResponseStore.js';
 
 describe('codexWebsocketRuntime', () => {
   let upstreamServer: WebSocketServer;
@@ -25,6 +26,7 @@ describe('codexWebsocketRuntime', () => {
   });
 
   beforeEach(() => {
+    resetCodexSessionResponseStore();
     upstreamConnectionCount = 0;
     upstreamRequests = [];
     upstreamMessageHandler = (socket, parsed, requestIndex) => {

src/server/proxy-core/runtime/codexWebsocketRuntime.ts

@@ -1,10 +1,22 @@
 import type { IncomingMessage } from 'node:http';
 import WebSocket from 'ws';
+import {
+  extractResponsesTerminalResponseId,
+  isResponsesPreviousResponseNotFoundError,
+  shouldInferResponsesPreviousResponseId,
+  stripResponsesPreviousResponseId,
+  withResponsesPreviousResponseId,
+} from '../../transformers/openai/responses/continuation.js';
 import {
   buildCodexWebsocketHandshakeHeaders,
   buildCodexWebsocketRequestBody,
   toCodexWebsocketUrl,
 } from './codexWebsocketHeaders.js';
+import {
+  clearCodexSessionResponseId,
+  getCodexSessionResponseId,
+  setCodexSessionResponseId,
+} from './codexSessionResponseStore.js';
 import { createCodexWebsocketSessionStore } from './codexWebsocketSessionStore.js';
 import type {
   CodexWebsocketRuntimeResult,
@@ -190,6 +202,23 @@ function clearSessionSocket(session: CodexWebsocketSession, socket: WebSocket):
   session.socketUrl = null;
 }
 
+function buildContinuationAwareRuntimeBody(
+  sessionId: string,
+  body: Record<string, unknown>,
+): Record<string, unknown> {
+  const rememberedResponseId = getCodexSessionResponseId(sessionId);
+  if (!shouldInferResponsesPreviousResponseId(body, rememberedResponseId)) {
+    return body;
+  }
+  return withResponsesPreviousResponseId(body, rememberedResponseId);
+}
+
+function rememberSessionResponseId(sessionId: string, payload: unknown): void {
+  const responseId = extractResponsesTerminalResponseId(payload);
+  if (!responseId) return;
+  setCodexSessionResponseId(sessionId, responseId);
+}
+
 async function ensureSessionSocket(
   session: CodexWebsocketSession,
   input: CodexWebsocketRuntimeSendInput,
@@ -232,9 +261,11 @@ async function ensureSessionSocket(
   };
 }
 
-async function sendSessionRequest(
+async function sendSessionRequestAttempt(
   session: CodexWebsocketSession,
-  input: CodexWebsocketRuntimeSendInput,
+  input: CodexWebsocketRuntimeSendInput & {
+    body: Record<string, unknown>;
+  },
 ): Promise<CodexWebsocketRuntimeResult> {
   const { socket, reusedSession } = await ensureSessionSocket(session, input);
   const events: Array<Record<string, unknown>> = [];
@@ -262,7 +293,13 @@ async function sendSessionRequest(
         events.push(parsed);
         if (!isTerminalEvent(parsed)) return;
         if (settled) return;
-        if (isRuntimeErrorEvent(parsed)) {
+        if (
+          isRuntimeErrorEvent(parsed)
+          || isResponsesPreviousResponseNotFoundError({
+            payload: parsed,
+            rawErrText: extractTerminalErrorMessage(parsed),
+          })
+        ) {
           settled = true;
           cleanup();
           clearSessionSocket(session, socket);
@@ -274,6 +311,7 @@ async function sendSessionRequest(
           }));
           return;
         }
+        rememberSessionResponseId(session.sessionId, parsed);
         settled = true;
         cleanup();
         resolve({
@@ -307,6 +345,43 @@ async function sendSessionRequest(
   });
 }
 
+async function sendSessionRequest(
+  session: CodexWebsocketSession,
+  input: CodexWebsocketRuntimeSendInput,
+): Promise<CodexWebsocketRuntimeResult> {
+  let currentBody = buildContinuationAwareRuntimeBody(session.sessionId, input.body);
+  let previousResponseRecoveryTried = false;
+
+  for (;;) {
+    try {
+      return await sendSessionRequestAttempt(session, {
+        ...input,
+        body: currentBody,
+      });
+    } catch (error) {
+      if (
+        previousResponseRecoveryTried
+        || !(error instanceof CodexWebsocketRuntimeError)
+        || !isResponsesPreviousResponseNotFoundError({
+          payload: error.payload ?? error.events[error.events.length - 1],
+          rawErrText: error.message,
+        })
+      ) {
+        throw error;
+      }
+
+      const previousResponseRecovery = stripResponsesPreviousResponseId(currentBody);
+      if (!previousResponseRecovery.removed) {
+        throw error;
+      }
+
+      previousResponseRecoveryTried = true;
+      clearCodexSessionResponseId(session.sessionId);
+      currentBody = previousResponseRecovery.body;
+    }
+  }
+}
+
 export function createCodexWebsocketRuntime(input?: {
   sessionStore?: CodexWebsocketSessionStore;
 }) {
@@ -334,6 +409,7 @@ export function createCodexWebsocketRuntime(input?: {
       await closeSocket(session.socket);
       session.socket = null;
       session.socketUrl = null;
+      clearCodexSessionResponseId(sessionId);
     },
 
     async closeAllSessions(): Promise<void> {