feat: add site headers editor and scheduled log cleanup

2026-03-12 08:12:29 +00:00
parent 1da3608d42
commit b6da71b1d6
34 changed files with 1572 additions and 210 deletions
@@ -43,6 +43,11 @@ describe('settings and auth events', () => {
    config.systemProxyUrl = '';
    config.checkinCron = '0 8 * * *';
    config.balanceRefreshCron = '0 * * * *';
+    config.logCleanupConfigured = false;
+    config.logCleanupCron = '0 6 * * *';
+    config.logCleanupUsageLogsEnabled = false;
+    config.logCleanupProgramLogsEnabled = false;
+    config.logCleanupRetentionDays = 30;
    config.routingFallbackUnitCost = 1;
  });

@@ -219,6 +224,80 @@ describe('settings and auth events', () => {
    expect(runtime.systemProxyUrl).toBe('http://127.0.0.1:7890');
  });

+  it('persists and returns log cleanup settings from runtime settings', async () => {
+    const updateResponse = await app.inject({
+      method: 'PUT',
+      url: '/api/settings/runtime',
+      payload: {
+        logCleanupCron: '15 4 * * *',
+        logCleanupUsageLogsEnabled: true,
+        logCleanupProgramLogsEnabled: true,
+        logCleanupRetentionDays: 14,
+      },
+    });
+
+    expect(updateResponse.statusCode).toBe(200);
+    const updated = updateResponse.json() as {
+      logCleanupCron?: string;
+      logCleanupUsageLogsEnabled?: boolean;
+      logCleanupProgramLogsEnabled?: boolean;
+      logCleanupRetentionDays?: number;
+    };
+    expect(updated.logCleanupCron).toBe('15 4 * * *');
+    expect(updated.logCleanupUsageLogsEnabled).toBe(true);
+    expect(updated.logCleanupProgramLogsEnabled).toBe(true);
+    expect(updated.logCleanupRetentionDays).toBe(14);
+    expect(config.logCleanupCron).toBe('15 4 * * *');
+    expect(config.logCleanupUsageLogsEnabled).toBe(true);
+    expect(config.logCleanupProgramLogsEnabled).toBe(true);
+    expect(config.logCleanupRetentionDays).toBe(14);
+
+    const rows = await db.select().from(schema.settings).all();
+    const settingsMap = new Map(rows.map((row) => [row.key, row.value]));
+    expect(settingsMap.get('log_cleanup_cron')).toBe(JSON.stringify('15 4 * * *'));
+    expect(settingsMap.get('log_cleanup_usage_logs_enabled')).toBe(JSON.stringify(true));
+    expect(settingsMap.get('log_cleanup_program_logs_enabled')).toBe(JSON.stringify(true));
+    expect(settingsMap.get('log_cleanup_retention_days')).toBe(JSON.stringify(14));
+
+    const getResponse = await app.inject({
+      method: 'GET',
+      url: '/api/settings/runtime',
+    });
+    expect(getResponse.statusCode).toBe(200);
+    const runtime = getResponse.json() as {
+      logCleanupCron?: string;
+      logCleanupUsageLogsEnabled?: boolean;
+      logCleanupProgramLogsEnabled?: boolean;
+      logCleanupRetentionDays?: number;
+    };
+    expect(runtime.logCleanupCron).toBe('15 4 * * *');
+    expect(runtime.logCleanupUsageLogsEnabled).toBe(true);
+    expect(runtime.logCleanupProgramLogsEnabled).toBe(true);
+    expect(runtime.logCleanupRetentionDays).toBe(14);
+  });
+
+  it('rejects invalid log cleanup cron and retention days', async () => {
+    const invalidCronResponse = await app.inject({
+      method: 'PUT',
+      url: '/api/settings/runtime',
+      payload: {
+        logCleanupCron: 'invalid cron',
+      },
+    });
+    expect(invalidCronResponse.statusCode).toBe(400);
+    expect((invalidCronResponse.json() as { message?: string }).message).toContain('日志清理 Cron');
+
+    const invalidRetentionResponse = await app.inject({
+      method: 'PUT',
+      url: '/api/settings/runtime',
+      payload: {
+        logCleanupRetentionDays: 0,
+      },
+    });
+    expect(invalidRetentionResponse.statusCode).toBe(400);
+    expect((invalidRetentionResponse.json() as { message?: string }).message).toContain('保留天数');
+  });
+
  it('invalidates cached site proxy resolution when system proxy url changes', async () => {
    await db.insert(schema.sites).values({
      name: 'proxy-site',
@@ -4,7 +4,7 @@ import { config } from '../../config.js';
 import { db, runtimeDbDialect, schema } from '../../db/index.js';
 import { upsertSetting } from '../../db/upsertSetting.js';
 import { refreshModelsAndRebuildRoutes } from '../../services/modelService.js';
-import { updateBalanceRefreshCron, updateCheckinCron } from '../../services/checkinScheduler.js';
+import { updateBalanceRefreshCron, updateCheckinCron, updateLogCleanupSettings } from '../../services/checkinScheduler.js';
 import { sendNotification } from '../../services/notifyService.js';
 import { exportBackup, importBackup, type BackupExportType } from '../../services/backupService.js';
 import { startBackgroundTask } from '../../services/backgroundTaskService.js';
@@ -19,6 +19,8 @@ import { formatUtcSqlDateTime } from '../../services/localTimeService.js';
 import { extractClientIp, isIpAllowed } from '../../middleware/auth.js';
 import { invalidateSiteProxyCache, normalizeSiteProxyUrl } from '../../services/siteProxy.js';
 import { performFactoryReset } from '../../services/factoryResetService.js';
+import { normalizeLogCleanupRetentionDays } from '../../services/logCleanupService.js';
+import { stopProxyLogRetentionService } from '../../services/proxyLogRetentionService.js';

 type RoutingWeights = typeof config.routingWeights;

@@ -27,6 +29,10 @@ interface RuntimeSettingsBody {
  systemProxyUrl?: string;
  checkinCron?: string;
  balanceRefreshCron?: string;
+  logCleanupCron?: string;
+  logCleanupUsageLogsEnabled?: boolean;
+  logCleanupProgramLogsEnabled?: boolean;
+  logCleanupRetentionDays?: number;
  webhookUrl?: string;
  barkUrl?: string;
  webhookEnabled?: boolean;
@@ -145,6 +151,35 @@ function applyImportedSettingToRuntime(key: string, value: unknown) {
      updateBalanceRefreshCron(value);
      return;
    }
+    case 'log_cleanup_cron': {
+      if (typeof value !== 'string' || !value || !cron.validate(value)) return;
+      config.logCleanupConfigured = true;
+      updateLogCleanupSettings({ cronExpr: value });
+      stopProxyLogRetentionService();
+      return;
+    }
+    case 'log_cleanup_usage_logs_enabled': {
+      if (typeof value !== 'boolean') return;
+      config.logCleanupConfigured = true;
+      updateLogCleanupSettings({ usageLogsEnabled: value });
+      stopProxyLogRetentionService();
+      return;
+    }
+    case 'log_cleanup_program_logs_enabled': {
+      if (typeof value !== 'boolean') return;
+      config.logCleanupConfigured = true;
+      updateLogCleanupSettings({ programLogsEnabled: value });
+      stopProxyLogRetentionService();
+      return;
+    }
+    case 'log_cleanup_retention_days': {
+      const retentionDays = Number(value);
+      if (!Number.isFinite(retentionDays) || retentionDays < 1) return;
+      config.logCleanupConfigured = true;
+      updateLogCleanupSettings({ retentionDays: Math.trunc(retentionDays) });
+      stopProxyLogRetentionService();
+      return;
+    }
    case 'proxy_token': {
      if (typeof value !== 'string') return;
      const nextToken = value.trim();
@@ -274,6 +309,10 @@ function getRuntimeSettingsResponse(currentAdminIp = '') {
  return {
    checkinCron: config.checkinCron,
    balanceRefreshCron: config.balanceRefreshCron,
+    logCleanupCron: config.logCleanupCron,
+    logCleanupUsageLogsEnabled: config.logCleanupUsageLogsEnabled,
+    logCleanupProgramLogsEnabled: config.logCleanupProgramLogsEnabled,
+    logCleanupRetentionDays: config.logCleanupRetentionDays,
    routingFallbackUnitCost: config.routingFallbackUnitCost,
    routingWeights: config.routingWeights,
    webhookUrl: config.webhookUrl,
@@ -458,6 +497,61 @@ export async function settingsRoutes(app: FastifyInstance) {
      upsertSetting('balance_refresh_cron', body.balanceRefreshCron);
    }

+    const logCleanupTouched =
+      body.logCleanupCron !== undefined
+      || body.logCleanupUsageLogsEnabled !== undefined
+      || body.logCleanupProgramLogsEnabled !== undefined
+      || body.logCleanupRetentionDays !== undefined;
+
+    if (logCleanupTouched) {
+      const nextLogCleanupCron = body.logCleanupCron !== undefined
+        ? String(body.logCleanupCron || '').trim()
+        : config.logCleanupCron;
+      if (!cron.validate(nextLogCleanupCron)) {
+        return reply.code(400).send({ success: false, message: '日志清理 Cron 表达式无效' });
+      }
+
+      const rawRetentionDays = body.logCleanupRetentionDays !== undefined
+        ? Number(body.logCleanupRetentionDays)
+        : config.logCleanupRetentionDays;
+      if (!Number.isFinite(rawRetentionDays) || rawRetentionDays < 1) {
+        return reply.code(400).send({ success: false, message: '日志清理保留天数必须是大于等于 1 的整数' });
+      }
+      const nextLogCleanupRetentionDays = normalizeLogCleanupRetentionDays(rawRetentionDays);
+      const nextUsageLogsEnabled = body.logCleanupUsageLogsEnabled !== undefined
+        ? !!body.logCleanupUsageLogsEnabled
+        : config.logCleanupUsageLogsEnabled;
+      const nextProgramLogsEnabled = body.logCleanupProgramLogsEnabled !== undefined
+        ? !!body.logCleanupProgramLogsEnabled
+        : config.logCleanupProgramLogsEnabled;
+
+      if (nextLogCleanupCron !== config.logCleanupCron) {
+        changedLabels.push(`日志清理 Cron（${config.logCleanupCron} -> ${nextLogCleanupCron}）`);
+      }
+      if (nextUsageLogsEnabled !== config.logCleanupUsageLogsEnabled) {
+        changedLabels.push(`自动清理使用日志（${config.logCleanupUsageLogsEnabled ? '开启' : '关闭'} -> ${nextUsageLogsEnabled ? '开启' : '关闭'}）`);
+      }
+      if (nextProgramLogsEnabled !== config.logCleanupProgramLogsEnabled) {
+        changedLabels.push(`自动清理程序日志（${config.logCleanupProgramLogsEnabled ? '开启' : '关闭'} -> ${nextProgramLogsEnabled ? '开启' : '关闭'}）`);
+      }
+      if (nextLogCleanupRetentionDays !== config.logCleanupRetentionDays) {
+        changedLabels.push(`日志清理保留天数（${config.logCleanupRetentionDays} -> ${nextLogCleanupRetentionDays}）`);
+      }
+
+      config.logCleanupConfigured = true;
+      updateLogCleanupSettings({
+        cronExpr: nextLogCleanupCron,
+        usageLogsEnabled: nextUsageLogsEnabled,
+        programLogsEnabled: nextProgramLogsEnabled,
+        retentionDays: nextLogCleanupRetentionDays,
+      });
+      stopProxyLogRetentionService();
+      upsertSetting('log_cleanup_cron', nextLogCleanupCron);
+      upsertSetting('log_cleanup_usage_logs_enabled', nextUsageLogsEnabled);
+      upsertSetting('log_cleanup_program_logs_enabled', nextProgramLogsEnabled);
+      upsertSetting('log_cleanup_retention_days', nextLogCleanupRetentionDays);
+    }
+
    if (body.proxyToken !== undefined) {
      const proxyToken = String(body.proxyToken).trim();
      if (!proxyToken.startsWith(PROXY_TOKEN_PREFIX)) {
@@ -36,7 +36,7 @@ describe('sites system proxy settings', () => {
    delete process.env.DATA_DIR;
  });

-  it('stores useSystemProxy and external checkin url when creating a site', async () => {
+  it('stores useSystemProxy, external checkin url, and custom headers when creating a site', async () => {
    const response = await app.inject({
      method: 'POST',
      url: '/api/sites',
@@ -45,6 +45,10 @@ describe('sites system proxy settings', () => {
        url: 'https://proxy-site.example.com',
        platform: 'new-api',
        useSystemProxy: true,
+        customHeaders: JSON.stringify({
+          'cf-access-client-id': 'site-client-id',
+          'x-site-scope': 'internal',
+        }),
        externalCheckinUrl: 'https://checkin.example.com/welfare',
        globalWeight: 1.5,
      },
@@ -53,10 +57,12 @@ describe('sites system proxy settings', () => {
    expect(response.statusCode).toBe(200);
    const payload = response.json() as {
      useSystemProxy?: boolean;
+      customHeaders?: string | null;
      externalCheckinUrl?: string | null;
      globalWeight?: number;
    };
    expect(payload.useSystemProxy).toBe(true);
+    expect(payload.customHeaders).toBe('{"cf-access-client-id":"site-client-id","x-site-scope":"internal"}');
    expect(payload.externalCheckinUrl).toBe('https://checkin.example.com/welfare');
    expect(payload.globalWeight).toBe(1.5);
  });
@@ -134,4 +140,38 @@ describe('sites system proxy settings', () => {
    expect(response.statusCode).toBe(200);
    expect((response.json() as { useSystemProxy?: boolean }).useSystemProxy).toBe(true);
  });
+
+  it('rejects invalid custom headers json', async () => {
+    const response = await app.inject({
+      method: 'POST',
+      url: '/api/sites',
+      payload: {
+        name: 'headers-site',
+        url: 'https://headers-site.example.com',
+        platform: 'new-api',
+        customHeaders: '{invalid-json}',
+      },
+    });
+
+    expect(response.statusCode).toBe(400);
+    expect((response.json() as { error?: string }).error).toContain('Invalid customHeaders');
+  });
+
+  it('rejects custom headers with non-string values', async () => {
+    const response = await app.inject({
+      method: 'POST',
+      url: '/api/sites',
+      payload: {
+        name: 'headers-site',
+        url: 'https://headers-site.example.com',
+        platform: 'new-api',
+        customHeaders: JSON.stringify({
+          'x-site-scope': true,
+        }),
+      },
+    });
+
+    expect(response.statusCode).toBe(400);
+    expect((response.json() as { error?: string }).error).toContain('must use a string value');
+  });
 });
@@ -5,6 +5,7 @@ import { detectSite } from '../../services/siteDetector.js';
 import { invalidateSiteProxyCache } from '../../services/siteProxy.js';
 import { formatUtcSqlDateTime } from '../../services/localTimeService.js';
 import { invalidateTokenRouterCache } from '../../services/tokenRouter.js';
+import { parseSiteCustomHeadersInput } from '../../services/siteCustomHeaders.js';

 function normalizeSiteStatus(input: unknown): 'active' | 'disabled' | null {
  if (input === undefined || input === null) return null;
@@ -160,13 +161,14 @@ export async function sitesRoutes(app: FastifyInstance) {
    url: string;
    platform?: string;
    useSystemProxy?: boolean;
+    customHeaders?: string | null;
    externalCheckinUrl?: string | null;
    status?: string;
    isPinned?: boolean;
    sortOrder?: number;
    globalWeight?: number;
  } }>('/api/sites', async (request, reply) => {
-    const { name, url, platform, useSystemProxy, externalCheckinUrl, status, isPinned, sortOrder, globalWeight } = request.body;
+    const { name, url, platform, useSystemProxy, customHeaders, externalCheckinUrl, status, isPinned, sortOrder, globalWeight } = request.body;
    const normalizedStatus = normalizeSiteStatus(status);
    if (status !== undefined && !normalizedStatus) {
      return reply.code(400).send({ error: 'Invalid site status. Expected active or disabled.' });
@@ -191,6 +193,10 @@ export async function sitesRoutes(app: FastifyInstance) {
    if (globalWeight !== undefined && normalizedGlobalWeight === null) {
      return reply.code(400).send({ error: 'Invalid globalWeight value. Expected a positive number.' });
    }
+    const normalizedCustomHeaders = parseSiteCustomHeadersInput(customHeaders);
+    if (!normalizedCustomHeaders.valid) {
+      return reply.code(400).send({ error: normalizedCustomHeaders.error || 'Invalid customHeaders.' });
+    }

    const existingSites = await db.select().from(schema.sites).all();
    const maxSortOrder = existingSites.reduce((max, site) => Math.max(max, site.sortOrder || 0), -1);
@@ -208,6 +214,7 @@ export async function sitesRoutes(app: FastifyInstance) {
      url: url.replace(/\/+$/, ''),
      platform: detectedPlatform,
      useSystemProxy: normalizedUseSystemProxy ?? false,
+      customHeaders: normalizedCustomHeaders.customHeaders,
      externalCheckinUrl: normalizedExternalCheckinUrl.url,
      status: normalizedStatus ?? 'active',
      isPinned: normalizedPinned ?? false,
@@ -232,6 +239,7 @@ export async function sitesRoutes(app: FastifyInstance) {
    url?: string;
    platform?: string;
    useSystemProxy?: boolean;
+    customHeaders?: string | null;
    externalCheckinUrl?: string | null;
    status?: string;
    isPinned?: boolean;
@@ -274,11 +282,16 @@ export async function sitesRoutes(app: FastifyInstance) {
    if (body.globalWeight !== undefined && normalizedGlobalWeight === null) {
      return reply.code(400).send({ error: 'Invalid globalWeight value. Expected a positive number.' });
    }
+    const normalizedCustomHeaders = parseSiteCustomHeadersInput(body.customHeaders);
+    if (!normalizedCustomHeaders.valid) {
+      return reply.code(400).send({ error: normalizedCustomHeaders.error || 'Invalid customHeaders.' });
+    }

    if (body.name !== undefined) updates.name = body.name;
    if (body.url !== undefined) updates.url = body.url.replace(/\/+$/, '');
    if (body.platform !== undefined) updates.platform = body.platform;
    if (body.useSystemProxy !== undefined) updates.useSystemProxy = normalizedUseSystemProxy;
+    if (normalizedCustomHeaders.present) updates.customHeaders = normalizedCustomHeaders.customHeaders;
    if (normalizedExternalCheckinUrl.present) updates.externalCheckinUrl = normalizedExternalCheckinUrl.url;
    if (body.status !== undefined) updates.status = normalizedStatus;
    if (body.isPinned !== undefined) updates.isPinned = normalizedPinned;
@@ -1,5 +1,5 @@
 import { fetch } from 'undici';
-import { withExplicitProxyRequestInit } from '../../services/siteProxy.js';
+import { withSiteProxyRequestInit } from '../../services/siteProxy.js';
 import { summarizeUpstreamError } from './upstreamError.js';
 import type { UpstreamEndpoint } from './upstreamEndpoint.js';

@@ -68,7 +68,7 @@ export async function executeEndpointFlow(input: ExecuteEndpointFlowInput): Prom
    const request = input.buildRequest(endpoint, endpointIndex);
    const targetUrl = `${input.siteUrl}${request.path}`;

-    let response = await fetch(targetUrl, withExplicitProxyRequestInit(input.proxyUrl, {
+    let response = await fetch(targetUrl, await withSiteProxyRequestInit(targetUrl, {
      method: 'POST',
      headers: request.headers,
      body: JSON.stringify(request.body),
@@ -7,7 +7,7 @@ import { isTokenExpiredError } from '../../services/alertRules.js';
 import { estimateProxyCost } from '../../services/modelPricingService.js';
 import { shouldRetryProxyRequest } from '../../services/proxyRetryPolicy.js';
 import { ensureModelAllowedForDownstreamKey, getDownstreamRoutingPolicy, recordDownstreamCostUsage } from './downstreamPolicy.js';
-import { withSiteRecordProxyRequestInit } from '../../services/siteProxy.js';
+import { withSiteProxyRequestInit, withSiteRecordProxyRequestInit } from '../../services/siteProxy.js';
 import { cloneFormDataWithOverrides, ensureMultipartBufferParser, parseMultipartFormData } from './multipart.js';
 import {
  deleteProxyVideoTaskByPublicId,
@@ -180,12 +180,13 @@ export async function videosProxyRoute(app: FastifyInstance) {
      });
    }

-    const upstream = await fetch(`${mapping.siteUrl}/v1/videos/${encodeURIComponent(mapping.upstreamVideoId)}`, {
+    const targetUrl = `${mapping.siteUrl}/v1/videos/${encodeURIComponent(mapping.upstreamVideoId)}`;
+    const upstream = await fetch(targetUrl, await withSiteProxyRequestInit(targetUrl, {
      method: 'GET',
      headers: {
        Authorization: `Bearer ${mapping.tokenValue}`,
      },
-    });
+    }));
    const text = await upstream.text();
    try {
      const data = JSON.parse(text);
@@ -210,12 +211,13 @@ export async function videosProxyRoute(app: FastifyInstance) {
      });
    }

-    const upstream = await fetch(`${mapping.siteUrl}/v1/videos/${encodeURIComponent(mapping.upstreamVideoId)}`, {
+    const targetUrl = `${mapping.siteUrl}/v1/videos/${encodeURIComponent(mapping.upstreamVideoId)}`;
+    const upstream = await fetch(targetUrl, await withSiteProxyRequestInit(targetUrl, {
      method: 'DELETE',
      headers: {
        Authorization: `Bearer ${mapping.tokenValue}`,
      },
-    });
+    }));
    if (upstream.ok) {
      await deleteProxyVideoTaskByPublicId(mapping.publicId);
      return reply.code(upstream.status).send();