Merge pull request #138 from cita-777/fix/sqlite-sites-unique-migration

fix: recover site unique index bootstrap

src/server/db/migrate.test.ts

@@ -384,4 +384,84 @@ describe('sqlite migrate bootstrap', () => {
 
     verified.close();
   });
+
+  it('deduplicates legacy duplicate sites before applying the oauth site unique index', async () => {
+    const dataDir = mkdtempSync(join(tmpdir(), 'metapi-migrate-duplicate-sites-'));
+    const dbPath = join(dataDir, 'hub.db');
+    const sqlite = new Database(dbPath);
+    const journalEntries = readMigrationJournalEntries();
+    const appliedEntries = journalEntries.filter((entry) => entry.tag !== '0013_oauth_multi_provider');
+
+    for (const entry of appliedEntries) {
+      const sqlText = readFileSync(join(migrationsDir, `${entry.tag}.sql`), 'utf8');
+      applyMigrationSql(sqlite, sqlText);
+    }
+    recordAppliedMigrations(sqlite, appliedEntries);
+
+    sqlite.exec(`
+      INSERT INTO sites (id, name, url, platform, status, is_pinned, sort_order, global_weight)
+      VALUES
+        (101, 'Primary Codex', 'https://chatgpt.com/backend-api/codex', 'codex', 'active', 0, 0, 1),
+        (202, 'Duplicate Codex', 'https://chatgpt.com/backend-api/codex', 'codex', 'disabled', 1, 9, 3);
+
+      INSERT INTO accounts (site_id, username, access_token, status, checkin_enabled)
+      VALUES
+        (101, 'first@example.com', 'token-a', 'active', 0),
+        (202, 'second@example.com', 'token-b', 'disabled', 0);
+
+      INSERT INTO site_disabled_models (site_id, model_name)
+      VALUES
+        (101, 'gpt-5'),
+        (202, 'gpt-5'),
+        (202, 'gpt-5-mini');
+    `);
+
+    sqlite.close();
+
+    process.env.DATA_DIR = dataDir;
+    vi.resetModules();
+
+    await expect(import('./migrate.js')).resolves.toMatchObject({
+      runSqliteMigrations: expect.any(Function),
+    });
+
+    const verified = new Database(dbPath, { readonly: true });
+    const sites = verified
+      .prepare('SELECT id, name, url, platform, status, is_pinned, sort_order, global_weight FROM sites ORDER BY id ASC')
+      .all() as Array<{
+      id: number;
+      name: string;
+      url: string;
+      platform: string;
+      status: string;
+      is_pinned: number;
+      sort_order: number;
+      global_weight: number;
+    }>;
+    const accounts = verified
+      .prepare('SELECT username, site_id FROM accounts ORDER BY username ASC')
+      .all() as Array<{ username: string; site_id: number }>;
+    const disabledModels = verified
+      .prepare('SELECT site_id, model_name FROM site_disabled_models ORDER BY site_id ASC, model_name ASC')
+      .all() as Array<{ site_id: number; model_name: string }>;
+
+    expect(sites).toEqual([
+      expect.objectContaining({
+        id: 101,
+        name: 'Primary Codex',
+        url: 'https://chatgpt.com/backend-api/codex',
+        platform: 'codex',
+      }),
+    ]);
+    expect(accounts).toEqual([
+      { username: 'first@example.com', site_id: 101 },
+      { username: 'second@example.com', site_id: 101 },
+    ]);
+    expect(disabledModels).toEqual([
+      { site_id: 101, model_name: 'gpt-5' },
+      { site_id: 101, model_name: 'gpt-5-mini' },
+    ]);
+
+    verified.close();
+  });
 });

src/server/db/migrate.ts

@@ -34,6 +34,12 @@ type RecoveryMigration = RecoveryMigrationRecord & {
   statements: string[];
 };
 
+type LegacySiteRow = {
+  id: number;
+  platform: string;
+  url: string;
+};
+
 const VERIFIED_BOOTSTRAP_TAG = '0012_account_token_value_status';
 const VERIFIED_SCHEMA_MARKERS: SchemaMarker[] = [
   { table: 'sites' },
@@ -311,6 +317,21 @@ function isRecoverableSchemaConflictError(error: unknown): boolean {
     || lowered.includes('already exists');
 }
 
+function isSitesPlatformUrlUniqueConflictError(error: unknown): boolean {
+  const lowered = normalizeSchemaErrorMessage(error).toLowerCase();
+  if (!lowered.includes('unique constraint failed: sites.platform, sites.url')) {
+    return false;
+  }
+
+  const failedSqlText = extractFailedSqlFromError(error);
+  if (!failedSqlText) {
+    return true;
+  }
+
+  return normalizeSqlForMatch(failedSqlText)
+    === normalizeSqlForMatch('CREATE UNIQUE INDEX `sites_platform_url_unique` ON `sites` (`platform`,`url`);');
+}
+
 function getLatestRecordedMigrationCreatedAt(sqlite: Database.Database): number | null {
   if (!tableExists(sqlite, '__drizzle_migrations')) return null;
   const row = sqlite
@@ -384,6 +405,107 @@ function tryRecoverDuplicateColumnMigrationError(
   return recovered;
 }
 
+function rewriteDownstreamSiteWeightMultipliers(
+  sqlite: Database.Database,
+  siteIdMapping: Map<number, number>,
+): void {
+  if (siteIdMapping.size <= 0) return;
+  if (!tableExists(sqlite, 'downstream_api_keys')) return;
+  if (!columnExists(sqlite, 'downstream_api_keys', 'site_weight_multipliers')) return;
+
+  const rows = sqlite.prepare(`
+    SELECT id, site_weight_multipliers
+    FROM downstream_api_keys
+    WHERE site_weight_multipliers IS NOT NULL
+      AND TRIM(site_weight_multipliers) <> ''
+  `).all() as Array<{ id: number; site_weight_multipliers: string | null }>;
+
+  const update = sqlite.prepare('UPDATE downstream_api_keys SET site_weight_multipliers = ? WHERE id = ?');
+  for (const row of rows) {
+    if (!row.site_weight_multipliers) continue;
+
+    let parsed: unknown;
+    try {
+      parsed = JSON.parse(row.site_weight_multipliers);
+    } catch {
+      continue;
+    }
+
+    if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) continue;
+    const nextValue = { ...(parsed as Record<string, unknown>) };
+    let changed = false;
+
+    for (const [fromSiteId, toSiteId] of siteIdMapping.entries()) {
+      const fromKey = String(fromSiteId);
+      const toKey = String(toSiteId);
+      if (!(fromKey in nextValue)) continue;
+      if (!(toKey in nextValue)) {
+        nextValue[toKey] = nextValue[fromKey];
+      }
+      delete nextValue[fromKey];
+      changed = true;
+    }
+
+    if (!changed) continue;
+    update.run(JSON.stringify(nextValue), row.id);
+  }
+}
+
+function deduplicateLegacySitesForUniqueIndex(sqlite: Database.Database): boolean {
+  const duplicateGroups = sqlite.prepare(`
+    SELECT platform, url
+    FROM sites
+    GROUP BY platform, url
+    HAVING COUNT(*) > 1
+  `).all() as Array<{ platform: string; url: string }>;
+
+  if (duplicateGroups.length <= 0) {
+    return false;
+  }
+
+  const selectSitesByIdentity = sqlite.prepare(`
+    SELECT id, platform, url
+    FROM sites
+    WHERE platform = ? AND url = ?
+    ORDER BY id ASC
+  `);
+  const rebindAccounts = sqlite.prepare('UPDATE accounts SET site_id = ? WHERE site_id = ?');
+  const mergeDisabledModels = sqlite.prepare(`
+    INSERT OR IGNORE INTO site_disabled_models (site_id, model_name, created_at)
+    SELECT ?, model_name, created_at
+    FROM site_disabled_models
+    WHERE site_id = ?
+  `);
+  const deleteDisabledModels = sqlite.prepare('DELETE FROM site_disabled_models WHERE site_id = ?');
+  const deleteSite = sqlite.prepare('DELETE FROM sites WHERE id = ?');
+
+  const siteIdMapping = new Map<number, number>();
+
+  const transaction = sqlite.transaction(() => {
+    for (const group of duplicateGroups) {
+      const sites = selectSitesByIdentity.all(group.platform, group.url) as LegacySiteRow[];
+      if (sites.length <= 1) continue;
+
+      const canonicalSiteId = sites[0]!.id;
+      for (const site of sites.slice(1)) {
+        mergeDisabledModels.run(canonicalSiteId, site.id);
+        deleteDisabledModels.run(site.id);
+        rebindAccounts.run(canonicalSiteId, site.id);
+        siteIdMapping.set(site.id, canonicalSiteId);
+        deleteSite.run(site.id);
+      }
+    }
+
+    rewriteDownstreamSiteWeightMultipliers(sqlite, siteIdMapping);
+  });
+
+  transaction();
+  if (siteIdMapping.size > 0) {
+    console.warn(`[db] Deduplicated ${siteIdMapping.size} legacy site entries before applying sites_platform_url_unique.`);
+  }
+  return siteIdMapping.size > 0;
+}
+
 export const __migrateTestUtils = {
   splitMigrationStatements,
   normalizeSqlForMatch,
@@ -395,6 +517,8 @@ export const __migrateTestUtils = {
   markMigrationRecordIfMissing,
   recoverMigrationSequence,
   tryRecoverDuplicateColumnMigrationError,
+  isSitesPlatformUrlUniqueConflictError,
+  deduplicateLegacySitesForUniqueIndex,
 };
 
 function bootstrapLegacyDrizzleMigrations(sqlite: Database.Database, migrationsFolder: string): boolean {
@@ -437,7 +561,13 @@ export function runSqliteMigrations(): void {
   try {
     migrate(drizzle(sqlite), { migrationsFolder });
   } catch (error) {
-    if (!tryRecoverDuplicateColumnMigrationError(sqlite, migrationsFolder, error)) {
+    const recoveredDuplicateColumns = tryRecoverDuplicateColumnMigrationError(sqlite, migrationsFolder, error);
+    const recoveredDuplicateSites = (
+      !recoveredDuplicateColumns
+      && isSitesPlatformUrlUniqueConflictError(error)
+      && deduplicateLegacySitesForUniqueIndex(sqlite)
+    );
+    if (!recoveredDuplicateColumns && !recoveredDuplicateSites) {
       sqlite.close();
       throw error;
     }

src/server/db/runtimeSchemaBootstrap.test.ts

@@ -231,4 +231,74 @@ describe('runtime schema bootstrap', () => {
       'CREATE INDEX `proxy_logs_downstream_api_key_created_at_idx` ON `proxy_logs` (`downstream_api_key_id`, `created_at`(191))',
     );
   });
+
+  it('does not add mysql text prefixes when live indexed text columns are varchar-backed', async () => {
+    const executedSql: string[] = [];
+    const minimalContract: SchemaContract = {
+      tables: {
+        sites: {
+          columns: {
+            platform: makeColumn({ logicalType: 'text', notNull: true }),
+            url: makeColumn({ logicalType: 'text', notNull: true }),
+          },
+        },
+      },
+      indexes: [],
+      uniques: [
+        {
+          name: 'sites_platform_url_unique',
+          table: 'sites',
+          columns: ['platform', 'url'],
+        },
+      ],
+      foreignKeys: [],
+    };
+
+    await ensureRuntimeDatabaseSchema({
+      ...createStubClient('mysql', executedSql),
+      execute: async (sqlText: string) => {
+        if (sqlText.includes('FROM information_schema.columns')) {
+          return [[
+            {
+              table_name: 'sites',
+              column_name: 'platform',
+              data_type: 'varchar',
+              column_type: 'varchar(32)',
+            },
+            {
+              table_name: 'sites',
+              column_name: 'url',
+              data_type: 'varchar',
+              column_type: 'varchar(255)',
+            },
+          ]];
+        }
+
+        executedSql.push(sqlText);
+        return [];
+      },
+      queryScalar: async (sqlText: string, params: unknown[] = []) => {
+        if (sqlText.includes('information_schema.tables')) {
+          return params[0] === 'sites' ? 1 : 0;
+        }
+        if (sqlText.includes('information_schema.columns')) {
+          return 1;
+        }
+        return 0;
+      },
+    }, {
+      currentContract: minimalContract,
+      liveContract: {
+        ...minimalContract,
+        uniques: [],
+      },
+    });
+
+    expect(executedSql).toContain(
+      'CREATE UNIQUE INDEX `sites_platform_url_unique` ON `sites` (`platform`, `url`)',
+    );
+    expect(executedSql).not.toContain(
+      'CREATE UNIQUE INDEX `sites_platform_url_unique` ON `sites` (`platform`(191), `url`(191))',
+    );
+  });
 });

src/server/db/runtimeSchemaBootstrap.ts

@@ -312,12 +312,8 @@ async function resolveMySqlIndexPrefixRequirements(
     }
 
     const declaredType = String(row.column_type || row.data_type || '');
-    if (!requiresMysqlIndexPrefixForColumnType(declaredType)) {
-      continue;
-    }
-
     requirements[tableName] ??= {};
-    requirements[tableName][columnName] = true;
+    requirements[tableName][columnName] = requiresMysqlIndexPrefixForColumnType(declaredType);
   }
 
   return requirements;