[daily] cap sqlite migration recovery retries (#410)

* [daily] cap sqlite migration recovery retries

* fix typecheck for sqlite retry budget guard

src/server/db/migrate.test.ts

@@ -584,4 +584,47 @@ describe('sqlite migrate bootstrap', () => {
 
     verified.close();
   });
+
+  it('fails fast when duplicate-column recovery exceeds the retry budget', async () => {
+    const dataDir = mkdtempSync(join(tmpdir(), 'metapi-migrate-retry-budget-'));
+    process.env.DATA_DIR = dataDir;
+    vi.resetModules();
+
+    const migrateModule = await import('./migrate.js');
+    const { __migrateTestUtils } = migrateModule as {
+      __migrateTestUtils: Record<string, unknown>;
+    };
+
+    const runSqliteMigrationRecoveryLoop = __migrateTestUtils
+      .runSqliteMigrationRecoveryLoop as ((input: {
+        runMigrate: () => void;
+        recoverDuplicateColumnMigrationError: (error: unknown) => { tag: string; recoveredCount: number } | null;
+        isSitesPlatformUrlUniqueConflictError: (error: unknown) => boolean;
+        deduplicateLegacySitesForUniqueIndex: () => boolean;
+        closeSqlite: () => void;
+      }) => void) | undefined;
+    const retryBudget = __migrateTestUtils.sqliteMigrationRecoveryRetryBudget as number | undefined;
+
+    const runMigrate = vi.fn(() => {
+      throw new Error('duplicate column name: token_group');
+    });
+    const closeSqlite = vi.fn();
+
+    expect(runSqliteMigrationRecoveryLoop).toBeTypeOf('function');
+    expect(typeof retryBudget).toBe('number');
+
+    expect(() => runSqliteMigrationRecoveryLoop!({
+      runMigrate,
+      recoverDuplicateColumnMigrationError: () => ({
+        tag: '0007_account_token_group',
+        recoveredCount: 1,
+      }),
+      isSitesPlatformUrlUniqueConflictError: () => false,
+      deduplicateLegacySitesForUniqueIndex: () => false,
+      closeSqlite,
+    })).toThrow(/retry budget/i);
+
+    expect(runMigrate).toHaveBeenCalledTimes((retryBudget ?? 0) + 1);
+    expect(closeSqlite).toHaveBeenCalledTimes(1);
+  });
 });

src/server/db/migrate.ts

@@ -34,6 +34,15 @@ type RecoveryMigration = RecoveryMigrationRecord & {
   statements: string[];
 };
 
+type SqliteMigrationRecoveryLoopInput = {
+  runMigrate: () => void;
+  recoverDuplicateColumnMigrationError: (error: unknown) => DuplicateColumnRecoveryResult | null;
+  isSitesPlatformUrlUniqueConflictError: (error: unknown) => boolean;
+  deduplicateLegacySitesForUniqueIndex: () => boolean;
+  closeSqlite: () => void;
+  retryBudget?: number;
+};
+
 type LegacySiteRow = {
   id: number;
   platform: string;
@@ -41,6 +50,7 @@ type LegacySiteRow = {
 };
 
 const VERIFIED_BOOTSTRAP_TAG = '0012_account_token_value_status';
+const SQLITE_MIGRATION_RECOVERY_RETRY_BUDGET = 64;
 const VERIFIED_SCHEMA_MARKERS: SchemaMarker[] = [
   { table: 'sites' },
   { table: 'settings' },
@@ -463,6 +473,57 @@ function recoverDuplicateColumnMigrationError(
   };
 }
 
+function buildSqliteMigrationRetryBudgetError(error: unknown, retryBudget: number): Error {
+  const detail = normalizeSchemaErrorMessage(error);
+  return new Error(
+    detail
+      ? `[db] Migration recovery exceeded retry budget (${retryBudget} attempts): ${detail}`
+      : `[db] Migration recovery exceeded retry budget (${retryBudget} attempts).`,
+  );
+}
+
+function runSqliteMigrationRecoveryLoop(input: SqliteMigrationRecoveryLoopInput): void {
+  const retryBudget = Math.max(1, Math.trunc(input.retryBudget ?? SQLITE_MIGRATION_RECOVERY_RETRY_BUDGET));
+  let recoveryRetries = 0;
+
+  while (true) {
+    try {
+      input.runMigrate();
+      return;
+    } catch (error) {
+      const duplicateColumnRecovery = input.recoverDuplicateColumnMigrationError(error);
+      if (duplicateColumnRecovery && duplicateColumnRecovery.recoveredCount > 0) {
+        recoveryRetries += 1;
+        if (recoveryRetries > retryBudget) {
+          input.closeSqlite();
+          throw buildSqliteMigrationRetryBudgetError(error, retryBudget);
+        }
+        continue;
+      }
+      if (duplicateColumnRecovery) {
+        input.closeSqlite();
+        throw error;
+      }
+
+      const recoveredDuplicateSites = (
+        input.isSitesPlatformUrlUniqueConflictError(error)
+        && input.deduplicateLegacySitesForUniqueIndex()
+      );
+      if (recoveredDuplicateSites) {
+        recoveryRetries += 1;
+        if (recoveryRetries > retryBudget) {
+          input.closeSqlite();
+          throw buildSqliteMigrationRetryBudgetError(error, retryBudget);
+        }
+        continue;
+      }
+
+      input.closeSqlite();
+      throw error;
+    }
+  }
+}
+
 function tryRecoverDuplicateColumnMigrationError(
   sqlite: Database.Database,
   migrationsFolder: string,
@@ -586,6 +647,8 @@ export const __migrateTestUtils = {
   tryRecoverDuplicateColumnMigrationError,
   isSitesPlatformUrlUniqueConflictError,
   deduplicateLegacySitesForUniqueIndex,
+  runSqliteMigrationRecoveryLoop,
+  sqliteMigrationRecoveryRetryBudget: SQLITE_MIGRATION_RECOVERY_RETRY_BUDGET,
 };
 
 function bootstrapLegacyDrizzleMigrations(sqlite: Database.Database, migrationsFolder: string): boolean {
@@ -626,32 +689,17 @@ export function runSqliteMigrations(): void {
   bootstrapLegacyDrizzleMigrations(sqlite, migrationsFolder);
   backfillMissingRecordedMigrations(sqlite, migrationsFolder);
 
-  while (true) {
+  runSqliteMigrationRecoveryLoop({
-    try {
+    runMigrate: () => {
       migrate(drizzle(sqlite), { migrationsFolder });
-      break;
+    },
-    } catch (error) {
+    recoverDuplicateColumnMigrationError: (error) => (
-      const duplicateColumnRecovery = recoverDuplicateColumnMigrationError(sqlite, migrationsFolder, error);
+      recoverDuplicateColumnMigrationError(sqlite, migrationsFolder, error)
-      if (duplicateColumnRecovery) {
+    ),
-        if (duplicateColumnRecovery.recoveredCount > 0) {
+    isSitesPlatformUrlUniqueConflictError,
-          continue;
+    deduplicateLegacySitesForUniqueIndex: () => deduplicateLegacySitesForUniqueIndex(sqlite),
-        }
+    closeSqlite: () => sqlite.close(),
-        sqlite.close();
+  });
-        throw error;
-      }
-
-      const recoveredDuplicateSites = (
-        isSitesPlatformUrlUniqueConflictError(error)
-        && deduplicateLegacySitesForUniqueIndex(sqlite)
-      );
-      if (recoveredDuplicateSites) {
-        continue;
-      }
-
-      sqlite.close();
-      throw error;
-    }
-  }
 
   sqlite.close();
   console.log('Migration complete.');