feat(migrate): implement recovery for duplicate-column migrations and enhance migration handling

src/server/db/migrate.test.ts

@@ -1,5 +1,6 @@
 import Database from 'better-sqlite3';
 import { afterEach, describe, expect, it, vi } from 'vitest';
+import { createHash } from 'node:crypto';
 import { mkdirSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs';
 import { tmpdir } from 'node:os';
 import { dirname, join, resolve } from 'node:path';
@@ -29,6 +30,26 @@ function applyMigrationSql(sqlite: Database.Database, sqlText: string) {
   }
 }
 
+function recordAppliedMigrations(
+  sqlite: Database.Database,
+  journalEntries: MigrationJournalEntry[],
+) {
+  sqlite.exec(`
+    CREATE TABLE IF NOT EXISTS "__drizzle_migrations" (
+      id SERIAL PRIMARY KEY,
+      hash text NOT NULL,
+      created_at numeric
+    )
+  `);
+
+  const insert = sqlite.prepare('INSERT INTO "__drizzle_migrations" ("hash", "created_at") VALUES (?, ?)');
+  for (const entry of journalEntries) {
+    const sqlText = readFileSync(join(migrationsDir, `${entry.tag}.sql`), 'utf8');
+    const hash = createHash('sha256').update(sqlText).digest('hex');
+    insert.run(hash, entry.when);
+  }
+}
+
 describe('sqlite migrate bootstrap', () => {
   afterEach(() => {
     delete process.env.DATA_DIR;
@@ -186,4 +207,44 @@ describe('sqlite migrate bootstrap', () => {
 
     sqlite.close();
   });
+
+  it('replays missing migrations before marking a duplicate-column migration as applied', async () => {
+    const dataDir = mkdtempSync(join(tmpdir(), 'metapi-migrate-partial-journal-'));
+    const dbPath = join(dataDir, 'hub.db');
+    const sqlite = new Database(dbPath);
+    const journalEntries = readMigrationJournalEntries();
+    const appliedEntries = journalEntries.filter((entry) => entry.tag !== '0006_site_disabled_models' && entry.tag !== '0007_account_token_group');
+
+    for (const entry of appliedEntries) {
+      const sqlText = readFileSync(join(migrationsDir, `${entry.tag}.sql`), 'utf8');
+      applyMigrationSql(sqlite, sqlText);
+    }
+    recordAppliedMigrations(sqlite, appliedEntries);
+
+    // Simulate legacy compatibility code adding token_group before the formal 0007 migration ran.
+    sqlite.exec('ALTER TABLE account_tokens ADD COLUMN token_group text;');
+    sqlite.close();
+
+    process.env.DATA_DIR = dataDir;
+    vi.resetModules();
+
+    await expect(import('./migrate.js')).resolves.toMatchObject({
+      runSqliteMigrations: expect.any(Function),
+    });
+
+    const verified = new Database(dbPath, { readonly: true });
+    const appliedRows = verified
+      .prepare('SELECT created_at FROM __drizzle_migrations ORDER BY created_at ASC')
+      .all() as Array<{ created_at: number }>;
+    const disabledModelsTable = verified
+      .prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'site_disabled_models'")
+      .get() as { name?: string } | undefined;
+
+    expect(disabledModelsTable?.name).toBe('site_disabled_models');
+    expect(appliedRows.map((row) => Number(row.created_at))).toEqual(
+      journalEntries.map((entry) => entry.when),
+    );
+
+    verified.close();
+  });
 });

src/server/db/migrate.ts

@@ -30,6 +30,10 @@ type RecoveryMigrationRecord = MigrationRecord & {
   tag: string;
 };
 
+type RecoveryMigration = RecoveryMigrationRecord & {
+  statements: string[];
+};
+
 const VERIFIED_BOOTSTRAP_TAG = '0007_account_token_group';
 const VERIFIED_SCHEMA_MARKERS: SchemaMarker[] = [
   { table: 'sites' },
@@ -170,6 +174,21 @@ function findMatchingSingleStatementMigration(
   return null;
 }
 
+function readRecoveryMigrations(migrationsFolder: string): RecoveryMigration[] {
+  const journalPath = resolve(migrationsFolder, 'meta', '_journal.json');
+  const journal = JSON.parse(readFileSync(journalPath, 'utf8')) as MigrationJournalFile;
+
+  return (journal.entries ?? []).map((entry) => {
+    const migrationSql = readFileSync(resolve(migrationsFolder, `${entry.tag}.sql`), 'utf8');
+    return {
+      tag: entry.tag,
+      createdAt: Number(entry.when),
+      hash: createHash('sha256').update(migrationSql).digest('hex'),
+      statements: splitMigrationStatements(migrationSql),
+    };
+  });
+}
+
 function ensureDrizzleMigrationsTable(sqlite: Database.Database): void {
   sqlite.exec(`
     CREATE TABLE IF NOT EXISTS "__drizzle_migrations" (
@@ -232,6 +251,61 @@ function isDuplicateColumnError(error: unknown): boolean {
     || lowered.includes('duplicate column name');
 }
 
+function isRecoverableSchemaConflictError(error: unknown): boolean {
+  const lowered = normalizeSchemaErrorMessage(error).toLowerCase();
+  return lowered.includes('duplicate column')
+    || lowered.includes('duplicate column name')
+    || lowered.includes('already exists');
+}
+
+function getLatestRecordedMigrationCreatedAt(sqlite: Database.Database): number | null {
+  if (!tableExists(sqlite, '__drizzle_migrations')) return null;
+  const row = sqlite
+    .prepare('SELECT created_at FROM "__drizzle_migrations" ORDER BY created_at DESC LIMIT 1')
+    .get() as { created_at?: number } | undefined;
+  if (!row || row.created_at === undefined || row.created_at === null) {
+    return null;
+  }
+  return Number(row.created_at);
+}
+
+function replayMigrationStatements(sqlite: Database.Database, statements: string[]): void {
+  for (const statement of statements) {
+    try {
+      sqlite.exec(statement);
+    } catch (error) {
+      if (!isRecoverableSchemaConflictError(error)) {
+        throw error;
+      }
+    }
+  }
+}
+
+function recoverMigrationSequence(
+  sqlite: Database.Database,
+  migrationsFolder: string,
+  failedMigrationTag: string,
+): boolean {
+  const migrations = readRecoveryMigrations(migrationsFolder);
+  const failedMigrationIndex = migrations.findIndex((migration) => migration.tag === failedMigrationTag);
+  if (failedMigrationIndex < 0) {
+    return false;
+  }
+
+  let latestRecordedCreatedAt = getLatestRecordedMigrationCreatedAt(sqlite);
+  for (const migration of migrations.slice(0, failedMigrationIndex + 1)) {
+    if (latestRecordedCreatedAt !== null && latestRecordedCreatedAt >= migration.createdAt) {
+      continue;
+    }
+
+    replayMigrationStatements(sqlite, migration.statements);
+    markMigrationRecordIfMissing(sqlite, migration);
+    latestRecordedCreatedAt = migration.createdAt;
+  }
+
+  return true;
+}
+
 function tryRecoverDuplicateColumnMigrationError(
   sqlite: Database.Database,
   migrationsFolder: string,
@@ -251,11 +325,11 @@ function tryRecoverDuplicateColumnMigrationError(
     return false;
   }
 
-  const inserted = markMigrationRecordIfMissing(sqlite, matchedMigration);
-  if (inserted) {
-    console.warn(`[db] Recovered duplicate-column migration by marking ${matchedMigration.tag} as applied.`);
+  const recovered = recoverMigrationSequence(sqlite, migrationsFolder, matchedMigration.tag);
+  if (recovered) {
+    console.warn(`[db] Recovered duplicate-column migration sequence through ${matchedMigration.tag}.`);
   }
-  return true;
+  return recovered;
 }
 
 export const __migrateTestUtils = {
@@ -263,7 +337,9 @@ export const __migrateTestUtils = {
   normalizeSqlForMatch,
   extractFailedSqlFromError,
   findMatchingSingleStatementMigration,
+  readRecoveryMigrations,
   markMigrationRecordIfMissing,
+  recoverMigrationSequence,
   tryRecoverDuplicateColumnMigrationError,
 };