aibo/BoosAPI
1
0
Fork 0
forked from boos/BoosAPI
Code Pull Requests Activity

Revert "Merge pull request #77 from cita-777/database-parity"

Browse Source 
This reverts commit d2210b8ffb, reversing
changes made to b834a9bef8.
This commit is contained in:
ciat-777
2026-03-15 01:11:48 +08:00
parent d2210b8ffb
commit 064af9e0bb
29 changed files with 143 additions and 7466 deletions
Download Patch File Download Diff File Expand all files Collapse all files
drizzle/0008_sqlite_schema_backfill.sql
-214
View File
@@ -1,214 +0,0 @@
ALTER TABLE `sites` ADD `proxy_url` text;
--> statement-breakpoint
ALTER TABLE `sites` ADD `use_system_proxy` integer DEFAULT false;
--> statement-breakpoint
UPDATE `sites`
SET `use_system_proxy` = false
WHERE `use_system_proxy` IS NULL;
--> statement-breakpoint
ALTER TABLE `sites` ADD `custom_headers` text;
--> statement-breakpoint
ALTER TABLE `sites` ADD `external_checkin_url` text;
--> statement-breakpoint
ALTER TABLE `sites` ADD `global_weight` real DEFAULT 1;
--> statement-breakpoint
UPDATE `sites`
SET `global_weight` = 1
WHERE `global_weight` IS NULL
OR `global_weight` <= 0;
--> statement-breakpoint
ALTER TABLE `token_routes` ADD `display_name` text;
--> statement-breakpoint
ALTER TABLE `token_routes` ADD `display_icon` text;
--> statement-breakpoint
ALTER TABLE `token_routes` ADD `decision_snapshot` text;
--> statement-breakpoint
ALTER TABLE `token_routes` ADD `decision_refreshed_at` text;
--> statement-breakpoint
ALTER TABLE `token_routes` ADD `routing_strategy` text DEFAULT 'weighted';
--> statement-breakpoint
ALTER TABLE `route_channels` ADD `source_model` text;
--> statement-breakpoint
ALTER TABLE `route_channels` ADD `last_selected_at` text;
--> statement-breakpoint
ALTER TABLE `route_channels` ADD `consecutive_fail_count` integer NOT NULL DEFAULT 0;
--> statement-breakpoint
ALTER TABLE `route_channels` ADD `cooldown_level` integer NOT NULL DEFAULT 0;
--> statement-breakpoint
CREATE TABLE IF NOT EXISTS `downstream_api_keys` (
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
`name` text NOT NULL,
`key` text NOT NULL,
`description` text,
`enabled` integer DEFAULT true,
`expires_at` text,
`max_cost` real,
`used_cost` real DEFAULT 0,
`max_requests` integer,
`used_requests` integer DEFAULT 0,
`supported_models` text,
`allowed_route_ids` text,
`site_weight_multipliers` text,
`last_used_at` text,
`created_at` text DEFAULT (datetime('now')),
`updated_at` text DEFAULT (datetime('now'))
);
--> statement-breakpoint
CREATE UNIQUE INDEX IF NOT EXISTS `downstream_api_keys_key_unique`
ON `downstream_api_keys` (`key`);
--> statement-breakpoint
CREATE INDEX IF NOT EXISTS `downstream_api_keys_name_idx`
ON `downstream_api_keys` (`name`);
--> statement-breakpoint
CREATE INDEX IF NOT EXISTS `downstream_api_keys_enabled_idx`
ON `downstream_api_keys` (`enabled`);
--> statement-breakpoint
CREATE INDEX IF NOT EXISTS `downstream_api_keys_expires_at_idx`
ON `downstream_api_keys` (`expires_at`);
--> statement-breakpoint
CREATE TABLE IF NOT EXISTS `proxy_files` (
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
`public_id` text NOT NULL,
`owner_type` text NOT NULL,
`owner_id` text NOT NULL,
`filename` text NOT NULL,
`mime_type` text NOT NULL,
`purpose` text,
`byte_size` integer NOT NULL,
`sha256` text NOT NULL,
`content_base64` text NOT NULL,
`created_at` text DEFAULT (datetime('now')),
`updated_at` text DEFAULT (datetime('now')),
`deleted_at` text
);
--> statement-breakpoint
CREATE UNIQUE INDEX IF NOT EXISTS `proxy_files_public_id_unique`
ON `proxy_files` (`public_id`);
--> statement-breakpoint
CREATE INDEX IF NOT EXISTS `proxy_files_owner_lookup_idx`
ON `proxy_files` (`owner_type`, `owner_id`, `deleted_at`);
--> statement-breakpoint
CREATE TABLE IF NOT EXISTS `proxy_video_tasks` (
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
`public_id` text NOT NULL,
`upstream_video_id` text NOT NULL,
`site_url` text NOT NULL,
`token_value` text NOT NULL,
`requested_model` text,
`actual_model` text,
`channel_id` integer,
`account_id` integer,
`status_snapshot` text,
`upstream_response_meta` text,
`last_upstream_status` integer,
`last_polled_at` text,
`created_at` text DEFAULT (datetime('now')),
`updated_at` text DEFAULT (datetime('now'))
);
--> statement-breakpoint
CREATE UNIQUE INDEX IF NOT EXISTS `proxy_video_tasks_public_id_unique`
ON `proxy_video_tasks` (`public_id`);
--> statement-breakpoint
CREATE INDEX IF NOT EXISTS `proxy_video_tasks_upstream_video_id_idx`
ON `proxy_video_tasks` (`upstream_video_id`);
--> statement-breakpoint
CREATE INDEX IF NOT EXISTS `proxy_video_tasks_created_at_idx`
ON `proxy_video_tasks` (`created_at`);
--> statement-breakpoint
CREATE UNIQUE INDEX IF NOT EXISTS `model_availability_account_model_unique`
ON `model_availability` (`account_id`, `model_name`);
--> statement-breakpoint
PRAGMA foreign_keys=OFF;
--> statement-breakpoint
CREATE TABLE `__new_route_channels` (
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
`route_id` integer NOT NULL,
`account_id` integer NOT NULL,
`token_id` integer,
`source_model` text,
`priority` integer DEFAULT 0,
`weight` integer DEFAULT 10,
`enabled` integer DEFAULT true,
`manual_override` integer DEFAULT false,
`success_count` integer DEFAULT 0,
`fail_count` integer DEFAULT 0,
`total_latency_ms` integer DEFAULT 0,
`total_cost` real DEFAULT 0,
`last_used_at` text,
`last_selected_at` text,
`last_fail_at` text,
`consecutive_fail_count` integer NOT NULL DEFAULT 0,
`cooldown_level` integer NOT NULL DEFAULT 0,
`cooldown_until` text,
FOREIGN KEY (`route_id`) REFERENCES `token_routes`(`id`) ON UPDATE no action ON DELETE cascade,
FOREIGN KEY (`account_id`) REFERENCES `accounts`(`id`) ON UPDATE no action ON DELETE cascade,
FOREIGN KEY (`token_id`) REFERENCES `account_tokens`(`id`) ON UPDATE no action ON DELETE set null
);
--> statement-breakpoint
INSERT INTO `__new_route_channels` (
`id`,
`route_id`,
`account_id`,
`token_id`,
`source_model`,
`priority`,
`weight`,
`enabled`,
`manual_override`,
`success_count`,
`fail_count`,
`total_latency_ms`,
`total_cost`,
`last_used_at`,
`last_selected_at`,
`last_fail_at`,
`consecutive_fail_count`,
`cooldown_level`,
`cooldown_until`
)
SELECT
`id`,
`route_id`,
`account_id`,
CASE
WHEN `token_id` IS NULL THEN NULL
WHEN EXISTS (SELECT 1 FROM `account_tokens` WHERE `account_tokens`.`id` = `route_channels`.`token_id`) THEN `token_id`
ELSE NULL
END,
`source_model`,
`priority`,
`weight`,
`enabled`,
`manual_override`,
`success_count`,
`fail_count`,
`total_latency_ms`,
`total_cost`,
`last_used_at`,
`last_selected_at`,
`last_fail_at`,
`consecutive_fail_count`,
`cooldown_level`,
`cooldown_until`
FROM `route_channels`;
--> statement-breakpoint
DROP TABLE `route_channels`;
--> statement-breakpoint
ALTER TABLE `__new_route_channels` RENAME TO `route_channels`;
--> statement-breakpoint
CREATE INDEX IF NOT EXISTS `route_channels_route_id_idx`
ON `route_channels` (`route_id`);
--> statement-breakpoint
CREATE INDEX IF NOT EXISTS `route_channels_account_id_idx`
ON `route_channels` (`account_id`);
--> statement-breakpoint
CREATE INDEX IF NOT EXISTS `route_channels_token_id_idx`
ON `route_channels` (`token_id`);
--> statement-breakpoint
CREATE INDEX IF NOT EXISTS `route_channels_route_enabled_idx`
ON `route_channels` (`route_id`, `enabled`);
--> statement-breakpoint
CREATE INDEX IF NOT EXISTS `route_channels_route_token_idx`
ON `route_channels` (`route_id`, `token_id`);
--> statement-breakpoint
PRAGMA foreign_keys=ON;
drizzle/meta/0008_snapshot.json
-1847
View File
File diff suppressed because it is too large Load Diff
drizzle/meta/_journal.json
-7
View File
@@ -50,13 +50,6 @@
"when": 1773500400000, "when": 1773500400000,
"tag": "0007_account_token_group", "tag": "0007_account_token_group",
"breakpoints": true "breakpoints": true
},
{
"idx": 7,
"version": "6",
"when": 1773500500000,
"tag": "0008_sqlite_schema_backfill",
"breakpoints": true
} }
] ]
} }
package.json
-5
View File
@@ -41,11 +41,6 @@
"start:desktop": "electron dist/desktop/main.js", "start:desktop": "electron dist/desktop/main.js",
"db:generate": "drizzle-kit generate", "db:generate": "drizzle-kit generate",
"db:migrate": "tsx src/server/db/migrate.ts", "db:migrate": "tsx src/server/db/migrate.ts",
"schema:contract": "tsx scripts/dev/generate-schema-contract.ts",
"schema:generate": "npm run db:generate && npm run schema:contract",
"test:schema:unit": "vitest run --root . src/server/db/schemaContract.test.ts src/server/db/schemaArtifactGenerator.test.ts src/server/db/schemaIntrospection.test.ts src/server/db/schemaParity.test.ts",
"test:schema:parity": "vitest run --root . src/server/db/schemaParity.live.test.ts",
"test:schema:upgrade": "vitest run --root . src/server/db/schemaUpgrade.live.test.ts",
"smoke:db": "tsx scripts/dev/db-smoke.ts", "smoke:db": "tsx scripts/dev/db-smoke.ts",
"smoke:db:sqlite": "tsx scripts/dev/db-smoke.ts --db-type sqlite", "smoke:db:sqlite": "tsx scripts/dev/db-smoke.ts --db-type sqlite",
"smoke:db:mysql": "tsx scripts/dev/db-smoke.ts --db-type mysql", "smoke:db:mysql": "tsx scripts/dev/db-smoke.ts --db-type mysql",
scripts/dev/generate-schema-contract.ts
-23
View File
@@ -1,23 +0,0 @@
import { existsSync, readFileSync } from 'node:fs';
import { writeDialectArtifactFiles } from '../../src/server/db/schemaArtifactGenerator.js';
import {
resolveGeneratedSchemaContractPath,
type SchemaContract,
writeSchemaContractFile,
} from '../../src/server/db/schemaContract.js';
function readPreviousSchemaContract(): SchemaContract | null {
const contractPath = resolveGeneratedSchemaContractPath();
if (!existsSync(contractPath)) {
return null;
}
return JSON.parse(readFileSync(contractPath, 'utf8')) as SchemaContract;
}
const previousContract = readPreviousSchemaContract();
const contract = writeSchemaContractFile();
writeDialectArtifactFiles(contract, previousContract);
const tableCount = Object.keys(contract.tables).length;
console.log(`[schema:contract] wrote ${tableCount} tables and dialect artifacts`);
scripts/dev/generate-upgrade-fixture.ts
-113
View File
@@ -1,113 +0,0 @@
import { execFileSync } from 'node:child_process';
import { mkdirSync, readFileSync, writeFileSync } from 'node:fs';
import { dirname, resolve } from 'node:path';
import {
resolveGeneratedSchemaContractPath,
type SchemaContract,
} from '../../src/server/db/schemaContract.js';
interface FixtureScriptOptions {
fromRef?: string;
outputPath: string;
dropTables: string[];
dropColumns: string[];
}
function parseArgs(argv: string[]): FixtureScriptOptions {
const options: FixtureScriptOptions = {
outputPath: 'src/server/db/generated/fixtures/2026-03-14-baseline.schemaContract.json',
dropTables: [],
dropColumns: [],
};
for (let index = 0; index < argv.length; index += 1) {
const argument = argv[index];
if (argument === '--from-ref') {
options.fromRef = argv[index + 1];
index += 1;
continue;
}
if (argument === '--output') {
options.outputPath = argv[index + 1];
index += 1;
continue;
}
if (argument === '--drop-table') {
options.dropTables.push(argv[index + 1]);
index += 1;
continue;
}
if (argument === '--drop-column') {
options.dropColumns.push(argv[index + 1]);
index += 1;
}
}
return options;
}
function pruneContract(contract: SchemaContract, options: FixtureScriptOptions): SchemaContract {
const droppedTables = new Set(options.dropTables);
const droppedColumns = new Map<string, Set<string>>();
for (const entry of options.dropColumns) {
const [tableName, columnName] = entry.split('.', 2);
if (!tableName || !columnName) {
throw new Error(`invalid --drop-column value: ${entry}`);
}
const tableColumns = droppedColumns.get(tableName) ?? new Set<string>();
tableColumns.add(columnName);
droppedColumns.set(tableName, tableColumns);
}
const tables = Object.fromEntries(
Object.entries(contract.tables)
.filter(([tableName]) => !droppedTables.has(tableName))
.map(([tableName, table]) => {
const removedColumns = droppedColumns.get(tableName) ?? new Set<string>();
const columns = Object.fromEntries(
Object.entries(table.columns).filter(([columnName]) => !removedColumns.has(columnName)),
);
return [tableName, { columns }];
}),
);
const hasColumn = (tableName: string, columnName: string): boolean =>
!!tables[tableName]?.columns[columnName];
return {
tables,
indexes: contract.indexes.filter((index) =>
!!tables[index.table] && index.columns.every((columnName) => hasColumn(index.table, columnName))),
uniques: contract.uniques.filter((unique) =>
!!tables[unique.table] && unique.columns.every((columnName) => hasColumn(unique.table, columnName))),
foreignKeys: contract.foreignKeys.filter((foreignKey) =>
!!tables[foreignKey.table]
&& !!tables[foreignKey.referencedTable]
&& foreignKey.columns.every((columnName) => hasColumn(foreignKey.table, columnName))
&& foreignKey.referencedColumns.every((columnName) => hasColumn(foreignKey.referencedTable, columnName))),
};
}
function readContractFromRef(fromRef: string): SchemaContract {
const contractJson = execFileSync(
'git',
['-c', 'safe.directory=.', 'show', `${fromRef}:src/server/db/generated/schemaContract.json`],
{ encoding: 'utf8' },
);
return JSON.parse(contractJson) as SchemaContract;
}
function readCurrentContract(): SchemaContract {
return JSON.parse(readFileSync(resolveGeneratedSchemaContractPath(), 'utf8')) as SchemaContract;
}
const options = parseArgs(process.argv.slice(2));
const sourceContract = options.fromRef ? readContractFromRef(options.fromRef) : readCurrentContract();
const contract = pruneContract(sourceContract, options);
const outputPath = resolve(process.cwd(), options.outputPath);
mkdirSync(dirname(outputPath), { recursive: true });
writeFileSync(outputPath, `${JSON.stringify(contract, null, 2)}\n`, 'utf8');
console.log(`[schema:upgrade-fixture] wrote ${outputPath}`);
src/server/db/generated/fixtures/2026-03-14-baseline.schemaContract.json
-1120
View File
File diff suppressed because it is too large Load Diff
src/server/db/generated/mysql.bootstrap.sql
-57
View File
@@ -1,57 +0,0 @@
CREATE TABLE IF NOT EXISTS `sites` (`id` INT AUTO_INCREMENT NOT NULL PRIMARY KEY, `name` TEXT NOT NULL, `url` TEXT NOT NULL, `platform` TEXT NOT NULL, `status` VARCHAR(191) NOT NULL DEFAULT 'active', `api_key` TEXT, `created_at` DATETIME DEFAULT CURRENT_TIMESTAMP, `updated_at` DATETIME DEFAULT CURRENT_TIMESTAMP, `is_pinned` BOOLEAN DEFAULT false, `sort_order` INT DEFAULT 0, `proxy_url` TEXT, `use_system_proxy` BOOLEAN DEFAULT false, `custom_headers` JSON, `external_checkin_url` TEXT, `global_weight` DOUBLE DEFAULT 1);
CREATE TABLE IF NOT EXISTS `accounts` (`id` INT AUTO_INCREMENT NOT NULL PRIMARY KEY, `site_id` INT NOT NULL, `username` TEXT, `access_token` TEXT NOT NULL, `api_token` TEXT, `balance` DOUBLE DEFAULT 0, `balance_used` DOUBLE DEFAULT 0, `quota` DOUBLE DEFAULT 0, `unit_cost` DOUBLE, `value_score` DOUBLE DEFAULT 0, `status` VARCHAR(191) DEFAULT 'active', `checkin_enabled` BOOLEAN DEFAULT true, `last_checkin_at` DATETIME, `last_balance_refresh` DATETIME, `extra_config` JSON, `created_at` DATETIME DEFAULT CURRENT_TIMESTAMP, `updated_at` DATETIME DEFAULT CURRENT_TIMESTAMP, `is_pinned` BOOLEAN DEFAULT false, `sort_order` INT DEFAULT 0, FOREIGN KEY (`site_id`) REFERENCES `sites`(`id`) ON DELETE CASCADE);
CREATE TABLE IF NOT EXISTS `account_tokens` (`id` INT AUTO_INCREMENT NOT NULL PRIMARY KEY, `account_id` INT NOT NULL, `name` TEXT NOT NULL, `token` TEXT NOT NULL, `source` VARCHAR(191) DEFAULT 'manual', `enabled` BOOLEAN DEFAULT true, `is_default` BOOLEAN DEFAULT false, `created_at` DATETIME DEFAULT CURRENT_TIMESTAMP, `updated_at` DATETIME DEFAULT CURRENT_TIMESTAMP, `token_group` TEXT, FOREIGN KEY (`account_id`) REFERENCES `accounts`(`id`) ON DELETE CASCADE);
CREATE TABLE IF NOT EXISTS `checkin_logs` (`id` INT AUTO_INCREMENT NOT NULL PRIMARY KEY, `account_id` INT NOT NULL, `status` TEXT NOT NULL, `message` TEXT, `reward` TEXT, `created_at` DATETIME DEFAULT CURRENT_TIMESTAMP, FOREIGN KEY (`account_id`) REFERENCES `accounts`(`id`) ON DELETE CASCADE);
CREATE TABLE IF NOT EXISTS `downstream_api_keys` (`id` INT AUTO_INCREMENT NOT NULL PRIMARY KEY, `name` TEXT NOT NULL, `key` TEXT NOT NULL, `description` TEXT, `enabled` BOOLEAN DEFAULT true, `expires_at` DATETIME, `max_cost` DOUBLE, `used_cost` DOUBLE DEFAULT 0, `max_requests` INT, `used_requests` INT DEFAULT 0, `supported_models` JSON, `allowed_route_ids` JSON, `site_weight_multipliers` JSON, `last_used_at` DATETIME, `created_at` DATETIME DEFAULT CURRENT_TIMESTAMP, `updated_at` DATETIME DEFAULT CURRENT_TIMESTAMP);
CREATE TABLE IF NOT EXISTS `events` (`id` INT AUTO_INCREMENT NOT NULL PRIMARY KEY, `type` TEXT NOT NULL, `title` TEXT NOT NULL, `message` TEXT, `level` VARCHAR(191) NOT NULL DEFAULT 'info', `read` BOOLEAN DEFAULT false, `related_id` INT, `related_type` TEXT, `created_at` DATETIME DEFAULT CURRENT_TIMESTAMP);
CREATE TABLE IF NOT EXISTS `model_availability` (`id` INT AUTO_INCREMENT NOT NULL PRIMARY KEY, `account_id` INT NOT NULL, `model_name` TEXT NOT NULL, `available` BOOLEAN, `latency_ms` INT, `checked_at` DATETIME DEFAULT CURRENT_TIMESTAMP, FOREIGN KEY (`account_id`) REFERENCES `accounts`(`id`) ON DELETE CASCADE);
CREATE TABLE IF NOT EXISTS `proxy_files` (`id` INT AUTO_INCREMENT NOT NULL PRIMARY KEY, `public_id` TEXT NOT NULL, `owner_type` TEXT NOT NULL, `owner_id` TEXT NOT NULL, `filename` TEXT NOT NULL, `mime_type` TEXT NOT NULL, `purpose` TEXT, `byte_size` INT NOT NULL, `sha256` TEXT NOT NULL, `content_base64` TEXT NOT NULL, `created_at` DATETIME DEFAULT CURRENT_TIMESTAMP, `updated_at` DATETIME DEFAULT CURRENT_TIMESTAMP, `deleted_at` DATETIME);
CREATE TABLE IF NOT EXISTS `proxy_logs` (`id` INT AUTO_INCREMENT NOT NULL PRIMARY KEY, `route_id` INT, `channel_id` INT, `account_id` INT, `model_requested` TEXT, `model_actual` TEXT, `status` TEXT, `http_status` INT, `latency_ms` INT, `prompt_tokens` INT, `completion_tokens` INT, `total_tokens` INT, `estimated_cost` DOUBLE, `error_message` TEXT, `retry_count` INT DEFAULT 0, `created_at` DATETIME DEFAULT CURRENT_TIMESTAMP, `billing_details` JSON);
CREATE TABLE IF NOT EXISTS `proxy_video_tasks` (`id` INT AUTO_INCREMENT NOT NULL PRIMARY KEY, `public_id` TEXT NOT NULL, `upstream_video_id` TEXT NOT NULL, `site_url` TEXT NOT NULL, `token_value` TEXT NOT NULL, `requested_model` TEXT, `actual_model` TEXT, `channel_id` INT, `account_id` INT, `status_snapshot` JSON, `upstream_response_meta` JSON, `last_upstream_status` INT, `last_polled_at` DATETIME, `created_at` DATETIME DEFAULT CURRENT_TIMESTAMP, `updated_at` DATETIME DEFAULT CURRENT_TIMESTAMP);
CREATE TABLE IF NOT EXISTS `token_routes` (`id` INT AUTO_INCREMENT NOT NULL PRIMARY KEY, `model_pattern` TEXT NOT NULL, `model_mapping` JSON, `enabled` BOOLEAN DEFAULT true, `created_at` DATETIME DEFAULT CURRENT_TIMESTAMP, `updated_at` DATETIME DEFAULT CURRENT_TIMESTAMP, `display_name` TEXT, `display_icon` TEXT, `decision_snapshot` JSON, `decision_refreshed_at` DATETIME, `routing_strategy` VARCHAR(191) DEFAULT 'weighted');
CREATE TABLE IF NOT EXISTS `route_channels` (`id` INT AUTO_INCREMENT NOT NULL PRIMARY KEY, `route_id` INT NOT NULL, `account_id` INT NOT NULL, `token_id` INT, `source_model` TEXT, `priority` INT DEFAULT 0, `weight` INT DEFAULT 10, `enabled` BOOLEAN DEFAULT true, `manual_override` BOOLEAN DEFAULT false, `success_count` INT DEFAULT 0, `fail_count` INT DEFAULT 0, `total_latency_ms` INT DEFAULT 0, `total_cost` DOUBLE DEFAULT 0, `last_used_at` DATETIME, `last_selected_at` DATETIME, `last_fail_at` DATETIME, `consecutive_fail_count` INT NOT NULL DEFAULT 0, `cooldown_level` INT NOT NULL DEFAULT 0, `cooldown_until` DATETIME, FOREIGN KEY (`account_id`) REFERENCES `accounts`(`id`) ON DELETE CASCADE, FOREIGN KEY (`route_id`) REFERENCES `token_routes`(`id`) ON DELETE CASCADE, FOREIGN KEY (`token_id`) REFERENCES `account_tokens`(`id`) ON DELETE SET NULL);
CREATE TABLE IF NOT EXISTS `settings` (`key` TEXT NOT NULL PRIMARY KEY, `value` TEXT);
CREATE TABLE IF NOT EXISTS `site_disabled_models` (`id` INT AUTO_INCREMENT NOT NULL PRIMARY KEY, `site_id` INT NOT NULL, `model_name` TEXT NOT NULL, `created_at` DATETIME DEFAULT CURRENT_TIMESTAMP, FOREIGN KEY (`site_id`) REFERENCES `sites`(`id`) ON DELETE CASCADE);
CREATE TABLE IF NOT EXISTS `token_model_availability` (`id` INT AUTO_INCREMENT NOT NULL PRIMARY KEY, `token_id` INT NOT NULL, `model_name` TEXT NOT NULL, `available` BOOLEAN, `latency_ms` INT, `checked_at` DATETIME DEFAULT CURRENT_TIMESTAMP, FOREIGN KEY (`token_id`) REFERENCES `account_tokens`(`id`) ON DELETE CASCADE);
CREATE UNIQUE INDEX `downstream_api_keys_key_unique` ON `downstream_api_keys` (`key`(191));
CREATE UNIQUE INDEX `model_availability_account_model_unique` ON `model_availability` (`account_id`, `model_name`(191));
CREATE UNIQUE INDEX `proxy_files_public_id_unique` ON `proxy_files` (`public_id`(191));
CREATE UNIQUE INDEX `proxy_video_tasks_public_id_unique` ON `proxy_video_tasks` (`public_id`(191));
CREATE UNIQUE INDEX `site_disabled_models_site_model_unique` ON `site_disabled_models` (`site_id`, `model_name`(191));
CREATE UNIQUE INDEX `token_model_availability_token_model_unique` ON `token_model_availability` (`token_id`, `model_name`(191));
CREATE INDEX `account_tokens_account_enabled_idx` ON `account_tokens` (`account_id`, `enabled`);
CREATE INDEX `account_tokens_account_id_idx` ON `account_tokens` (`account_id`);
CREATE INDEX `account_tokens_enabled_idx` ON `account_tokens` (`enabled`);
CREATE INDEX `accounts_site_id_idx` ON `accounts` (`site_id`);
CREATE INDEX `accounts_site_status_idx` ON `accounts` (`site_id`, `status`(191));
CREATE INDEX `accounts_status_idx` ON `accounts` (`status`(191));
CREATE INDEX `checkin_logs_account_created_at_idx` ON `checkin_logs` (`account_id`, `created_at`(191));
CREATE INDEX `checkin_logs_created_at_idx` ON `checkin_logs` (`created_at`(191));
CREATE INDEX `checkin_logs_status_idx` ON `checkin_logs` (`status`(191));
CREATE INDEX `downstream_api_keys_enabled_idx` ON `downstream_api_keys` (`enabled`);
CREATE INDEX `downstream_api_keys_expires_at_idx` ON `downstream_api_keys` (`expires_at`(191));
CREATE INDEX `downstream_api_keys_name_idx` ON `downstream_api_keys` (`name`(191));
CREATE INDEX `events_created_at_idx` ON `events` (`created_at`(191));
CREATE INDEX `events_read_created_at_idx` ON `events` (`read`, `created_at`(191));
CREATE INDEX `events_type_created_at_idx` ON `events` (`type`(191), `created_at`(191));
CREATE INDEX `model_availability_account_available_idx` ON `model_availability` (`account_id`, `available`);
CREATE INDEX `model_availability_model_name_idx` ON `model_availability` (`model_name`(191));
CREATE INDEX `proxy_files_owner_lookup_idx` ON `proxy_files` (`owner_type`(191), `owner_id`(191), `deleted_at`(191));
CREATE INDEX `proxy_logs_account_created_at_idx` ON `proxy_logs` (`account_id`, `created_at`(191));
CREATE INDEX `proxy_logs_created_at_idx` ON `proxy_logs` (`created_at`(191));
CREATE INDEX `proxy_logs_model_actual_created_at_idx` ON `proxy_logs` (`model_actual`(191), `created_at`(191));
CREATE INDEX `proxy_logs_status_created_at_idx` ON `proxy_logs` (`status`(191), `created_at`(191));
CREATE INDEX `proxy_video_tasks_created_at_idx` ON `proxy_video_tasks` (`created_at`(191));
CREATE INDEX `proxy_video_tasks_upstream_video_id_idx` ON `proxy_video_tasks` (`upstream_video_id`(191));
CREATE INDEX `route_channels_account_id_idx` ON `route_channels` (`account_id`);
CREATE INDEX `route_channels_route_enabled_idx` ON `route_channels` (`route_id`, `enabled`);
CREATE INDEX `route_channels_route_id_idx` ON `route_channels` (`route_id`);
CREATE INDEX `route_channels_route_token_idx` ON `route_channels` (`route_id`, `token_id`);
CREATE INDEX `route_channels_token_id_idx` ON `route_channels` (`token_id`);
CREATE INDEX `site_disabled_models_site_id_idx` ON `site_disabled_models` (`site_id`);
CREATE INDEX `sites_status_idx` ON `sites` (`status`(191));
CREATE INDEX `token_model_availability_available_idx` ON `token_model_availability` (`available`);
CREATE INDEX `token_model_availability_model_name_idx` ON `token_model_availability` (`model_name`(191));
CREATE INDEX `token_model_availability_token_available_idx` ON `token_model_availability` (`token_id`, `available`);
CREATE INDEX `token_routes_enabled_idx` ON `token_routes` (`enabled`);
CREATE INDEX `token_routes_model_pattern_idx` ON `token_routes` (`model_pattern`(191));
src/server/db/generated/mysql.upgrade.sql
-1
View File
@@ -1 +0,0 @@
-- no schema changes detected for mysql
src/server/db/generated/postgres.bootstrap.sql
-57
View File
@@ -1,57 +0,0 @@
CREATE TABLE IF NOT EXISTS "sites" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL PRIMARY KEY, "name" TEXT NOT NULL, "url" TEXT NOT NULL, "platform" TEXT NOT NULL, "status" TEXT NOT NULL DEFAULT 'active', "api_key" TEXT, "created_at" TIMESTAMP DEFAULT CURRENT_TIMESTAMP, "updated_at" TIMESTAMP DEFAULT CURRENT_TIMESTAMP, "is_pinned" BOOLEAN DEFAULT false, "sort_order" INTEGER DEFAULT 0, "proxy_url" TEXT, "use_system_proxy" BOOLEAN DEFAULT false, "custom_headers" JSONB, "external_checkin_url" TEXT, "global_weight" DOUBLE PRECISION DEFAULT 1);
CREATE TABLE IF NOT EXISTS "accounts" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL PRIMARY KEY, "site_id" INTEGER NOT NULL, "username" TEXT, "access_token" TEXT NOT NULL, "api_token" TEXT, "balance" DOUBLE PRECISION DEFAULT 0, "balance_used" DOUBLE PRECISION DEFAULT 0, "quota" DOUBLE PRECISION DEFAULT 0, "unit_cost" DOUBLE PRECISION, "value_score" DOUBLE PRECISION DEFAULT 0, "status" TEXT DEFAULT 'active', "checkin_enabled" BOOLEAN DEFAULT true, "last_checkin_at" TIMESTAMP, "last_balance_refresh" TIMESTAMP, "extra_config" JSONB, "created_at" TIMESTAMP DEFAULT CURRENT_TIMESTAMP, "updated_at" TIMESTAMP DEFAULT CURRENT_TIMESTAMP, "is_pinned" BOOLEAN DEFAULT false, "sort_order" INTEGER DEFAULT 0, FOREIGN KEY ("site_id") REFERENCES "sites"("id") ON DELETE CASCADE);
CREATE TABLE IF NOT EXISTS "account_tokens" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL PRIMARY KEY, "account_id" INTEGER NOT NULL, "name" TEXT NOT NULL, "token" TEXT NOT NULL, "source" TEXT DEFAULT 'manual', "enabled" BOOLEAN DEFAULT true, "is_default" BOOLEAN DEFAULT false, "created_at" TIMESTAMP DEFAULT CURRENT_TIMESTAMP, "updated_at" TIMESTAMP DEFAULT CURRENT_TIMESTAMP, "token_group" TEXT, FOREIGN KEY ("account_id") REFERENCES "accounts"("id") ON DELETE CASCADE);
CREATE TABLE IF NOT EXISTS "checkin_logs" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL PRIMARY KEY, "account_id" INTEGER NOT NULL, "status" TEXT NOT NULL, "message" TEXT, "reward" TEXT, "created_at" TIMESTAMP DEFAULT CURRENT_TIMESTAMP, FOREIGN KEY ("account_id") REFERENCES "accounts"("id") ON DELETE CASCADE);
CREATE TABLE IF NOT EXISTS "downstream_api_keys" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL PRIMARY KEY, "name" TEXT NOT NULL, "key" TEXT NOT NULL, "description" TEXT, "enabled" BOOLEAN DEFAULT true, "expires_at" TIMESTAMP, "max_cost" DOUBLE PRECISION, "used_cost" DOUBLE PRECISION DEFAULT 0, "max_requests" INTEGER, "used_requests" INTEGER DEFAULT 0, "supported_models" JSONB, "allowed_route_ids" JSONB, "site_weight_multipliers" JSONB, "last_used_at" TIMESTAMP, "created_at" TIMESTAMP DEFAULT CURRENT_TIMESTAMP, "updated_at" TIMESTAMP DEFAULT CURRENT_TIMESTAMP);
CREATE TABLE IF NOT EXISTS "events" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL PRIMARY KEY, "type" TEXT NOT NULL, "title" TEXT NOT NULL, "message" TEXT, "level" TEXT NOT NULL DEFAULT 'info', "read" BOOLEAN DEFAULT false, "related_id" INTEGER, "related_type" TEXT, "created_at" TIMESTAMP DEFAULT CURRENT_TIMESTAMP);
CREATE TABLE IF NOT EXISTS "model_availability" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL PRIMARY KEY, "account_id" INTEGER NOT NULL, "model_name" TEXT NOT NULL, "available" BOOLEAN, "latency_ms" INTEGER, "checked_at" TIMESTAMP DEFAULT CURRENT_TIMESTAMP, FOREIGN KEY ("account_id") REFERENCES "accounts"("id") ON DELETE CASCADE);
CREATE TABLE IF NOT EXISTS "proxy_files" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL PRIMARY KEY, "public_id" TEXT NOT NULL, "owner_type" TEXT NOT NULL, "owner_id" TEXT NOT NULL, "filename" TEXT NOT NULL, "mime_type" TEXT NOT NULL, "purpose" TEXT, "byte_size" INTEGER NOT NULL, "sha256" TEXT NOT NULL, "content_base64" TEXT NOT NULL, "created_at" TIMESTAMP DEFAULT CURRENT_TIMESTAMP, "updated_at" TIMESTAMP DEFAULT CURRENT_TIMESTAMP, "deleted_at" TIMESTAMP);
CREATE TABLE IF NOT EXISTS "proxy_logs" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL PRIMARY KEY, "route_id" INTEGER, "channel_id" INTEGER, "account_id" INTEGER, "model_requested" TEXT, "model_actual" TEXT, "status" TEXT, "http_status" INTEGER, "latency_ms" INTEGER, "prompt_tokens" INTEGER, "completion_tokens" INTEGER, "total_tokens" INTEGER, "estimated_cost" DOUBLE PRECISION, "error_message" TEXT, "retry_count" INTEGER DEFAULT 0, "created_at" TIMESTAMP DEFAULT CURRENT_TIMESTAMP, "billing_details" JSONB);
CREATE TABLE IF NOT EXISTS "proxy_video_tasks" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL PRIMARY KEY, "public_id" TEXT NOT NULL, "upstream_video_id" TEXT NOT NULL, "site_url" TEXT NOT NULL, "token_value" TEXT NOT NULL, "requested_model" TEXT, "actual_model" TEXT, "channel_id" INTEGER, "account_id" INTEGER, "status_snapshot" JSONB, "upstream_response_meta" JSONB, "last_upstream_status" INTEGER, "last_polled_at" TIMESTAMP, "created_at" TIMESTAMP DEFAULT CURRENT_TIMESTAMP, "updated_at" TIMESTAMP DEFAULT CURRENT_TIMESTAMP);
CREATE TABLE IF NOT EXISTS "token_routes" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL PRIMARY KEY, "model_pattern" TEXT NOT NULL, "model_mapping" JSONB, "enabled" BOOLEAN DEFAULT true, "created_at" TIMESTAMP DEFAULT CURRENT_TIMESTAMP, "updated_at" TIMESTAMP DEFAULT CURRENT_TIMESTAMP, "display_name" TEXT, "display_icon" TEXT, "decision_snapshot" JSONB, "decision_refreshed_at" TIMESTAMP, "routing_strategy" TEXT DEFAULT 'weighted');
CREATE TABLE IF NOT EXISTS "route_channels" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL PRIMARY KEY, "route_id" INTEGER NOT NULL, "account_id" INTEGER NOT NULL, "token_id" INTEGER, "source_model" TEXT, "priority" INTEGER DEFAULT 0, "weight" INTEGER DEFAULT 10, "enabled" BOOLEAN DEFAULT true, "manual_override" BOOLEAN DEFAULT false, "success_count" INTEGER DEFAULT 0, "fail_count" INTEGER DEFAULT 0, "total_latency_ms" INTEGER DEFAULT 0, "total_cost" DOUBLE PRECISION DEFAULT 0, "last_used_at" TIMESTAMP, "last_selected_at" TIMESTAMP, "last_fail_at" TIMESTAMP, "consecutive_fail_count" INTEGER NOT NULL DEFAULT 0, "cooldown_level" INTEGER NOT NULL DEFAULT 0, "cooldown_until" TIMESTAMP, FOREIGN KEY ("account_id") REFERENCES "accounts"("id") ON DELETE CASCADE, FOREIGN KEY ("route_id") REFERENCES "token_routes"("id") ON DELETE CASCADE, FOREIGN KEY ("token_id") REFERENCES "account_tokens"("id") ON DELETE SET NULL);
CREATE TABLE IF NOT EXISTS "settings" ("key" TEXT NOT NULL PRIMARY KEY, "value" TEXT);
CREATE TABLE IF NOT EXISTS "site_disabled_models" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL PRIMARY KEY, "site_id" INTEGER NOT NULL, "model_name" TEXT NOT NULL, "created_at" TIMESTAMP DEFAULT CURRENT_TIMESTAMP, FOREIGN KEY ("site_id") REFERENCES "sites"("id") ON DELETE CASCADE);
CREATE TABLE IF NOT EXISTS "token_model_availability" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL PRIMARY KEY, "token_id" INTEGER NOT NULL, "model_name" TEXT NOT NULL, "available" BOOLEAN, "latency_ms" INTEGER, "checked_at" TIMESTAMP DEFAULT CURRENT_TIMESTAMP, FOREIGN KEY ("token_id") REFERENCES "account_tokens"("id") ON DELETE CASCADE);
CREATE UNIQUE INDEX "downstream_api_keys_key_unique" ON "downstream_api_keys" ("key");
CREATE UNIQUE INDEX "model_availability_account_model_unique" ON "model_availability" ("account_id", "model_name");
CREATE UNIQUE INDEX "proxy_files_public_id_unique" ON "proxy_files" ("public_id");
CREATE UNIQUE INDEX "proxy_video_tasks_public_id_unique" ON "proxy_video_tasks" ("public_id");
CREATE UNIQUE INDEX "site_disabled_models_site_model_unique" ON "site_disabled_models" ("site_id", "model_name");
CREATE UNIQUE INDEX "token_model_availability_token_model_unique" ON "token_model_availability" ("token_id", "model_name");
CREATE INDEX "account_tokens_account_enabled_idx" ON "account_tokens" ("account_id", "enabled");
CREATE INDEX "account_tokens_account_id_idx" ON "account_tokens" ("account_id");
CREATE INDEX "account_tokens_enabled_idx" ON "account_tokens" ("enabled");
CREATE INDEX "accounts_site_id_idx" ON "accounts" ("site_id");
CREATE INDEX "accounts_site_status_idx" ON "accounts" ("site_id", "status");
CREATE INDEX "accounts_status_idx" ON "accounts" ("status");
CREATE INDEX "checkin_logs_account_created_at_idx" ON "checkin_logs" ("account_id", "created_at");
CREATE INDEX "checkin_logs_created_at_idx" ON "checkin_logs" ("created_at");
CREATE INDEX "checkin_logs_status_idx" ON "checkin_logs" ("status");
CREATE INDEX "downstream_api_keys_enabled_idx" ON "downstream_api_keys" ("enabled");
CREATE INDEX "downstream_api_keys_expires_at_idx" ON "downstream_api_keys" ("expires_at");
CREATE INDEX "downstream_api_keys_name_idx" ON "downstream_api_keys" ("name");
CREATE INDEX "events_created_at_idx" ON "events" ("created_at");
CREATE INDEX "events_read_created_at_idx" ON "events" ("read", "created_at");
CREATE INDEX "events_type_created_at_idx" ON "events" ("type", "created_at");
CREATE INDEX "model_availability_account_available_idx" ON "model_availability" ("account_id", "available");
CREATE INDEX "model_availability_model_name_idx" ON "model_availability" ("model_name");
CREATE INDEX "proxy_files_owner_lookup_idx" ON "proxy_files" ("owner_type", "owner_id", "deleted_at");
CREATE INDEX "proxy_logs_account_created_at_idx" ON "proxy_logs" ("account_id", "created_at");
CREATE INDEX "proxy_logs_created_at_idx" ON "proxy_logs" ("created_at");
CREATE INDEX "proxy_logs_model_actual_created_at_idx" ON "proxy_logs" ("model_actual", "created_at");
CREATE INDEX "proxy_logs_status_created_at_idx" ON "proxy_logs" ("status", "created_at");
CREATE INDEX "proxy_video_tasks_created_at_idx" ON "proxy_video_tasks" ("created_at");
CREATE INDEX "proxy_video_tasks_upstream_video_id_idx" ON "proxy_video_tasks" ("upstream_video_id");
CREATE INDEX "route_channels_account_id_idx" ON "route_channels" ("account_id");
CREATE INDEX "route_channels_route_enabled_idx" ON "route_channels" ("route_id", "enabled");
CREATE INDEX "route_channels_route_id_idx" ON "route_channels" ("route_id");
CREATE INDEX "route_channels_route_token_idx" ON "route_channels" ("route_id", "token_id");
CREATE INDEX "route_channels_token_id_idx" ON "route_channels" ("token_id");
CREATE INDEX "site_disabled_models_site_id_idx" ON "site_disabled_models" ("site_id");
CREATE INDEX "sites_status_idx" ON "sites" ("status");
CREATE INDEX "token_model_availability_available_idx" ON "token_model_availability" ("available");
CREATE INDEX "token_model_availability_model_name_idx" ON "token_model_availability" ("model_name");
CREATE INDEX "token_model_availability_token_available_idx" ON "token_model_availability" ("token_id", "available");
CREATE INDEX "token_routes_enabled_idx" ON "token_routes" ("enabled");
CREATE INDEX "token_routes_model_pattern_idx" ON "token_routes" ("model_pattern");
src/server/db/generated/postgres.upgrade.sql
-1
View File
@@ -1 +0,0 @@
-- no schema changes detected for postgres
src/server/db/generated/schemaContract.json
-1575
View File
File diff suppressed because it is too large Load Diff
src/server/db/index.default-path.test.ts
-44
View File
@@ -1,44 +0,0 @@
import { afterEach, describe, expect, it, vi } from 'vitest';
import { tmpdir } from 'node:os';
import { resolve } from 'node:path';
type DbModule = typeof import('./index.js');
describe('sqlite default path resolution', () => {
let dbModule: DbModule | null = null;
afterEach(async () => {
if (dbModule) {
await dbModule.closeDbConnections();
dbModule = null;
}
delete process.env.DATA_DIR;
delete process.env.DB_URL;
vi.resetModules();
});
it('uses an isolated temp sqlite path under vitest when no db env is configured', async () => {
delete process.env.DATA_DIR;
delete process.env.DB_URL;
vi.resetModules();
dbModule = await import('./index.js');
const sqlitePath = dbModule.__dbProxyTestUtils.resolveSqlitePath();
const sharedRepoPath = resolve('./data/hub.db');
expect(sqlitePath).not.toBe(sharedRepoPath);
expect(sqlitePath).toContain(tmpdir());
expect(sqlitePath).toContain('metapi-vitest');
});
it('still honors explicit DATA_DIR when provided', async () => {
process.env.DATA_DIR = resolve(tmpdir(), 'metapi-explicit-data-dir');
delete process.env.DB_URL;
vi.resetModules();
dbModule = await import('./index.js');
const sqlitePath = dbModule.__dbProxyTestUtils.resolveSqlitePath();
expect(sqlitePath).toBe(resolve(process.env.DATA_DIR, 'hub.db'));
});
});
src/server/db/index.ts
+61 -101
View File
@@ -9,13 +9,10 @@ import { ensureSiteSchemaCompatibility, type SiteSchemaInspector } from './siteS
import { ensureRouteGroupingSchemaCompatibility } from './routeGroupingSchemaCompatibility.js'; import { ensureRouteGroupingSchemaCompatibility } from './routeGroupingSchemaCompatibility.js';
import { ensureProxyFileSchemaCompatibility } from './proxyFileSchemaCompatibility.js'; import { ensureProxyFileSchemaCompatibility } from './proxyFileSchemaCompatibility.js';
import { ensureSharedIndexSchemaCompatibility } from './sharedIndexSchemaCompatibility.js'; import { ensureSharedIndexSchemaCompatibility } from './sharedIndexSchemaCompatibility.js';
import { executeLegacyCompat, executeLegacyCompatSync } from './legacySchemaCompat.js';
import { config } from '../config.js'; import { config } from '../config.js';
import { ensureRuntimeDatabaseReady } from '../runtimeDatabaseBootstrap.js'; import { ensureRuntimeDatabaseReady } from '../runtimeDatabaseBootstrap.js';
import { mkdirSync } from 'fs'; import { mkdirSync } from 'fs';
import { tmpdir } from 'os';
import { dirname, resolve } from 'path'; import { dirname, resolve } from 'path';
import { threadId } from 'worker_threads';
export type RuntimeDbDialect = 'sqlite' | 'mysql' | 'postgres'; export type RuntimeDbDialect = 'sqlite' | 'mysql' | 'postgres';
type SqlMethod = 'all' | 'get' | 'run' | 'values' | 'execute'; type SqlMethod = 'all' | 'get' | 'run' | 'values' | 'execute';
@@ -45,13 +42,7 @@ let proxyLogBillingDetailsColumnAvailable: boolean | null = null;
function resolveSqlitePath(): string { function resolveSqlitePath(): string {
const raw = (config.dbUrl || '').trim(); const raw = (config.dbUrl || '').trim();
if (!raw) { if (!raw) return resolve(`${config.dataDir}/hub.db`);
const isolatedVitestPath = resolveVitestSqlitePath();
if (isolatedVitestPath) {
return isolatedVitestPath;
}
return resolve(`${config.dataDir}/hub.db`);
}
if (raw === ':memory:') return raw; if (raw === ':memory:') return raw;
if (raw.startsWith('file://')) { if (raw.startsWith('file://')) {
const parsed = new URL(raw); const parsed = new URL(raw);
@@ -63,35 +54,6 @@ function resolveSqlitePath(): string {
return resolve(raw); return resolve(raw);
} }
function isVitestRuntime(): boolean {
if ((process.env.VITEST_POOL_ID || '').trim()) {
return true;
}
if ((process.env.VITEST_WORKER_ID || '').trim()) {
return true;
}
const runtimeArgs = [...process.argv, ...process.execArgv]
.map((value) => String(value || '').toLowerCase());
return runtimeArgs.some((value) => value.includes('vitest'));
}
function resolveVitestSqlitePath(): string | null {
if (!isVitestRuntime()) {
return null;
}
if ((process.env.DB_URL || '').trim()) {
return null;
}
if ((process.env.DATA_DIR || '').trim()) {
return null;
}
const workerTag = process.env.VITEST_POOL_ID
|| process.env.VITEST_WORKER_ID
|| `${process.pid}-${threadId}`;
return resolve(tmpdir(), `metapi-vitest-${workerTag}`, 'hub.db');
}
function requireSqliteConnection(): Database.Database { function requireSqliteConnection(): Database.Database {
if (!sqliteConnection) { if (!sqliteConnection) {
throw new Error('SQLite connection is not initialized'); throw new Error('SQLite connection is not initialized');
@@ -112,19 +74,13 @@ function tableColumnExists(table: string, column: string): boolean {
return rows.some((row) => row.name === column); return rows.some((row) => row.name === column);
} }
function execSqliteStatement(sqlText: string): void {
requireSqliteConnection().exec(sqlText);
}
function execSqliteLegacyCompat(sqlText: string): void {
executeLegacyCompatSync(execSqliteStatement, sqlText);
}
function ensureTokenManagementSchema() { function ensureTokenManagementSchema() {
const sqlite = requireSqliteConnection();
if (!tableExists('accounts') || !tableExists('route_channels')) { if (!tableExists('accounts') || !tableExists('route_channels')) {
return; return;
} }
execSqliteLegacyCompat(`
sqlite.exec(`
CREATE TABLE IF NOT EXISTS account_tokens ( CREATE TABLE IF NOT EXISTS account_tokens (
id integer PRIMARY KEY AUTOINCREMENT NOT NULL, id integer PRIMARY KEY AUTOINCREMENT NOT NULL,
account_id integer NOT NULL, account_id integer NOT NULL,
@@ -139,15 +95,16 @@ function ensureTokenManagementSchema() {
FOREIGN KEY (account_id) REFERENCES accounts(id) ON DELETE cascade FOREIGN KEY (account_id) REFERENCES accounts(id) ON DELETE cascade
); );
`); `);
if (!tableColumnExists('route_channels', 'token_id')) { if (!tableColumnExists('route_channels', 'token_id')) {
execSqliteLegacyCompat('ALTER TABLE route_channels ADD COLUMN token_id integer;'); sqlite.exec('ALTER TABLE route_channels ADD COLUMN token_id integer;');
} }
if (!tableColumnExists('account_tokens', 'token_group')) { if (!tableColumnExists('account_tokens', 'token_group')) {
execSqliteLegacyCompat('ALTER TABLE account_tokens ADD COLUMN token_group text;'); sqlite.exec('ALTER TABLE account_tokens ADD COLUMN token_group text;');
} }
execSqliteStatement(` sqlite.exec(`
INSERT INTO account_tokens (account_id, name, token, source, enabled, is_default, created_at, updated_at) INSERT INTO account_tokens (account_id, name, token, source, enabled, is_default, created_at, updated_at)
SELECT SELECT
a.id, a.id,
@@ -169,7 +126,7 @@ function ensureTokenManagementSchema() {
); );
`); `);
execSqliteLegacyCompat(` sqlite.exec(`
CREATE TABLE IF NOT EXISTS token_model_availability ( CREATE TABLE IF NOT EXISTS token_model_availability (
id integer PRIMARY KEY AUTOINCREMENT NOT NULL, id integer PRIMARY KEY AUTOINCREMENT NOT NULL,
token_id integer NOT NULL, token_id integer NOT NULL,
@@ -181,14 +138,15 @@ function ensureTokenManagementSchema() {
); );
`); `);
execSqliteLegacyCompat(` sqlite.exec(`
CREATE UNIQUE INDEX IF NOT EXISTS token_model_availability_token_model_unique CREATE UNIQUE INDEX IF NOT EXISTS token_model_availability_token_model_unique
ON token_model_availability(token_id, model_name); ON token_model_availability(token_id, model_name);
`); `);
} }
function ensureProxyVideoTaskSchema() { function ensureProxyVideoTaskSchema() {
execSqliteLegacyCompat(` const sqlite = requireSqliteConnection();
sqlite.exec(`
CREATE TABLE IF NOT EXISTS proxy_video_tasks ( CREATE TABLE IF NOT EXISTS proxy_video_tasks (
id integer PRIMARY KEY AUTOINCREMENT NOT NULL, id integer PRIMARY KEY AUTOINCREMENT NOT NULL,
public_id text NOT NULL, public_id text NOT NULL,
@@ -208,29 +166,30 @@ function ensureProxyVideoTaskSchema() {
); );
`); `);
if (!tableColumnExists('proxy_video_tasks', 'status_snapshot')) { if (!tableColumnExists('proxy_video_tasks', 'status_snapshot')) {
execSqliteLegacyCompat('ALTER TABLE proxy_video_tasks ADD COLUMN status_snapshot text;'); sqlite.exec('ALTER TABLE proxy_video_tasks ADD COLUMN status_snapshot text;');
} }
if (!tableColumnExists('proxy_video_tasks', 'upstream_response_meta')) { if (!tableColumnExists('proxy_video_tasks', 'upstream_response_meta')) {
execSqliteLegacyCompat('ALTER TABLE proxy_video_tasks ADD COLUMN upstream_response_meta text;'); sqlite.exec('ALTER TABLE proxy_video_tasks ADD COLUMN upstream_response_meta text;');
} }
if (!tableColumnExists('proxy_video_tasks', 'last_upstream_status')) { if (!tableColumnExists('proxy_video_tasks', 'last_upstream_status')) {
execSqliteLegacyCompat('ALTER TABLE proxy_video_tasks ADD COLUMN last_upstream_status integer;'); sqlite.exec('ALTER TABLE proxy_video_tasks ADD COLUMN last_upstream_status integer;');
} }
if (!tableColumnExists('proxy_video_tasks', 'last_polled_at')) { if (!tableColumnExists('proxy_video_tasks', 'last_polled_at')) {
execSqliteLegacyCompat('ALTER TABLE proxy_video_tasks ADD COLUMN last_polled_at text;'); sqlite.exec('ALTER TABLE proxy_video_tasks ADD COLUMN last_polled_at text;');
} }
execSqliteLegacyCompat(` sqlite.exec(`
CREATE UNIQUE INDEX IF NOT EXISTS proxy_video_tasks_public_id_unique CREATE UNIQUE INDEX IF NOT EXISTS proxy_video_tasks_public_id_unique
ON proxy_video_tasks(public_id); ON proxy_video_tasks(public_id);
`); `);
execSqliteLegacyCompat(` sqlite.exec(`
CREATE INDEX IF NOT EXISTS proxy_video_tasks_upstream_video_id_idx CREATE INDEX IF NOT EXISTS proxy_video_tasks_upstream_video_id_idx
ON proxy_video_tasks(upstream_video_id); ON proxy_video_tasks(upstream_video_id);
`); `);
} }
function ensureProxyFileSchema() { function ensureProxyFileSchema() {
execSqliteLegacyCompat(` const sqlite = requireSqliteConnection();
sqlite.exec(`
CREATE TABLE IF NOT EXISTS proxy_files ( CREATE TABLE IF NOT EXISTS proxy_files (
id integer PRIMARY KEY AUTOINCREMENT NOT NULL, id integer PRIMARY KEY AUTOINCREMENT NOT NULL,
public_id text NOT NULL, public_id text NOT NULL,
@@ -247,26 +206,27 @@ function ensureProxyFileSchema() {
deleted_at text deleted_at text
); );
`); `);
execSqliteLegacyCompat(` sqlite.exec(`
CREATE UNIQUE INDEX IF NOT EXISTS proxy_files_public_id_unique CREATE UNIQUE INDEX IF NOT EXISTS proxy_files_public_id_unique
ON proxy_files(public_id); ON proxy_files(public_id);
`); `);
execSqliteLegacyCompat(` sqlite.exec(`
CREATE INDEX IF NOT EXISTS proxy_files_owner_lookup_idx CREATE INDEX IF NOT EXISTS proxy_files_owner_lookup_idx
ON proxy_files(owner_type, owner_id, deleted_at); ON proxy_files(owner_type, owner_id, deleted_at);
`); `);
} }
function ensureSiteStatusSchema() { function ensureSiteStatusSchema() {
const sqlite = requireSqliteConnection();
if (!tableExists('sites')) { if (!tableExists('sites')) {
return; return;
} }
if (!tableColumnExists('sites', 'status')) { if (!tableColumnExists('sites', 'status')) {
execSqliteLegacyCompat(`ALTER TABLE sites ADD COLUMN status text DEFAULT 'active';`); sqlite.exec(`ALTER TABLE sites ADD COLUMN status text DEFAULT 'active';`);
} }
execSqliteStatement(` sqlite.exec(`
UPDATE sites UPDATE sites
SET status = lower(trim(status)) SET status = lower(trim(status))
WHERE status IS NOT NULL WHERE status IS NOT NULL
@@ -274,7 +234,7 @@ function ensureSiteStatusSchema() {
AND status != lower(trim(status)); AND status != lower(trim(status));
`); `);
execSqliteStatement(` sqlite.exec(`
UPDATE sites UPDATE sites
SET status = 'active' SET status = 'active'
WHERE status IS NULL WHERE status IS NULL
@@ -284,25 +244,27 @@ function ensureSiteStatusSchema() {
} }
function ensureSiteProxySchema() { function ensureSiteProxySchema() {
const sqlite = requireSqliteConnection();
if (!tableExists('sites')) { if (!tableExists('sites')) {
return; return;
} }
if (!tableColumnExists('sites', 'proxy_url')) { if (!tableColumnExists('sites', 'proxy_url')) {
execSqliteLegacyCompat(`ALTER TABLE sites ADD COLUMN proxy_url text;`); sqlite.exec(`ALTER TABLE sites ADD COLUMN proxy_url text;`);
} }
} }
function ensureSiteUseSystemProxySchema() { function ensureSiteUseSystemProxySchema() {
const sqlite = requireSqliteConnection();
if (!tableExists('sites')) { if (!tableExists('sites')) {
return; return;
} }
if (!tableColumnExists('sites', 'use_system_proxy')) { if (!tableColumnExists('sites', 'use_system_proxy')) {
execSqliteLegacyCompat(`ALTER TABLE sites ADD COLUMN use_system_proxy integer DEFAULT 0;`); sqlite.exec(`ALTER TABLE sites ADD COLUMN use_system_proxy integer DEFAULT 0;`);
} }
execSqliteLegacyCompat(` sqlite.exec(`
UPDATE sites UPDATE sites
SET use_system_proxy = 0 SET use_system_proxy = 0
WHERE use_system_proxy IS NULL; WHERE use_system_proxy IS NULL;
@@ -310,35 +272,38 @@ function ensureSiteUseSystemProxySchema() {
} }
function ensureSiteCustomHeadersSchema() { function ensureSiteCustomHeadersSchema() {
const sqlite = requireSqliteConnection();
if (!tableExists('sites')) { if (!tableExists('sites')) {
return; return;
} }
if (!tableColumnExists('sites', 'custom_headers')) { if (!tableColumnExists('sites', 'custom_headers')) {
execSqliteLegacyCompat(`ALTER TABLE sites ADD COLUMN custom_headers text;`); sqlite.exec(`ALTER TABLE sites ADD COLUMN custom_headers text;`);
} }
} }
function ensureSiteExternalCheckinUrlSchema() { function ensureSiteExternalCheckinUrlSchema() {
const sqlite = requireSqliteConnection();
if (!tableExists('sites')) { if (!tableExists('sites')) {
return; return;
} }
if (!tableColumnExists('sites', 'external_checkin_url')) { if (!tableColumnExists('sites', 'external_checkin_url')) {
execSqliteLegacyCompat(`ALTER TABLE sites ADD COLUMN external_checkin_url text;`); sqlite.exec(`ALTER TABLE sites ADD COLUMN external_checkin_url text;`);
} }
} }
function ensureSiteGlobalWeightSchema() { function ensureSiteGlobalWeightSchema() {
const sqlite = requireSqliteConnection();
if (!tableExists('sites')) { if (!tableExists('sites')) {
return; return;
} }
if (!tableColumnExists('sites', 'global_weight')) { if (!tableColumnExists('sites', 'global_weight')) {
execSqliteLegacyCompat(`ALTER TABLE sites ADD COLUMN global_weight real DEFAULT 1;`); sqlite.exec(`ALTER TABLE sites ADD COLUMN global_weight real DEFAULT 1;`);
} }
execSqliteLegacyCompat(` sqlite.exec(`
UPDATE sites UPDATE sites
SET global_weight = 1 SET global_weight = 1
WHERE global_weight IS NULL WHERE global_weight IS NULL
@@ -359,7 +324,7 @@ function createSqliteSchemaInspector(): RuntimeSchemaInspector {
tableExists: async (table) => tableExists(table), tableExists: async (table) => tableExists(table),
columnExists: async (table, column) => tableColumnExists(table, column), columnExists: async (table, column) => tableColumnExists(table, column),
execute: async (sqlText) => { execute: async (sqlText) => {
executeLegacyCompatSync(execSqliteStatement, sqlText); requireSqliteConnection().exec(sqlText);
}, },
}; };
} }
@@ -383,7 +348,7 @@ function createMysqlSchemaInspector(): RuntimeSchemaInspector | null {
return Array.isArray(rows) && rows.length > 0; return Array.isArray(rows) && rows.length > 0;
}, },
execute: async (sqlText) => { execute: async (sqlText) => {
await executeLegacyCompat((statement) => mysqlPool!.query(statement).then(() => undefined), sqlText); await mysqlPool!.query(sqlText);
}, },
}; };
} }
@@ -407,7 +372,7 @@ function createPostgresSchemaInspector(): RuntimeSchemaInspector | null {
return Number(result.rowCount || 0) > 0; return Number(result.rowCount || 0) > 0;
}, },
execute: async (sqlText) => { execute: async (sqlText) => {
await executeLegacyCompat((statement) => pgPool!.query(statement).then(() => undefined), sqlText); await pgPool!.query(sqlText);
}, },
}; };
} }
@@ -447,49 +412,51 @@ export async function ensureSharedIndexCompatibility(): Promise<void> {
} }
function ensureRouteGroupingSchema() { function ensureRouteGroupingSchema() {
const sqlite = requireSqliteConnection();
if (!tableExists('token_routes') || !tableExists('route_channels')) { if (!tableExists('token_routes') || !tableExists('route_channels')) {
return; return;
} }
if (!tableColumnExists('token_routes', 'display_name')) { if (!tableColumnExists('token_routes', 'display_name')) {
execSqliteLegacyCompat(`ALTER TABLE token_routes ADD COLUMN display_name text;`); sqlite.exec(`ALTER TABLE token_routes ADD COLUMN display_name text;`);
} }
if (!tableColumnExists('token_routes', 'display_icon')) { if (!tableColumnExists('token_routes', 'display_icon')) {
execSqliteLegacyCompat(`ALTER TABLE token_routes ADD COLUMN display_icon text;`); sqlite.exec(`ALTER TABLE token_routes ADD COLUMN display_icon text;`);
} }
if (!tableColumnExists('token_routes', 'decision_snapshot')) { if (!tableColumnExists('token_routes', 'decision_snapshot')) {
execSqliteLegacyCompat(`ALTER TABLE token_routes ADD COLUMN decision_snapshot text;`); sqlite.exec(`ALTER TABLE token_routes ADD COLUMN decision_snapshot text;`);
} }
if (!tableColumnExists('token_routes', 'decision_refreshed_at')) { if (!tableColumnExists('token_routes', 'decision_refreshed_at')) {
execSqliteLegacyCompat(`ALTER TABLE token_routes ADD COLUMN decision_refreshed_at text;`); sqlite.exec(`ALTER TABLE token_routes ADD COLUMN decision_refreshed_at text;`);
} }
if (!tableColumnExists('token_routes', 'routing_strategy')) { if (!tableColumnExists('token_routes', 'routing_strategy')) {
execSqliteLegacyCompat(`ALTER TABLE token_routes ADD COLUMN routing_strategy text DEFAULT 'weighted';`); sqlite.exec(`ALTER TABLE token_routes ADD COLUMN routing_strategy text DEFAULT 'weighted';`);
} }
if (!tableColumnExists('route_channels', 'source_model')) { if (!tableColumnExists('route_channels', 'source_model')) {
execSqliteLegacyCompat(`ALTER TABLE route_channels ADD COLUMN source_model text;`); sqlite.exec(`ALTER TABLE route_channels ADD COLUMN source_model text;`);
} }
if (!tableColumnExists('route_channels', 'last_selected_at')) { if (!tableColumnExists('route_channels', 'last_selected_at')) {
execSqliteLegacyCompat(`ALTER TABLE route_channels ADD COLUMN last_selected_at text;`); sqlite.exec(`ALTER TABLE route_channels ADD COLUMN last_selected_at text;`);
} }
if (!tableColumnExists('route_channels', 'consecutive_fail_count')) { if (!tableColumnExists('route_channels', 'consecutive_fail_count')) {
execSqliteLegacyCompat(`ALTER TABLE route_channels ADD COLUMN consecutive_fail_count integer NOT NULL DEFAULT 0;`); sqlite.exec(`ALTER TABLE route_channels ADD COLUMN consecutive_fail_count integer NOT NULL DEFAULT 0;`);
} }
if (!tableColumnExists('route_channels', 'cooldown_level')) { if (!tableColumnExists('route_channels', 'cooldown_level')) {
execSqliteLegacyCompat(`ALTER TABLE route_channels ADD COLUMN cooldown_level integer NOT NULL DEFAULT 0;`); sqlite.exec(`ALTER TABLE route_channels ADD COLUMN cooldown_level integer NOT NULL DEFAULT 0;`);
} }
} }
function ensureDownstreamApiKeySchema() { function ensureDownstreamApiKeySchema() {
execSqliteLegacyCompat(` const sqlite = requireSqliteConnection();
sqlite.exec(`
CREATE TABLE IF NOT EXISTS downstream_api_keys ( CREATE TABLE IF NOT EXISTS downstream_api_keys (
id integer PRIMARY KEY AUTOINCREMENT NOT NULL, id integer PRIMARY KEY AUTOINCREMENT NOT NULL,
name text NOT NULL, name text NOT NULL,
@@ -510,31 +477,32 @@ function ensureDownstreamApiKeySchema() {
); );
`); `);
execSqliteLegacyCompat(` sqlite.exec(`
CREATE UNIQUE INDEX IF NOT EXISTS downstream_api_keys_key_unique CREATE UNIQUE INDEX IF NOT EXISTS downstream_api_keys_key_unique
ON downstream_api_keys(key); ON downstream_api_keys(key);
`); `);
execSqliteLegacyCompat(` sqlite.exec(`
CREATE INDEX IF NOT EXISTS downstream_api_keys_name_idx CREATE INDEX IF NOT EXISTS downstream_api_keys_name_idx
ON downstream_api_keys(name); ON downstream_api_keys(name);
`); `);
execSqliteLegacyCompat(` sqlite.exec(`
CREATE INDEX IF NOT EXISTS downstream_api_keys_enabled_idx CREATE INDEX IF NOT EXISTS downstream_api_keys_enabled_idx
ON downstream_api_keys(enabled); ON downstream_api_keys(enabled);
`); `);
execSqliteLegacyCompat(` sqlite.exec(`
CREATE INDEX IF NOT EXISTS downstream_api_keys_expires_at_idx CREATE INDEX IF NOT EXISTS downstream_api_keys_expires_at_idx
ON downstream_api_keys(expires_at); ON downstream_api_keys(expires_at);
`); `);
} }
function ensureProxyLogBillingDetailsSchema() { function ensureProxyLogBillingDetailsSchema() {
const sqlite = requireSqliteConnection();
if (!tableExists('proxy_logs')) { if (!tableExists('proxy_logs')) {
return; return;
} }
if (!tableColumnExists('proxy_logs', 'billing_details')) { if (!tableColumnExists('proxy_logs', 'billing_details')) {
execSqliteLegacyCompat('ALTER TABLE proxy_logs ADD COLUMN billing_details text;'); sqlite.exec('ALTER TABLE proxy_logs ADD COLUMN billing_details text;');
} }
proxyLogBillingDetailsColumnAvailable = true; proxyLogBillingDetailsColumnAvailable = true;
@@ -594,16 +562,10 @@ export async function ensureProxyLogBillingDetailsColumn(): Promise<boolean> {
try { try {
if (runtimeDbDialect === 'mysql') { if (runtimeDbDialect === 'mysql') {
if (!mysqlPool) return false; if (!mysqlPool) return false;
await executeLegacyCompat( await mysqlPool.query('ALTER TABLE `proxy_logs` ADD COLUMN `billing_details` TEXT NULL');
(statement) => mysqlPool!.query(statement).then(() => undefined),
'ALTER TABLE `proxy_logs` ADD COLUMN `billing_details` TEXT NULL',
);
} else { } else {
if (!pgPool) return false; if (!pgPool) return false;
await executeLegacyCompat( await pgPool.query('ALTER TABLE "proxy_logs" ADD COLUMN "billing_details" TEXT');
(statement) => pgPool!.query(statement).then(() => undefined),
'ALTER TABLE "proxy_logs" ADD COLUMN "billing_details" TEXT',
);
} }
proxyLogBillingDetailsColumnAvailable = true; proxyLogBillingDetailsColumnAvailable = true;
return true; return true;
@@ -1031,6 +993,4 @@ export async function switchRuntimeDatabase(nextDialect: RuntimeDbDialect, nextD
export const __dbProxyTestUtils = { export const __dbProxyTestUtils = {
wrapQueryLike, wrapQueryLike,
shouldWrapObject, shouldWrapObject,
resolveSqlitePath,
resolveVitestSqlitePath,
}; };
src/server/db/legacySchemaCompat.test.ts
-11
View File
@@ -1,11 +0,0 @@
import { describe, expect, it } from 'vitest';
import { classifyLegacyCompatMutation } from './legacySchemaCompat.js';
describe('legacy schema compat boundary', () => {
it('allows only explicitly registered legacy upgrade shims', () => {
expect(classifyLegacyCompatMutation('ALTER TABLE proxy_logs ADD COLUMN billing_details text;')).toBe('legacy');
expect(classifyLegacyCompatMutation('UPDATE "sites" SET "use_system_proxy" = FALSE WHERE "use_system_proxy" IS NULL')).toBe('legacy');
expect(classifyLegacyCompatMutation('ALTER TABLE sites ADD COLUMN brand_new_column text;')).toBe('forbidden');
expect(classifyLegacyCompatMutation('UPDATE "sites" SET "brand_new_column" = 1')).toBe('forbidden');
});
});
src/server/db/legacySchemaCompat.ts
-162
View File
@@ -1,162 +0,0 @@
import {
ensureAccountTokenSchemaCompatibility,
type AccountTokenSchemaInspector,
} from './accountTokenSchemaCompatibility.js';
import {
ensureProxyFileSchemaCompatibility,
type ProxyFileSchemaInspector,
} from './proxyFileSchemaCompatibility.js';
import {
ensureRouteGroupingSchemaCompatibility,
type RouteGroupingSchemaInspector,
} from './routeGroupingSchemaCompatibility.js';
import {
ensureSharedIndexSchemaCompatibility,
SHARED_INDEX_COMPATIBILITY_SPECS,
type SharedIndexSchemaInspector,
} from './sharedIndexSchemaCompatibility.js';
import {
ensureSiteSchemaCompatibility,
type SiteSchemaInspector,
} from './siteSchemaCompatibility.js';
export type LegacySchemaCompatClassification = 'legacy' | 'forbidden';
export interface LegacySchemaCompatInspector extends
SiteSchemaInspector,
RouteGroupingSchemaInspector,
ProxyFileSchemaInspector,
AccountTokenSchemaInspector,
SharedIndexSchemaInspector {}
const LEGACY_COMPAT_TABLES = new Set([
'account_tokens',
'token_model_availability',
'proxy_video_tasks',
'proxy_files',
'downstream_api_keys',
'site_disabled_models',
]);
const LEGACY_COMPAT_COLUMNS = new Set([
'sites.status',
'sites.proxy_url',
'sites.use_system_proxy',
'sites.custom_headers',
'sites.external_checkin_url',
'sites.global_weight',
'account_tokens.token_group',
'token_routes.display_name',
'token_routes.display_icon',
'token_routes.decision_snapshot',
'token_routes.decision_refreshed_at',
'token_routes.routing_strategy',
'route_channels.token_id',
'route_channels.source_model',
'route_channels.last_selected_at',
'route_channels.consecutive_fail_count',
'route_channels.cooldown_level',
'proxy_video_tasks.status_snapshot',
'proxy_video_tasks.upstream_response_meta',
'proxy_video_tasks.last_upstream_status',
'proxy_video_tasks.last_polled_at',
'proxy_logs.billing_details',
]);
const LEGACY_COMPAT_INDEXES = new Set([
'site_disabled_models_site_model_unique',
'site_disabled_models_site_id_idx',
'token_model_availability_token_model_unique',
'proxy_video_tasks_public_id_unique',
'proxy_video_tasks_upstream_video_id_idx',
'proxy_files_public_id_unique',
'proxy_files_owner_lookup_idx',
'downstream_api_keys_key_unique',
'downstream_api_keys_name_idx',
'downstream_api_keys_enabled_idx',
'downstream_api_keys_expires_at_idx',
...SHARED_INDEX_COMPATIBILITY_SPECS.map((spec) => spec.indexName),
]);
function normalizeSqlText(sqlText: string): string {
return sqlText.trim().replace(/\s+/g, ' ').toLowerCase();
}
const LEGACY_COMPAT_UPDATES = new Set([
'UPDATE sites SET use_system_proxy = 0 WHERE use_system_proxy IS NULL;',
'UPDATE `sites` SET `use_system_proxy` = FALSE WHERE `use_system_proxy` IS NULL',
'UPDATE "sites" SET "use_system_proxy" = FALSE WHERE "use_system_proxy" IS NULL',
'UPDATE sites SET global_weight = 1 WHERE global_weight IS NULL OR global_weight <= 0;',
'UPDATE `sites` SET `global_weight` = 1 WHERE `global_weight` IS NULL OR `global_weight` <= 0',
'UPDATE "sites" SET "global_weight" = 1 WHERE "global_weight" IS NULL OR "global_weight" <= 0',
].map((sqlText) => normalizeSqlText(sqlText)));
export function classifyLegacyCompatMutation(sqlText: string): LegacySchemaCompatClassification {
const normalized = normalizeSqlText(sqlText);
if (LEGACY_COMPAT_UPDATES.has(normalized)) {
return 'legacy';
}
const createTableMatch = normalized.match(/^create table if not exists [`"]?([a-z0-9_]+)[`"]?/i);
if (createTableMatch) {
return LEGACY_COMPAT_TABLES.has(createTableMatch[1]) ? 'legacy' : 'forbidden';
}
const alterTableMatch = normalized.match(
/^alter table [`"]?([a-z0-9_]+)[`"]? add column [`"]?([a-z0-9_]+)[`"]?/i,
);
if (alterTableMatch) {
const [, tableName, columnName] = alterTableMatch;
return LEGACY_COMPAT_COLUMNS.has(`${tableName}.${columnName}`) ? 'legacy' : 'forbidden';
}
const createIndexMatch = normalized.match(
/^create (?:unique )?index(?: if not exists)? [`"]?([a-z0-9_]+)[`"]?/i,
);
if (createIndexMatch) {
return LEGACY_COMPAT_INDEXES.has(createIndexMatch[1]) ? 'legacy' : 'forbidden';
}
return 'forbidden';
}
function assertLegacyCompatMutation(sqlText: string): void {
if (classifyLegacyCompatMutation(sqlText) === 'forbidden') {
throw new Error(`Forbidden legacy schema mutation: ${sqlText}`);
}
}
export async function executeLegacyCompat(
execute: (sqlText: string) => Promise<void>,
sqlText: string,
): Promise<void> {
assertLegacyCompatMutation(sqlText);
await execute(sqlText);
}
export function executeLegacyCompatSync(
execute: (sqlText: string) => void,
sqlText: string,
): void {
assertLegacyCompatMutation(sqlText);
execute(sqlText);
}
function wrapLegacyCompatInspector(inspector: LegacySchemaCompatInspector): LegacySchemaCompatInspector {
return {
...inspector,
execute: async (sqlText: string) => {
await executeLegacyCompat((statement) => inspector.execute(statement), sqlText);
},
};
}
export async function ensureLegacySchemaCompatibility(inspector: LegacySchemaCompatInspector): Promise<void> {
const wrappedInspector = wrapLegacyCompatInspector(inspector);
await ensureSiteSchemaCompatibility(wrappedInspector);
await ensureRouteGroupingSchemaCompatibility(wrappedInspector);
await ensureProxyFileSchemaCompatibility(wrappedInspector);
await ensureAccountTokenSchemaCompatibility(wrappedInspector);
await ensureSharedIndexSchemaCompatibility(wrappedInspector);
}
src/server/db/migrate.test.ts
+1 -75
View File
@@ -208,82 +208,12 @@ describe('sqlite migrate bootstrap', () => {
sqlite.close(); sqlite.close();
}); });
it('recovers duplicate-column errors inside multi-statement migrations by replaying the full migration', async () => {
const dataDir = mkdtempSync(join(tmpdir(), 'metapi-migrate-recover-multi-'));
process.env.DATA_DIR = dataDir;
vi.resetModules();
const migrateModule = await import('./migrate.js');
const { __migrateTestUtils } = migrateModule;
const sqlite = new Database(':memory:');
sqlite.exec(`
CREATE TABLE account_tokens (
id integer PRIMARY KEY AUTOINCREMENT NOT NULL,
token_group text
);
`);
const tempMigrationsDir = mkdtempSync(join(tmpdir(), 'metapi-migration-files-multi-'));
mkdirSync(join(tempMigrationsDir, 'meta'), { recursive: true });
writeFileSync(
join(tempMigrationsDir, 'meta', '_journal.json'),
JSON.stringify({
entries: [
{
tag: '0008_sqlite_schema_backfill',
when: 1772600000000,
},
],
}),
);
writeFileSync(
join(tempMigrationsDir, '0008_sqlite_schema_backfill.sql'),
[
'CREATE TABLE IF NOT EXISTS `downstream_api_keys` (`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL);',
'ALTER TABLE `account_tokens` ADD `token_group` text;',
].join('\n--> statement-breakpoint\n'),
);
const duplicateColumnError = new Error(
"DrizzleError: Failed to run the query 'ALTER TABLE `account_tokens` ADD `token_group` text;\n' duplicate column name: token_group",
);
const recovered = __migrateTestUtils.tryRecoverDuplicateColumnMigrationError(
sqlite,
tempMigrationsDir,
duplicateColumnError,
);
expect(recovered).toBe(true);
const createdTable = sqlite
.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'downstream_api_keys'")
.get() as { name?: string } | undefined;
const applied = sqlite
.prepare('SELECT hash, created_at FROM __drizzle_migrations')
.all() as Array<{ hash: string; created_at: number }>;
expect(createdTable?.name).toBe('downstream_api_keys');
expect(applied).toHaveLength(1);
expect(Number(applied[0].created_at)).toBe(1772600000000);
sqlite.close();
});
it('replays missing migrations before marking a duplicate-column migration as applied', async () => { it('replays missing migrations before marking a duplicate-column migration as applied', async () => {
const dataDir = mkdtempSync(join(tmpdir(), 'metapi-migrate-partial-journal-')); const dataDir = mkdtempSync(join(tmpdir(), 'metapi-migrate-partial-journal-'));
const dbPath = join(dataDir, 'hub.db'); const dbPath = join(dataDir, 'hub.db');
const sqlite = new Database(dbPath); const sqlite = new Database(dbPath);
const journalEntries = readMigrationJournalEntries(); const journalEntries = readMigrationJournalEntries();
const missingTags = new Set([ const appliedEntries = journalEntries.filter((entry) => entry.tag !== '0006_site_disabled_models' && entry.tag !== '0007_account_token_group');
'0006_site_disabled_models',
'0007_account_token_group',
'0008_sqlite_schema_backfill',
]);
const appliedEntries = journalEntries.filter((entry) => !missingTags.has(entry.tag));
for (const entry of appliedEntries) { for (const entry of appliedEntries) {
const sqlText = readFileSync(join(migrationsDir, `${entry.tag}.sql`), 'utf8'); const sqlText = readFileSync(join(migrationsDir, `${entry.tag}.sql`), 'utf8');
@@ -309,12 +239,8 @@ describe('sqlite migrate bootstrap', () => {
const disabledModelsTable = verified const disabledModelsTable = verified
.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'site_disabled_models'") .prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'site_disabled_models'")
.get() as { name?: string } | undefined; .get() as { name?: string } | undefined;
const downstreamApiKeysTable = verified
.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'downstream_api_keys'")
.get() as { name?: string } | undefined;
expect(disabledModelsTable?.name).toBe('site_disabled_models'); expect(disabledModelsTable?.name).toBe('site_disabled_models');
expect(downstreamApiKeysTable?.name).toBe('downstream_api_keys');
expect(appliedRows.map((row) => Number(row.created_at))).toEqual( expect(appliedRows.map((row) => Number(row.created_at))).toEqual(
journalEntries.map((entry) => entry.when), journalEntries.map((entry) => entry.when),
); );
src/server/db/migrate.ts
+1 -24
View File
@@ -174,28 +174,6 @@ function findMatchingSingleStatementMigration(
return null; return null;
} }
function findMatchingMigrationByStatement(
migrationsFolder: string,
failedSqlText: string,
): RecoveryMigrationRecord | null {
const normalizedFailedSql = normalizeSqlForMatch(failedSqlText);
const migrations = readRecoveryMigrations(migrationsFolder);
for (const migration of migrations) {
if (!migration.statements.some((statement) => normalizeSqlForMatch(statement) === normalizedFailedSql)) {
continue;
}
return {
tag: migration.tag,
createdAt: migration.createdAt,
hash: migration.hash,
};
}
return null;
}
function readRecoveryMigrations(migrationsFolder: string): RecoveryMigration[] { function readRecoveryMigrations(migrationsFolder: string): RecoveryMigration[] {
const journalPath = resolve(migrationsFolder, 'meta', '_journal.json'); const journalPath = resolve(migrationsFolder, 'meta', '_journal.json');
const journal = JSON.parse(readFileSync(journalPath, 'utf8')) as MigrationJournalFile; const journal = JSON.parse(readFileSync(journalPath, 'utf8')) as MigrationJournalFile;
@@ -342,7 +320,7 @@ function tryRecoverDuplicateColumnMigrationError(
return false; return false;
} }
const matchedMigration = findMatchingMigrationByStatement(migrationsFolder, failedSqlText); const matchedMigration = findMatchingSingleStatementMigration(migrationsFolder, failedSqlText);
if (!matchedMigration) { if (!matchedMigration) {
return false; return false;
} }
@@ -359,7 +337,6 @@ export const __migrateTestUtils = {
normalizeSqlForMatch, normalizeSqlForMatch,
extractFailedSqlFromError, extractFailedSqlFromError,
findMatchingSingleStatementMigration, findMatchingSingleStatementMigration,
findMatchingMigrationByStatement,
readRecoveryMigrations, readRecoveryMigrations,
markMigrationRecordIfMissing, markMigrationRecordIfMissing,
recoverMigrationSequence, recoverMigrationSequence,
src/server/db/runtimeSchemaBootstrap.test.ts
-41
View File
@@ -1,41 +0,0 @@
import { describe, expect, it } from 'vitest';
import {
__runtimeSchemaBootstrapTestUtils,
ensureRuntimeDatabaseSchema,
type RuntimeSchemaClient,
type RuntimeSchemaDialect,
} from './runtimeSchemaBootstrap.js';
function createStubClient(dialect: RuntimeSchemaDialect, executedSql: string[]): RuntimeSchemaClient {
return {
dialect,
begin: async () => {},
commit: async () => {},
rollback: async () => {},
execute: async (sqlText: string) => {
executedSql.push(sqlText);
return [];
},
queryScalar: async (sqlText: string, params: unknown[] = []) => {
if (sqlText.includes('information_schema') || sqlText.includes('sqlite_master') || sqlText.includes('pragma_table_info')) {
return 1;
}
if (params.length > 0) {
return 1;
}
return 0;
},
close: async () => {},
};
}
describe('runtime schema bootstrap', () => {
it.each(['mysql', 'postgres'] as const)('loads generated bootstrap statements for %s', async (dialect) => {
const executedSql: string[] = [];
const expectedBootstrapSql = __runtimeSchemaBootstrapTestUtils.readGeneratedBootstrapStatements(dialect);
await ensureRuntimeDatabaseSchema(createStubClient(dialect, executedSql));
expect(executedSql.slice(0, expectedBootstrapSql.length)).toEqual(expectedBootstrapSql);
});
});
src/server/db/runtimeSchemaBootstrap.ts
+62 -37
View File
@@ -1,14 +1,13 @@
import Database from 'better-sqlite3'; import Database from 'better-sqlite3';
import mysql from 'mysql2/promise'; import mysql from 'mysql2/promise';
import pg from 'pg'; import pg from 'pg';
import { mkdirSync, readFileSync } from 'node:fs'; import { mkdirSync } from 'node:fs';
import { dirname, resolve } from 'node:path'; import { dirname, resolve } from 'node:path';
import { import { ensureSiteSchemaCompatibility, type SiteSchemaInspector } from './siteSchemaCompatibility.js';
ensureLegacySchemaCompatibility, import { ensureRouteGroupingSchemaCompatibility } from './routeGroupingSchemaCompatibility.js';
type LegacySchemaCompatInspector, import { ensureProxyFileSchemaCompatibility } from './proxyFileSchemaCompatibility.js';
} from './legacySchemaCompat.js'; import { ensureAccountTokenSchemaCompatibility } from './accountTokenSchemaCompatibility.js';
import { generateBootstrapSql, resolveGeneratedArtifactPath } from './schemaArtifactGenerator.js'; import { ensureSharedIndexSchemaCompatibility } from './sharedIndexSchemaCompatibility.js';
import type { SchemaContract } from './schemaContract.js';
export type RuntimeSchemaDialect = 'sqlite' | 'mysql' | 'postgres'; export type RuntimeSchemaDialect = 'sqlite' | 'mysql' | 'postgres';
@@ -35,7 +34,7 @@ function validateIdentifier(identifier: string): string {
return identifier; return identifier;
} }
function createLegacySchemaInspector(client: RuntimeSchemaClient): LegacySchemaCompatInspector { function createSiteSchemaInspector(client: RuntimeSchemaClient): SiteSchemaInspector {
if (client.dialect === 'sqlite') { if (client.dialect === 'sqlite') {
return { return {
dialect: 'sqlite', dialect: 'sqlite',
@@ -99,27 +98,6 @@ function createLegacySchemaInspector(client: RuntimeSchemaClient): LegacySchemaC
}; };
} }
function splitSqlStatements(sqlText: string): string[] {
const withoutCommentLines = sqlText
.split(/\r?\n/g)
.filter((line) => !line.trim().startsWith('--'))
.join('\n');
return withoutCommentLines
.split(/;\s*(?:\r?\n|$)/g)
.map((statement) => statement.trim())
.filter((statement) => statement.length > 0);
}
function readSchemaContract(): SchemaContract {
return JSON.parse(readFileSync(resolveGeneratedArtifactPath('schemaContract.json'), 'utf8')) as SchemaContract;
}
function readGeneratedBootstrapStatements(dialect: Exclude<RuntimeSchemaDialect, 'sqlite'>): string[] {
const filename = dialect === 'mysql' ? 'mysql.bootstrap.sql' : 'postgres.bootstrap.sql';
return splitSqlStatements(readFileSync(resolveGeneratedArtifactPath(filename), 'utf8'));
}
async function createPostgresClient(connectionString: string, ssl: boolean): Promise<RuntimeSchemaClient> { async function createPostgresClient(connectionString: string, ssl: boolean): Promise<RuntimeSchemaClient> {
const clientOptions: pg.ClientConfig = { connectionString }; const clientOptions: pg.ClientConfig = { connectionString };
if (ssl) { if (ssl) {
@@ -207,15 +185,66 @@ export async function createRuntimeSchemaClient(input: RuntimeSchemaConnectionIn
} }
export async function ensureRuntimeDatabaseSchema(client: RuntimeSchemaClient): Promise<void> { export async function ensureRuntimeDatabaseSchema(client: RuntimeSchemaClient): Promise<void> {
const statements = client.dialect === 'sqlite' const statements = client.dialect === 'postgres'
? splitSqlStatements(generateBootstrapSql('sqlite', readSchemaContract())) ? [
: readGeneratedBootstrapStatements(client.dialect); `CREATE TABLE IF NOT EXISTS "sites" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, "name" TEXT NOT NULL, "url" TEXT NOT NULL, "external_checkin_url" TEXT, "platform" TEXT NOT NULL, "proxy_url" TEXT, "use_system_proxy" BOOLEAN DEFAULT FALSE, "custom_headers" TEXT, "status" TEXT DEFAULT 'active', "is_pinned" BOOLEAN DEFAULT FALSE, "sort_order" INTEGER DEFAULT 0, "global_weight" DOUBLE PRECISION DEFAULT 1, "api_key" TEXT, "created_at" TEXT, "updated_at" TEXT)`,
`CREATE TABLE IF NOT EXISTS "accounts" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, "site_id" INTEGER NOT NULL REFERENCES "sites"("id") ON DELETE CASCADE, "username" TEXT, "access_token" TEXT NOT NULL, "api_token" TEXT, "balance" DOUBLE PRECISION DEFAULT 0, "balance_used" DOUBLE PRECISION DEFAULT 0, "quota" DOUBLE PRECISION DEFAULT 0, "unit_cost" DOUBLE PRECISION, "value_score" DOUBLE PRECISION DEFAULT 0, "status" TEXT DEFAULT 'active', "is_pinned" BOOLEAN DEFAULT FALSE, "sort_order" INTEGER DEFAULT 0, "checkin_enabled" BOOLEAN DEFAULT TRUE, "last_checkin_at" TEXT, "last_balance_refresh" TEXT, "extra_config" TEXT, "created_at" TEXT, "updated_at" TEXT)`,
`CREATE TABLE IF NOT EXISTS "account_tokens" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, "account_id" INTEGER NOT NULL REFERENCES "accounts"("id") ON DELETE CASCADE, "name" TEXT NOT NULL, "token" TEXT NOT NULL, "token_group" TEXT, "source" TEXT DEFAULT 'manual', "enabled" BOOLEAN DEFAULT TRUE, "is_default" BOOLEAN DEFAULT FALSE, "created_at" TEXT, "updated_at" TEXT)`,
`CREATE TABLE IF NOT EXISTS "checkin_logs" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, "account_id" INTEGER NOT NULL REFERENCES "accounts"("id") ON DELETE CASCADE, "status" TEXT NOT NULL, "message" TEXT, "reward" TEXT, "created_at" TEXT)`,
`CREATE TABLE IF NOT EXISTS "model_availability" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, "account_id" INTEGER NOT NULL REFERENCES "accounts"("id") ON DELETE CASCADE, "model_name" TEXT NOT NULL, "available" BOOLEAN, "latency_ms" INTEGER, "checked_at" TEXT)`,
`CREATE TABLE IF NOT EXISTS "token_model_availability" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, "token_id" INTEGER NOT NULL REFERENCES "account_tokens"("id") ON DELETE CASCADE, "model_name" TEXT NOT NULL, "available" BOOLEAN, "latency_ms" INTEGER, "checked_at" TEXT)`,
`CREATE TABLE IF NOT EXISTS "token_routes" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, "model_pattern" TEXT NOT NULL, "display_name" TEXT, "display_icon" TEXT, "model_mapping" TEXT, "decision_snapshot" TEXT, "decision_refreshed_at" TEXT, "routing_strategy" TEXT DEFAULT 'weighted', "enabled" BOOLEAN DEFAULT TRUE, "created_at" TEXT, "updated_at" TEXT)`,
`CREATE TABLE IF NOT EXISTS "route_channels" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, "route_id" INTEGER NOT NULL REFERENCES "token_routes"("id") ON DELETE CASCADE, "account_id" INTEGER NOT NULL REFERENCES "accounts"("id") ON DELETE CASCADE, "token_id" INTEGER REFERENCES "account_tokens"("id") ON DELETE SET NULL, "source_model" TEXT, "priority" INTEGER DEFAULT 0, "weight" INTEGER DEFAULT 10, "enabled" BOOLEAN DEFAULT TRUE, "manual_override" BOOLEAN DEFAULT FALSE, "success_count" INTEGER DEFAULT 0, "fail_count" INTEGER DEFAULT 0, "total_latency_ms" INTEGER DEFAULT 0, "total_cost" DOUBLE PRECISION DEFAULT 0, "last_used_at" TEXT, "last_selected_at" TEXT, "last_fail_at" TEXT, "consecutive_fail_count" INTEGER NOT NULL DEFAULT 0, "cooldown_level" INTEGER NOT NULL DEFAULT 0, "cooldown_until" TEXT)`,
`CREATE TABLE IF NOT EXISTS "proxy_logs" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, "route_id" INTEGER, "channel_id" INTEGER, "account_id" INTEGER, "model_requested" TEXT, "model_actual" TEXT, "status" TEXT, "http_status" INTEGER, "latency_ms" INTEGER, "prompt_tokens" INTEGER, "completion_tokens" INTEGER, "total_tokens" INTEGER, "estimated_cost" DOUBLE PRECISION, "billing_details" TEXT, "error_message" TEXT, "retry_count" INTEGER DEFAULT 0, "created_at" TEXT)`,
`CREATE TABLE IF NOT EXISTS "proxy_video_tasks" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, "public_id" TEXT NOT NULL, "upstream_video_id" TEXT NOT NULL, "site_url" TEXT NOT NULL, "token_value" TEXT NOT NULL, "requested_model" TEXT, "actual_model" TEXT, "channel_id" INTEGER, "account_id" INTEGER, "status_snapshot" TEXT, "upstream_response_meta" TEXT, "last_upstream_status" INTEGER, "last_polled_at" TEXT, "created_at" TEXT, "updated_at" TEXT, CONSTRAINT "proxy_video_tasks_public_id_unique" UNIQUE ("public_id"))`,
`CREATE TABLE IF NOT EXISTS "proxy_files" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, "public_id" TEXT NOT NULL UNIQUE, "owner_type" TEXT NOT NULL, "owner_id" TEXT NOT NULL, "filename" TEXT NOT NULL, "mime_type" TEXT NOT NULL, "purpose" TEXT, "byte_size" INTEGER NOT NULL, "sha256" TEXT NOT NULL, "content_base64" TEXT NOT NULL, "created_at" TEXT, "updated_at" TEXT, "deleted_at" TEXT)`,
`CREATE TABLE IF NOT EXISTS "downstream_api_keys" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, "name" TEXT NOT NULL, "key" TEXT NOT NULL, "description" TEXT, "enabled" BOOLEAN DEFAULT TRUE, "expires_at" TEXT, "max_cost" DOUBLE PRECISION, "used_cost" DOUBLE PRECISION DEFAULT 0, "max_requests" INTEGER, "used_requests" INTEGER DEFAULT 0, "supported_models" TEXT, "allowed_route_ids" TEXT, "site_weight_multipliers" TEXT, "last_used_at" TEXT, "created_at" TEXT, "updated_at" TEXT, CONSTRAINT "downstream_api_keys_key_unique" UNIQUE ("key"))`,
`CREATE TABLE IF NOT EXISTS "events" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, "type" TEXT NOT NULL, "title" TEXT NOT NULL, "message" TEXT, "level" TEXT DEFAULT 'info', "read" BOOLEAN DEFAULT FALSE, "related_id" INTEGER, "related_type" TEXT, "created_at" TEXT)`,
`CREATE TABLE IF NOT EXISTS "settings" ("key" TEXT PRIMARY KEY, "value" TEXT)`,
]
: client.dialect === 'mysql'
? [
`CREATE TABLE IF NOT EXISTS \`sites\` (\`id\` INT AUTO_INCREMENT PRIMARY KEY, \`name\` TEXT NOT NULL, \`url\` TEXT NOT NULL, \`external_checkin_url\` TEXT NULL, \`platform\` VARCHAR(64) NOT NULL, \`proxy_url\` TEXT NULL, \`use_system_proxy\` BOOLEAN DEFAULT FALSE, \`custom_headers\` TEXT NULL, \`status\` VARCHAR(32) DEFAULT 'active', \`is_pinned\` BOOLEAN DEFAULT FALSE, \`sort_order\` INT DEFAULT 0, \`global_weight\` DOUBLE DEFAULT 1, \`api_key\` TEXT NULL, \`created_at\` TEXT NULL, \`updated_at\` TEXT NULL)`,
`CREATE TABLE IF NOT EXISTS \`accounts\` (\`id\` INT AUTO_INCREMENT PRIMARY KEY, \`site_id\` INT NOT NULL, \`username\` TEXT NULL, \`access_token\` TEXT NOT NULL, \`api_token\` TEXT NULL, \`balance\` DOUBLE DEFAULT 0, \`balance_used\` DOUBLE DEFAULT 0, \`quota\` DOUBLE DEFAULT 0, \`unit_cost\` DOUBLE NULL, \`value_score\` DOUBLE DEFAULT 0, \`status\` VARCHAR(32) DEFAULT 'active', \`is_pinned\` BOOLEAN DEFAULT FALSE, \`sort_order\` INT DEFAULT 0, \`checkin_enabled\` BOOLEAN DEFAULT TRUE, \`last_checkin_at\` TEXT NULL, \`last_balance_refresh\` TEXT NULL, \`extra_config\` TEXT NULL, \`created_at\` TEXT NULL, \`updated_at\` TEXT NULL, CONSTRAINT \`accounts_site_fk\` FOREIGN KEY (\`site_id\`) REFERENCES \`sites\`(\`id\`) ON DELETE CASCADE)`,
`CREATE TABLE IF NOT EXISTS \`account_tokens\` (\`id\` INT AUTO_INCREMENT PRIMARY KEY, \`account_id\` INT NOT NULL, \`name\` TEXT NOT NULL, \`token\` TEXT NOT NULL, \`token_group\` TEXT NULL, \`source\` VARCHAR(32) DEFAULT 'manual', \`enabled\` BOOLEAN DEFAULT TRUE, \`is_default\` BOOLEAN DEFAULT FALSE, \`created_at\` TEXT NULL, \`updated_at\` TEXT NULL, CONSTRAINT \`account_tokens_account_fk\` FOREIGN KEY (\`account_id\`) REFERENCES \`accounts\`(\`id\`) ON DELETE CASCADE)`,
`CREATE TABLE IF NOT EXISTS \`checkin_logs\` (\`id\` INT AUTO_INCREMENT PRIMARY KEY, \`account_id\` INT NOT NULL, \`status\` VARCHAR(32) NOT NULL, \`message\` TEXT NULL, \`reward\` TEXT NULL, \`created_at\` TEXT NULL, CONSTRAINT \`checkin_logs_account_fk\` FOREIGN KEY (\`account_id\`) REFERENCES \`accounts\`(\`id\`) ON DELETE CASCADE)`,
`CREATE TABLE IF NOT EXISTS \`model_availability\` (\`id\` INT AUTO_INCREMENT PRIMARY KEY, \`account_id\` INT NOT NULL, \`model_name\` VARCHAR(191) NOT NULL, \`available\` BOOLEAN NULL, \`latency_ms\` INT NULL, \`checked_at\` TEXT NULL, CONSTRAINT \`model_availability_account_fk\` FOREIGN KEY (\`account_id\`) REFERENCES \`accounts\`(\`id\`) ON DELETE CASCADE)`,
`CREATE TABLE IF NOT EXISTS \`token_model_availability\` (\`id\` INT AUTO_INCREMENT PRIMARY KEY, \`token_id\` INT NOT NULL, \`model_name\` VARCHAR(191) NOT NULL, \`available\` BOOLEAN NULL, \`latency_ms\` INT NULL, \`checked_at\` TEXT NULL, CONSTRAINT \`token_model_availability_token_fk\` FOREIGN KEY (\`token_id\`) REFERENCES \`account_tokens\`(\`id\`) ON DELETE CASCADE)`,
`CREATE TABLE IF NOT EXISTS \`token_routes\` (\`id\` INT AUTO_INCREMENT PRIMARY KEY, \`model_pattern\` TEXT NOT NULL, \`display_name\` TEXT NULL, \`display_icon\` TEXT NULL, \`model_mapping\` TEXT NULL, \`decision_snapshot\` TEXT NULL, \`decision_refreshed_at\` TEXT NULL, \`routing_strategy\` VARCHAR(32) NULL DEFAULT 'weighted', \`enabled\` BOOLEAN DEFAULT TRUE, \`created_at\` TEXT NULL, \`updated_at\` TEXT NULL)`,
`CREATE TABLE IF NOT EXISTS \`route_channels\` (\`id\` INT AUTO_INCREMENT PRIMARY KEY, \`route_id\` INT NOT NULL, \`account_id\` INT NOT NULL, \`token_id\` INT NULL, \`source_model\` TEXT NULL, \`priority\` INT DEFAULT 0, \`weight\` INT DEFAULT 10, \`enabled\` BOOLEAN DEFAULT TRUE, \`manual_override\` BOOLEAN DEFAULT FALSE, \`success_count\` INT DEFAULT 0, \`fail_count\` INT DEFAULT 0, \`total_latency_ms\` INT DEFAULT 0, \`total_cost\` DOUBLE DEFAULT 0, \`last_used_at\` TEXT NULL, \`last_selected_at\` TEXT NULL, \`last_fail_at\` TEXT NULL, \`consecutive_fail_count\` INT NOT NULL DEFAULT 0, \`cooldown_level\` INT NOT NULL DEFAULT 0, \`cooldown_until\` TEXT NULL, CONSTRAINT \`route_channels_route_fk\` FOREIGN KEY (\`route_id\`) REFERENCES \`token_routes\`(\`id\`) ON DELETE CASCADE, CONSTRAINT \`route_channels_account_fk\` FOREIGN KEY (\`account_id\`) REFERENCES \`accounts\`(\`id\`) ON DELETE CASCADE, CONSTRAINT \`route_channels_token_fk\` FOREIGN KEY (\`token_id\`) REFERENCES \`account_tokens\`(\`id\`) ON DELETE SET NULL)`,
`CREATE TABLE IF NOT EXISTS \`proxy_logs\` (\`id\` INT AUTO_INCREMENT PRIMARY KEY, \`route_id\` INT NULL, \`channel_id\` INT NULL, \`account_id\` INT NULL, \`model_requested\` TEXT NULL, \`model_actual\` TEXT NULL, \`status\` VARCHAR(32) NULL, \`http_status\` INT NULL, \`latency_ms\` INT NULL, \`prompt_tokens\` INT NULL, \`completion_tokens\` INT NULL, \`total_tokens\` INT NULL, \`estimated_cost\` DOUBLE NULL, \`billing_details\` TEXT NULL, \`error_message\` TEXT NULL, \`retry_count\` INT DEFAULT 0, \`created_at\` TEXT NULL)`,
`CREATE TABLE IF NOT EXISTS \`proxy_video_tasks\` (\`id\` INT AUTO_INCREMENT PRIMARY KEY, \`public_id\` VARCHAR(191) NOT NULL, \`upstream_video_id\` TEXT NOT NULL, \`site_url\` TEXT NOT NULL, \`token_value\` TEXT NOT NULL, \`requested_model\` TEXT NULL, \`actual_model\` TEXT NULL, \`channel_id\` INT NULL, \`account_id\` INT NULL, \`status_snapshot\` TEXT NULL, \`upstream_response_meta\` TEXT NULL, \`last_upstream_status\` INT NULL, \`last_polled_at\` TEXT NULL, \`created_at\` TEXT NULL, \`updated_at\` TEXT NULL, UNIQUE KEY \`proxy_video_tasks_public_id_unique\` (\`public_id\`))`,
`CREATE TABLE IF NOT EXISTS \`proxy_files\` (\`id\` INT AUTO_INCREMENT PRIMARY KEY, \`public_id\` VARCHAR(191) NOT NULL UNIQUE, \`owner_type\` VARCHAR(64) NOT NULL, \`owner_id\` VARCHAR(191) NOT NULL, \`filename\` TEXT NOT NULL, \`mime_type\` VARCHAR(191) NOT NULL, \`purpose\` TEXT NULL, \`byte_size\` INT NOT NULL, \`sha256\` VARCHAR(191) NOT NULL, \`content_base64\` LONGTEXT NOT NULL, \`created_at\` TEXT NULL, \`updated_at\` TEXT NULL, \`deleted_at\` TEXT NULL)`,
`CREATE TABLE IF NOT EXISTS \`downstream_api_keys\` (\`id\` INT AUTO_INCREMENT PRIMARY KEY, \`name\` TEXT NOT NULL, \`key\` VARCHAR(191) NOT NULL, \`description\` TEXT NULL, \`enabled\` BOOLEAN DEFAULT TRUE, \`expires_at\` TEXT NULL, \`max_cost\` DOUBLE NULL, \`used_cost\` DOUBLE DEFAULT 0, \`max_requests\` INT NULL, \`used_requests\` INT DEFAULT 0, \`supported_models\` TEXT NULL, \`allowed_route_ids\` TEXT NULL, \`site_weight_multipliers\` TEXT NULL, \`last_used_at\` TEXT NULL, \`created_at\` TEXT NULL, \`updated_at\` TEXT NULL, UNIQUE KEY \`downstream_api_keys_key_unique\` (\`key\`))`,
`CREATE TABLE IF NOT EXISTS \`events\` (\`id\` INT AUTO_INCREMENT PRIMARY KEY, \`type\` VARCHAR(32) NOT NULL, \`title\` TEXT NOT NULL, \`message\` TEXT NULL, \`level\` VARCHAR(16) DEFAULT 'info', \`read\` BOOLEAN DEFAULT FALSE, \`related_id\` INT NULL, \`related_type\` VARCHAR(32) NULL, \`created_at\` TEXT NULL)`,
`CREATE TABLE IF NOT EXISTS \`settings\` (\`key\` VARCHAR(191) PRIMARY KEY, \`value\` TEXT NULL)`,
]
: [
`CREATE TABLE IF NOT EXISTS "sites" ("id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, "name" TEXT NOT NULL, "url" TEXT NOT NULL, "external_checkin_url" TEXT, "platform" TEXT NOT NULL, "proxy_url" TEXT, "use_system_proxy" INTEGER DEFAULT 0, "custom_headers" TEXT, "status" TEXT DEFAULT 'active', "is_pinned" INTEGER DEFAULT 0, "sort_order" INTEGER DEFAULT 0, "global_weight" REAL DEFAULT 1, "api_key" TEXT, "created_at" TEXT, "updated_at" TEXT)`,
`CREATE TABLE IF NOT EXISTS "accounts" ("id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, "site_id" INTEGER NOT NULL REFERENCES "sites"("id") ON DELETE CASCADE, "username" TEXT, "access_token" TEXT NOT NULL, "api_token" TEXT, "balance" REAL DEFAULT 0, "balance_used" REAL DEFAULT 0, "quota" REAL DEFAULT 0, "unit_cost" REAL, "value_score" REAL DEFAULT 0, "status" TEXT DEFAULT 'active', "is_pinned" INTEGER DEFAULT 0, "sort_order" INTEGER DEFAULT 0, "checkin_enabled" INTEGER DEFAULT 1, "last_checkin_at" TEXT, "last_balance_refresh" TEXT, "extra_config" TEXT, "created_at" TEXT, "updated_at" TEXT)`,
`CREATE TABLE IF NOT EXISTS "account_tokens" ("id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, "account_id" INTEGER NOT NULL REFERENCES "accounts"("id") ON DELETE CASCADE, "name" TEXT NOT NULL, "token" TEXT NOT NULL, "token_group" TEXT, "source" TEXT DEFAULT 'manual', "enabled" INTEGER DEFAULT 1, "is_default" INTEGER DEFAULT 0, "created_at" TEXT, "updated_at" TEXT)`,
`CREATE TABLE IF NOT EXISTS "checkin_logs" ("id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, "account_id" INTEGER NOT NULL REFERENCES "accounts"("id") ON DELETE CASCADE, "status" TEXT NOT NULL, "message" TEXT, "reward" TEXT, "created_at" TEXT)`,
`CREATE TABLE IF NOT EXISTS "model_availability" ("id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, "account_id" INTEGER NOT NULL REFERENCES "accounts"("id") ON DELETE CASCADE, "model_name" TEXT NOT NULL, "available" INTEGER, "latency_ms" INTEGER, "checked_at" TEXT)`,
`CREATE TABLE IF NOT EXISTS "token_model_availability" ("id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, "token_id" INTEGER NOT NULL REFERENCES "account_tokens"("id") ON DELETE CASCADE, "model_name" TEXT NOT NULL, "available" INTEGER, "latency_ms" INTEGER, "checked_at" TEXT)`,
`CREATE TABLE IF NOT EXISTS "token_routes" ("id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, "model_pattern" TEXT NOT NULL, "display_name" TEXT, "display_icon" TEXT, "model_mapping" TEXT, "decision_snapshot" TEXT, "decision_refreshed_at" TEXT, "routing_strategy" TEXT DEFAULT 'weighted', "enabled" INTEGER DEFAULT 1, "created_at" TEXT, "updated_at" TEXT)`,
`CREATE TABLE IF NOT EXISTS "route_channels" ("id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, "route_id" INTEGER NOT NULL REFERENCES "token_routes"("id") ON DELETE CASCADE, "account_id" INTEGER NOT NULL REFERENCES "accounts"("id") ON DELETE CASCADE, "token_id" INTEGER REFERENCES "account_tokens"("id") ON DELETE SET NULL, "source_model" TEXT, "priority" INTEGER DEFAULT 0, "weight" INTEGER DEFAULT 10, "enabled" INTEGER DEFAULT 1, "manual_override" INTEGER DEFAULT 0, "success_count" INTEGER DEFAULT 0, "fail_count" INTEGER DEFAULT 0, "total_latency_ms" INTEGER DEFAULT 0, "total_cost" REAL DEFAULT 0, "last_used_at" TEXT, "last_selected_at" TEXT, "last_fail_at" TEXT, "consecutive_fail_count" INTEGER NOT NULL DEFAULT 0, "cooldown_level" INTEGER NOT NULL DEFAULT 0, "cooldown_until" TEXT)`,
`CREATE TABLE IF NOT EXISTS "proxy_logs" ("id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, "route_id" INTEGER, "channel_id" INTEGER, "account_id" INTEGER, "model_requested" TEXT, "model_actual" TEXT, "status" TEXT, "http_status" INTEGER, "latency_ms" INTEGER, "prompt_tokens" INTEGER, "completion_tokens" INTEGER, "total_tokens" INTEGER, "estimated_cost" REAL, "billing_details" TEXT, "error_message" TEXT, "retry_count" INTEGER DEFAULT 0, "created_at" TEXT)`,
`CREATE TABLE IF NOT EXISTS "proxy_video_tasks" ("id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, "public_id" TEXT NOT NULL, "upstream_video_id" TEXT NOT NULL, "site_url" TEXT NOT NULL, "token_value" TEXT NOT NULL, "requested_model" TEXT, "actual_model" TEXT, "channel_id" INTEGER, "account_id" INTEGER, "status_snapshot" TEXT, "upstream_response_meta" TEXT, "last_upstream_status" INTEGER, "last_polled_at" TEXT, "created_at" TEXT, "updated_at" TEXT, CONSTRAINT "proxy_video_tasks_public_id_unique" UNIQUE ("public_id"))`,
`CREATE TABLE IF NOT EXISTS "proxy_files" ("id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, "public_id" TEXT NOT NULL UNIQUE, "owner_type" TEXT NOT NULL, "owner_id" TEXT NOT NULL, "filename" TEXT NOT NULL, "mime_type" TEXT NOT NULL, "purpose" TEXT, "byte_size" INTEGER NOT NULL, "sha256" TEXT NOT NULL, "content_base64" TEXT NOT NULL, "created_at" TEXT, "updated_at" TEXT, "deleted_at" TEXT)`,
`CREATE TABLE IF NOT EXISTS "downstream_api_keys" ("id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, "name" TEXT NOT NULL, "key" TEXT NOT NULL, "description" TEXT, "enabled" INTEGER DEFAULT 1, "expires_at" TEXT, "max_cost" REAL, "used_cost" REAL DEFAULT 0, "max_requests" INTEGER, "used_requests" INTEGER DEFAULT 0, "supported_models" TEXT, "allowed_route_ids" TEXT, "site_weight_multipliers" TEXT, "last_used_at" TEXT, "created_at" TEXT, "updated_at" TEXT, CONSTRAINT "downstream_api_keys_key_unique" UNIQUE ("key"))`,
`CREATE TABLE IF NOT EXISTS "events" ("id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, "type" TEXT NOT NULL, "title" TEXT NOT NULL, "message" TEXT, "level" TEXT DEFAULT 'info', "read" INTEGER DEFAULT 0, "related_id" INTEGER, "related_type" TEXT, "created_at" TEXT)`,
`CREATE TABLE IF NOT EXISTS "settings" ("key" TEXT PRIMARY KEY, "value" TEXT)`,
];
for (const sqlText of statements) { for (const sqlText of statements) {
await client.execute(sqlText); await client.execute(sqlText);
} }
await ensureLegacySchemaCompatibility(createLegacySchemaInspector(client)); await ensureSiteSchemaCompatibility(createSiteSchemaInspector(client));
await ensureRouteGroupingSchemaCompatibility(createSiteSchemaInspector(client));
await ensureProxyFileSchemaCompatibility(createSiteSchemaInspector(client));
await ensureAccountTokenSchemaCompatibility(createSiteSchemaInspector(client));
await ensureSharedIndexSchemaCompatibility(createSiteSchemaInspector(client));
} }
export async function bootstrapRuntimeDatabaseSchema(input: RuntimeSchemaConnectionInput): Promise<void> { export async function bootstrapRuntimeDatabaseSchema(input: RuntimeSchemaConnectionInput): Promise<void> {
@@ -226,7 +255,3 @@ export async function bootstrapRuntimeDatabaseSchema(input: RuntimeSchemaConnect
await client.close(); await client.close();
} }
} }
export const __runtimeSchemaBootstrapTestUtils = {
readGeneratedBootstrapStatements,
};
src/server/db/schemaArtifactGenerator.test.ts
-118
View File
@@ -1,118 +0,0 @@
import { readFileSync } from 'node:fs';
import { dirname, resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
import { describe, expect, it } from 'vitest';
import { generateDialectArtifacts, type GeneratedDialectArtifacts } from './schemaArtifactGenerator.js';
import type { SchemaContract, SchemaContractColumn } from './schemaContract.js';
const dbDir = dirname(fileURLToPath(import.meta.url));
const schemaContractPath = resolve(dbDir, 'generated/schemaContract.json');
function readSchemaContract(): SchemaContract {
return JSON.parse(readFileSync(schemaContractPath, 'utf8')) as SchemaContract;
}
function makeColumn(overrides: Partial<SchemaContractColumn> = {}): SchemaContractColumn {
return {
logicalType: 'text',
notNull: false,
defaultValue: null,
primaryKey: false,
...overrides,
};
}
describe('schema artifact generator', () => {
it('generates bootstrap sql for mysql and postgres from the contract', () => {
const artifacts = generateDialectArtifacts(readSchemaContract());
expect(artifacts.mysqlBootstrap).toContain('CREATE TABLE IF NOT EXISTS `sites`');
expect(artifacts.mysqlBootstrap).toContain('CREATE TABLE IF NOT EXISTS `site_disabled_models`');
expect(artifacts.postgresBootstrap).toContain('CREATE TABLE IF NOT EXISTS "account_tokens"');
expect(artifacts.postgresBootstrap).toContain('"token_group"');
});
it('generates additive upgrade sql for newly added tables, columns, indexes, and uniques', () => {
const previousContract: SchemaContract = {
tables: {
sites: {
columns: {
id: makeColumn({ logicalType: 'integer', notNull: true, primaryKey: true }),
},
},
},
indexes: [],
uniques: [],
foreignKeys: [],
};
const currentContract: SchemaContract = {
tables: {
sites: {
columns: {
id: makeColumn({ logicalType: 'integer', notNull: true, primaryKey: true }),
status: makeColumn({ notNull: true, defaultValue: "'active'" }),
},
},
accounts: {
columns: {
id: makeColumn({ logicalType: 'integer', notNull: true, primaryKey: true }),
site_id: makeColumn({ logicalType: 'integer', notNull: true }),
email: makeColumn({ notNull: true }),
},
},
},
indexes: [
{ name: 'sites_status_idx', table: 'sites', columns: ['status'], unique: false },
],
uniques: [
{ name: 'accounts_site_email_unique', table: 'accounts', columns: ['site_id', 'email'] },
],
foreignKeys: [
{
table: 'accounts',
columns: ['site_id'],
referencedTable: 'sites',
referencedColumns: ['id'],
onDelete: 'cascade',
},
],
};
const artifacts: GeneratedDialectArtifacts = generateDialectArtifacts(currentContract, previousContract);
expect(artifacts.mysqlUpgrade).toContain('CREATE TABLE IF NOT EXISTS `accounts`');
expect(artifacts.mysqlUpgrade).toContain('ALTER TABLE `sites` ADD COLUMN `status`');
expect(artifacts.mysqlUpgrade).toContain('CREATE INDEX `sites_status_idx` ON `sites`');
expect(artifacts.mysqlUpgrade).toContain('CREATE UNIQUE INDEX `accounts_site_email_unique` ON `accounts`');
expect(artifacts.postgresUpgrade).toContain('CREATE TABLE IF NOT EXISTS "accounts"');
expect(artifacts.postgresUpgrade).toContain('ALTER TABLE "sites" ADD COLUMN "status"');
expect(artifacts.postgresUpgrade).toContain('CREATE INDEX "sites_status_idx" ON "sites"');
expect(artifacts.postgresUpgrade).toContain('CREATE UNIQUE INDEX "accounts_site_email_unique" ON "accounts"');
});
it('orders dependent tables safely and emits executable datetime defaults for external dialects', () => {
const artifacts = generateDialectArtifacts(readSchemaContract());
expect(
artifacts.mysqlBootstrap.indexOf('CREATE TABLE IF NOT EXISTS `accounts`'),
).toBeLessThan(
artifacts.mysqlBootstrap.indexOf('CREATE TABLE IF NOT EXISTS `account_tokens`'),
);
expect(
artifacts.postgresBootstrap.indexOf('CREATE TABLE IF NOT EXISTS "accounts"'),
).toBeLessThan(
artifacts.postgresBootstrap.indexOf('CREATE TABLE IF NOT EXISTS "account_tokens"'),
);
expect(artifacts.mysqlBootstrap).toContain('`created_at` DATETIME DEFAULT CURRENT_TIMESTAMP');
expect(artifacts.postgresBootstrap).toContain('"created_at" TIMESTAMP DEFAULT CURRENT_TIMESTAMP');
});
it('rejects destructive diffs when generating additive upgrades', () => {
const current = readSchemaContract();
const previous = structuredClone(current);
delete current.tables.sites.columns.status;
expect(() => generateDialectArtifacts(current, previous)).toThrow(/non-additive schema diff/i);
});
});
src/server/db/schemaArtifactGenerator.ts
-410
View File
@@ -1,410 +0,0 @@
import { mkdirSync, writeFileSync } from 'node:fs';
import { dirname, resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
import type {
LogicalColumnType,
SchemaContract,
SchemaContractColumn,
SchemaContractForeignKey,
SchemaContractIndex,
SchemaContractUnique,
} from './schemaContract.js';
export interface GeneratedDialectArtifacts {
mysqlBootstrap: string;
postgresBootstrap: string;
mysqlUpgrade: string;
postgresUpgrade: string;
}
type Dialect = 'mysql' | 'postgres';
type SqlDialect = 'sqlite' | Dialect;
function resolveDbDir(): string {
return dirname(fileURLToPath(import.meta.url));
}
export function resolveGeneratedArtifactPath(filename: string): string {
return resolve(resolveDbDir(), 'generated', filename);
}
function quoteIdentifier(dialect: SqlDialect, identifier: string): string {
return dialect === 'mysql' ? `\`${identifier}\`` : `"${identifier}"`;
}
function escapeMysqlTextPrefix(columnType: LogicalColumnType): string {
return columnType === 'text' || columnType === 'datetime' || columnType === 'json' ? '(191)' : '';
}
function mapColumnType(dialect: SqlDialect, columnName: string, column: SchemaContractColumn): string {
if (dialect === 'sqlite') {
switch (column.logicalType) {
case 'boolean':
case 'integer':
return 'INTEGER';
case 'real':
return 'REAL';
case 'datetime':
case 'json':
case 'text':
default:
return 'TEXT';
}
}
if (dialect === 'mysql') {
switch (column.logicalType) {
case 'boolean':
return 'BOOLEAN';
case 'integer':
return column.primaryKey && columnName === 'id' ? 'INT AUTO_INCREMENT' : 'INT';
case 'real':
return 'DOUBLE';
case 'datetime':
return 'DATETIME';
case 'json':
return 'JSON';
case 'text':
return column.defaultValue != null ? 'VARCHAR(191)' : 'TEXT';
default:
return 'TEXT';
}
}
switch (column.logicalType) {
case 'boolean':
return 'BOOLEAN';
case 'integer':
return column.primaryKey && columnName === 'id' ? 'INTEGER GENERATED BY DEFAULT AS IDENTITY' : 'INTEGER';
case 'real':
return 'DOUBLE PRECISION';
case 'datetime':
return 'TIMESTAMP';
case 'json':
return 'JSONB';
case 'text':
default:
return 'TEXT';
}
}
function formatDefaultValue(dialect: SqlDialect, column: SchemaContractColumn): string {
if (column.defaultValue == null || column.primaryKey) {
return '';
}
if (column.logicalType === 'datetime') {
if (dialect === 'sqlite') {
return " DEFAULT (datetime('now'))";
}
return ' DEFAULT CURRENT_TIMESTAMP';
}
if (column.logicalType === 'boolean') {
return ` DEFAULT ${column.defaultValue.toLowerCase()}`;
}
return ` DEFAULT ${column.defaultValue}`;
}
function buildColumnDefinition(
dialect: SqlDialect,
columnName: string,
column: SchemaContractColumn,
): string {
const sqlType = mapColumnType(dialect, columnName, column);
const notNull = column.notNull ? ' NOT NULL' : '';
const defaultValue = formatDefaultValue(dialect, column);
const primaryKey = column.primaryKey ? ' PRIMARY KEY' : '';
return `${quoteIdentifier(dialect, columnName)} ${sqlType}${notNull}${defaultValue}${primaryKey}`;
}
function buildForeignKeyClause(dialect: SqlDialect, foreignKey: SchemaContractForeignKey): string {
const sourceColumns = foreignKey.columns.map((column) => quoteIdentifier(dialect, column)).join(', ');
const targetColumns = foreignKey.referencedColumns.map((column) => quoteIdentifier(dialect, column)).join(', ');
const onDelete = foreignKey.onDelete ? ` ON DELETE ${foreignKey.onDelete.toUpperCase()}` : '';
return `FOREIGN KEY (${sourceColumns}) REFERENCES ${quoteIdentifier(dialect, foreignKey.referencedTable)}(${targetColumns})${onDelete}`;
}
function sortTableNamesForCreation(contract: SchemaContract): string[] {
const ordered: string[] = [];
const visited = new Set<string>();
const visiting = new Set<string>();
const tableNames = Object.keys(contract.tables).sort((left, right) => left.localeCompare(right, 'en'));
const dependencies = new Map<string, string[]>();
for (const tableName of tableNames) {
const referencedTables = contract.foreignKeys
.filter((foreignKey) => foreignKey.table === tableName)
.map((foreignKey) => foreignKey.referencedTable)
.filter((referencedTable) => referencedTable !== tableName && tableNames.includes(referencedTable))
.sort((left, right) => left.localeCompare(right, 'en'));
dependencies.set(tableName, [...new Set(referencedTables)]);
}
function visit(tableName: string): void {
if (visited.has(tableName) || visiting.has(tableName)) {
return;
}
visiting.add(tableName);
for (const dependency of dependencies.get(tableName) ?? []) {
visit(dependency);
}
visiting.delete(tableName);
visited.add(tableName);
ordered.push(tableName);
}
for (const tableName of tableNames) {
visit(tableName);
}
return ordered;
}
function buildCreateTableStatement(
dialect: SqlDialect,
tableName: string,
contract: SchemaContract,
): string {
const table = contract.tables[tableName];
const columnEntries = Object.entries(table.columns);
const foreignKeys = contract.foreignKeys.filter((foreignKey) => foreignKey.table === tableName);
const parts = [
...columnEntries.map(([columnName, column]) => buildColumnDefinition(dialect, columnName, column)),
...foreignKeys.map((foreignKey) => buildForeignKeyClause(dialect, foreignKey)),
];
return `CREATE TABLE IF NOT EXISTS ${quoteIdentifier(dialect, tableName)} (${parts.join(', ')})`;
}
function buildUniqueIndexStatement(dialect: SqlDialect, uniqueIndex: SchemaContractUnique, contract: SchemaContract): string {
const columns = uniqueIndex.columns
.map((columnName) => {
const column = contract.tables[uniqueIndex.table]?.columns[columnName];
const suffix = dialect === 'mysql' && column ? escapeMysqlTextPrefix(column.logicalType) : '';
return `${quoteIdentifier(dialect, columnName)}${suffix}`;
})
.join(', ');
return `CREATE UNIQUE INDEX ${quoteIdentifier(dialect, uniqueIndex.name)} ON ${quoteIdentifier(dialect, uniqueIndex.table)} (${columns})`;
}
function buildIndexStatement(dialect: SqlDialect, index: SchemaContractIndex, contract: SchemaContract): string {
const columns = index.columns
.map((columnName) => {
const column = contract.tables[index.table]?.columns[columnName];
const suffix = dialect === 'mysql' && column ? escapeMysqlTextPrefix(column.logicalType) : '';
return `${quoteIdentifier(dialect, columnName)}${suffix}`;
})
.join(', ');
return `CREATE INDEX ${quoteIdentifier(dialect, index.name)} ON ${quoteIdentifier(dialect, index.table)} (${columns})`;
}
function serializeColumn(column: SchemaContractColumn): string {
return [
column.logicalType,
column.notNull ? 'not-null' : 'nullable',
column.defaultValue ?? 'default:null',
column.primaryKey ? 'pk' : 'non-pk',
].join('|');
}
function serializeIndex(index: SchemaContractIndex): string {
return [index.table, index.columns.join(','), index.unique ? 'unique' : 'non-unique'].join('|');
}
function serializeUnique(unique: SchemaContractUnique): string {
return [unique.table, unique.columns.join(',')].join('|');
}
function serializeForeignKey(foreignKey: SchemaContractForeignKey): string {
return [
foreignKey.table,
foreignKey.columns.join(','),
foreignKey.referencedTable,
foreignKey.referencedColumns.join(','),
foreignKey.onDelete ?? 'null',
].join('|');
}
function assertAdditiveSchemaDiff(currentContract: SchemaContract, previousContract: SchemaContract): void {
const violations: string[] = [];
for (const [tableName, previousTable] of Object.entries(previousContract.tables)) {
const currentTable = currentContract.tables[tableName];
if (!currentTable) {
violations.push(`removed table ${tableName}`);
continue;
}
for (const [columnName, previousColumn] of Object.entries(previousTable.columns)) {
const currentColumn = currentTable.columns[columnName];
if (!currentColumn) {
violations.push(`removed column ${tableName}.${columnName}`);
continue;
}
if (serializeColumn(currentColumn) !== serializeColumn(previousColumn)) {
violations.push(`changed column ${tableName}.${columnName}`);
}
}
}
const currentIndexes = new Map(currentContract.indexes.map((index) => [index.name, index]));
for (const previousIndex of previousContract.indexes) {
const currentIndex = currentIndexes.get(previousIndex.name);
if (!currentIndex) {
violations.push(`removed index ${previousIndex.name}`);
continue;
}
if (serializeIndex(currentIndex) !== serializeIndex(previousIndex)) {
violations.push(`changed index ${previousIndex.name}`);
}
}
const currentUniques = new Map(currentContract.uniques.map((unique) => [unique.name, unique]));
for (const previousUnique of previousContract.uniques) {
const currentUnique = currentUniques.get(previousUnique.name);
if (!currentUnique) {
violations.push(`removed unique ${previousUnique.name}`);
continue;
}
if (serializeUnique(currentUnique) !== serializeUnique(previousUnique)) {
violations.push(`changed unique ${previousUnique.name}`);
}
}
const currentForeignKeys = new Set(currentContract.foreignKeys.map(serializeForeignKey));
for (const previousForeignKey of previousContract.foreignKeys) {
if (!currentForeignKeys.has(serializeForeignKey(previousForeignKey))) {
violations.push(`removed foreign key ${previousForeignKey.table}(${previousForeignKey.columns.join(',')})`);
}
}
if (violations.length > 0) {
throw new Error(`Non-additive schema diff detected:\n- ${violations.join('\n- ')}`);
}
}
export function generateBootstrapSql(dialect: SqlDialect, contract: SchemaContract): string {
const tableStatements = sortTableNamesForCreation(contract)
.map((tableName) => buildCreateTableStatement(dialect, tableName, contract));
const uniqueStatements = contract.uniques
.slice()
.sort((left, right) => left.name.localeCompare(right.name, 'en'))
.map((uniqueIndex) => buildUniqueIndexStatement(dialect, uniqueIndex, contract));
const knownUniqueNames = new Set(contract.uniques.map((uniqueIndex) => uniqueIndex.name));
const indexStatements = contract.indexes
.filter((index) => !knownUniqueNames.has(index.name))
.slice()
.sort((left, right) => left.name.localeCompare(right.name, 'en'))
.map((index) => buildIndexStatement(dialect, index, contract));
return `${[...tableStatements, ...uniqueStatements, ...indexStatements].join(';\n')};\n`;
}
function buildAddColumnStatement(
dialect: SqlDialect,
tableName: string,
columnName: string,
column: SchemaContractColumn,
): string {
return `ALTER TABLE ${quoteIdentifier(dialect, tableName)} ADD COLUMN ${buildColumnDefinition(dialect, columnName, column)}`;
}
export function generateUpgradeSql(
dialect: SqlDialect,
currentContract: SchemaContract,
previousContract?: SchemaContract | null,
): string {
if (!previousContract) {
return `-- no previous schema contract available for ${dialect} additive upgrade generation\n`;
}
assertAdditiveSchemaDiff(currentContract, previousContract);
const previousTableNames = new Set(Object.keys(previousContract.tables));
const currentTableNames = Object.keys(currentContract.tables).sort((left, right) => left.localeCompare(right, 'en'));
const addedTableNames = currentTableNames.filter((tableName) => !previousTableNames.has(tableName));
const addedTablesContract: SchemaContract = {
tables: Object.fromEntries(addedTableNames.map((tableName) => [tableName, currentContract.tables[tableName]])),
indexes: currentContract.indexes.filter((index) => addedTableNames.includes(index.table)),
uniques: currentContract.uniques.filter((unique) => addedTableNames.includes(unique.table)),
foreignKeys: currentContract.foreignKeys.filter((foreignKey) => addedTableNames.includes(foreignKey.table)),
};
const addedTableStatements = sortTableNamesForCreation(addedTablesContract)
.map((tableName) => buildCreateTableStatement(dialect, tableName, currentContract));
const addColumnStatements: string[] = [];
for (const tableName of currentTableNames) {
if (!previousTableNames.has(tableName)) {
continue;
}
const previousColumns = previousContract.tables[tableName]?.columns ?? {};
const currentColumns = currentContract.tables[tableName]?.columns ?? {};
for (const [columnName, column] of Object.entries(currentColumns)) {
if (previousColumns[columnName]) {
continue;
}
addColumnStatements.push(buildAddColumnStatement(dialect, tableName, columnName, column));
}
}
const previousUniqueNames = new Set(previousContract.uniques.map((unique) => unique.name));
const uniqueStatements = currentContract.uniques
.filter((unique) => !previousUniqueNames.has(unique.name))
.slice()
.sort((left, right) => left.name.localeCompare(right.name, 'en'))
.map((unique) => buildUniqueIndexStatement(dialect, unique, currentContract));
const currentUniqueNames = new Set(currentContract.uniques.map((unique) => unique.name));
const previousIndexNames = new Set(previousContract.indexes.map((index) => index.name));
const indexStatements = currentContract.indexes
.filter((index) => !currentUniqueNames.has(index.name))
.filter((index) => !previousIndexNames.has(index.name))
.slice()
.sort((left, right) => left.name.localeCompare(right.name, 'en'))
.map((index) => buildIndexStatement(dialect, index, currentContract));
const statements = [...addedTableStatements, ...addColumnStatements, ...uniqueStatements, ...indexStatements];
if (statements.length === 0) {
return `-- no schema changes detected for ${dialect}\n`;
}
return `${statements.join(';\n')};\n`;
}
export function generateDialectArtifacts(
contract: SchemaContract,
previousContract?: SchemaContract | null,
): GeneratedDialectArtifacts {
return {
mysqlBootstrap: generateBootstrapSql('mysql', contract),
postgresBootstrap: generateBootstrapSql('postgres', contract),
mysqlUpgrade: generateUpgradeSql('mysql', contract, previousContract),
postgresUpgrade: generateUpgradeSql('postgres', contract, previousContract),
};
}
export function writeDialectArtifactFiles(
contract: SchemaContract,
previousContract?: SchemaContract | null,
): GeneratedDialectArtifacts {
const artifacts = generateDialectArtifacts(contract, previousContract);
const artifactEntries = [
['mysql.bootstrap.sql', artifacts.mysqlBootstrap],
['postgres.bootstrap.sql', artifacts.postgresBootstrap],
['mysql.upgrade.sql', artifacts.mysqlUpgrade],
['postgres.upgrade.sql', artifacts.postgresUpgrade],
] as const;
for (const [filename, content] of artifactEntries) {
const outputPath = resolveGeneratedArtifactPath(filename);
mkdirSync(dirname(outputPath), { recursive: true });
writeFileSync(outputPath, content, 'utf8');
}
return artifacts;
}
src/server/db/schemaContract.test.ts
-67
View File
@@ -1,67 +0,0 @@
import { describe, expect, it } from 'vitest';
import { buildSchemaContractFromSqliteMigrations } from './schemaContract.js';
describe('schema contract generation', () => {
it('captures the current schema shape from sqlite migrations', () => {
const contract = buildSchemaContractFromSqliteMigrations();
expect(contract.tables.sites.columns.status).toMatchObject({
logicalType: 'text',
notNull: true,
primaryKey: false,
});
expect(contract.tables.account_tokens.columns.token_group).toBeDefined();
expect(contract.tables.site_disabled_models).toBeDefined();
expect(contract.tables.downstream_api_keys).toBeDefined();
expect(contract.tables.proxy_files).toBeDefined();
expect(contract.tables.proxy_video_tasks).toBeDefined();
expect(contract.tables.route_channels.columns.source_model).toBeDefined();
expect(contract.tables.route_channels.columns.last_selected_at).toBeDefined();
expect(contract.tables.route_channels.columns.consecutive_fail_count).toMatchObject({
logicalType: 'integer',
notNull: true,
defaultValue: '0',
});
expect(contract.tables.sites.columns.use_system_proxy).toMatchObject({
logicalType: 'boolean',
defaultValue: 'false',
});
expect(contract.tables.token_routes.columns.routing_strategy).toMatchObject({
logicalType: 'text',
defaultValue: "'weighted'",
});
expect(contract.indexes).toContainEqual(
expect.objectContaining({ name: 'sites_status_idx', table: 'sites', unique: false }),
);
expect(contract.uniques).toContainEqual(
expect.objectContaining({
name: 'site_disabled_models_site_model_unique',
table: 'site_disabled_models',
columns: ['site_id', 'model_name'],
}),
);
expect(contract.uniques).toContainEqual(
expect.objectContaining({
name: 'model_availability_account_model_unique',
table: 'model_availability',
columns: ['account_id', 'model_name'],
}),
);
expect(contract.foreignKeys).toContainEqual(
expect.objectContaining({
table: 'site_disabled_models',
columns: ['site_id'],
referencedTable: 'sites',
referencedColumns: ['id'],
}),
);
expect(contract.foreignKeys).toContainEqual(
expect.objectContaining({
table: 'route_channels',
columns: ['token_id'],
referencedTable: 'account_tokens',
referencedColumns: ['id'],
}),
);
});
});
src/server/db/schemaContract.ts
-323
View File
@@ -1,323 +0,0 @@
import Database from 'better-sqlite3';
import { mkdirSync, readFileSync, readdirSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { dirname, join, resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
export type LogicalColumnType =
| 'integer'
| 'real'
| 'text'
| 'boolean'
| 'datetime'
| 'json';
export interface SchemaContractColumn {
logicalType: LogicalColumnType;
notNull: boolean;
defaultValue: string | null;
primaryKey: boolean;
}
export interface SchemaContractTable {
columns: Record<string, SchemaContractColumn>;
}
export interface SchemaContractIndex {
name: string;
table: string;
columns: string[];
unique: boolean;
}
export interface SchemaContractUnique {
name: string;
table: string;
columns: string[];
}
export interface SchemaContractForeignKey {
table: string;
columns: string[];
referencedTable: string;
referencedColumns: string[];
onDelete: string | null;
}
export interface SchemaContract {
tables: Record<string, SchemaContractTable>;
indexes: SchemaContractIndex[];
uniques: SchemaContractUnique[];
foreignKeys: SchemaContractForeignKey[];
}
type TableInfoRow = {
name: string;
type: string;
notnull: number;
dflt_value: string | null;
pk: number;
};
type IndexListRow = {
name: string;
unique: number;
origin: string;
partial: number;
};
type IndexInfoRow = {
seqno: number;
cid: number;
name: string;
};
type ForeignKeyRow = {
id: number;
seq: number;
table: string;
from: string;
to: string;
on_update: string;
on_delete: string;
match: string;
};
function resolveDbDir(): string {
return dirname(fileURLToPath(import.meta.url));
}
export function resolveMigrationsFolder(): string {
return resolve(resolveDbDir(), '../../../drizzle');
}
export function resolveGeneratedSchemaContractPath(): string {
return resolve(resolveDbDir(), 'generated/schemaContract.json');
}
function resolveMigrationFiles(migrationsFolder: string): string[] {
return readdirSync(migrationsFolder)
.filter((entry) => entry.endsWith('.sql'))
.sort((left, right) => left.localeCompare(right, 'en'));
}
function splitMigrationStatements(sqlText: string): string[] {
return sqlText
.split('--> statement-breakpoint')
.map((statement) => statement.trim())
.filter((statement) => statement.length > 0);
}
function applySqliteMigrations(sqlite: Database.Database, migrationsFolder: string): void {
for (const migrationFile of resolveMigrationFiles(migrationsFolder)) {
const sqlText = readFileSync(join(migrationsFolder, migrationFile), 'utf8');
for (const statement of splitMigrationStatements(sqlText)) {
sqlite.exec(statement);
}
}
}
function normalizeDefaultValue(defaultValue: string | null): string | null {
if (defaultValue == null) return null;
return String(defaultValue).trim() || null;
}
function isBooleanLikeColumn(columnName: string, defaultValue: string | null): boolean {
const normalizedColumn = columnName.toLowerCase();
const normalizedDefault = (defaultValue || '').trim().toLowerCase();
if (normalizedDefault === 'true' || normalizedDefault === 'false') {
return true;
}
return normalizedColumn.startsWith('is_')
|| normalizedColumn.startsWith('use_')
|| normalizedColumn.startsWith('has_')
|| normalizedColumn.endsWith('_enabled')
|| normalizedColumn.endsWith('_available')
|| normalizedColumn === 'read'
|| normalizedColumn === 'enabled'
|| normalizedColumn === 'available'
|| normalizedColumn === 'manual_override';
}
function isDateTimeLikeColumn(columnName: string, defaultValue: string | null): boolean {
const normalizedColumn = columnName.toLowerCase();
const normalizedDefault = (defaultValue || '').toLowerCase();
return normalizedColumn.endsWith('_at')
|| normalizedColumn.endsWith('_until')
|| normalizedColumn.endsWith('_refresh')
|| normalizedDefault.includes('datetime(')
|| normalizedDefault.includes('current_timestamp');
}
function isJsonLikeColumn(columnName: string): boolean {
const normalizedColumn = columnName.toLowerCase();
return normalizedColumn.endsWith('_json')
|| normalizedColumn.includes('snapshot')
|| normalizedColumn.includes('mapping')
|| normalizedColumn.includes('headers')
|| normalizedColumn.includes('config')
|| normalizedColumn.includes('details')
|| normalizedColumn.includes('meta')
|| normalizedColumn.includes('models')
|| normalizedColumn.includes('route_ids')
|| normalizedColumn.includes('multipliers');
}
function normalizeLogicalType(columnName: string, declaredType: string, defaultValue: string | null): LogicalColumnType {
const normalizedType = declaredType.trim().toLowerCase();
if (normalizedType.includes('int')) {
return isBooleanLikeColumn(columnName, defaultValue) ? 'boolean' : 'integer';
}
if (normalizedType.includes('real') || normalizedType.includes('double') || normalizedType.includes('float')) {
return 'real';
}
if (normalizedType.includes('text') || normalizedType.includes('char') || normalizedType.includes('clob')) {
if (isDateTimeLikeColumn(columnName, defaultValue)) return 'datetime';
if (isJsonLikeColumn(columnName)) return 'json';
return 'text';
}
if (isDateTimeLikeColumn(columnName, defaultValue)) return 'datetime';
return 'text';
}
function readTables(sqlite: Database.Database): Record<string, SchemaContractTable> {
const tables = sqlite.prepare(`
SELECT name
FROM sqlite_master
WHERE type = 'table'
AND name NOT LIKE 'sqlite_%'
ORDER BY name ASC
`).all() as Array<{ name: string }>;
const result: Record<string, SchemaContractTable> = {};
for (const table of tables) {
const rows = sqlite.prepare(`PRAGMA table_info("${table.name}")`).all() as TableInfoRow[];
const columns = Object.fromEntries(rows.map((row) => {
const defaultValue = normalizeDefaultValue(row.dflt_value);
return [row.name, {
logicalType: normalizeLogicalType(row.name, row.type, defaultValue),
notNull: row.notnull === 1,
defaultValue,
primaryKey: row.pk > 0,
} satisfies SchemaContractColumn];
}));
result[table.name] = { columns };
}
return result;
}
function readIndexes(sqlite: Database.Database, tables: Record<string, SchemaContractTable>): {
indexes: SchemaContractIndex[];
uniques: SchemaContractUnique[];
} {
const indexes: SchemaContractIndex[] = [];
const uniques: SchemaContractUnique[] = [];
for (const tableName of Object.keys(tables).sort((left, right) => left.localeCompare(right, 'en'))) {
const rows = sqlite.prepare(`PRAGMA index_list("${tableName}")`).all() as IndexListRow[];
for (const row of rows) {
if (!row.name || row.name.startsWith('sqlite_autoindex')) {
continue;
}
const indexColumns = (sqlite.prepare(`PRAGMA index_info("${row.name}")`).all() as IndexInfoRow[])
.sort((left, right) => left.seqno - right.seqno)
.map((item) => item.name)
.filter((name) => !!name);
const index: SchemaContractIndex = {
name: row.name,
table: tableName,
columns: indexColumns,
unique: row.unique === 1,
};
indexes.push(index);
if (row.unique === 1) {
uniques.push({
name: row.name,
table: tableName,
columns: indexColumns,
});
}
}
}
indexes.sort((left, right) => left.name.localeCompare(right.name, 'en'));
uniques.sort((left, right) => left.name.localeCompare(right.name, 'en'));
return { indexes, uniques };
}
function readForeignKeys(sqlite: Database.Database, tables: Record<string, SchemaContractTable>): SchemaContractForeignKey[] {
const foreignKeys: SchemaContractForeignKey[] = [];
for (const tableName of Object.keys(tables).sort((left, right) => left.localeCompare(right, 'en'))) {
const rows = sqlite.prepare(`PRAGMA foreign_key_list("${tableName}")`).all() as ForeignKeyRow[];
const grouped = new Map<number, SchemaContractForeignKey>();
for (const row of rows) {
const existing = grouped.get(row.id);
if (existing) {
existing.columns.push(row.from);
existing.referencedColumns.push(row.to);
continue;
}
grouped.set(row.id, {
table: tableName,
columns: [row.from],
referencedTable: row.table,
referencedColumns: [row.to],
onDelete: row.on_delete || null,
});
}
foreignKeys.push(...grouped.values());
}
foreignKeys.sort((left, right) => {
const leftKey = `${left.table}:${left.columns.join(',')}`;
const rightKey = `${right.table}:${right.columns.join(',')}`;
return leftKey.localeCompare(rightKey, 'en');
});
return foreignKeys;
}
export function buildSchemaContractFromSqliteMigrations(migrationsFolder = resolveMigrationsFolder()): SchemaContract {
const sqlite = new Database(':memory:');
sqlite.pragma('foreign_keys = ON');
try {
applySqliteMigrations(sqlite, migrationsFolder);
const tables = readTables(sqlite);
const { indexes, uniques } = readIndexes(sqlite, tables);
const foreignKeys = readForeignKeys(sqlite, tables);
return {
tables,
indexes,
uniques,
foreignKeys,
};
} finally {
sqlite.close();
}
}
export function writeSchemaContractFile(outputPath = resolveGeneratedSchemaContractPath()): SchemaContract {
const contract = buildSchemaContractFromSqliteMigrations();
mkdirSync(dirname(outputPath), { recursive: true });
writeFileSync(outputPath, `${JSON.stringify(contract, null, 2)}\n`, 'utf8');
return contract;
}
export const __schemaContractTestUtils = {
splitMigrationStatements,
normalizeDefaultValue,
normalizeLogicalType,
resolveMigrationsFolder,
};
src/server/db/schemaIntrospection.test.ts
-16
View File
@@ -1,16 +0,0 @@
import { describe, expect, it } from 'vitest';
import { normalizeDefaultValue, normalizeSqlType } from './schemaIntrospection.js';
describe('schema introspection normalization', () => {
it('normalizes booleans consistently across dialects', () => {
expect(normalizeSqlType('sqlite', 'INTEGER', 'use_system_proxy')).toBe('boolean');
expect(normalizeSqlType('mysql', 'tinyint', 'use_system_proxy')).toBe('boolean');
expect(normalizeSqlType('postgres', 'boolean', 'use_system_proxy')).toBe('boolean');
});
it('normalizes common default values', () => {
expect(normalizeDefaultValue("DEFAULT 'active'")).toBe("'active'");
expect(normalizeDefaultValue('DEFAULT FALSE')).toBe('false');
expect(normalizeDefaultValue("datetime('now')")).toBe("datetime('now')");
});
});
src/server/db/schemaIntrospection.ts
-910
View File
@@ -1,910 +0,0 @@
import Database from 'better-sqlite3';
import { mkdtempSync, mkdirSync, readFileSync, readdirSync } from 'node:fs';
import mysql from 'mysql2/promise';
import { tmpdir } from 'node:os';
import pg from 'pg';
import { join, resolve } from 'node:path';
import { randomUUID } from 'node:crypto';
import {
generateBootstrapSql,
generateUpgradeSql,
resolveGeneratedArtifactPath,
} from './schemaArtifactGenerator.js';
import type {
LogicalColumnType,
SchemaContract,
SchemaContractColumn,
SchemaContractForeignKey,
SchemaContractIndex,
SchemaContractTable,
SchemaContractUnique,
} from './schemaContract.js';
import { resolveMigrationsFolder } from './schemaContract.js';
export type SchemaIntrospectionDialect = 'sqlite' | 'mysql' | 'postgres';
export interface SchemaIntrospectionInput {
dialect: SchemaIntrospectionDialect;
connectionString: string;
ssl?: boolean;
}
export interface MaterializeFreshSchemaOptions {
connectionString?: string;
ssl?: boolean;
}
export interface ApplyContractFixtureThenUpgradeOptions extends MaterializeFreshSchemaOptions {}
type SqliteTableInfoRow = {
name: string;
type: string;
notnull: number;
dflt_value: string | null;
pk: number;
};
type SqliteIndexListRow = {
name: string;
unique: number;
};
type SqliteIndexInfoRow = {
seqno: number;
name: string;
};
type SqliteForeignKeyRow = {
id: number;
table: string;
from: string;
to: string;
on_delete: string;
};
type MySqlColumnRow = {
table_name: string;
column_name: string;
data_type: string;
column_type: string;
is_nullable: 'YES' | 'NO';
column_default: string | null;
};
type MySqlPrimaryKeyRow = {
table_name: string;
column_name: string;
};
type MySqlIndexRow = {
table_name: string;
index_name: string;
column_name: string;
non_unique: number;
};
type MySqlForeignKeyRow = {
table_name: string;
constraint_name: string;
column_name: string;
referenced_table_name: string;
referenced_column_name: string;
delete_rule: string | null;
};
type PostgresColumnRow = {
table_name: string;
column_name: string;
data_type: string;
udt_name: string;
is_nullable: 'YES' | 'NO';
column_default: string | null;
};
type PostgresPrimaryKeyRow = {
table_name: string;
column_name: string;
};
type PostgresIndexRow = {
table_name: string;
index_name: string;
is_unique: boolean;
column_name: string;
};
type PostgresForeignKeyRow = {
table_name: string;
constraint_name: string;
column_name: string;
referenced_table_name: string;
referenced_column_name: string;
delete_rule: string | null;
};
function splitMigrationStatements(sqlText: string): string[] {
return sqlText
.split('--> statement-breakpoint')
.map((statement) => statement.trim())
.filter((statement) => statement.length > 0);
}
function splitSqlStatements(sqlText: string): string[] {
const withoutCommentLines = sqlText
.split(/\r?\n/g)
.filter((line) => !line.trim().startsWith('--'))
.join('\n');
return withoutCommentLines
.split(/;\s*(?:\r?\n|$)/g)
.map((statement) => statement.trim())
.filter((statement) => statement.length > 0);
}
function hasBalancedParentheses(value: string): boolean {
let depth = 0;
for (const char of value) {
if (char === '(') depth += 1;
if (char === ')') depth -= 1;
if (depth < 0) return false;
}
return depth === 0;
}
function unwrapSurroundingParentheses(value: string): string {
let normalized = value.trim();
while (normalized.startsWith('(') && normalized.endsWith(')')) {
const inner = normalized.slice(1, -1).trim();
if (!hasBalancedParentheses(inner)) {
break;
}
normalized = inner;
}
return normalized;
}
function normalizeDeleteRule(value: string | null | undefined): string | null {
if (!value) return null;
return value.trim().replace(/\s+/g, ' ').toUpperCase();
}
function isBooleanLikeColumn(columnName: string, defaultValue: string | null): boolean {
const normalizedColumn = columnName.toLowerCase();
const normalizedDefault = (defaultValue || '').trim().toLowerCase();
if (normalizedDefault === 'true' || normalizedDefault === 'false') {
return true;
}
return normalizedColumn.startsWith('is_')
|| normalizedColumn.startsWith('use_')
|| normalizedColumn.startsWith('has_')
|| normalizedColumn.endsWith('_enabled')
|| normalizedColumn.endsWith('_available')
|| normalizedColumn === 'read'
|| normalizedColumn === 'enabled'
|| normalizedColumn === 'available'
|| normalizedColumn === 'manual_override';
}
function isDateTimeLikeColumn(columnName: string, defaultValue: string | null): boolean {
const normalizedColumn = columnName.toLowerCase();
const normalizedDefault = (defaultValue || '').toLowerCase();
return normalizedColumn.endsWith('_at')
|| normalizedColumn.endsWith('_until')
|| normalizedColumn.endsWith('_refresh')
|| normalizedDefault.includes("datetime('now')")
|| normalizedDefault.includes('current_timestamp')
|| normalizedDefault.includes('now()');
}
function isJsonLikeColumn(columnName: string): boolean {
const normalizedColumn = columnName.toLowerCase();
return normalizedColumn.endsWith('_json')
|| normalizedColumn.includes('snapshot')
|| normalizedColumn.includes('mapping')
|| normalizedColumn.includes('headers')
|| normalizedColumn.includes('config')
|| normalizedColumn.includes('details')
|| normalizedColumn.includes('meta')
|| normalizedColumn.includes('models')
|| normalizedColumn.includes('route_ids')
|| normalizedColumn.includes('multipliers');
}
export function normalizeSqlType(
dialect: SchemaIntrospectionDialect,
declaredType: string,
columnName: string,
rawDefaultValue: string | null = null,
): LogicalColumnType {
const normalizedType = declaredType.trim().toLowerCase();
const normalizedDefault = normalizeDefaultValue(rawDefaultValue);
if (dialect === 'mysql') {
if (normalizedType.includes('tinyint(1)') || normalizedType.includes('boolean') || normalizedType.includes('bool')) {
return 'boolean';
}
if (normalizedType.includes('json')) {
return 'json';
}
if (normalizedType.includes('timestamp') || normalizedType.includes('datetime') || normalizedType === 'date') {
return 'datetime';
}
if (normalizedType.includes('int')) {
return isBooleanLikeColumn(columnName, normalizedDefault) ? 'boolean' : 'integer';
}
if (normalizedType.includes('double') || normalizedType.includes('float') || normalizedType.includes('real') || normalizedType.includes('decimal')) {
return 'real';
}
if (normalizedType.includes('char') || normalizedType.includes('text')) {
if (isDateTimeLikeColumn(columnName, normalizedDefault)) return 'datetime';
if (isJsonLikeColumn(columnName)) return 'json';
return 'text';
}
}
if (dialect === 'postgres') {
if (normalizedType.includes('bool')) {
return 'boolean';
}
if (normalizedType.includes('json')) {
return 'json';
}
if (normalizedType.includes('timestamp') || normalizedType === 'date') {
return 'datetime';
}
if (normalizedType.includes('int')) {
return isBooleanLikeColumn(columnName, normalizedDefault) ? 'boolean' : 'integer';
}
if (normalizedType.includes('double') || normalizedType.includes('real') || normalizedType.includes('numeric') || normalizedType.includes('decimal')) {
return 'real';
}
if (normalizedType.includes('char') || normalizedType.includes('text')) {
if (isDateTimeLikeColumn(columnName, normalizedDefault)) return 'datetime';
if (isJsonLikeColumn(columnName)) return 'json';
return 'text';
}
}
if (normalizedType.includes('int')) {
return isBooleanLikeColumn(columnName, normalizedDefault) ? 'boolean' : 'integer';
}
if (normalizedType.includes('real') || normalizedType.includes('double') || normalizedType.includes('float') || normalizedType.includes('decimal')) {
return 'real';
}
if (normalizedType.includes('json')) {
return 'json';
}
if (normalizedType.includes('timestamp') || normalizedType.includes('datetime') || normalizedType === 'date') {
return 'datetime';
}
if (normalizedType.includes('text') || normalizedType.includes('char') || normalizedType.includes('clob')) {
if (isDateTimeLikeColumn(columnName, normalizedDefault)) return 'datetime';
if (isJsonLikeColumn(columnName)) return 'json';
return 'text';
}
if (isDateTimeLikeColumn(columnName, normalizedDefault)) return 'datetime';
return 'text';
}
function normalizeDefaultValueForColumn(
rawDefaultValue: string | null | undefined,
logicalType: LogicalColumnType | null,
): string | null {
if (rawDefaultValue == null) return null;
let normalized = String(rawDefaultValue).trim();
if (!normalized) return null;
normalized = normalized.replace(/^default\s+/i, '').trim();
normalized = unwrapSurroundingParentheses(normalized);
normalized = normalized.replace(/::[\w\s.\[\]"]+/g, '').trim();
normalized = unwrapSurroundingParentheses(normalized);
const lowered = normalized.toLowerCase();
if (lowered === 'null') return null;
if (logicalType === 'datetime' || lowered === 'current_timestamp' || lowered === 'current_timestamp()' || lowered === 'now()' || lowered.includes("datetime('now')")) {
return "datetime('now')";
}
if (logicalType === 'boolean') {
if (lowered === '1' || lowered === 'true' || lowered === "b'1'") return 'true';
if (lowered === '0' || lowered === 'false' || lowered === "b'0'") return 'false';
}
if (lowered === 'true' || lowered === 'false') return lowered;
if (/^-?\d+(?:\.\d+)?$/.test(normalized)) return normalized;
if (/^'.*'$/.test(normalized)) return normalized;
if (/^[a-z_][a-z0-9_]*$/i.test(normalized)) return `'${normalized}'`;
return normalized;
}
export function normalizeDefaultValue(rawDefaultValue: string | null | undefined): string | null {
return normalizeDefaultValueForColumn(rawDefaultValue, null);
}
function sortForeignKeys(foreignKeys: SchemaContractForeignKey[]): SchemaContractForeignKey[] {
return foreignKeys.sort((left, right) => {
const leftKey = `${left.table}:${left.columns.join(',')}`;
const rightKey = `${right.table}:${right.columns.join(',')}`;
return leftKey.localeCompare(rightKey, 'en');
});
}
function buildSqliteTables(sqlite: Database.Database): Record<string, SchemaContractTable> {
const tableRows = sqlite.prepare(`
SELECT name
FROM sqlite_master
WHERE type = 'table'
AND name NOT LIKE 'sqlite_%'
ORDER BY name ASC
`).all() as Array<{ name: string }>;
const tables: Record<string, SchemaContractTable> = {};
for (const { name: tableName } of tableRows) {
const rows = sqlite.prepare(`PRAGMA table_info("${tableName}")`).all() as SqliteTableInfoRow[];
const columns = Object.fromEntries(rows.map((row) => {
const logicalType = normalizeSqlType('sqlite', row.type, row.name, row.dflt_value);
return [row.name, {
logicalType,
notNull: row.notnull === 1,
defaultValue: normalizeDefaultValueForColumn(row.dflt_value, logicalType),
primaryKey: row.pk > 0,
} satisfies SchemaContractColumn];
}));
tables[tableName] = { columns };
}
return tables;
}
function buildSqliteIndexes(sqlite: Database.Database, tables: Record<string, SchemaContractTable>): {
indexes: SchemaContractIndex[];
uniques: SchemaContractUnique[];
} {
const indexes: SchemaContractIndex[] = [];
const uniques: SchemaContractUnique[] = [];
for (const tableName of Object.keys(tables).sort((left, right) => left.localeCompare(right, 'en'))) {
const rows = sqlite.prepare(`PRAGMA index_list("${tableName}")`).all() as SqliteIndexListRow[];
for (const row of rows) {
if (!row.name || row.name.startsWith('sqlite_autoindex')) {
continue;
}
const columns = (sqlite.prepare(`PRAGMA index_info("${row.name}")`).all() as SqliteIndexInfoRow[])
.sort((left, right) => left.seqno - right.seqno)
.map((item) => item.name)
.filter(Boolean);
indexes.push({
name: row.name,
table: tableName,
columns,
unique: row.unique === 1,
});
if (row.unique === 1) {
uniques.push({
name: row.name,
table: tableName,
columns,
});
}
}
}
indexes.sort((left, right) => left.name.localeCompare(right.name, 'en'));
uniques.sort((left, right) => left.name.localeCompare(right.name, 'en'));
return { indexes, uniques };
}
function buildSqliteForeignKeys(sqlite: Database.Database, tables: Record<string, SchemaContractTable>): SchemaContractForeignKey[] {
const foreignKeys: SchemaContractForeignKey[] = [];
for (const tableName of Object.keys(tables).sort((left, right) => left.localeCompare(right, 'en'))) {
const rows = sqlite.prepare(`PRAGMA foreign_key_list("${tableName}")`).all() as SqliteForeignKeyRow[];
const grouped = new Map<number, SchemaContractForeignKey>();
for (const row of rows) {
const existing = grouped.get(row.id);
if (existing) {
existing.columns.push(row.from);
existing.referencedColumns.push(row.to);
continue;
}
grouped.set(row.id, {
table: tableName,
columns: [row.from],
referencedTable: row.table,
referencedColumns: [row.to],
onDelete: normalizeDeleteRule(row.on_delete),
});
}
foreignKeys.push(...grouped.values());
}
return sortForeignKeys(foreignKeys);
}
async function introspectSqliteSchema(connectionString: string): Promise<SchemaContract> {
const sqlitePath = connectionString === ':memory:' ? ':memory:' : resolve(connectionString);
const sqlite = new Database(sqlitePath, { readonly: true });
sqlite.pragma('foreign_keys = ON');
try {
const tables = buildSqliteTables(sqlite);
const { indexes, uniques } = buildSqliteIndexes(sqlite, tables);
const foreignKeys = buildSqliteForeignKeys(sqlite, tables);
return { tables, indexes, uniques, foreignKeys };
} finally {
sqlite.close();
}
}
async function introspectMySqlSchema(input: SchemaIntrospectionInput): Promise<SchemaContract> {
const connectionOptions: mysql.ConnectionOptions = { uri: input.connectionString };
if (input.ssl) {
connectionOptions.ssl = { rejectUnauthorized: false };
}
const connection = await mysql.createConnection(connectionOptions);
try {
const [tableRows] = await connection.query(`
SELECT table_name
FROM information_schema.tables
WHERE table_schema = DATABASE()
AND table_type = 'BASE TABLE'
ORDER BY table_name ASC
`);
const tableNames = (tableRows as Array<{ table_name: string }>).map((row) => row.table_name);
const [primaryKeyRows] = await connection.query(`
SELECT kcu.table_name, kcu.column_name
FROM information_schema.table_constraints tc
JOIN information_schema.key_column_usage kcu
ON tc.constraint_schema = kcu.constraint_schema
AND tc.constraint_name = kcu.constraint_name
AND tc.table_name = kcu.table_name
WHERE tc.table_schema = DATABASE()
AND tc.constraint_type = 'PRIMARY KEY'
`);
const primaryKeys = new Set((primaryKeyRows as MySqlPrimaryKeyRow[]).map((row) => `${row.table_name}.${row.column_name}`));
const [columnRows] = await connection.query(`
SELECT table_name, column_name, data_type, column_type, is_nullable, column_default
FROM information_schema.columns
WHERE table_schema = DATABASE()
ORDER BY table_name ASC, ordinal_position ASC
`);
const tableMap = new Map<string, SchemaContractTable>();
for (const tableName of tableNames) {
tableMap.set(tableName, { columns: {} });
}
for (const row of columnRows as MySqlColumnRow[]) {
const logicalType = normalizeSqlType('mysql', row.column_type || row.data_type, row.column_name, row.column_default);
tableMap.get(row.table_name)!.columns[row.column_name] = {
logicalType,
notNull: row.is_nullable === 'NO',
defaultValue: primaryKeys.has(`${row.table_name}.${row.column_name}`)
? null
: normalizeDefaultValueForColumn(row.column_default, logicalType),
primaryKey: primaryKeys.has(`${row.table_name}.${row.column_name}`),
};
}
const tables = Object.fromEntries(tableNames.map((tableName) => [tableName, tableMap.get(tableName)!]));
const [indexRows] = await connection.query(`
SELECT table_name, index_name, column_name, non_unique
FROM information_schema.statistics
WHERE table_schema = DATABASE()
AND index_name <> 'PRIMARY'
ORDER BY table_name ASC, index_name ASC, seq_in_index ASC
`);
const indexGroups = new Map<string, SchemaContractIndex>();
for (const row of indexRows as MySqlIndexRow[]) {
const key = `${row.table_name}:${row.index_name}`;
const existing = indexGroups.get(key);
if (existing) {
existing.columns.push(row.column_name);
continue;
}
indexGroups.set(key, {
name: row.index_name,
table: row.table_name,
columns: [row.column_name],
unique: Number(row.non_unique) === 0,
});
}
const indexes = [...indexGroups.values()].sort((left, right) => left.name.localeCompare(right.name, 'en'));
const uniques = indexes
.filter((index) => index.unique)
.map((index) => ({ name: index.name, table: index.table, columns: [...index.columns] }))
.sort((left, right) => left.name.localeCompare(right.name, 'en'));
const [foreignKeyRows] = await connection.query(`
SELECT
kcu.table_name,
kcu.constraint_name,
kcu.column_name,
kcu.referenced_table_name,
kcu.referenced_column_name,
rc.delete_rule
FROM information_schema.key_column_usage kcu
JOIN information_schema.referential_constraints rc
ON rc.constraint_schema = kcu.constraint_schema
AND rc.constraint_name = kcu.constraint_name
WHERE kcu.table_schema = DATABASE()
AND kcu.referenced_table_name IS NOT NULL
ORDER BY kcu.table_name ASC, kcu.constraint_name ASC, kcu.ordinal_position ASC
`);
const foreignKeyGroups = new Map<string, SchemaContractForeignKey>();
for (const row of foreignKeyRows as MySqlForeignKeyRow[]) {
const key = `${row.table_name}:${row.constraint_name}`;
const existing = foreignKeyGroups.get(key);
if (existing) {
existing.columns.push(row.column_name);
existing.referencedColumns.push(row.referenced_column_name);
continue;
}
foreignKeyGroups.set(key, {
table: row.table_name,
columns: [row.column_name],
referencedTable: row.referenced_table_name,
referencedColumns: [row.referenced_column_name],
onDelete: normalizeDeleteRule(row.delete_rule),
});
}
return {
tables,
indexes,
uniques,
foreignKeys: sortForeignKeys([...foreignKeyGroups.values()]),
};
} finally {
await connection.end();
}
}
async function introspectPostgresSchema(input: SchemaIntrospectionInput): Promise<SchemaContract> {
const clientOptions: pg.ClientConfig = { connectionString: input.connectionString };
if (input.ssl) {
clientOptions.ssl = { rejectUnauthorized: false };
}
const client = new pg.Client(clientOptions);
await client.connect();
try {
const tableResult = await client.query(`
SELECT table_name
FROM information_schema.tables
WHERE table_schema = current_schema()
AND table_type = 'BASE TABLE'
ORDER BY table_name ASC
`);
const tableNames = tableResult.rows.map((row) => String(row.table_name));
const primaryKeyResult = await client.query(`
SELECT kcu.table_name, kcu.column_name
FROM information_schema.table_constraints tc
JOIN information_schema.key_column_usage kcu
ON tc.constraint_schema = kcu.constraint_schema
AND tc.constraint_name = kcu.constraint_name
AND tc.table_name = kcu.table_name
WHERE tc.table_schema = current_schema()
AND tc.constraint_type = 'PRIMARY KEY'
`);
const primaryKeys = new Set((primaryKeyResult.rows as PostgresPrimaryKeyRow[]).map((row) => `${row.table_name}.${row.column_name}`));
const columnResult = await client.query(`
SELECT table_name, column_name, data_type, udt_name, is_nullable, column_default
FROM information_schema.columns
WHERE table_schema = current_schema()
ORDER BY table_name ASC, ordinal_position ASC
`);
const tableMap = new Map<string, SchemaContractTable>();
for (const tableName of tableNames) {
tableMap.set(tableName, { columns: {} });
}
for (const row of columnResult.rows as PostgresColumnRow[]) {
const logicalType = normalizeSqlType('postgres', `${row.data_type} ${row.udt_name}`, row.column_name, row.column_default);
tableMap.get(row.table_name)!.columns[row.column_name] = {
logicalType,
notNull: row.is_nullable === 'NO',
defaultValue: primaryKeys.has(`${row.table_name}.${row.column_name}`)
? null
: normalizeDefaultValueForColumn(row.column_default, logicalType),
primaryKey: primaryKeys.has(`${row.table_name}.${row.column_name}`),
};
}
const tables = Object.fromEntries(tableNames.map((tableName) => [tableName, tableMap.get(tableName)!]));
const indexResult = await client.query(`
SELECT
t.relname AS table_name,
i.relname AS index_name,
ix.indisunique AS is_unique,
a.attname AS column_name
FROM pg_class t
JOIN pg_namespace ns ON ns.oid = t.relnamespace
JOIN pg_index ix ON t.oid = ix.indrelid
JOIN pg_class i ON i.oid = ix.indexrelid
JOIN LATERAL unnest(ix.indkey) WITH ORDINALITY AS ord(attnum, n) ON TRUE
JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = ord.attnum
WHERE ns.nspname = current_schema()
AND t.relkind = 'r'
AND NOT ix.indisprimary
ORDER BY t.relname ASC, i.relname ASC, ord.n ASC
`);
const indexGroups = new Map<string, SchemaContractIndex>();
for (const row of indexResult.rows as PostgresIndexRow[]) {
const key = `${row.table_name}:${row.index_name}`;
const existing = indexGroups.get(key);
if (existing) {
existing.columns.push(row.column_name);
continue;
}
indexGroups.set(key, {
name: row.index_name,
table: row.table_name,
columns: [row.column_name],
unique: !!row.is_unique,
});
}
const indexes = [...indexGroups.values()].sort((left, right) => left.name.localeCompare(right.name, 'en'));
const uniques = indexes
.filter((index) => index.unique)
.map((index) => ({ name: index.name, table: index.table, columns: [...index.columns] }))
.sort((left, right) => left.name.localeCompare(right.name, 'en'));
const foreignKeyResult = await client.query(`
SELECT
tc.table_name,
tc.constraint_name,
kcu.column_name,
ccu.table_name AS referenced_table_name,
ccu.column_name AS referenced_column_name,
rc.delete_rule
FROM information_schema.table_constraints tc
JOIN information_schema.key_column_usage kcu
ON tc.constraint_schema = kcu.constraint_schema
AND tc.constraint_name = kcu.constraint_name
AND tc.table_name = kcu.table_name
JOIN information_schema.referential_constraints rc
ON tc.constraint_schema = rc.constraint_schema
AND tc.constraint_name = rc.constraint_name
JOIN information_schema.constraint_column_usage ccu
ON rc.unique_constraint_schema = ccu.constraint_schema
AND rc.unique_constraint_name = ccu.constraint_name
WHERE tc.table_schema = current_schema()
AND tc.constraint_type = 'FOREIGN KEY'
ORDER BY tc.table_name ASC, tc.constraint_name ASC, kcu.ordinal_position ASC
`);
const foreignKeyGroups = new Map<string, SchemaContractForeignKey>();
for (const row of foreignKeyResult.rows as PostgresForeignKeyRow[]) {
const key = `${row.table_name}:${row.constraint_name}`;
const existing = foreignKeyGroups.get(key);
if (existing) {
existing.columns.push(row.column_name);
existing.referencedColumns.push(row.referenced_column_name);
continue;
}
foreignKeyGroups.set(key, {
table: row.table_name,
columns: [row.column_name],
referencedTable: row.referenced_table_name,
referencedColumns: [row.referenced_column_name],
onDelete: normalizeDeleteRule(row.delete_rule),
});
}
return {
tables,
indexes,
uniques,
foreignKeys: sortForeignKeys([...foreignKeyGroups.values()]),
};
} finally {
await client.end();
}
}
export async function introspectLiveSchema(input: SchemaIntrospectionInput): Promise<SchemaContract> {
if (input.dialect === 'sqlite') {
return introspectSqliteSchema(input.connectionString);
}
if (input.dialect === 'mysql') {
return introspectMySqlSchema(input);
}
return introspectPostgresSchema(input);
}
function readBootstrapSql(dialect: Exclude<SchemaIntrospectionDialect, 'sqlite'>): string {
const filename = dialect === 'mysql' ? 'mysql.bootstrap.sql' : 'postgres.bootstrap.sql';
return readFileSync(resolveGeneratedArtifactPath(filename), 'utf8');
}
async function resetMySqlSchema(connection: mysql.Connection): Promise<void> {
await connection.query('SET FOREIGN_KEY_CHECKS = 0');
const [rows] = await connection.query(`
SELECT table_name
FROM information_schema.tables
WHERE table_schema = DATABASE()
AND table_type = 'BASE TABLE'
`);
for (const row of rows as Array<{ table_name: string }>) {
await connection.query(`DROP TABLE IF EXISTS \`${row.table_name}\``);
}
await connection.query('SET FOREIGN_KEY_CHECKS = 1');
}
async function resetPostgresSchema(client: pg.Client): Promise<void> {
const result = await client.query(`
SELECT tablename
FROM pg_tables
WHERE schemaname = current_schema()
ORDER BY tablename ASC
`);
for (const row of result.rows as Array<{ tablename: string }>) {
await client.query(`DROP TABLE IF EXISTS "${row.tablename}" CASCADE`);
}
}
function applySqliteMigrations(sqlite: Database.Database): void {
const migrationsFolder = resolveMigrationsFolder();
const migrationFiles = readdirSync(migrationsFolder)
.filter((entry) => entry.endsWith('.sql'))
.sort((left, right) => left.localeCompare(right, 'en'));
for (const migrationFile of migrationFiles) {
const sqlText = readFileSync(join(migrationsFolder, migrationFile), 'utf8');
for (const statement of splitMigrationStatements(sqlText)) {
sqlite.exec(statement);
}
}
}
function createTemporarySqlitePath(): string {
const tempDir = mkdtempSync(join(tmpdir(), 'metapi-schema-parity-'));
return resolve(tempDir, `${randomUUID()}.db`);
}
function applySqliteStatements(sqlitePath: string, statements: string[]): void {
const sqlite = new Database(sqlitePath);
sqlite.pragma('foreign_keys = ON');
try {
for (const statement of statements) {
sqlite.exec(statement);
}
} finally {
sqlite.close();
}
}
async function applyMySqlStatements(
connectionString: string,
ssl: boolean | undefined,
statements: string[],
resetSchema = false,
): Promise<void> {
const connectionOptions: mysql.ConnectionOptions = { uri: connectionString };
if (ssl) {
connectionOptions.ssl = { rejectUnauthorized: false };
}
const connection = await mysql.createConnection(connectionOptions);
try {
if (resetSchema) {
await resetMySqlSchema(connection);
}
for (const statement of statements) {
await connection.query(statement);
}
} finally {
await connection.end();
}
}
async function applyPostgresStatements(
connectionString: string,
ssl: boolean | undefined,
statements: string[],
resetSchema = false,
): Promise<void> {
const clientOptions: pg.ClientConfig = { connectionString };
if (ssl) {
clientOptions.ssl = { rejectUnauthorized: false };
}
const client = new pg.Client(clientOptions);
await client.connect();
try {
if (resetSchema) {
await resetPostgresSchema(client);
}
for (const statement of statements) {
await client.query(statement);
}
} finally {
await client.end();
}
}
export async function materializeFreshSchema(
dialect: SchemaIntrospectionDialect,
options: MaterializeFreshSchemaOptions = {},
): Promise<string> {
if (dialect === 'sqlite') {
const sqlitePath = createTemporarySqlitePath();
const sqlite = new Database(sqlitePath);
sqlite.pragma('foreign_keys = ON');
try {
applySqliteMigrations(sqlite);
} finally {
sqlite.close();
}
return sqlitePath;
}
if (!options.connectionString) {
throw new Error(`connectionString is required to materialize ${dialect} parity schema`);
}
const bootstrapStatements = splitSqlStatements(readBootstrapSql(dialect));
if (dialect === 'mysql') {
await applyMySqlStatements(options.connectionString, options.ssl, bootstrapStatements, true);
return options.connectionString;
}
await applyPostgresStatements(options.connectionString, options.ssl, bootstrapStatements, true);
return options.connectionString;
}
export async function applyContractFixtureThenUpgrade(
dialect: SchemaIntrospectionDialect,
baselineContract: SchemaContract,
currentContract: SchemaContract,
options: ApplyContractFixtureThenUpgradeOptions = {},
): Promise<string> {
const bootstrapStatements = splitSqlStatements(generateBootstrapSql(dialect, baselineContract));
const upgradeStatements = splitSqlStatements(generateUpgradeSql(dialect, currentContract, baselineContract));
if (dialect === 'sqlite') {
const sqlitePath = createTemporarySqlitePath();
applySqliteStatements(sqlitePath, bootstrapStatements);
if (upgradeStatements.length > 0) {
applySqliteStatements(sqlitePath, upgradeStatements);
}
return sqlitePath;
}
if (!options.connectionString) {
throw new Error(`connectionString is required to upgrade ${dialect} parity schema`);
}
if (dialect === 'mysql') {
await applyMySqlStatements(options.connectionString, options.ssl, bootstrapStatements, true);
if (upgradeStatements.length > 0) {
await applyMySqlStatements(options.connectionString, options.ssl, upgradeStatements, false);
}
return options.connectionString;
}
await applyPostgresStatements(options.connectionString, options.ssl, bootstrapStatements, true);
if (upgradeStatements.length > 0) {
await applyPostgresStatements(options.connectionString, options.ssl, upgradeStatements, false);
}
return options.connectionString;
}
export const __schemaIntrospectionTestUtils = {
splitMigrationStatements,
splitSqlStatements,
};
src/server/db/schemaParity.live.test.ts
-37
View File
@@ -1,37 +0,0 @@
import { readFileSync } from 'node:fs';
import { dirname, resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
import { describe, expect, it } from 'vitest';
import type { SchemaContract } from './schemaContract.js';
import { introspectLiveSchema, materializeFreshSchema } from './schemaIntrospection.js';
const dbDir = dirname(fileURLToPath(import.meta.url));
const schemaContractPath = resolve(dbDir, 'generated/schemaContract.json');
const contract = JSON.parse(readFileSync(schemaContractPath, 'utf8')) as SchemaContract;
const mysqlParity = process.env.DB_PARITY_MYSQL_URL ? it : it.skip;
const postgresParity = process.env.DB_PARITY_POSTGRES_URL ? it : it.skip;
describe('live schema parity', () => {
it('matches the contract for sqlite', async () => {
const sqliteUrl = await materializeFreshSchema('sqlite');
const live = await introspectLiveSchema({ dialect: 'sqlite', connectionString: sqliteUrl });
expect(live).toEqual(contract);
});
mysqlParity('matches the contract for mysql', async () => {
const mysqlUrl = await materializeFreshSchema('mysql', {
connectionString: process.env.DB_PARITY_MYSQL_URL!,
});
const live = await introspectLiveSchema({ dialect: 'mysql', connectionString: mysqlUrl });
expect(live).toEqual(contract);
});
postgresParity('matches the contract for postgres', async () => {
const postgresUrl = await materializeFreshSchema('postgres', {
connectionString: process.env.DB_PARITY_POSTGRES_URL!,
});
const live = await introspectLiveSchema({ dialect: 'postgres', connectionString: postgresUrl });
expect(live).toEqual(contract);
});
});
src/server/db/schemaParity.test.ts
+17 -38
View File
@@ -1,11 +1,10 @@
import { existsSync, readFileSync } from 'node:fs'; import { readFileSync } from 'node:fs';
import { dirname, resolve } from 'node:path'; import { dirname, resolve } from 'node:path';
import { fileURLToPath } from 'node:url'; import { fileURLToPath } from 'node:url';
import { describe, expect, it } from 'vitest'; import { describe, expect, it } from 'vitest';
import type { SchemaContract } from './schemaContract.js';
const dbDir = dirname(fileURLToPath(import.meta.url)); const dbDir = dirname(fileURLToPath(import.meta.url));
const generatedDir = resolve(dbDir, 'generated'); const schemaPath = resolve(dbDir, 'schema.ts');
const supportPaths = [ const supportPaths = [
resolve(dbDir, 'runtimeSchemaBootstrap.ts'), resolve(dbDir, 'runtimeSchemaBootstrap.ts'),
resolve(dbDir, 'siteSchemaCompatibility.ts'), resolve(dbDir, 'siteSchemaCompatibility.ts'),
@@ -14,53 +13,33 @@ const supportPaths = [
resolve(dbDir, 'accountTokenSchemaCompatibility.ts'), resolve(dbDir, 'accountTokenSchemaCompatibility.ts'),
resolve(dbDir, 'sharedIndexSchemaCompatibility.ts'), resolve(dbDir, 'sharedIndexSchemaCompatibility.ts'),
]; ];
const schemaContractPath = resolve(generatedDir, 'schemaContract.json');
function extractAllMatches(content: string, pattern: RegExp): string[] { function extractAllMatches(content: string, pattern: RegExp): string[] {
return Array.from(content.matchAll(pattern), (match) => match[1]); return Array.from(content.matchAll(pattern), (match) => match[1]);
} }
describe('database schema parity', () => { describe('database schema parity', () => {
it('keeps generated schema artifacts present', () => { it('keeps external runtime table support in sync with schema.ts', () => {
const artifactPaths = [ const schemaContent = readFileSync(schemaPath, 'utf8');
schemaContractPath,
resolve(generatedDir, 'mysql.bootstrap.sql'),
resolve(generatedDir, 'mysql.upgrade.sql'),
resolve(generatedDir, 'postgres.bootstrap.sql'),
resolve(generatedDir, 'postgres.upgrade.sql'),
];
for (const artifactPath of artifactPaths) {
expect(existsSync(artifactPath), artifactPath).toBe(true);
expect(readFileSync(artifactPath, 'utf8').trim().length).toBeGreaterThan(0);
}
});
it('keeps runtime support modules scoped to contract-defined tables and indexes', () => {
const contract = JSON.parse(readFileSync(schemaContractPath, 'utf8')) as SchemaContract;
const supportContent = supportPaths const supportContent = supportPaths
.map((filePath) => readFileSync(filePath, 'utf8')) .map((filePath) => readFileSync(filePath, 'utf8'))
.join('\n'); .join('\n');
const knownTables = new Set(Object.keys(contract.tables)); const schemaTables = extractAllMatches(schemaContent, /sqliteTable\('([^']+)'/g);
const knownIndexes = new Set([ const missingTables = schemaTables.filter((tableName) => !supportContent.includes(tableName));
...contract.indexes.map((index) => index.name),
...contract.uniques.map((unique) => unique.name),
]);
const supportTables = extractAllMatches( expect(missingTables).toEqual([]);
supportContent, });
/(?:CREATE TABLE IF NOT EXISTS|ALTER TABLE|INSERT INTO)\s+["`]?([a-z_][a-z0-9_]*)["`]?/gi,
);
const supportIndexes = extractAllMatches(
supportContent,
/(?:CREATE UNIQUE INDEX(?: IF NOT EXISTS)?|CREATE INDEX(?: IF NOT EXISTS)?|indexName:\s*')["`]?([a-z_][a-z0-9_]*)/gi,
);
const unknownTables = [...new Set(supportTables)].filter((tableName) => !knownTables.has(tableName)).sort(); it('keeps external runtime index support in sync with schema.ts', () => {
const unknownIndexes = [...new Set(supportIndexes)].filter((indexName) => !knownIndexes.has(indexName)).sort(); const schemaContent = readFileSync(schemaPath, 'utf8');
const supportContent = supportPaths
.map((filePath) => readFileSync(filePath, 'utf8'))
.join('\n');
expect(unknownTables).toEqual([]); const schemaIndexes = extractAllMatches(schemaContent, /(?:uniqueIndex|index)\('([^']+)'/g);
expect(unknownIndexes).toEqual([]); const missingIndexes = schemaIndexes.filter((indexName) => !supportContent.includes(indexName));
expect(missingIndexes).toEqual([]);
}); });
}); });
src/server/db/schemaUpgrade.live.test.ts
-31
View File
@@ -1,31 +0,0 @@
import baselineContract from './generated/fixtures/2026-03-14-baseline.schemaContract.json' with { type: 'json' };
import currentContract from './generated/schemaContract.json' with { type: 'json' };
import { applyContractFixtureThenUpgrade, introspectLiveSchema } from './schemaIntrospection.js';
import { describe, expect, it } from 'vitest';
const mysqlUpgrade = process.env.DB_PARITY_MYSQL_URL ? it : it.skip;
const postgresUpgrade = process.env.DB_PARITY_POSTGRES_URL ? it : it.skip;
describe('schema upgrade parity', () => {
it('upgrades sqlite to the current contract', async () => {
const sqliteUrl = await applyContractFixtureThenUpgrade('sqlite', baselineContract, currentContract);
const live = await introspectLiveSchema({ dialect: 'sqlite', connectionString: sqliteUrl });
expect(live).toEqual(currentContract);
});
mysqlUpgrade('upgrades mysql to the current contract', async () => {
const mysqlUrl = await applyContractFixtureThenUpgrade('mysql', baselineContract, currentContract, {
connectionString: process.env.DB_PARITY_MYSQL_URL!,
});
const live = await introspectLiveSchema({ dialect: 'mysql', connectionString: mysqlUrl });
expect(live).toEqual(currentContract);
});
postgresUpgrade('upgrades postgres to the current contract', async () => {
const postgresUrl = await applyContractFixtureThenUpgrade('postgres', baselineContract, currentContract, {
connectionString: process.env.DB_PARITY_POSTGRES_URL!,
});
const live = await introspectLiveSchema({ dialect: 'postgres', connectionString: postgresUrl });
expect(live).toEqual(currentContract);
});
});