fix: recover sqlite migration gaps and harden route regex
This commit is contained in:
+48
-16
@@ -274,6 +274,14 @@ function markMigrationRecordIfMissing(sqlite: Database.Database, record: Migrati
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function hasMigrationRecord(sqlite: Database.Database, record: MigrationRecord): boolean {
|
||||||
|
if (!tableExists(sqlite, '__drizzle_migrations')) return false;
|
||||||
|
const row = sqlite
|
||||||
|
.prepare('SELECT 1 FROM "__drizzle_migrations" WHERE "hash" = ? LIMIT 1')
|
||||||
|
.get(record.hash);
|
||||||
|
return !!row;
|
||||||
|
}
|
||||||
|
|
||||||
function normalizeSchemaErrorMessage(error: unknown): string {
|
function normalizeSchemaErrorMessage(error: unknown): string {
|
||||||
if (!error || typeof error !== 'object') {
|
if (!error || typeof error !== 'object') {
|
||||||
return String(error || '');
|
return String(error || '');
|
||||||
@@ -332,25 +340,28 @@ function isSitesPlatformUrlUniqueConflictError(error: unknown): boolean {
|
|||||||
=== normalizeSqlForMatch('CREATE UNIQUE INDEX `sites_platform_url_unique` ON `sites` (`platform`,`url`);');
|
=== normalizeSqlForMatch('CREATE UNIQUE INDEX `sites_platform_url_unique` ON `sites` (`platform`,`url`);');
|
||||||
}
|
}
|
||||||
|
|
||||||
function getLatestRecordedMigrationCreatedAt(sqlite: Database.Database): number | null {
|
|
||||||
if (!tableExists(sqlite, '__drizzle_migrations')) return null;
|
|
||||||
const row = sqlite
|
|
||||||
.prepare('SELECT created_at FROM "__drizzle_migrations" ORDER BY created_at DESC LIMIT 1')
|
|
||||||
.get() as { created_at?: number } | undefined;
|
|
||||||
if (!row || row.created_at === undefined || row.created_at === null) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
return Number(row.created_at);
|
|
||||||
}
|
|
||||||
|
|
||||||
function replayMigrationStatements(sqlite: Database.Database, statements: string[]): void {
|
function replayMigrationStatements(sqlite: Database.Database, statements: string[]): void {
|
||||||
for (const statement of statements) {
|
for (const statement of statements) {
|
||||||
try {
|
try {
|
||||||
sqlite.exec(statement);
|
sqlite.exec(statement);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
if (!isRecoverableSchemaConflictError(error)) {
|
if (isRecoverableSchemaConflictError(error)) {
|
||||||
throw error;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (isSitesPlatformUrlUniqueConflictError(error) && deduplicateLegacySitesForUniqueIndex(sqlite)) {
|
||||||
|
try {
|
||||||
|
sqlite.exec(statement);
|
||||||
|
continue;
|
||||||
|
} catch (retryError) {
|
||||||
|
if (isRecoverableSchemaConflictError(retryError)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
throw retryError;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
throw error;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -366,20 +377,40 @@ function recoverMigrationSequence(
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
let latestRecordedCreatedAt = getLatestRecordedMigrationCreatedAt(sqlite);
|
|
||||||
for (const migration of migrations.slice(0, failedMigrationIndex + 1)) {
|
for (const migration of migrations.slice(0, failedMigrationIndex + 1)) {
|
||||||
if (latestRecordedCreatedAt !== null && latestRecordedCreatedAt >= migration.createdAt) {
|
if (hasMigrationRecord(sqlite, migration)) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
replayMigrationStatements(sqlite, migration.statements);
|
replayMigrationStatements(sqlite, migration.statements);
|
||||||
markMigrationRecordIfMissing(sqlite, migration);
|
markMigrationRecordIfMissing(sqlite, migration);
|
||||||
latestRecordedCreatedAt = migration.createdAt;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function backfillMissingRecordedMigrations(sqlite: Database.Database, migrationsFolder: string): number {
|
||||||
|
if (!tableExists(sqlite, '__drizzle_migrations')) return 0;
|
||||||
|
|
||||||
|
let recoveredCount = 0;
|
||||||
|
for (const migration of readRecoveryMigrations(migrationsFolder)) {
|
||||||
|
if (hasMigrationRecord(sqlite, migration)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
replayMigrationStatements(sqlite, migration.statements);
|
||||||
|
if (markMigrationRecordIfMissing(sqlite, migration)) {
|
||||||
|
recoveredCount += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (recoveredCount > 0) {
|
||||||
|
console.warn(`[db] Backfilled ${recoveredCount} missing drizzle migration record(s).`);
|
||||||
|
}
|
||||||
|
|
||||||
|
return recoveredCount;
|
||||||
|
}
|
||||||
|
|
||||||
function tryRecoverDuplicateColumnMigrationError(
|
function tryRecoverDuplicateColumnMigrationError(
|
||||||
sqlite: Database.Database,
|
sqlite: Database.Database,
|
||||||
migrationsFolder: string,
|
migrationsFolder: string,
|
||||||
@@ -557,6 +588,7 @@ export function runSqliteMigrations(): void {
|
|||||||
|
|
||||||
const sqlite = new Database(dbPath);
|
const sqlite = new Database(dbPath);
|
||||||
bootstrapLegacyDrizzleMigrations(sqlite, migrationsFolder);
|
bootstrapLegacyDrizzleMigrations(sqlite, migrationsFolder);
|
||||||
|
backfillMissingRecordedMigrations(sqlite, migrationsFolder);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
migrate(drizzle(sqlite), { migrationsFolder });
|
migrate(drizzle(sqlite), { migrationsFolder });
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
import { describe, expect, it } from 'vitest';
|
import { describe, expect, it } from 'vitest';
|
||||||
import { filterRecentlyFailedCandidates, isChannelRecentlyFailed } from './tokenRouter.js';
|
import { filterRecentlyFailedCandidates, isChannelRecentlyFailed, matchesModelPattern, parseRegexModelPattern } from './tokenRouter.js';
|
||||||
|
|
||||||
type Candidate = {
|
type Candidate = {
|
||||||
channel: {
|
channel: {
|
||||||
@@ -10,6 +10,11 @@ type Candidate = {
|
|||||||
};
|
};
|
||||||
|
|
||||||
describe('filterRecentlyFailedCandidates', () => {
|
describe('filterRecentlyFailedCandidates', () => {
|
||||||
|
it('rejects unsafe nested-quantifier regex route patterns', () => {
|
||||||
|
expect(parseRegexModelPattern('re:^(a+)+$')).toBeNull();
|
||||||
|
expect(matchesModelPattern('aaaa', 're:^(a+)+$')).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
it('uses a short default recent-failure window', () => {
|
it('uses a short default recent-failure window', () => {
|
||||||
const nowMs = Date.now();
|
const nowMs = Date.now();
|
||||||
expect(isChannelRecentlyFailed({
|
expect(isChannelRecentlyFailed({
|
||||||
|
|||||||
@@ -43,6 +43,7 @@ const FAILURE_BACKOFF_BASE_SEC = 15;
|
|||||||
const MIN_EFFECTIVE_UNIT_COST = 1e-6;
|
const MIN_EFFECTIVE_UNIT_COST = 1e-6;
|
||||||
const ROUND_ROBIN_FAILURE_THRESHOLD = 3;
|
const ROUND_ROBIN_FAILURE_THRESHOLD = 3;
|
||||||
const ROUND_ROBIN_COOLDOWN_LEVELS_SEC = [0, 10 * 60, 60 * 60, 24 * 60 * 60] as const;
|
const ROUND_ROBIN_COOLDOWN_LEVELS_SEC = [0, 10 * 60, 60 * 60, 24 * 60 * 60] as const;
|
||||||
|
const MAX_ROUTE_REGEX_BODY_LENGTH = 256;
|
||||||
|
|
||||||
function fibonacciNumber(index: number): number {
|
function fibonacciNumber(index: number): number {
|
||||||
if (index <= 2) return 1;
|
if (index <= 2) return 1;
|
||||||
@@ -289,10 +290,121 @@ export function isRegexModelPattern(pattern: string): boolean {
|
|||||||
return pattern.trim().toLowerCase().startsWith('re:');
|
return pattern.trim().toLowerCase().startsWith('re:');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function readRegexQuantifierLength(pattern: string, startIndex: number): number {
|
||||||
|
const ch = pattern[startIndex];
|
||||||
|
if (ch === '*' || ch === '+' || ch === '?') return 1;
|
||||||
|
if (ch !== '{') return 0;
|
||||||
|
|
||||||
|
let index = startIndex + 1;
|
||||||
|
let sawDigit = false;
|
||||||
|
while (index < pattern.length && /\d/.test(pattern[index])) {
|
||||||
|
sawDigit = true;
|
||||||
|
index += 1;
|
||||||
|
}
|
||||||
|
if (!sawDigit) return 0;
|
||||||
|
|
||||||
|
if (pattern[index] === ',') {
|
||||||
|
index += 1;
|
||||||
|
while (index < pattern.length && /\d/.test(pattern[index])) {
|
||||||
|
index += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (pattern[index] !== '}') return 0;
|
||||||
|
return index - startIndex + 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
function isSafeRegexPatternBody(body: string): boolean {
|
||||||
|
if (!body || body.length > MAX_ROUTE_REGEX_BODY_LENGTH) return false;
|
||||||
|
if (!/^[a-z0-9\s.^$|()[\]{}+*?\\:_/-]+$/i.test(body)) return false;
|
||||||
|
if (body.includes('(?=') || body.includes('(?!') || body.includes('(?<=') || body.includes('(?<!') || body.includes('(?<')) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (/(^|[^\\])\\[1-9]/.test(body)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
const groupStack: Array<{ hasInnerQuantifier: boolean; hasAlternation: boolean }> = [];
|
||||||
|
let escaped = false;
|
||||||
|
let inCharClass = false;
|
||||||
|
|
||||||
|
for (let index = 0; index < body.length; index += 1) {
|
||||||
|
const ch = body[index];
|
||||||
|
|
||||||
|
if (escaped) {
|
||||||
|
escaped = false;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (ch === '\\') {
|
||||||
|
escaped = true;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (inCharClass) {
|
||||||
|
if (ch === ']') inCharClass = false;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (ch === '[') {
|
||||||
|
inCharClass = true;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (ch === '(') {
|
||||||
|
if (body[index + 1] === '?') {
|
||||||
|
if (body[index + 2] !== ':') {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
groupStack.push({ hasInnerQuantifier: false, hasAlternation: false });
|
||||||
|
index += 2;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
groupStack.push({ hasInnerQuantifier: false, hasAlternation: false });
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (ch === '|') {
|
||||||
|
if (groupStack.length > 0) {
|
||||||
|
groupStack[groupStack.length - 1].hasAlternation = true;
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (ch === ')') {
|
||||||
|
const group = groupStack.pop();
|
||||||
|
if (!group) return false;
|
||||||
|
|
||||||
|
const quantifierLength = readRegexQuantifierLength(body, index + 1);
|
||||||
|
if (quantifierLength > 0 && (group.hasInnerQuantifier || group.hasAlternation)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
const parent = groupStack[groupStack.length - 1];
|
||||||
|
if (parent && (group.hasInnerQuantifier || quantifierLength > 0)) {
|
||||||
|
parent.hasInnerQuantifier = true;
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
const quantifierLength = readRegexQuantifierLength(body, index);
|
||||||
|
if (quantifierLength > 0) {
|
||||||
|
if (groupStack.length > 0) {
|
||||||
|
groupStack[groupStack.length - 1].hasInnerQuantifier = true;
|
||||||
|
}
|
||||||
|
index += quantifierLength - 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return !escaped && !inCharClass && groupStack.length === 0;
|
||||||
|
}
|
||||||
|
|
||||||
export function parseRegexModelPattern(pattern: string): RegExp | null {
|
export function parseRegexModelPattern(pattern: string): RegExp | null {
|
||||||
if (!isRegexModelPattern(pattern)) return null;
|
if (!isRegexModelPattern(pattern)) return null;
|
||||||
const body = pattern.trim().slice(3).trim();
|
const body = pattern.trim().slice(3).trim();
|
||||||
if (!body) return null;
|
if (!body) return null;
|
||||||
|
if (!isSafeRegexPatternBody(body)) return null;
|
||||||
try {
|
try {
|
||||||
return new RegExp(body);
|
return new RegExp(body);
|
||||||
} catch {
|
} catch {
|
||||||
|
|||||||
Reference in New Issue
Block a user