forked from boos/BoosAPI
feat: implement proxy input file resolution and storage
- Added `inputFiles.ts` for handling input file resolution, including validation and normalization of file types. - Introduced `proxyFileStore.ts` for managing file storage, retrieval, and deletion with owner-based isolation. - Created tests for proxy file storage functionality in `proxyFileStore.test.ts`. - Developed `proxyInputFileResolver.ts` to resolve input files in OpenAI and response formats. - Added utility functions for handling reasoning signatures in `reasoningTransport.ts` and corresponding tests. - Enhanced shared input file transformers in `inputFile.ts` for better file handling and type inference.
This commit is contained in:
cita
@@ -7,6 +7,7 @@ import { drizzle as drizzlePgProxy } from 'drizzle-orm/pg-proxy';
|
||||
import * as schema from './schema.js';
|
||||
import { ensureSiteSchemaCompatibility, type SiteSchemaInspector } from './siteSchemaCompatibility.js';
|
||||
import { ensureRouteGroupingSchemaCompatibility } from './routeGroupingSchemaCompatibility.js';
|
||||
import { ensureProxyFileSchemaCompatibility } from './proxyFileSchemaCompatibility.js';
|
||||
import { config } from '../config.js';
|
||||
import { mkdirSync } from 'fs';
|
||||
import { dirname, resolve } from 'path';
|
||||
@@ -25,6 +26,7 @@ const TABLES_WITH_NUMERIC_ID = new Set([
|
||||
'route_channels',
|
||||
'proxy_logs',
|
||||
'proxy_video_tasks',
|
||||
'proxy_files',
|
||||
'downstream_api_keys',
|
||||
'events',
|
||||
]);
|
||||
@@ -183,6 +185,35 @@ function ensureProxyVideoTaskSchema() {
|
||||
`);
|
||||
}
|
||||
|
||||
function ensureProxyFileSchema() {
|
||||
const sqlite = requireSqliteConnection();
|
||||
sqlite.exec(`
|
||||
CREATE TABLE IF NOT EXISTS proxy_files (
|
||||
id integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||
public_id text NOT NULL,
|
||||
owner_type text NOT NULL,
|
||||
owner_id text NOT NULL,
|
||||
filename text NOT NULL,
|
||||
mime_type text NOT NULL,
|
||||
purpose text,
|
||||
byte_size integer NOT NULL,
|
||||
sha256 text NOT NULL,
|
||||
content_base64 text NOT NULL,
|
||||
created_at text DEFAULT (datetime('now')),
|
||||
updated_at text DEFAULT (datetime('now')),
|
||||
deleted_at text
|
||||
);
|
||||
`);
|
||||
sqlite.exec(`
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS proxy_files_public_id_unique
|
||||
ON proxy_files(public_id);
|
||||
`);
|
||||
sqlite.exec(`
|
||||
CREATE INDEX IF NOT EXISTS proxy_files_owner_lookup_idx
|
||||
ON proxy_files(owner_type, owner_id, deleted_at);
|
||||
`);
|
||||
}
|
||||
|
||||
function ensureSiteStatusSchema() {
|
||||
const sqlite = requireSqliteConnection();
|
||||
if (!tableExists('sites')) {
|
||||
@@ -355,6 +386,12 @@ export async function ensureRouteGroupingCompatibilityColumns(): Promise<void> {
|
||||
await ensureRouteGroupingSchemaCompatibility(inspector);
|
||||
}
|
||||
|
||||
export async function ensureProxyFileCompatibilityColumns(): Promise<void> {
|
||||
const inspector = createRuntimeSchemaInspector();
|
||||
if (!inspector) return;
|
||||
await ensureProxyFileSchemaCompatibility(inspector);
|
||||
}
|
||||
|
||||
function ensureRouteGroupingSchema() {
|
||||
const sqlite = requireSqliteConnection();
|
||||
if (!tableExists('token_routes') || !tableExists('route_channels')) {
|
||||
@@ -770,6 +807,7 @@ function initSqliteDb() {
|
||||
ensureDownstreamApiKeySchema();
|
||||
ensureProxyLogBillingDetailsSchema();
|
||||
ensureProxyVideoTaskSchema();
|
||||
ensureProxyFileSchema();
|
||||
|
||||
const rawDb = drizzleSqliteProxy(
|
||||
(sqlText, params, method) => sqliteProxyQuery(sqlText, params, method as SqlMethod),
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
import {
|
||||
ensureProxyFileSchemaCompatibility,
|
||||
type ProxyFileSchemaInspector,
|
||||
} from './proxyFileSchemaCompatibility.js';
|
||||
|
||||
function createInspector(
|
||||
dialect: ProxyFileSchemaInspector['dialect'],
|
||||
options?: {
|
||||
hasTable?: boolean;
|
||||
existingColumns?: string[];
|
||||
},
|
||||
) {
|
||||
const executedSql: string[] = [];
|
||||
const hasTable = options?.hasTable ?? false;
|
||||
const existingColumns = new Set(options?.existingColumns ?? []);
|
||||
|
||||
const inspector: ProxyFileSchemaInspector = {
|
||||
dialect,
|
||||
async tableExists(table) {
|
||||
return table === 'proxy_files' && hasTable;
|
||||
},
|
||||
async columnExists(table, column) {
|
||||
return table === 'proxy_files' && existingColumns.has(column);
|
||||
},
|
||||
async execute(sqlText) {
|
||||
executedSql.push(sqlText);
|
||||
},
|
||||
};
|
||||
|
||||
return { inspector, executedSql };
|
||||
}
|
||||
|
||||
describe('ensureProxyFileSchemaCompatibility', () => {
|
||||
it.each([
|
||||
{
|
||||
dialect: 'postgres' as const,
|
||||
createPattern: /create table/i,
|
||||
uniqueIndexPattern: /create unique index/i,
|
||||
ownerIndexPattern: /owner_lookup_idx/i,
|
||||
},
|
||||
{
|
||||
dialect: 'mysql' as const,
|
||||
createPattern: /create table/i,
|
||||
uniqueIndexPattern: /create unique index/i,
|
||||
ownerIndexPattern: /owner_lookup_idx/i,
|
||||
},
|
||||
{
|
||||
dialect: 'sqlite' as const,
|
||||
createPattern: /create table/i,
|
||||
uniqueIndexPattern: /create unique index/i,
|
||||
ownerIndexPattern: /owner_lookup_idx/i,
|
||||
},
|
||||
])('creates proxy_files table and indexes for $dialect', async ({ dialect, createPattern, uniqueIndexPattern, ownerIndexPattern }) => {
|
||||
const { inspector, executedSql } = createInspector(dialect);
|
||||
|
||||
await ensureProxyFileSchemaCompatibility(inspector);
|
||||
|
||||
expect(executedSql.some((sqlText) => createPattern.test(sqlText) && /proxy_files/i.test(sqlText))).toBe(true);
|
||||
expect(executedSql.some((sqlText) => uniqueIndexPattern.test(sqlText) && /proxy_files_public_id_unique/i.test(sqlText))).toBe(true);
|
||||
expect(executedSql.some((sqlText) => ownerIndexPattern.test(sqlText))).toBe(true);
|
||||
});
|
||||
|
||||
it('adds missing columns on existing table before ensuring indexes', async () => {
|
||||
const { inspector, executedSql } = createInspector('postgres', {
|
||||
hasTable: true,
|
||||
existingColumns: ['public_id', 'owner_type', 'owner_id'],
|
||||
});
|
||||
|
||||
await ensureProxyFileSchemaCompatibility(inspector);
|
||||
|
||||
expect(executedSql.some((sqlText) => sqlText.includes('ADD COLUMN "purpose"'))).toBe(true);
|
||||
expect(executedSql.some((sqlText) => sqlText.includes('ADD COLUMN "filename"'))).toBe(true);
|
||||
expect(executedSql.some((sqlText) => sqlText.includes('ADD COLUMN "deleted_at"'))).toBe(true);
|
||||
expect(executedSql.some((sqlText) => sqlText.includes('proxy_files_public_id_unique'))).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,170 @@
|
||||
export type ProxyFileSchemaDialect = 'sqlite' | 'mysql' | 'postgres';
|
||||
|
||||
export interface ProxyFileSchemaInspector {
|
||||
dialect: ProxyFileSchemaDialect;
|
||||
tableExists(table: string): Promise<boolean>;
|
||||
columnExists(table: string, column: string): Promise<boolean>;
|
||||
execute(sqlText: string): Promise<void>;
|
||||
}
|
||||
|
||||
type ProxyFileColumnCompatibilitySpec = {
|
||||
column: string;
|
||||
addSql: Record<ProxyFileSchemaDialect, string>;
|
||||
};
|
||||
|
||||
const CREATE_TABLE_SQL: Record<ProxyFileSchemaDialect, string> = {
|
||||
sqlite: 'CREATE TABLE IF NOT EXISTS "proxy_files" ("id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, "public_id" TEXT NOT NULL, "owner_type" TEXT NOT NULL, "owner_id" TEXT NOT NULL, "filename" TEXT NOT NULL, "mime_type" TEXT NOT NULL, "purpose" TEXT, "byte_size" INTEGER NOT NULL, "sha256" TEXT NOT NULL, "content_base64" TEXT NOT NULL, "created_at" TEXT, "updated_at" TEXT, "deleted_at" TEXT)',
|
||||
postgres: 'CREATE TABLE IF NOT EXISTS "proxy_files" ("id" SERIAL PRIMARY KEY, "public_id" TEXT NOT NULL, "owner_type" TEXT NOT NULL, "owner_id" TEXT NOT NULL, "filename" TEXT NOT NULL, "mime_type" TEXT NOT NULL, "purpose" TEXT, "byte_size" INTEGER NOT NULL, "sha256" TEXT NOT NULL, "content_base64" TEXT NOT NULL, "created_at" TEXT, "updated_at" TEXT, "deleted_at" TEXT)',
|
||||
mysql: 'CREATE TABLE IF NOT EXISTS `proxy_files` (`id` INTEGER NOT NULL AUTO_INCREMENT PRIMARY KEY, `public_id` TEXT NOT NULL, `owner_type` TEXT NOT NULL, `owner_id` TEXT NOT NULL, `filename` TEXT NOT NULL, `mime_type` TEXT NOT NULL, `purpose` TEXT NULL, `byte_size` INTEGER NOT NULL, `sha256` TEXT NOT NULL, `content_base64` LONGTEXT NOT NULL, `created_at` TEXT NULL, `updated_at` TEXT NULL, `deleted_at` TEXT NULL)',
|
||||
};
|
||||
|
||||
const COLUMN_COMPATIBILITY_SPECS: ProxyFileColumnCompatibilitySpec[] = [
|
||||
{
|
||||
column: 'public_id',
|
||||
addSql: {
|
||||
sqlite: 'ALTER TABLE "proxy_files" ADD COLUMN "public_id" TEXT',
|
||||
postgres: 'ALTER TABLE "proxy_files" ADD COLUMN "public_id" TEXT',
|
||||
mysql: 'ALTER TABLE `proxy_files` ADD COLUMN `public_id` TEXT NULL',
|
||||
},
|
||||
},
|
||||
{
|
||||
column: 'owner_type',
|
||||
addSql: {
|
||||
sqlite: 'ALTER TABLE "proxy_files" ADD COLUMN "owner_type" TEXT',
|
||||
postgres: 'ALTER TABLE "proxy_files" ADD COLUMN "owner_type" TEXT',
|
||||
mysql: 'ALTER TABLE `proxy_files` ADD COLUMN `owner_type` TEXT NULL',
|
||||
},
|
||||
},
|
||||
{
|
||||
column: 'owner_id',
|
||||
addSql: {
|
||||
sqlite: 'ALTER TABLE "proxy_files" ADD COLUMN "owner_id" TEXT',
|
||||
postgres: 'ALTER TABLE "proxy_files" ADD COLUMN "owner_id" TEXT',
|
||||
mysql: 'ALTER TABLE `proxy_files` ADD COLUMN `owner_id` TEXT NULL',
|
||||
},
|
||||
},
|
||||
{
|
||||
column: 'filename',
|
||||
addSql: {
|
||||
sqlite: 'ALTER TABLE "proxy_files" ADD COLUMN "filename" TEXT',
|
||||
postgres: 'ALTER TABLE "proxy_files" ADD COLUMN "filename" TEXT',
|
||||
mysql: 'ALTER TABLE `proxy_files` ADD COLUMN `filename` TEXT NULL',
|
||||
},
|
||||
},
|
||||
{
|
||||
column: 'mime_type',
|
||||
addSql: {
|
||||
sqlite: 'ALTER TABLE "proxy_files" ADD COLUMN "mime_type" TEXT',
|
||||
postgres: 'ALTER TABLE "proxy_files" ADD COLUMN "mime_type" TEXT',
|
||||
mysql: 'ALTER TABLE `proxy_files` ADD COLUMN `mime_type` TEXT NULL',
|
||||
},
|
||||
},
|
||||
{
|
||||
column: 'purpose',
|
||||
addSql: {
|
||||
sqlite: 'ALTER TABLE "proxy_files" ADD COLUMN "purpose" TEXT',
|
||||
postgres: 'ALTER TABLE "proxy_files" ADD COLUMN "purpose" TEXT',
|
||||
mysql: 'ALTER TABLE `proxy_files` ADD COLUMN `purpose` TEXT NULL',
|
||||
},
|
||||
},
|
||||
{
|
||||
column: 'byte_size',
|
||||
addSql: {
|
||||
sqlite: 'ALTER TABLE "proxy_files" ADD COLUMN "byte_size" INTEGER',
|
||||
postgres: 'ALTER TABLE "proxy_files" ADD COLUMN "byte_size" INTEGER',
|
||||
mysql: 'ALTER TABLE `proxy_files` ADD COLUMN `byte_size` INTEGER NULL',
|
||||
},
|
||||
},
|
||||
{
|
||||
column: 'sha256',
|
||||
addSql: {
|
||||
sqlite: 'ALTER TABLE "proxy_files" ADD COLUMN "sha256" TEXT',
|
||||
postgres: 'ALTER TABLE "proxy_files" ADD COLUMN "sha256" TEXT',
|
||||
mysql: 'ALTER TABLE `proxy_files` ADD COLUMN `sha256` TEXT NULL',
|
||||
},
|
||||
},
|
||||
{
|
||||
column: 'content_base64',
|
||||
addSql: {
|
||||
sqlite: 'ALTER TABLE "proxy_files" ADD COLUMN "content_base64" TEXT',
|
||||
postgres: 'ALTER TABLE "proxy_files" ADD COLUMN "content_base64" TEXT',
|
||||
mysql: 'ALTER TABLE `proxy_files` ADD COLUMN `content_base64` LONGTEXT NULL',
|
||||
},
|
||||
},
|
||||
{
|
||||
column: 'created_at',
|
||||
addSql: {
|
||||
sqlite: 'ALTER TABLE "proxy_files" ADD COLUMN "created_at" TEXT',
|
||||
postgres: 'ALTER TABLE "proxy_files" ADD COLUMN "created_at" TEXT',
|
||||
mysql: 'ALTER TABLE `proxy_files` ADD COLUMN `created_at` TEXT NULL',
|
||||
},
|
||||
},
|
||||
{
|
||||
column: 'updated_at',
|
||||
addSql: {
|
||||
sqlite: 'ALTER TABLE "proxy_files" ADD COLUMN "updated_at" TEXT',
|
||||
postgres: 'ALTER TABLE "proxy_files" ADD COLUMN "updated_at" TEXT',
|
||||
mysql: 'ALTER TABLE `proxy_files` ADD COLUMN `updated_at` TEXT NULL',
|
||||
},
|
||||
},
|
||||
{
|
||||
column: 'deleted_at',
|
||||
addSql: {
|
||||
sqlite: 'ALTER TABLE "proxy_files" ADD COLUMN "deleted_at" TEXT',
|
||||
postgres: 'ALTER TABLE "proxy_files" ADD COLUMN "deleted_at" TEXT',
|
||||
mysql: 'ALTER TABLE `proxy_files` ADD COLUMN `deleted_at` TEXT NULL',
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
const CREATE_INDEX_SQL: Record<ProxyFileSchemaDialect, string[]> = {
|
||||
sqlite: [
|
||||
'CREATE UNIQUE INDEX IF NOT EXISTS "proxy_files_public_id_unique" ON "proxy_files" ("public_id")',
|
||||
'CREATE INDEX IF NOT EXISTS "proxy_files_owner_lookup_idx" ON "proxy_files" ("owner_type", "owner_id", "deleted_at")',
|
||||
],
|
||||
postgres: [
|
||||
'CREATE UNIQUE INDEX IF NOT EXISTS "proxy_files_public_id_unique" ON "proxy_files" ("public_id")',
|
||||
'CREATE INDEX IF NOT EXISTS "proxy_files_owner_lookup_idx" ON "proxy_files" ("owner_type", "owner_id", "deleted_at")',
|
||||
],
|
||||
mysql: [
|
||||
'CREATE UNIQUE INDEX `proxy_files_public_id_unique` ON `proxy_files` (`public_id`(191))',
|
||||
'CREATE INDEX `proxy_files_owner_lookup_idx` ON `proxy_files` (`owner_type`(191), `owner_id`(191), `deleted_at`(191))',
|
||||
],
|
||||
};
|
||||
|
||||
function normalizeSchemaErrorMessage(error: unknown): string {
|
||||
if (typeof error === 'object' && error && 'message' in error) {
|
||||
return String((error as { message?: unknown }).message || '');
|
||||
}
|
||||
return String(error || '');
|
||||
}
|
||||
|
||||
function isDuplicateSchemaError(error: unknown): boolean {
|
||||
const lowered = normalizeSchemaErrorMessage(error).toLowerCase();
|
||||
return lowered.includes('already exists') || lowered.includes('duplicate') || lowered.includes('relation');
|
||||
}
|
||||
|
||||
async function executeIgnoreDuplicate(inspector: ProxyFileSchemaInspector, sqlText: string): Promise<void> {
|
||||
try {
|
||||
await inspector.execute(sqlText);
|
||||
} catch (error) {
|
||||
if (!isDuplicateSchemaError(error)) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function ensureProxyFileSchemaCompatibility(inspector: ProxyFileSchemaInspector): Promise<void> {
|
||||
if (!await inspector.tableExists('proxy_files')) {
|
||||
await executeIgnoreDuplicate(inspector, CREATE_TABLE_SQL[inspector.dialect]);
|
||||
} else {
|
||||
for (const spec of COLUMN_COMPATIBILITY_SPECS) {
|
||||
if (await inspector.columnExists('proxy_files', spec.column)) continue;
|
||||
await executeIgnoreDuplicate(inspector, spec.addSql[inspector.dialect]);
|
||||
}
|
||||
}
|
||||
|
||||
for (const sqlText of CREATE_INDEX_SQL[inspector.dialect]) {
|
||||
await executeIgnoreDuplicate(inspector, sqlText);
|
||||
}
|
||||
}
|
||||
@@ -191,6 +191,25 @@ export const proxyVideoTasks = sqliteTable('proxy_video_tasks', {
|
||||
createdAtIdx: index('proxy_video_tasks_created_at_idx').on(table.createdAt),
|
||||
}));
|
||||
|
||||
export const proxyFiles = sqliteTable('proxy_files', {
|
||||
id: integer('id').primaryKey({ autoIncrement: true }),
|
||||
publicId: text('public_id').notNull(),
|
||||
ownerType: text('owner_type').notNull(),
|
||||
ownerId: text('owner_id').notNull(),
|
||||
filename: text('filename').notNull(),
|
||||
mimeType: text('mime_type').notNull(),
|
||||
purpose: text('purpose'),
|
||||
byteSize: integer('byte_size').notNull(),
|
||||
sha256: text('sha256').notNull(),
|
||||
contentBase64: text('content_base64').notNull(),
|
||||
createdAt: text('created_at').default(sql`(datetime('now'))`),
|
||||
updatedAt: text('updated_at').default(sql`(datetime('now'))`),
|
||||
deletedAt: text('deleted_at'),
|
||||
}, (table) => ({
|
||||
publicIdUnique: uniqueIndex('proxy_files_public_id_unique').on(table.publicId),
|
||||
ownerLookupIdx: index('proxy_files_owner_lookup_idx').on(table.ownerType, table.ownerId, table.deletedAt),
|
||||
}));
|
||||
|
||||
export const settings = sqliteTable('settings', {
|
||||
key: text('key').primaryKey(),
|
||||
value: text('value'), // JSON
|
||||
|
||||
Reference in New Issue
Block a user