boos/BoosAPI
2
0
Fork 2
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: sync upstream analytics schema support

Browse Source
This commit is contained in:
boos
2026-06-06 16:07:13 +08:00
parent 0d231ec467
commit 339f91d014
7 changed files with 1233 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/db/mod.rs
+1
View File
@@ -8,6 +8,7 @@ use sqlx::{
pub mod models;
pub mod queries;
pub mod schema_compat;
pub mod sqlite_to_postgres;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
src/db/schema_compat.rs
+351
View File
@@ -0,0 +1,351 @@
use anyhow::Context;
use sqlx::{Row, SqlitePool};
use std::collections::HashSet;
pub async fn ensure_sqlite_upstream_compat(pool: &SqlitePool) -> anyhow::Result<()> {
ensure_columns(
pool,
"sites",
&[
("post_refresh_probe_enabled", "INTEGER DEFAULT false"),
("post_refresh_probe_model", "TEXT DEFAULT ''"),
("post_refresh_probe_scope", "TEXT DEFAULT 'single'"),
(
"post_refresh_probe_latency_threshold_ms",
"INTEGER DEFAULT 0",
),
],
)
.await?;
ensure_columns(
pool,
"downstream_api_keys",
&[
("excluded_site_ids", "TEXT"),
("excluded_credential_refs", "TEXT"),
],
)
.await?;
ensure_columns(
pool,
"route_channels",
&[("oauth_route_unit_id", "INTEGER")],
)
.await?;
ensure_columns(
pool,
"proxy_logs",
&[
("client_family", "TEXT"),
("client_app_id", "TEXT"),
("client_app_name", "TEXT"),
("client_confidence", "TEXT"),
("is_stream", "INTEGER"),
("first_byte_latency_ms", "INTEGER"),
],
)
.await?;
run_compat_ddl(pool).await?;
Ok(())
}
async fn ensure_columns(
pool: &SqlitePool,
table: &str,
columns: &[(&str, &str)],
) -> anyhow::Result<()> {
let existing = table_columns(pool, table).await?;
if existing.is_empty() {
return Ok(());
}
for (name, definition) in columns {
if existing.contains(*name) {
continue;
}
let sql = format!("ALTER TABLE {table} ADD {name} {definition}");
sqlx::query(&sql)
.execute(pool)
.await
.with_context(|| format!("add column {table}.{name}"))?;
}
Ok(())
}
async fn table_columns(pool: &SqlitePool, table: &str) -> anyhow::Result<HashSet<String>> {
let sql = format!("PRAGMA table_info({table})");
let rows = sqlx::query(&sql).fetch_all(pool).await?;
Ok(rows
.into_iter()
.filter_map(|row| row.try_get::<String, _>("name").ok())
.collect())
}
async fn run_compat_ddl(pool: &SqlitePool) -> anyhow::Result<()> {
for statement in SQLITE_COMPAT_DDL {
sqlx::query(statement)
.execute(pool)
.await
.with_context(|| format!("run SQLite compat DDL: {statement}"))?;
}
Ok(())
}
const SQLITE_COMPAT_DDL: &[&str] = &[
"CREATE INDEX IF NOT EXISTS proxy_logs_client_app_id_created_at_idx
ON proxy_logs (client_app_id, created_at)",
"CREATE INDEX IF NOT EXISTS proxy_logs_client_family_created_at_idx
ON proxy_logs (client_family, created_at)",
"CREATE INDEX IF NOT EXISTS route_channels_oauth_route_unit_id_idx
ON route_channels (oauth_route_unit_id)",
"CREATE TABLE IF NOT EXISTS site_api_endpoints (
id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
site_id INTEGER NOT NULL,
url TEXT NOT NULL,
enabled INTEGER DEFAULT true,
sort_order INTEGER DEFAULT 0,
cooldown_until TEXT,
last_selected_at TEXT,
last_failed_at TEXT,
last_failure_reason TEXT,
created_at TEXT DEFAULT (datetime('now')),
updated_at TEXT DEFAULT (datetime('now')),
FOREIGN KEY (site_id) REFERENCES sites(id) ON DELETE CASCADE
)",
"CREATE UNIQUE INDEX IF NOT EXISTS site_api_endpoints_site_url_unique
ON site_api_endpoints (site_id, url)",
"CREATE INDEX IF NOT EXISTS site_api_endpoints_site_enabled_sort_idx
ON site_api_endpoints (site_id, enabled, sort_order)",
"CREATE INDEX IF NOT EXISTS site_api_endpoints_site_cooldown_idx
ON site_api_endpoints (site_id, cooldown_until)",
"CREATE TABLE IF NOT EXISTS proxy_debug_traces (
id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
downstream_path TEXT NOT NULL,
client_kind TEXT,
session_id TEXT,
trace_hint TEXT,
requested_model TEXT,
downstream_api_key_id INTEGER,
request_headers_json TEXT,
request_body_json TEXT,
sticky_session_key TEXT,
sticky_hit_channel_id INTEGER,
selected_channel_id INTEGER,
selected_route_id INTEGER,
selected_account_id INTEGER,
selected_site_id INTEGER,
selected_site_platform TEXT,
endpoint_candidates_json TEXT,
endpoint_runtime_state_json TEXT,
decision_summary_json TEXT,
final_status TEXT,
final_http_status INTEGER,
final_upstream_path TEXT,
final_response_headers_json TEXT,
final_response_body_json TEXT,
created_at TEXT DEFAULT (datetime('now')),
updated_at TEXT DEFAULT (datetime('now'))
)",
"CREATE INDEX IF NOT EXISTS proxy_debug_traces_created_at_idx
ON proxy_debug_traces (created_at)",
"CREATE INDEX IF NOT EXISTS proxy_debug_traces_session_created_at_idx
ON proxy_debug_traces (session_id, created_at)",
"CREATE INDEX IF NOT EXISTS proxy_debug_traces_model_created_at_idx
ON proxy_debug_traces (requested_model, created_at)",
"CREATE INDEX IF NOT EXISTS proxy_debug_traces_final_status_created_at_idx
ON proxy_debug_traces (final_status, created_at)",
"CREATE TABLE IF NOT EXISTS proxy_debug_attempts (
id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
trace_id INTEGER NOT NULL,
attempt_index INTEGER NOT NULL,
endpoint TEXT NOT NULL,
request_path TEXT NOT NULL,
target_url TEXT NOT NULL,
runtime_executor TEXT,
request_headers_json TEXT,
request_body_json TEXT,
response_status INTEGER,
response_headers_json TEXT,
response_body_json TEXT,
raw_error_text TEXT,
recover_applied INTEGER DEFAULT false,
downgrade_decision INTEGER DEFAULT false,
downgrade_reason TEXT,
memory_write_json TEXT,
created_at TEXT DEFAULT (datetime('now')),
FOREIGN KEY (trace_id) REFERENCES proxy_debug_traces(id) ON DELETE CASCADE
)",
"CREATE UNIQUE INDEX IF NOT EXISTS proxy_debug_attempts_trace_attempt_unique
ON proxy_debug_attempts (trace_id, attempt_index)",
"CREATE INDEX IF NOT EXISTS proxy_debug_attempts_trace_created_at_idx
ON proxy_debug_attempts (trace_id, created_at)",
"CREATE TABLE IF NOT EXISTS oauth_route_units (
id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
site_id INTEGER NOT NULL,
provider TEXT NOT NULL,
name TEXT NOT NULL,
strategy TEXT NOT NULL DEFAULT 'round_robin',
enabled INTEGER DEFAULT true,
created_at TEXT DEFAULT (datetime('now')),
updated_at TEXT DEFAULT (datetime('now')),
FOREIGN KEY (site_id) REFERENCES sites(id) ON DELETE CASCADE
)",
"CREATE INDEX IF NOT EXISTS oauth_route_units_site_provider_idx
ON oauth_route_units (site_id, provider)",
"CREATE INDEX IF NOT EXISTS oauth_route_units_enabled_idx
ON oauth_route_units (enabled)",
"CREATE TABLE IF NOT EXISTS oauth_route_unit_members (
id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
unit_id INTEGER NOT NULL,
account_id INTEGER NOT NULL,
sort_order INTEGER DEFAULT 0,
success_count INTEGER DEFAULT 0,
fail_count INTEGER DEFAULT 0,
total_latency_ms INTEGER DEFAULT 0,
total_cost REAL DEFAULT 0,
last_used_at TEXT,
last_selected_at TEXT,
last_fail_at TEXT,
consecutive_fail_count INTEGER NOT NULL DEFAULT 0,
cooldown_level INTEGER NOT NULL DEFAULT 0,
cooldown_until TEXT,
created_at TEXT DEFAULT (datetime('now')),
updated_at TEXT DEFAULT (datetime('now')),
FOREIGN KEY (unit_id) REFERENCES oauth_route_units(id) ON DELETE CASCADE,
FOREIGN KEY (account_id) REFERENCES accounts(id) ON DELETE CASCADE
)",
"CREATE UNIQUE INDEX IF NOT EXISTS oauth_route_unit_members_unit_account_unique
ON oauth_route_unit_members (unit_id, account_id)",
"CREATE UNIQUE INDEX IF NOT EXISTS oauth_route_unit_members_account_unique
ON oauth_route_unit_members (account_id)",
"CREATE INDEX IF NOT EXISTS oauth_route_unit_members_unit_sort_idx
ON oauth_route_unit_members (unit_id, sort_order)",
"CREATE INDEX IF NOT EXISTS oauth_route_unit_members_unit_cooldown_idx
ON oauth_route_unit_members (unit_id, cooldown_until)",
"CREATE TABLE IF NOT EXISTS admin_snapshots (
id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
namespace TEXT NOT NULL,
snapshot_key TEXT NOT NULL,
payload TEXT NOT NULL,
generated_at TEXT NOT NULL,
expires_at TEXT NOT NULL,
stale_until TEXT NOT NULL,
created_at TEXT DEFAULT (datetime('now')),
updated_at TEXT DEFAULT (datetime('now'))
)",
"CREATE UNIQUE INDEX IF NOT EXISTS admin_snapshots_namespace_key_unique
ON admin_snapshots (namespace, snapshot_key)",
"CREATE INDEX IF NOT EXISTS admin_snapshots_expires_at_idx
ON admin_snapshots (expires_at)",
"CREATE INDEX IF NOT EXISTS admin_snapshots_stale_until_idx
ON admin_snapshots (stale_until)",
"CREATE TABLE IF NOT EXISTS analytics_projection_checkpoints (
projector_key TEXT PRIMARY KEY NOT NULL,
time_zone TEXT NOT NULL DEFAULT 'Local',
last_proxy_log_id INTEGER NOT NULL DEFAULT 0,
watermark_created_at TEXT,
recompute_from_id INTEGER,
recompute_requested_at TEXT,
recompute_reason TEXT,
recompute_started_at TEXT,
recompute_completed_at TEXT,
last_projected_at TEXT,
last_successful_at TEXT,
last_error TEXT,
lease_owner TEXT,
lease_token TEXT,
lease_expires_at TEXT,
created_at TEXT DEFAULT (datetime('now')),
updated_at TEXT DEFAULT (datetime('now'))
)",
"CREATE INDEX IF NOT EXISTS analytics_projection_checkpoints_recompute_from_id_idx
ON analytics_projection_checkpoints (recompute_from_id)",
"CREATE INDEX IF NOT EXISTS analytics_projection_checkpoints_lease_expires_at_idx
ON analytics_projection_checkpoints (lease_expires_at)",
"CREATE TABLE IF NOT EXISTS model_day_usage (
id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
local_day TEXT NOT NULL,
site_id INTEGER NOT NULL,
model TEXT NOT NULL,
total_calls INTEGER NOT NULL DEFAULT 0,
success_calls INTEGER NOT NULL DEFAULT 0,
failed_calls INTEGER NOT NULL DEFAULT 0,
total_tokens INTEGER NOT NULL DEFAULT 0,
total_spend REAL NOT NULL DEFAULT 0,
total_latency_ms INTEGER NOT NULL DEFAULT 0,
latency_count INTEGER NOT NULL DEFAULT 0,
created_at TEXT DEFAULT (datetime('now')),
updated_at TEXT DEFAULT (datetime('now')),
FOREIGN KEY (site_id) REFERENCES sites(id) ON DELETE CASCADE,
CHECK (
total_calls >= 0 AND success_calls >= 0 AND failed_calls >= 0
AND total_tokens >= 0 AND total_spend >= 0
AND total_latency_ms >= 0 AND latency_count >= 0
)
)",
"CREATE UNIQUE INDEX IF NOT EXISTS model_day_usage_day_site_model_unique
ON model_day_usage (local_day, site_id, model)",
"CREATE INDEX IF NOT EXISTS model_day_usage_day_idx
ON model_day_usage (local_day)",
"CREATE INDEX IF NOT EXISTS model_day_usage_site_id_idx
ON model_day_usage (site_id)",
"CREATE INDEX IF NOT EXISTS model_day_usage_model_idx
ON model_day_usage (model)",
"CREATE TABLE IF NOT EXISTS site_day_usage (
id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
local_day TEXT NOT NULL,
site_id INTEGER NOT NULL,
total_calls INTEGER NOT NULL DEFAULT 0,
success_calls INTEGER NOT NULL DEFAULT 0,
failed_calls INTEGER NOT NULL DEFAULT 0,
total_tokens INTEGER NOT NULL DEFAULT 0,
total_summary_spend REAL NOT NULL DEFAULT 0,
total_site_spend REAL NOT NULL DEFAULT 0,
total_latency_ms INTEGER NOT NULL DEFAULT 0,
latency_count INTEGER NOT NULL DEFAULT 0,
created_at TEXT DEFAULT (datetime('now')),
updated_at TEXT DEFAULT (datetime('now')),
FOREIGN KEY (site_id) REFERENCES sites(id) ON DELETE CASCADE,
CHECK (
total_calls >= 0 AND success_calls >= 0 AND failed_calls >= 0
AND total_tokens >= 0 AND total_summary_spend >= 0
AND total_site_spend >= 0 AND total_latency_ms >= 0
AND latency_count >= 0
)
)",
"CREATE UNIQUE INDEX IF NOT EXISTS site_day_usage_day_site_unique
ON site_day_usage (local_day, site_id)",
"CREATE INDEX IF NOT EXISTS site_day_usage_day_idx
ON site_day_usage (local_day)",
"CREATE INDEX IF NOT EXISTS site_day_usage_site_id_idx
ON site_day_usage (site_id)",
"CREATE TABLE IF NOT EXISTS site_hour_usage (
id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
bucket_start_utc TEXT NOT NULL,
site_id INTEGER NOT NULL,
total_calls INTEGER NOT NULL DEFAULT 0,
success_calls INTEGER NOT NULL DEFAULT 0,
failed_calls INTEGER NOT NULL DEFAULT 0,
total_tokens INTEGER NOT NULL DEFAULT 0,
total_summary_spend REAL NOT NULL DEFAULT 0,
total_site_spend REAL NOT NULL DEFAULT 0,
total_latency_ms INTEGER NOT NULL DEFAULT 0,
latency_count INTEGER NOT NULL DEFAULT 0,
created_at TEXT DEFAULT (datetime('now')),
updated_at TEXT DEFAULT (datetime('now')),
FOREIGN KEY (site_id) REFERENCES sites(id) ON DELETE CASCADE,
CHECK (
total_calls >= 0 AND success_calls >= 0 AND failed_calls >= 0
AND total_tokens >= 0 AND total_summary_spend >= 0
AND total_site_spend >= 0 AND total_latency_ms >= 0
AND latency_count >= 0
)
)",
"CREATE UNIQUE INDEX IF NOT EXISTS site_hour_usage_hour_site_unique
ON site_hour_usage (bucket_start_utc, site_id)",
"CREATE INDEX IF NOT EXISTS site_hour_usage_hour_idx
ON site_hour_usage (bucket_start_utc)",
"CREATE INDEX IF NOT EXISTS site_hour_usage_site_id_idx
ON site_hour_usage (site_id)",
];
src/db/sqlite_to_postgres.rs
+194
View File
@@ -299,6 +299,200 @@ const TABLES: &[TableSpec] = &[
col!("read_at", text),
],
},
TableSpec {
name: "site_api_endpoints",
columns: &[
col!("id", i64),
col!("site_id", i64),
col!("url", text),
col!("enabled", bool),
col!("sort_order", i64),
col!("cooldown_until", text),
col!("last_selected_at", text),
col!("last_failed_at", text),
col!("last_failure_reason", text),
col!("created_at", text),
col!("updated_at", text),
],
},
TableSpec {
name: "oauth_route_units",
columns: &[
col!("id", i64),
col!("site_id", i64),
col!("provider", text),
col!("name", text),
col!("strategy", text),
col!("enabled", bool),
col!("created_at", text),
col!("updated_at", text),
],
},
TableSpec {
name: "oauth_route_unit_members",
columns: &[
col!("id", i64),
col!("unit_id", i64),
col!("account_id", i64),
col!("sort_order", i64),
col!("success_count", i64),
col!("fail_count", i64),
col!("total_latency_ms", i64),
col!("total_cost", f64),
col!("last_used_at", text),
col!("last_selected_at", text),
col!("last_fail_at", text),
col!("consecutive_fail_count", i64),
col!("cooldown_level", i64),
col!("cooldown_until", text),
col!("created_at", text),
col!("updated_at", text),
],
},
TableSpec {
name: "proxy_debug_traces",
columns: &[
col!("id", i64),
col!("downstream_path", text),
col!("client_kind", text),
col!("session_id", text),
col!("trace_hint", text),
col!("requested_model", text),
col!("downstream_api_key_id", i64),
col!("request_headers_json", text),
col!("request_body_json", text),
col!("sticky_session_key", text),
col!("sticky_hit_channel_id", i64),
col!("selected_channel_id", i64),
col!("selected_route_id", i64),
col!("selected_account_id", i64),
col!("selected_site_id", i64),
col!("selected_site_platform", text),
col!("endpoint_candidates_json", text),
col!("endpoint_runtime_state_json", text),
col!("decision_summary_json", text),
col!("final_status", text),
col!("final_http_status", i64),
col!("final_upstream_path", text),
col!("final_response_headers_json", text),
col!("final_response_body_json", text),
col!("created_at", text),
col!("updated_at", text),
],
},
TableSpec {
name: "proxy_debug_attempts",
columns: &[
col!("id", i64),
col!("trace_id", i64),
col!("attempt_index", i64),
col!("endpoint", text),
col!("request_path", text),
col!("target_url", text),
col!("runtime_executor", text),
col!("request_headers_json", text),
col!("request_body_json", text),
col!("response_status", i64),
col!("response_headers_json", text),
col!("response_body_json", text),
col!("raw_error_text", text),
col!("recover_applied", bool),
col!("downgrade_decision", bool),
col!("downgrade_reason", text),
col!("memory_write_json", text),
col!("created_at", text),
],
},
TableSpec {
name: "admin_snapshots",
columns: &[
col!("id", i64),
col!("namespace", text),
col!("snapshot_key", text),
col!("payload", text),
col!("generated_at", text),
col!("expires_at", text),
col!("stale_until", text),
col!("created_at", text),
col!("updated_at", text),
],
},
TableSpec {
name: "analytics_projection_checkpoints",
columns: &[
col!("projector_key", text),
col!("time_zone", text),
col!("last_proxy_log_id", i64),
col!("watermark_created_at", text),
col!("recompute_from_id", i64),
col!("recompute_requested_at", text),
col!("recompute_reason", text),
col!("recompute_started_at", text),
col!("recompute_completed_at", text),
col!("last_projected_at", text),
col!("last_successful_at", text),
col!("last_error", text),
col!("lease_owner", text),
col!("lease_token", text),
col!("lease_expires_at", text),
col!("created_at", text),
col!("updated_at", text),
],
},
TableSpec {
name: "model_day_usage",
columns: &[
col!("id", i64),
col!("local_day", text),
col!("site_id", i64),
col!("model", text),
col!("total_calls", i64),
col!("success_calls", i64),
col!("failed_calls", i64),
col!("total_tokens", i64),
col!("total_spend", f64),
col!("total_latency_ms", i64),
col!("latency_count", i64),
col!("created_at", text),
col!("updated_at", text),
],
},
TableSpec {
name: "site_day_usage",
columns: &[
col!("id", i64),
col!("local_day", text),
col!("site_id", i64),
col!("total_calls", i64),
col!("success_calls", i64),
col!("failed_calls", i64),
col!("total_tokens", i64),
col!("total_summary_spend", f64),
col!("total_site_spend", f64),
col!("total_latency_ms", i64),
col!("latency_count", i64),
col!("created_at", text),
col!("updated_at", text),
],
},
TableSpec {
name: "site_hour_usage",
columns: &[
col!("id", i64),
col!("bucket_start_utc", text),
col!("site_id", i64),
col!("total_calls", i64),
col!("success_calls", i64),
col!("failed_calls", i64),
col!("total_tokens", i64),
col!("total_summary_spend", f64),
col!("total_site_spend", f64),
col!("total_latency_ms", i64),
col!("latency_count", i64),
col!("created_at", text),
col!("updated_at", text),
],
},
];
pub async fn migrate(