feat: add site observability log drilldown

This commit is contained in:
bnvnvnv
2026-03-11 11:03:57 +00:00
parent 921c551a3a
commit bc3c731ed2
9 changed files with 1508 additions and 31 deletions
@@ -199,4 +199,99 @@ describe('stats proxy logs routes', () => {
usage: { promptTokens: 100, completionTokens: 20 },
});
});
it('filters proxy logs by site and time range', async () => {
const alphaSite = await db.insert(schema.sites).values({
name: 'alpha-site',
url: 'https://alpha.example.com',
platform: 'new-api',
}).returning().get();
const betaSite = await db.insert(schema.sites).values({
name: 'beta-site',
url: 'https://beta.example.com',
platform: 'new-api',
}).returning().get();
const alphaAccount = await db.insert(schema.accounts).values({
siteId: alphaSite.id,
username: 'alpha-user',
accessToken: 'alpha-token',
status: 'active',
}).returning().get();
const betaAccount = await db.insert(schema.accounts).values({
siteId: betaSite.id,
username: 'beta-user',
accessToken: 'beta-token',
status: 'active',
}).returning().get();
await db.insert(schema.proxyLogs).values([
{
accountId: alphaAccount.id,
modelRequested: 'gpt-4o',
modelActual: 'gpt-4o',
status: 'success',
totalTokens: 10,
estimatedCost: 0.11,
createdAt: formatUtcSqlDateTime(new Date('2026-03-09T08:15:00.000Z')),
},
{
accountId: alphaAccount.id,
modelRequested: 'gpt-4.1-mini',
modelActual: 'gpt-4.1-mini',
status: 'failed',
totalTokens: 20,
estimatedCost: 0.22,
createdAt: formatUtcSqlDateTime(new Date('2026-03-09T08:45:00.000Z')),
},
{
accountId: alphaAccount.id,
modelRequested: 'gpt-4.1',
modelActual: 'gpt-4.1',
status: 'success',
totalTokens: 30,
estimatedCost: 0.33,
createdAt: formatUtcSqlDateTime(new Date('2026-03-09T09:15:00.000Z')),
},
{
accountId: betaAccount.id,
modelRequested: 'claude-3-7-sonnet',
modelActual: 'claude-3-7-sonnet',
status: 'success',
totalTokens: 40,
estimatedCost: 0.44,
createdAt: formatUtcSqlDateTime(new Date('2026-03-09T08:30:00.000Z')),
},
]).run();
const response = await app.inject({
method: 'GET',
url: `/api/stats/proxy-logs?siteId=${alphaSite.id}&from=${encodeURIComponent('2026-03-09T08:00:00.000Z')}&to=${encodeURIComponent('2026-03-09T09:00:00.000Z')}`,
});
expect(response.statusCode).toBe(200);
const body = response.json() as {
items: Array<Record<string, unknown>>;
total: number;
summary: {
totalCount: number;
successCount: number;
failedCount: number;
totalCost: number;
totalTokensAll: number;
};
};
expect(body.total).toBe(2);
expect(body.items).toHaveLength(2);
expect(body.items.map((item) => item.siteId)).toEqual([alphaSite.id, alphaSite.id]);
expect(body.items.map((item) => item.siteName)).toEqual(['alpha-site', 'alpha-site']);
expect(body.summary).toEqual({
totalCount: 2,
successCount: 1,
failedCount: 1,
totalCost: 0.33,
totalTokensAll: 30,
});
});
});
@@ -242,4 +242,106 @@ describe('stats dashboard filters disabled sites', () => {
tokensPerMinute: 1800,
});
});
it('returns site availability buckets and average latency from per-request proxy logs', async () => {
const activeSite = await db.insert(schema.sites).values({
name: 'availability-site',
url: 'https://availability.example.com',
platform: 'new-api',
}).returning().get();
const disabledSite = await db.insert(schema.sites).values({
name: 'disabled-availability-site',
url: 'https://disabled-availability.example.com',
platform: 'new-api',
}).returning().get();
await db.run(sql`update sites set status = 'disabled' where id = ${disabledSite.id}`);
const activeAccount = await db.insert(schema.accounts).values({
siteId: activeSite.id,
username: 'availability-user',
accessToken: 'availability-token',
balance: 20,
status: 'active',
}).returning().get();
const disabledAccount = await db.insert(schema.accounts).values({
siteId: disabledSite.id,
username: 'disabled-availability-user',
accessToken: 'disabled-availability-token',
balance: 20,
status: 'active',
}).returning().get();
const now = Date.now();
const recentSuccess = formatUtcSqlDateTime(new Date(now - 5 * 60_000));
const recentFailure = formatUtcSqlDateTime(new Date(now - 65 * 60_000));
await db.insert(schema.proxyLogs).values([
{
accountId: activeAccount.id,
status: 'success',
latencyMs: 120,
createdAt: recentSuccess,
},
{
accountId: activeAccount.id,
status: 'failed',
latencyMs: 280,
createdAt: recentFailure,
},
{
accountId: disabledAccount.id,
status: 'success',
latencyMs: 999,
createdAt: recentSuccess,
},
]).run();
const response = await app.inject({
method: 'GET',
url: '/api/stats/dashboard',
});
expect(response.statusCode).toBe(200);
const body = response.json() as {
siteAvailability?: Array<{
siteId: number;
siteName: string;
totalRequests: number;
successCount: number;
failedCount: number;
availabilityPercent: number | null;
averageLatencyMs: number | null;
buckets: Array<{
totalRequests: number;
successCount: number;
failedCount: number;
}>;
}>;
};
expect(Array.isArray(body.siteAvailability)).toBe(true);
expect(body.siteAvailability).toHaveLength(1);
expect(body.siteAvailability?.[0]).toMatchObject({
siteId: activeSite.id,
siteName: 'availability-site',
totalRequests: 2,
successCount: 1,
failedCount: 1,
availabilityPercent: 50,
averageLatencyMs: 200,
});
expect(body.siteAvailability?.[0]?.buckets).toHaveLength(24);
expect(
body.siteAvailability?.[0]?.buckets.reduce((sum, bucket) => sum + bucket.totalRequests, 0),
).toBe(2);
expect(
body.siteAvailability?.[0]?.buckets.reduce((sum, bucket) => sum + bucket.successCount, 0),
).toBe(1);
expect(
body.siteAvailability?.[0]?.buckets.reduce((sum, bucket) => sum + bucket.failedCount, 0),
).toBe(1);
});
});
+235 -8
View File
@@ -19,9 +19,11 @@ import {
} from '../../services/proxyLogStore.js';
import { getCredentialModeFromExtraConfig } from '../../services/accountExtraConfig.js';
import {
formatLocalDateTime,
formatUtcSqlDateTime,
getLocalDayRangeUtc,
getLocalRangeStartUtc,
parseStoredUtcDateTime,
toLocalDayKeyFromStoredUtc,
} from '../../services/localTimeService.js';
@@ -105,6 +107,20 @@ function normalizeProxyLogSearch(raw?: string): string {
return (raw || '').trim().toLowerCase();
}
function normalizeProxyLogSiteId(raw?: string): number | null {
const parsed = Number.parseInt(raw || '', 10);
if (!Number.isFinite(parsed) || parsed <= 0) return null;
return parsed;
}
function normalizeProxyLogTimeBoundary(raw?: string): string | null {
const text = (raw || '').trim();
if (!text) return null;
const parsed = new Date(text);
if (Number.isNaN(parsed.getTime())) return null;
return formatUtcSqlDateTime(parsed);
}
function buildProxyLogSearchCondition(search: string) {
if (!search) return null;
const likeTerm = `%${search}%`;
@@ -127,10 +143,16 @@ function buildProxyLogStatusCondition(status: ProxyLogStatusFilter) {
function buildProxyLogWhereClause(params: {
status?: ProxyLogStatusFilter;
search?: string;
siteId?: number | null;
fromUtc?: string | null;
toUtc?: string | null;
}) {
const conditions = [
params.status ? buildProxyLogStatusCondition(params.status) : null,
params.search ? buildProxyLogSearchCondition(params.search) : null,
params.siteId ? eq(schema.sites.id, params.siteId) : null,
params.fromUtc ? gte(schema.proxyLogs.createdAt, params.fromUtc) : null,
params.toUtc ? lt(schema.proxyLogs.createdAt, params.toUtc) : null,
].filter((condition): condition is NonNullable<typeof condition> => condition !== null);
if (conditions.length === 0) return undefined;
@@ -141,11 +163,156 @@ function toRoundedMicroNumber(value: number | null | undefined): number {
return Math.round(Number(value || 0) * 1_000_000) / 1_000_000;
}
const SITE_AVAILABILITY_BUCKET_COUNT = 24;
const SITE_AVAILABILITY_BUCKET_MS = 60 * 60 * 1000;
type SiteAvailabilitySiteRow = {
id: number;
name: string;
url: string | null;
platform: string | null;
sortOrder: number | null;
isPinned: boolean | null;
};
type SiteAvailabilityLogRow = {
siteId: number | null;
createdAt: string | null;
status: string | null;
latencyMs: number | null;
};
type SiteAvailabilityBucketAccumulator = {
label: string;
totalRequests: number;
successCount: number;
failedCount: number;
latencyTotalMs: number;
latencyCount: number;
};
function roundPercent(value: number | null): number | null {
if (value == null || !Number.isFinite(value)) return null;
return Math.round(value * 10) / 10;
}
function buildSiteAvailabilitySummaries(
sites: SiteAvailabilitySiteRow[],
logs: SiteAvailabilityLogRow[],
now = new Date(),
) {
const endLocal = new Date(now.getFullYear(), now.getMonth(), now.getDate(), now.getHours(), 0, 0, 0);
const startLocal = new Date(endLocal.getTime() - (SITE_AVAILABILITY_BUCKET_COUNT - 1) * SITE_AVAILABILITY_BUCKET_MS);
const startMs = startLocal.getTime();
const rangeMs = SITE_AVAILABILITY_BUCKET_COUNT * SITE_AVAILABILITY_BUCKET_MS;
const createBucketTemplate = (): SiteAvailabilityBucketAccumulator[] => (
Array.from({ length: SITE_AVAILABILITY_BUCKET_COUNT }, (_, index) => {
const bucketStart = new Date(startMs + index * SITE_AVAILABILITY_BUCKET_MS);
return {
label: formatLocalDateTime(bucketStart),
totalRequests: 0,
successCount: 0,
failedCount: 0,
latencyTotalMs: 0,
latencyCount: 0,
};
})
);
const siteMap = new Map<number, {
site: SiteAvailabilitySiteRow;
totalRequests: number;
successCount: number;
failedCount: number;
latencyTotalMs: number;
latencyCount: number;
buckets: SiteAvailabilityBucketAccumulator[];
}>();
for (const site of sites) {
siteMap.set(site.id, {
site,
totalRequests: 0,
successCount: 0,
failedCount: 0,
latencyTotalMs: 0,
latencyCount: 0,
buckets: createBucketTemplate(),
});
}
for (const log of logs) {
if (log.siteId == null) continue;
const target = siteMap.get(log.siteId);
if (!target) continue;
const parsed = parseStoredUtcDateTime(log.createdAt);
if (!parsed) continue;
const timestampMs = parsed.getTime();
const diffMs = timestampMs - startMs;
if (diffMs < 0 || diffMs >= rangeMs) continue;
const bucketIndex = Math.floor(diffMs / SITE_AVAILABILITY_BUCKET_MS);
const bucket = target.buckets[bucketIndex];
const isSuccess = (log.status || '').trim().toLowerCase() === 'success';
target.totalRequests += 1;
bucket.totalRequests += 1;
if (isSuccess) {
target.successCount += 1;
bucket.successCount += 1;
} else {
target.failedCount += 1;
bucket.failedCount += 1;
}
const latencyMs = Number(log.latencyMs);
if (Number.isFinite(latencyMs) && latencyMs >= 0) {
target.latencyTotalMs += latencyMs;
target.latencyCount += 1;
bucket.latencyTotalMs += latencyMs;
bucket.latencyCount += 1;
}
}
return sites.map((site) => {
const aggregate = siteMap.get(site.id)!;
return {
siteId: site.id,
siteName: site.name,
siteUrl: site.url,
platform: site.platform,
totalRequests: aggregate.totalRequests,
successCount: aggregate.successCount,
failedCount: aggregate.failedCount,
availabilityPercent: aggregate.totalRequests > 0
? roundPercent((aggregate.successCount / aggregate.totalRequests) * 100)
: null,
averageLatencyMs: aggregate.latencyCount > 0
? Math.round(aggregate.latencyTotalMs / aggregate.latencyCount)
: null,
buckets: aggregate.buckets.map((bucket) => ({
label: bucket.label,
totalRequests: bucket.totalRequests,
successCount: bucket.successCount,
failedCount: bucket.failedCount,
availabilityPercent: bucket.totalRequests > 0
? roundPercent((bucket.successCount / bucket.totalRequests) * 100)
: null,
averageLatencyMs: bucket.latencyCount > 0
? Math.round(bucket.latencyTotalMs / bucket.latencyCount)
: null,
})),
};
});
}
function mapProxyLogRow(
row: {
proxy_logs: Record<string, unknown> & { billingDetails?: string | null };
accounts: { username?: string | null } | null;
sites: { name?: string | null; url?: string | null } | null;
sites: { id?: number | null; name?: string | null; url?: string | null } | null;
},
options?: { includeBillingDetails?: boolean },
) {
@@ -155,6 +322,7 @@ function mapProxyLogRow(
? { billingDetails: parseProxyLogBillingDetails(row.proxy_logs.billingDetails) }
: {}),
username: row.accounts?.username || null,
siteId: row.sites?.id || null,
siteName: row.sites?.name || null,
siteUrl: row.sites?.url || null,
};
@@ -271,6 +439,49 @@ export async function statsRoutes(app: FastifyInstance) {
extraConfig: account.extraConfig,
}), 0);
const modelAnalysis = buildModelAnalysis(recentProxyLogs, { days: 7 });
const activeSites = (await db.select({
id: schema.sites.id,
name: schema.sites.name,
url: schema.sites.url,
platform: schema.sites.platform,
sortOrder: schema.sites.sortOrder,
isPinned: schema.sites.isPinned,
}).from(schema.sites)
.where(eq(schema.sites.status, 'active'))
.all())
.sort((left: SiteAvailabilitySiteRow, right: SiteAvailabilitySiteRow) => {
const leftPinned = left.isPinned ? 1 : 0;
const rightPinned = right.isPinned ? 1 : 0;
if (leftPinned !== rightPinned) return rightPinned - leftPinned;
const leftOrder = Number(left.sortOrder || 0);
const rightOrder = Number(right.sortOrder || 0);
if (leftOrder !== rightOrder) return leftOrder - rightOrder;
return String(left.name || '').localeCompare(String(right.name || ''));
});
const siteAvailabilityNow = new Date();
const siteAvailabilitySinceUtc = formatUtcSqlDateTime(new Date(
siteAvailabilityNow.getFullYear(),
siteAvailabilityNow.getMonth(),
siteAvailabilityNow.getDate(),
siteAvailabilityNow.getHours() - (SITE_AVAILABILITY_BUCKET_COUNT - 1),
0,
0,
0,
));
const siteAvailabilityLogs = await db.select({
siteId: schema.sites.id,
createdAt: schema.proxyLogs.createdAt,
status: schema.proxyLogs.status,
latencyMs: schema.proxyLogs.latencyMs,
}).from(schema.proxyLogs)
.innerJoin(schema.accounts, eq(schema.proxyLogs.accountId, schema.accounts.id))
.innerJoin(schema.sites, eq(schema.accounts.siteId, schema.sites.id))
.where(and(
gte(schema.proxyLogs.createdAt, siteAvailabilitySinceUtc),
eq(schema.sites.status, 'active'),
))
.all();
const siteAvailability = buildSiteAvailabilitySummaries(activeSites, siteAvailabilityLogs, siteAvailabilityNow);
return {
totalBalance,
@@ -286,18 +497,30 @@ export async function statsRoutes(app: FastifyInstance) {
requestsPerMinute,
tokensPerMinute,
},
siteAvailability,
modelAnalysis,
};
});
// Proxy logs
app.get<{ Querystring: { limit?: string; offset?: string; status?: string; search?: string } }>('/api/stats/proxy-logs', async (request) => {
app.get<{ Querystring: {
limit?: string;
offset?: string;
status?: string;
search?: string;
siteId?: string;
from?: string;
to?: string;
} }>('/api/stats/proxy-logs', async (request) => {
const limit = normalizeProxyLogPageSize(request.query.limit);
const offset = normalizeProxyLogOffset(request.query.offset);
const status = normalizeProxyLogStatusFilter(request.query.status);
const search = normalizeProxyLogSearch(request.query.search);
const listWhere = buildProxyLogWhereClause({ status, search });
const summaryWhere = buildProxyLogWhereClause({ search });
const siteId = normalizeProxyLogSiteId(request.query.siteId);
const fromUtc = normalizeProxyLogTimeBoundary(request.query.from);
const toUtc = normalizeProxyLogTimeBoundary(request.query.to);
const listWhere = buildProxyLogWhereClause({ status, search, siteId, fromUtc, toUtc });
const summaryWhere = buildProxyLogWhereClause({ search, siteId, fromUtc, toUtc });
const listRows = await withProxyLogSelectFields(({ fields }) => {
let query = db.select({
@@ -320,12 +543,14 @@ export async function statsRoutes(app: FastifyInstance) {
}, { includeBillingDetails: false }) as Array<{
proxy_logs: Record<string, unknown> & { billingDetails?: string | null };
accounts: { username?: string | null } | null;
sites: { name?: string | null; url?: string | null } | null;
sites: { id?: number | null; name?: string | null; url?: string | null } | null;
}>;
let totalQuery = db.select({
total: sql<number>`count(*)`,
}).from(schema.proxyLogs);
}).from(schema.proxyLogs)
.leftJoin(schema.accounts, eq(schema.proxyLogs.accountId, schema.accounts.id))
.leftJoin(schema.sites, eq(schema.accounts.siteId, schema.sites.id));
if (listWhere) {
totalQuery = totalQuery.where(listWhere) as typeof totalQuery;
}
@@ -337,7 +562,9 @@ export async function statsRoutes(app: FastifyInstance) {
failedCount: sql<number>`coalesce(sum(case when coalesce(${schema.proxyLogs.status}, '') <> 'success' then 1 else 0 end), 0)`,
totalCost: sql<number>`coalesce(sum(coalesce(${schema.proxyLogs.estimatedCost}, 0)), 0)`,
totalTokensAll: sql<number>`coalesce(sum(coalesce(${schema.proxyLogs.totalTokens}, 0)), 0)`,
}).from(schema.proxyLogs);
}).from(schema.proxyLogs)
.leftJoin(schema.accounts, eq(schema.proxyLogs.accountId, schema.accounts.id))
.leftJoin(schema.sites, eq(schema.accounts.siteId, schema.sites.id));
if (summaryWhere) {
summaryQuery = summaryQuery.where(summaryWhere) as typeof summaryQuery;
}
@@ -377,7 +604,7 @@ export async function statsRoutes(app: FastifyInstance) {
), { includeBillingDetails: true }) as {
proxy_logs: Record<string, unknown> & { billingDetails?: string | null };
accounts: { username?: string | null } | null;
sites: { name?: string | null; url?: string | null } | null;
sites: { id?: number | null; name?: string | null; url?: string | null } | null;
} | undefined;
if (!row) {