boos/BoosAPI
2
0
Fork 2
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix model tester attachment replay and capability gating

Browse Source
This commit is contained in:
cita
2026-03-20 22:56:01 +08:00
parent 8c5ecc26d1
commit 4369aeb1cb
9 changed files with 646 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/server/transformers/anthropic/messages/index.test.ts
+51
View File
@@ -34,6 +34,57 @@ describe('anthropicMessagesTransformer protocol contract', () => {
});
});
it('preserves native image and document blocks when parsing into canonical envelopes', () => {
const result = anthropicMessagesTransformer.parseRequest({
model: 'claude-sonnet-4-5',
max_tokens: 256,
messages: [
{
role: 'user',
content: [
{ type: 'text', text: 'inspect both attachments' },
{
type: 'image',
source: {
type: 'base64',
media_type: 'image/png',
data: 'QUFBQQ==',
},
},
{
type: 'document',
source: {
type: 'base64',
media_type: 'application/pdf',
data: 'JVBERi0x',
},
title: 'brief.pdf',
},
],
},
],
});
expect(result.error).toBeUndefined();
expect(result.value).toMatchObject({
messages: [
{
role: 'user',
parts: [
{ type: 'text', text: 'inspect both attachments' },
{ type: 'image', url: 'data:image/png;base64,QUFBQQ==' },
{
type: 'file',
fileData: 'JVBERi0x',
filename: 'brief.pdf',
mimeType: 'application/pdf',
},
],
},
],
});
});
it('builds native messages requests from canonical envelopes', () => {
const body = anthropicMessagesTransformer.buildProtocolRequest({
operation: 'count_tokens',
src/server/transformers/gemini/generate-content/compatibility.ts
+58 -6
View File
@@ -1,3 +1,4 @@
import { toOpenAiChatFileBlock } from '../../shared/inputFile.js';
import type { NormalizedFinalResponse } from '../../shared/normalized.js';
import { extractReasoningMetadataFromGeminiRequest } from './convert.js';
@@ -30,6 +31,60 @@ function buildDataUrl(part: GeminiRecord): string | null {
return `data:${mimeType};base64,${data}`;
}
function buildInlineData(
part: GeminiRecord,
): { mimeType: string; data: string } | null {
const inlineData = isRecord(part.inlineData) ? part.inlineData : null;
if (!inlineData) return null;
const mimeType = asTrimmedString(inlineData.mime_type ?? inlineData.mimeType) || 'application/octet-stream';
const data = asTrimmedString(inlineData.data);
if (!data) return null;
return { mimeType, data };
}
function buildFileDataSource(
part: GeminiRecord,
): { fileUri: string; mimeType: string | null } | null {
const fileData = isRecord(part.fileData) ? part.fileData : null;
if (!fileData) return null;
const fileUri = asTrimmedString(fileData.fileUri ?? fileData.file_uri);
if (!fileUri) return null;
const mimeType = asTrimmedString(fileData.mimeType ?? fileData.mime_type) || null;
return { fileUri, mimeType };
}
function toOpenAiBlockFromGeminiPart(part: GeminiRecord): Record<string, unknown> | null {
const inlineData = buildInlineData(part);
if (inlineData) {
const normalizedMimeType = inlineData.mimeType.toLowerCase();
if (normalizedMimeType.startsWith('image/')) {
return {
type: 'image_url',
image_url: { url: buildDataUrl(part)! },
};
}
return toOpenAiChatFileBlock({
fileData: inlineData.data,
mimeType: inlineData.mimeType,
});
}
const fileData = buildFileDataSource(part);
if (!fileData) return null;
if (fileData.mimeType?.toLowerCase().startsWith('image/')) {
return {
type: 'image_url',
image_url: { url: fileData.fileUri },
};
}
return toOpenAiChatFileBlock({
fileUrl: fileData.fileUri,
mimeType: fileData.mimeType,
});
}
function toOpenAiContent(contentParts: GeminiRecord[]): string | Array<Record<string, unknown>> {
const blocks: Array<Record<string, unknown>> = [];
let textContent = '';
@@ -41,12 +96,9 @@ function toOpenAiContent(contentParts: GeminiRecord[]): string | Array<Record<st
continue;
}
const dataUrl = buildDataUrl(part);
if (dataUrl) {
blocks.push({
type: 'image_url',
image_url: { url: dataUrl },
});
const block = toOpenAiBlockFromGeminiPart(part);
if (block) {
blocks.push(block);
}
}
src/server/transformers/gemini/generate-content/index.test.ts
+43
View File
@@ -95,6 +95,49 @@ describe('geminiGenerateContentTransformer.inbound', () => {
});
});
it('compatibility preserves inline document parts as OpenAI file blocks', () => {
const body = geminiGenerateContentTransformer.compatibility.buildOpenAiBodyFromGeminiRequest({
modelName: 'gpt-5.4',
stream: false,
body: {
contents: [
{
role: 'user',
parts: [
{ text: 'summarize this file' },
{
inlineData: {
mimeType: 'application/pdf',
data: 'JVBERi0x',
},
},
],
},
],
},
});
expect(body).toEqual({
model: 'gpt-5.4',
stream: false,
messages: [
{
role: 'user',
content: [
{ type: 'text', text: 'summarize this file' },
{
type: 'file',
file: {
file_data: 'JVBERi0x',
mime_type: 'application/pdf',
},
},
],
},
],
});
});
it('preserves native Gemini request fields through normalization', () => {
const body = geminiGenerateContentTransformer.inbound.normalizeRequest({
contents: [
src/server/transformers/shared/chatFormatsCore.ts
+132 -11
View File
@@ -1,6 +1,7 @@
import {
decodeAnthropicReasoningSignature,
} from './reasoningTransport.js';
import { toOpenAiChatFileBlock } from './inputFile.js';
import {
consumeThinkTaggedText,
createThinkTagParserState,
@@ -86,6 +87,10 @@ function isNonEmptyString(value: unknown): value is string {
return typeof value === 'string' && value.trim().length > 0;
}
function asTrimmedString(value: unknown): string {
return typeof value === 'string' ? value.trim() : '';
}
function pickFiniteNumber(value: unknown): number | undefined {
return typeof value === 'number' && Number.isFinite(value) ? value : undefined;
}
@@ -464,6 +469,111 @@ function collectToolCallsFromResponsesPayload(payload: Record<string, unknown>):
return toolCalls;
}
function formatAnthropicBase64DataUrl(mimeType: string, data: string): string {
return `data:${mimeType};base64,${data}`;
}
function parseAnthropicBase64Source(
source: unknown,
): { mimeType: string; data: string } | null {
if (!isRecord(source)) return null;
const sourceType = asTrimmedString(source.type).toLowerCase();
if (sourceType !== 'base64') return null;
const mimeType = (
asTrimmedString(source.media_type)
|| asTrimmedString(source.mime_type)
|| asTrimmedString(source.mediaType)
|| asTrimmedString(source.mimeType)
|| 'application/octet-stream'
);
const data = asTrimmedString(source.data);
if (!data) return null;
return { mimeType, data };
}
function parseAnthropicUrlSource(
source: unknown,
): { url: string; mimeType: string | null } | null {
if (!isRecord(source)) return null;
const sourceType = asTrimmedString(source.type).toLowerCase();
if (sourceType !== 'url') return null;
const url = asTrimmedString(source.url);
if (!url) return null;
const mimeType = (
asTrimmedString(source.media_type)
|| asTrimmedString(source.mime_type)
|| asTrimmedString(source.mediaType)
|| asTrimmedString(source.mimeType)
|| null
);
return { url, mimeType };
}
function toOpenAiContentBlockFromClaudeBlock(
block: Record<string, unknown>,
): Record<string, unknown> | null {
const blockType = asTrimmedString(block.type).toLowerCase();
if (blockType === 'text') {
const text = asTrimmedString(block.text);
return text ? { type: 'text', text } : null;
}
if (blockType !== 'image' && blockType !== 'document') {
return null;
}
const title = asTrimmedString(block.title);
const base64Source = parseAnthropicBase64Source(block.source);
const urlSource = parseAnthropicUrlSource(block.source);
const mimeType = (base64Source?.mimeType || urlSource?.mimeType || '').toLowerCase();
const treatAsImage = blockType === 'image' || mimeType.startsWith('image/');
if (treatAsImage) {
const imageUrl = base64Source
? formatAnthropicBase64DataUrl(base64Source.mimeType, base64Source.data)
: (urlSource?.url || '');
return imageUrl
? {
type: 'image_url',
image_url: { url: imageUrl },
}
: null;
}
if (base64Source) {
return toOpenAiChatFileBlock({
fileData: base64Source.data,
filename: title || undefined,
mimeType: base64Source.mimeType,
});
}
if (urlSource) {
return toOpenAiChatFileBlock({
fileUrl: urlSource.url,
filename: title || undefined,
mimeType: urlSource.mimeType,
});
}
return null;
}
function collapseOpenAiContentBlocks(
blocks: Array<Record<string, unknown>>,
): string | Array<Record<string, unknown>> | null {
if (blocks.length <= 0) return null;
const textOnly = blocks.every((block) => block.type === 'text' && typeof block.text === 'string');
if (!textOnly) return blocks;
const text = blocks
.map((block) => (typeof block.text === 'string' ? block.text : ''))
.join('\n\n')
.trim();
return text || null;
}
function convertClaudeRequestToOpenAiBody(body: Record<string, unknown>): {
model: string;
stream: boolean;
@@ -521,14 +631,14 @@ function convertClaudeRequestToOpenAiBody(body: Record<string, unknown>): {
// Claude tool blocks need explicit OpenAI mapping:
// - assistant.tool_use -> assistant.tool_calls
// - user.tool_result -> tool messages with tool_call_id
const textParts: string[] = [];
const contentBlocks: Array<Record<string, unknown>> = [];
const toolCalls: Array<Record<string, unknown>> = [];
const flushTextAsMessage = () => {
const merged = textParts.map((item) => item.trim()).filter(Boolean).join('\n\n');
textParts.length = 0;
if (!merged) return;
messages.push({ role: mappedRole, content: merged });
const flushContentAsMessage = () => {
const contentPayload = collapseOpenAiContentBlocks(contentBlocks);
contentBlocks.length = 0;
if (contentPayload === null) return;
messages.push({ role: mappedRole, content: contentPayload });
};
for (const block of content) {
@@ -536,7 +646,7 @@ function convertClaudeRequestToOpenAiBody(body: Record<string, unknown>): {
const blockType = typeof block.type === 'string' ? block.type : '';
if (blockType === 'tool_result') {
flushTextAsMessage();
flushContentAsMessage();
appendToolResultMessage(block.tool_use_id, block.content);
continue;
}
@@ -564,11 +674,22 @@ function convertClaudeRequestToOpenAiBody(body: Record<string, unknown>): {
continue;
}
const text = parseClaudeMessageContent(block);
if (text) textParts.push(text);
const contentBlock = toOpenAiContentBlockFromClaudeBlock(block);
if (contentBlock) {
contentBlocks.push(contentBlock);
continue;
}
const merged = textParts.map((item) => item.trim()).filter(Boolean).join('\n\n');
const text = parseClaudeMessageContent(block);
if (text) {
contentBlocks.push({
type: 'text',
text,
});
}
}
const merged = collapseOpenAiContentBlocks(contentBlocks);
if (toolCalls.length > 0) {
const assistantMessage: Record<string, unknown> = {
role: 'assistant',
@@ -577,7 +698,7 @@ function convertClaudeRequestToOpenAiBody(body: Record<string, unknown>): {
// Keep textual assistant preface when present.
assistantMessage.content = merged || '';
messages.push(assistantMessage);
} else if (merged) {
} else if (merged !== null) {
messages.push({
role: mappedRole,
content: merged,
src/web/pages/ModelTester.tsx
+71 -35
View File
@@ -23,6 +23,7 @@ import {
createLoadingAssistantMessage,
createMessage,
createConversationUserMessage,
extractConversationUploadedFilesFromMessage,
filterModelTesterModelNames,
finalizeIncompleteMessage,
findLastLoadingAssistantIndex,
@@ -44,7 +45,12 @@ import {
type TestTargetFormat,
type TestChatPayload,
} from './helpers/modelTesterSession.js';
import { resolveConversationFileCapability } from './helpers/conversationFileCapabilities.js';
import {
buildConversationFileAccept,
buildConversationFileHint,
isConversationUploadedFileSupported,
resolveConversationFileCapability,
} from './helpers/conversationFileCapabilities.js';
import ModernSelect from '../components/ModernSelect.js';
import { useAnimatedVisibility } from '../components/useAnimatedVisibility.js';
import { tr } from '../i18n.js';
@@ -76,7 +82,6 @@ type ConversationFileState = UploadState & {
};
const POLL_INTERVAL_MS = 1200;
const CONVERSATION_FILE_ACCEPT = '.pdf,.txt,.md,.markdown,.json,image/*,audio/*';
const wait = (ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms));
const createConversationFileLocalId = () =>
@@ -689,6 +694,19 @@ export default function ModelTester() {
const restoredSessionRef = useRef<ReturnType<typeof parseModelTesterSession>>(null);
const streamAbortRef = useRef<AbortController | null>(null);
const streamStopRequestedRef = useRef(false);
const conversationFileCapability = useMemo(
() => resolveConversationFileCapability(inputs.protocol),
[inputs.protocol],
);
const conversationFileSupported = conversationFileCapability.supported;
const conversationFileAccept = useMemo(
() => buildConversationFileAccept(conversationFileCapability),
[conversationFileCapability],
);
const conversationFileHint = useMemo(
() => buildConversationFileHint(conversationFileCapability),
[conversationFileCapability],
);
const pushDebug = useCallback((level: DebugTimelineEntry['level'], text: string) => {
const now = new Date().toISOString();
@@ -884,34 +902,37 @@ export default function ModelTester() {
status: 'pending' as const,
errorMessage: null,
})));
setConversationFiles((prev) => [...prev, ...nextFiles]);
pushDebug('info', `已添加 ${nextFiles.length} 个会话附件。`);
const acceptedFiles = nextFiles.filter((file) => isConversationUploadedFileSupported(
conversationFileCapability,
{ filename: file.name, mimeType: file.mimeType },
));
const rejectedFiles = nextFiles.filter((file) => !isConversationUploadedFileSupported(
conversationFileCapability,
{ filename: file.name, mimeType: file.mimeType },
));
if (acceptedFiles.length > 0) {
setConversationFiles((prev) => [...prev, ...acceptedFiles]);
pushDebug('info', `已添加 ${acceptedFiles.length} 个会话附件。`);
}
if (rejectedFiles.length > 0) {
const message = `当前协议不支持这些会话附件：${rejectedFiles.map((file) => file.name).join('、')}。${conversationFileHint}`;
setError(message);
pushDebug('warn', message);
}
} catch (readError: any) {
const message = readError?.message || '读取附件失败';
setError(message);
pushDebug('error', message);
}
}, [pushDebug]);
}, [conversationFileCapability, conversationFileHint, pushDebug]);
const removeConversationFile = useCallback((localId: string) => {
if (sending) return;
setConversationFiles((prev) => prev.filter((item) => item.localId !== localId));
}, [sending]);
const extractUploadedFilesFromMessage = useCallback((message: ChatMessage): ConversationUploadedFile[] => {
const parts = Array.isArray(message.parts) ? message.parts : [];
return parts.flatMap((part) => {
if (part.type !== 'input_file') return [];
const fileId = typeof part.fileId === 'string' ? part.fileId.trim() : '';
if (!fileId) return [];
return [{
fileId,
filename: typeof part.filename === 'string' && part.filename.trim() ? part.filename.trim() : null,
mimeType: typeof part.mimeType === 'string' && part.mimeType.trim() ? part.mimeType.trim() : null,
}];
});
}, []);
const uploadConversationFiles = useCallback(async (): Promise<ConversationUploadedFile[]> => {
if (conversationFiles.length <= 0) return [];
@@ -979,6 +1000,20 @@ export default function ModelTester() {
data: item.dataUrl,
})), [conversationFiles]);
const ensureSupportedConversationFiles = useCallback((files: ConversationUploadedFile[]): boolean => {
const unsupported = files.filter((file) => !isConversationUploadedFileSupported(conversationFileCapability, file));
if (unsupported.length <= 0) return true;
const names = unsupported.map((file, index) => {
const filename = typeof file.filename === 'string' ? file.filename.trim() : '';
return filename || `附件${index + 1}`;
});
const message = `当前协议不支持这些会话附件：${names.join('、')}。${conversationFileHint}`;
setError(message);
pushDebug('warn', message);
return false;
}, [conversationFileCapability, conversationFileHint, pushDebug]);
const buildConversationMessagesWithSystem = useCallback((baseMessages: ChatMessage[]) => {
if (!inputs.systemPrompt.trim()) return baseMessages;
return [
@@ -1348,11 +1383,6 @@ export default function ModelTester() {
() => filteredModels.map((item) => ({ value: item, label: item })),
[filteredModels],
);
const conversationFileCapability = useMemo(
() => resolveConversationFileCapability(inputs.protocol),
[inputs.protocol],
);
const conversationFileSupported = conversationFileCapability.supported;
const canSend = useMemo(() => {
if (sending || pendingJobId || !inputs.model) return false;
if (inputs.mode !== 'conversation') {
@@ -1775,6 +1805,9 @@ export default function ModelTester() {
baseMessages: ChatMessage[],
files: ConversationUploadedFile[] = [],
) => {
if (!ensureSupportedConversationFiles(files)) {
return;
}
const userMessage = createConversationUserMessage(prompt, files);
const loadingAssistant = createLoadingAssistantMessage();
const nextMessages = [...baseMessages, userMessage, loadingAssistant];
@@ -1793,7 +1826,7 @@ export default function ModelTester() {
}
await dispatchPayload(nextMessages, payload, { syncedCustomBody });
}, [buildConversationProxyEnvelope, buildPayloadWithMessages, createConversationUserMessage, customRequestMode, dispatchPayload, dispatchProxyEnvelope, inputs.protocol, pushDebug]);
}, [buildConversationProxyEnvelope, buildPayloadWithMessages, createConversationUserMessage, customRequestMode, dispatchPayload, dispatchProxyEnvelope, ensureSupportedConversationFiles, inputs.protocol, pushDebug]);
const sendModeRequest = useCallback(async () => {
const envelope = buildModeProxyEnvelope();
@@ -1830,8 +1863,13 @@ export default function ModelTester() {
if (!customRequestMode && conversationFileSupported && conversationFiles.length > 0) {
setSending(true);
try {
const draftFiles = inlineConversationFiles();
if (!ensureSupportedConversationFiles(draftFiles)) {
setSending(false);
return;
}
const uploadedFiles = conversationFileCapability.documentMode === 'inline_only'
? inlineConversationFiles()
? draftFiles
: await uploadConversationFiles();
setInput('');
setConversationFiles([]);
@@ -1870,7 +1908,7 @@ export default function ModelTester() {
{ stream: inputs.stream, jobMode: !inputs.stream },
),
);
}, [canSend, conversationFileCapability, conversationFileSupported, conversationFiles.length, customRequestBody, customRequestMode, dispatchPayload, inlineConversationFiles, input, inputs.mode, messages, pushDebug, sendModeRequest, sendWithPrompt, uploadConversationFiles]);
}, [canSend, conversationFileCapability, conversationFileSupported, conversationFiles.length, customRequestBody, customRequestMode, dispatchPayload, ensureSupportedConversationFiles, inlineConversationFiles, input, inputs.mode, messages, pushDebug, sendModeRequest, sendWithPrompt, uploadConversationFiles]);
const retryPending = useCallback(async () => {
if (sending || pendingJobId || !pendingPayload) return;
@@ -2047,11 +2085,11 @@ export default function ModelTester() {
const base = messages.slice(0, userIndex);
const prompt = messages[userIndex].content;
const files = extractUploadedFilesFromMessage(messages[userIndex]);
const files = extractConversationUploadedFilesFromMessage(messages[userIndex]);
setEditingMessageId(null);
setEditValue('');
void sendWithPrompt(prompt, base, files);
}, [extractUploadedFilesFromMessage, messages, pendingJobId, sendWithPrompt, sending]);
}, [messages, pendingJobId, sendWithPrompt, sending]);
const startEditMessage = useCallback((target: ChatMessage) => {
if (sending) return;
@@ -2085,9 +2123,9 @@ export default function ModelTester() {
if (retry && target.role === 'user') {
const base = updated.slice(0, targetIndex);
void sendWithPrompt(nextContent, base, extractUploadedFilesFromMessage(target));
void sendWithPrompt(nextContent, base, extractConversationUploadedFilesFromMessage(target));
}
}, [cancelEditMessage, editValue, editingMessageId, extractUploadedFilesFromMessage, messages, sendWithPrompt]);
}, [cancelEditMessage, editValue, editingMessageId, messages, sendWithPrompt]);
const syncMessageToBody = useCallback(() => {
const nextBody = syncMessagesToCustomRequestBody(customRequestBody, messages, inputs);
@@ -2806,7 +2844,7 @@ export default function ModelTester() {
ref={conversationFileInputRef}
type="file"
multiple
accept={CONVERSATION_FILE_ACCEPT}
accept={conversationFileAccept}
style={{ display: 'none' }}
onChange={(event) => {
void handleConversationFilesChange(event.target.files);
@@ -2828,9 +2866,7 @@ export default function ModelTester() {
? '自定义请求模式不会自动上传这些附件；关闭自定义模式后可走标准 /v1/files 链路。'
: !conversationFileSupported
? (conversationFileCapability.reason || '当前协议暂不支持会话附件注入。')
: conversationFileCapability.documentMode === 'inline_only'
? '支持 PDF / TXT / Markdown / JSON / 图片 / 音频；发送时会以内联文档方式注入。'
: '支持 PDF / TXT / Markdown / JSON / 图片 / 音频；发送前会先上传到 /v1/files。'}
: conversationFileHint}
</span>
</div>
{conversationFiles.length > 0 && (
src/web/pages/helpers/conversationFileCapabilities.test.ts
+52 -4
View File
@@ -1,31 +1,79 @@
import { describe, expect, it } from 'vitest';
import { resolveConversationFileCapability } from './conversationFileCapabilities.js';
import {
buildConversationFileAccept,
buildConversationFileHint,
isConversationUploadedFileSupported,
resolveConversationFileCapability,
} from './conversationFileCapabilities.js';
describe('resolveConversationFileCapability', () => {
it('keeps OpenAI and Responses on native file references', () => {
expect(resolveConversationFileCapability('openai')).toEqual({
supported: true,
imageMode: 'native',
audioMode: 'native',
documentMode: 'native',
reason: '',
});
expect(resolveConversationFileCapability('responses')).toEqual({
supported: true,
imageMode: 'native',
audioMode: 'native',
documentMode: 'native',
reason: '',
});
});
it('marks Claude and Gemini as inline-only document transports', () => {
it('maps Claude and Gemini media support separately', () => {
expect(resolveConversationFileCapability('claude')).toEqual({
supported: true,
imageMode: 'native',
audioMode: 'unsupported',
documentMode: 'inline_only',
reason: '当前协议会以内联文档方式发送会话附件。',
reason: '',
});
expect(resolveConversationFileCapability('gemini')).toEqual({
supported: true,
imageMode: 'native',
audioMode: 'native',
documentMode: 'inline_only',
reason: '当前协议会以内联文档方式发送会话附件。',
reason: '',
});
});
it('builds the accept list from the supported file kinds', () => {
expect(buildConversationFileAccept(resolveConversationFileCapability('claude'))).toBe(
'.pdf,.txt,.md,.markdown,.json,image/*',
);
expect(buildConversationFileAccept(resolveConversationFileCapability('gemini'))).toBe(
'.pdf,.txt,.md,.markdown,.json,image/*,audio/*',
);
});
it('describes the actual per-kind transport behavior', () => {
expect(buildConversationFileHint(resolveConversationFileCapability('claude'))).toBe(
'支持 PDF / TXT / Markdown / JSON / 图片；文档会以内联数据注入，图片会按图片部件发送。',
);
expect(buildConversationFileHint(resolveConversationFileCapability('gemini'))).toBe(
'支持 PDF / TXT / Markdown / JSON / 图片 / 音频；文档会以内联数据注入，图片会按图片部件发送，音频会按音频部件发送。',
);
});
it('rejects unsupported file kinds before send', () => {
const claudeCapability = resolveConversationFileCapability('claude');
expect(isConversationUploadedFileSupported(claudeCapability, {
filename: 'paper.pdf',
mimeType: 'application/pdf',
})).toBe(true);
expect(isConversationUploadedFileSupported(claudeCapability, {
filename: 'chart.png',
mimeType: 'image/png',
})).toBe(true);
expect(isConversationUploadedFileSupported(claudeCapability, {
filename: 'voice.mp3',
mimeType: 'audio/mpeg',
})).toBe(false);
});
});
src/web/pages/helpers/conversationFileCapabilities.ts
+125 -3
View File
@@ -1,33 +1,155 @@
import type { PlaygroundProtocol } from './modelTesterSession.js';
export type ConversationFileTransportMode = 'native' | 'inline_only' | 'unsupported';
export type ConversationFileCapability = {
supported: boolean;
documentMode: 'native' | 'inline_only' | 'unsupported';
imageMode: ConversationFileTransportMode;
audioMode: ConversationFileTransportMode;
documentMode: ConversationFileTransportMode;
reason: string;
};
const DOCUMENT_ACCEPT_PARTS = ['.pdf', '.txt', '.md', '.markdown', '.json'] as const;
const DOCUMENT_MIME_TYPES = new Set([
'application/json',
'application/pdf',
'text/markdown',
'text/plain',
]);
const DOCUMENT_EXTENSIONS = ['.json', '.md', '.markdown', '.pdf', '.txt'] as const;
const IMAGE_EXTENSIONS = ['.avif', '.bmp', '.gif', '.jpeg', '.jpg', '.png', '.svg', '.webp'] as const;
const AUDIO_EXTENSIONS = ['.aac', '.flac', '.m4a', '.mp3', '.ogg', '.wav', '.weba'] as const;
type ConversationFileDescriptor = {
filename?: string | null;
mimeType?: string | null;
};
type ConversationFileKind = 'document' | 'image' | 'audio' | 'unknown';
const isSupportedMode = (mode: ConversationFileTransportMode): boolean => mode !== 'unsupported';
const normalizeName = (value: string | null | undefined): string => (value || '').trim().toLowerCase();
function detectKindFromFilename(filename: string): ConversationFileKind {
if (!filename) return 'unknown';
if (DOCUMENT_EXTENSIONS.some((extension) => filename.endsWith(extension))) return 'document';
if (IMAGE_EXTENSIONS.some((extension) => filename.endsWith(extension))) return 'image';
if (AUDIO_EXTENSIONS.some((extension) => filename.endsWith(extension))) return 'audio';
return 'unknown';
}
function resolveConversationFileKind(file: ConversationFileDescriptor): ConversationFileKind {
const mimeType = normalizeName(file.mimeType);
if (mimeType.startsWith('image/')) return 'image';
if (mimeType.startsWith('audio/')) return 'audio';
if (DOCUMENT_MIME_TYPES.has(mimeType)) return 'document';
return detectKindFromFilename(normalizeName(file.filename));
}
function buildSupportedTypeLabel(capability: ConversationFileCapability): string {
const labels: string[] = [];
if (isSupportedMode(capability.documentMode)) labels.push('PDF / TXT / Markdown / JSON');
if (isSupportedMode(capability.imageMode)) labels.push('图片');
if (isSupportedMode(capability.audioMode)) labels.push('音频');
return labels.join(' / ');
}
function buildTransportNotes(capability: ConversationFileCapability): string[] {
const notes: string[] = [];
if (capability.documentMode === 'inline_only') notes.push('文档会以内联数据注入');
if (capability.imageMode === 'native' && capability.documentMode === 'inline_only') {
notes.push('图片会按图片部件发送');
}
if (capability.audioMode === 'native' && capability.documentMode === 'inline_only') {
notes.push('音频会按音频部件发送');
}
return notes;
}
export function resolveConversationFileCapability(
protocol: PlaygroundProtocol,
): ConversationFileCapability {
if (protocol === 'openai' || protocol === 'responses') {
return {
supported: true,
imageMode: 'native',
audioMode: 'native',
documentMode: 'native',
reason: '',
};
}
if (protocol === 'claude' || protocol === 'gemini') {
if (protocol === 'claude') {
return {
supported: true,
imageMode: 'native',
audioMode: 'unsupported',
documentMode: 'inline_only',
reason: '当前协议会以内联文档方式发送会话附件。',
reason: '',
};
}
if (protocol === 'gemini') {
return {
supported: true,
imageMode: 'native',
audioMode: 'native',
documentMode: 'inline_only',
reason: '',
};
}
return {
supported: false,
imageMode: 'unsupported',
audioMode: 'unsupported',
documentMode: 'unsupported',
reason: '当前协议暂不支持会话附件。',
};
}
export function buildConversationFileAccept(capability: ConversationFileCapability): string {
const parts: string[] = [];
if (isSupportedMode(capability.documentMode)) parts.push(...DOCUMENT_ACCEPT_PARTS);
if (isSupportedMode(capability.imageMode)) parts.push('image/*');
if (isSupportedMode(capability.audioMode)) parts.push('audio/*');
return parts.join(',');
}
export function buildConversationFileHint(capability: ConversationFileCapability): string {
if (!capability.supported) {
return capability.reason || '当前协议暂不支持会话附件。';
}
const typeLabel = buildSupportedTypeLabel(capability);
if (!typeLabel) {
return capability.reason || '当前协议暂不支持会话附件。';
}
if (
capability.documentMode === 'native'
&& capability.imageMode === 'native'
&& capability.audioMode === 'native'
) {
return `支持 ${typeLabel}；发送前会先上传到 /v1/files。`;
}
const notes = buildTransportNotes(capability);
if (notes.length <= 0) {
return `支持 ${typeLabel}。`;
}
return `支持 ${typeLabel}；${notes.join('，')}。`;
}
export function isConversationUploadedFileSupported(
capability: ConversationFileCapability,
file: ConversationFileDescriptor,
): boolean {
const kind = resolveConversationFileKind(file);
if (kind === 'document') return isSupportedMode(capability.documentMode);
if (kind === 'image') return isSupportedMode(capability.imageMode);
if (kind === 'audio') return isSupportedMode(capability.audioMode);
return false;
}
src/web/pages/helpers/modelTesterSession.test.ts
+77
View File
@@ -15,6 +15,7 @@ import {
collectModelTesterModelNames,
countConversationTurns,
createConversationUserMessage,
extractConversationUploadedFilesFromMessage,
filterModelTesterModelNames,
parseCustomRequestBody,
parseModelTesterSession,
@@ -222,6 +223,34 @@ describe('modelTesterSession', () => {
]);
});
it('extracts both uploaded references and inline attachments from user messages', () => {
const message = createConversationUserMessage('请重新发送附件', [
{
fileId: 'file-metapi-123',
filename: 'paper.pdf',
mimeType: 'application/pdf',
},
{
filename: 'voice.mp3',
mimeType: 'audio/mpeg',
data: 'data:audio/mpeg;base64,QUFBQQ==',
},
]);
expect(extractConversationUploadedFilesFromMessage(message)).toEqual([
{
fileId: 'file-metapi-123',
filename: 'paper.pdf',
mimeType: 'application/pdf',
},
{
filename: 'voice.mp3',
mimeType: 'audio/mpeg',
data: 'data:audio/mpeg;base64,QUFBQQ==',
},
]);
});
it('preserves file reference parts when building responses conversation payloads', () => {
const payload = buildApiPayload(
[{
@@ -484,6 +513,54 @@ describe('modelTesterSession', () => {
});
});
it('builds Claude conversation payloads with image blocks for inline image attachments', () => {
const payload = buildApiPayload(
[{
id: 'u1',
role: 'user',
content: 'describe this image',
createAt: 1,
parts: [
{
type: 'input_file',
filename: 'chart.png',
mimeType: 'image/png',
data: 'data:image/png;base64,QUFBQQ==',
},
],
} as ChatMessage],
{
...DEFAULT_INPUTS,
model: 'claude-opus-4-6',
protocol: 'claude',
},
DEFAULT_PARAMETER_ENABLED,
);
expect(payload.jsonBody).toEqual({
model: 'claude-opus-4-6',
stream: false,
max_tokens: 4096,
messages: [
{
role: 'user',
content: [
{ type: 'text', text: 'describe this image' },
{
type: 'image',
source: {
type: 'base64',
media_type: 'image/png',
data: 'QUFBQQ==',
},
},
],
},
],
temperature: 0.7,
});
});
it('builds Gemini conversation payloads with inlineData document parts for conversation files', () => {
const payload = buildApiPayload(
[{
src/web/pages/helpers/modelTesterSession.ts
+34 -1
View File
@@ -355,11 +355,22 @@ const toClaudeContentPart = (part: ConversationContentPart): Record<string, unkn
if (part.type !== 'input_file' || typeof part.data !== 'string' || !part.data.trim()) return null;
const parsed = parseDataUrl(part.data);
if (!parsed) return null;
const mimeType = part.mimeType || parsed.mimeType || 'application/octet-stream';
if (mimeType.toLowerCase().startsWith('image/')) {
return {
type: 'image',
source: {
type: 'base64',
media_type: mimeType,
data: parsed.data,
},
};
}
return {
type: 'document',
source: {
type: 'base64',
media_type: part.mimeType || parsed.mimeType || 'application/octet-stream',
media_type: mimeType,
data: parsed.data,
},
...(typeof part.filename === 'string' && part.filename.trim()
@@ -901,6 +912,28 @@ export const createConversationUserMessage = (
});
};
export const extractConversationUploadedFilesFromMessage = (
message: ChatMessage,
): ConversationUploadedFile[] => {
const parts = Array.isArray(message.parts) ? message.parts : [];
return parts.flatMap((part) => {
if (part.type !== 'input_file') return [];
const fileId = typeof part.fileId === 'string' ? part.fileId.trim() : '';
const data = typeof part.data === 'string' ? part.data.trim() : '';
if (!fileId && !data) return [];
return [{
...(fileId ? { fileId } : {}),
...(typeof part.filename === 'string' && part.filename.trim()
? { filename: part.filename.trim() }
: {}),
...(typeof part.mimeType === 'string' && part.mimeType.trim()
? { mimeType: part.mimeType.trim() }
: {}),
...(data ? { data } : {}),
}];
});
};
export const createLoadingAssistantMessage = (): ChatMessage =>
createMessage('assistant', '', {
status: MESSAGE_STATUS.LOADING,