Nvidia/BossAI
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix model tester attachment replay and capability gating

Browse Source
This commit is contained in:
cita
2026-03-20 22:56:01 +08:00
parent 8c5ecc26d1
commit 4369aeb1cb
9 changed files with 646 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/web/pages/helpers/conversationFileCapabilities.test.ts
+52 -4
View File
@@ -1,31 +1,79 @@
import { describe, expect, it } from 'vitest';
import { resolveConversationFileCapability } from './conversationFileCapabilities.js';
import {
buildConversationFileAccept,
buildConversationFileHint,
isConversationUploadedFileSupported,
resolveConversationFileCapability,
} from './conversationFileCapabilities.js';
describe('resolveConversationFileCapability', () => {
it('keeps OpenAI and Responses on native file references', () => {
expect(resolveConversationFileCapability('openai')).toEqual({
supported: true,
imageMode: 'native',
audioMode: 'native',
documentMode: 'native',
reason: '',
});
expect(resolveConversationFileCapability('responses')).toEqual({
supported: true,
imageMode: 'native',
audioMode: 'native',
documentMode: 'native',
reason: '',
});
});
it('marks Claude and Gemini as inline-only document transports', () => {
it('maps Claude and Gemini media support separately', () => {
expect(resolveConversationFileCapability('claude')).toEqual({
supported: true,
imageMode: 'native',
audioMode: 'unsupported',
documentMode: 'inline_only',
reason: '当前协议会以内联文档方式发送会话附件。',
reason: '',
});
expect(resolveConversationFileCapability('gemini')).toEqual({
supported: true,
imageMode: 'native',
audioMode: 'native',
documentMode: 'inline_only',
reason: '当前协议会以内联文档方式发送会话附件。',
reason: '',
});
});
it('builds the accept list from the supported file kinds', () => {
expect(buildConversationFileAccept(resolveConversationFileCapability('claude'))).toBe(
'.pdf,.txt,.md,.markdown,.json,image/*',
);
expect(buildConversationFileAccept(resolveConversationFileCapability('gemini'))).toBe(
'.pdf,.txt,.md,.markdown,.json,image/*,audio/*',
);
});
it('describes the actual per-kind transport behavior', () => {
expect(buildConversationFileHint(resolveConversationFileCapability('claude'))).toBe(
'支持 PDF / TXT / Markdown / JSON / 图片；文档会以内联数据注入，图片会按图片部件发送。',
);
expect(buildConversationFileHint(resolveConversationFileCapability('gemini'))).toBe(
'支持 PDF / TXT / Markdown / JSON / 图片 / 音频；文档会以内联数据注入，图片会按图片部件发送，音频会按音频部件发送。',
);
});
it('rejects unsupported file kinds before send', () => {
const claudeCapability = resolveConversationFileCapability('claude');
expect(isConversationUploadedFileSupported(claudeCapability, {
filename: 'paper.pdf',
mimeType: 'application/pdf',
})).toBe(true);
expect(isConversationUploadedFileSupported(claudeCapability, {
filename: 'chart.png',
mimeType: 'image/png',
})).toBe(true);
expect(isConversationUploadedFileSupported(claudeCapability, {
filename: 'voice.mp3',
mimeType: 'audio/mpeg',
})).toBe(false);
});
});
src/web/pages/helpers/conversationFileCapabilities.ts
+125 -3
View File
@@ -1,33 +1,155 @@
import type { PlaygroundProtocol } from './modelTesterSession.js';
export type ConversationFileTransportMode = 'native' | 'inline_only' | 'unsupported';
export type ConversationFileCapability = {
supported: boolean;
documentMode: 'native' | 'inline_only' | 'unsupported';
imageMode: ConversationFileTransportMode;
audioMode: ConversationFileTransportMode;
documentMode: ConversationFileTransportMode;
reason: string;
};
const DOCUMENT_ACCEPT_PARTS = ['.pdf', '.txt', '.md', '.markdown', '.json'] as const;
const DOCUMENT_MIME_TYPES = new Set([
'application/json',
'application/pdf',
'text/markdown',
'text/plain',
]);
const DOCUMENT_EXTENSIONS = ['.json', '.md', '.markdown', '.pdf', '.txt'] as const;
const IMAGE_EXTENSIONS = ['.avif', '.bmp', '.gif', '.jpeg', '.jpg', '.png', '.svg', '.webp'] as const;
const AUDIO_EXTENSIONS = ['.aac', '.flac', '.m4a', '.mp3', '.ogg', '.wav', '.weba'] as const;
type ConversationFileDescriptor = {
filename?: string | null;
mimeType?: string | null;
};
type ConversationFileKind = 'document' | 'image' | 'audio' | 'unknown';
const isSupportedMode = (mode: ConversationFileTransportMode): boolean => mode !== 'unsupported';
const normalizeName = (value: string | null | undefined): string => (value || '').trim().toLowerCase();
function detectKindFromFilename(filename: string): ConversationFileKind {
if (!filename) return 'unknown';
if (DOCUMENT_EXTENSIONS.some((extension) => filename.endsWith(extension))) return 'document';
if (IMAGE_EXTENSIONS.some((extension) => filename.endsWith(extension))) return 'image';
if (AUDIO_EXTENSIONS.some((extension) => filename.endsWith(extension))) return 'audio';
return 'unknown';
}
function resolveConversationFileKind(file: ConversationFileDescriptor): ConversationFileKind {
const mimeType = normalizeName(file.mimeType);
if (mimeType.startsWith('image/')) return 'image';
if (mimeType.startsWith('audio/')) return 'audio';
if (DOCUMENT_MIME_TYPES.has(mimeType)) return 'document';
return detectKindFromFilename(normalizeName(file.filename));
}
function buildSupportedTypeLabel(capability: ConversationFileCapability): string {
const labels: string[] = [];
if (isSupportedMode(capability.documentMode)) labels.push('PDF / TXT / Markdown / JSON');
if (isSupportedMode(capability.imageMode)) labels.push('图片');
if (isSupportedMode(capability.audioMode)) labels.push('音频');
return labels.join(' / ');
}
function buildTransportNotes(capability: ConversationFileCapability): string[] {
const notes: string[] = [];
if (capability.documentMode === 'inline_only') notes.push('文档会以内联数据注入');
if (capability.imageMode === 'native' && capability.documentMode === 'inline_only') {
notes.push('图片会按图片部件发送');
}
if (capability.audioMode === 'native' && capability.documentMode === 'inline_only') {
notes.push('音频会按音频部件发送');
}
return notes;
}
export function resolveConversationFileCapability(
protocol: PlaygroundProtocol,
): ConversationFileCapability {
if (protocol === 'openai' || protocol === 'responses') {
return {
supported: true,
imageMode: 'native',
audioMode: 'native',
documentMode: 'native',
reason: '',
};
}
if (protocol === 'claude' || protocol === 'gemini') {
if (protocol === 'claude') {
return {
supported: true,
imageMode: 'native',
audioMode: 'unsupported',
documentMode: 'inline_only',
reason: '当前协议会以内联文档方式发送会话附件。',
reason: '',
};
}
if (protocol === 'gemini') {
return {
supported: true,
imageMode: 'native',
audioMode: 'native',
documentMode: 'inline_only',
reason: '',
};
}
return {
supported: false,
imageMode: 'unsupported',
audioMode: 'unsupported',
documentMode: 'unsupported',
reason: '当前协议暂不支持会话附件。',
};
}
export function buildConversationFileAccept(capability: ConversationFileCapability): string {
const parts: string[] = [];
if (isSupportedMode(capability.documentMode)) parts.push(...DOCUMENT_ACCEPT_PARTS);
if (isSupportedMode(capability.imageMode)) parts.push('image/*');
if (isSupportedMode(capability.audioMode)) parts.push('audio/*');
return parts.join(',');
}
export function buildConversationFileHint(capability: ConversationFileCapability): string {
if (!capability.supported) {
return capability.reason || '当前协议暂不支持会话附件。';
}
const typeLabel = buildSupportedTypeLabel(capability);
if (!typeLabel) {
return capability.reason || '当前协议暂不支持会话附件。';
}
if (
capability.documentMode === 'native'
&& capability.imageMode === 'native'
&& capability.audioMode === 'native'
) {
return `支持 ${typeLabel}；发送前会先上传到 /v1/files。`;
}
const notes = buildTransportNotes(capability);
if (notes.length <= 0) {
return `支持 ${typeLabel}。`;
}
return `支持 ${typeLabel}；${notes.join('，')}。`;
}
export function isConversationUploadedFileSupported(
capability: ConversationFileCapability,
file: ConversationFileDescriptor,
): boolean {
const kind = resolveConversationFileKind(file);
if (kind === 'document') return isSupportedMode(capability.documentMode);
if (kind === 'image') return isSupportedMode(capability.imageMode);
if (kind === 'audio') return isSupportedMode(capability.audioMode);
return false;
}
src/web/pages/helpers/modelTesterSession.test.ts
+77
View File
@@ -15,6 +15,7 @@ import {
collectModelTesterModelNames,
countConversationTurns,
createConversationUserMessage,
extractConversationUploadedFilesFromMessage,
filterModelTesterModelNames,
parseCustomRequestBody,
parseModelTesterSession,
@@ -222,6 +223,34 @@ describe('modelTesterSession', () => {
]);
});
it('extracts both uploaded references and inline attachments from user messages', () => {
const message = createConversationUserMessage('请重新发送附件', [
{
fileId: 'file-metapi-123',
filename: 'paper.pdf',
mimeType: 'application/pdf',
},
{
filename: 'voice.mp3',
mimeType: 'audio/mpeg',
data: 'data:audio/mpeg;base64,QUFBQQ==',
},
]);
expect(extractConversationUploadedFilesFromMessage(message)).toEqual([
{
fileId: 'file-metapi-123',
filename: 'paper.pdf',
mimeType: 'application/pdf',
},
{
filename: 'voice.mp3',
mimeType: 'audio/mpeg',
data: 'data:audio/mpeg;base64,QUFBQQ==',
},
]);
});
it('preserves file reference parts when building responses conversation payloads', () => {
const payload = buildApiPayload(
[{
@@ -484,6 +513,54 @@ describe('modelTesterSession', () => {
});
});
it('builds Claude conversation payloads with image blocks for inline image attachments', () => {
const payload = buildApiPayload(
[{
id: 'u1',
role: 'user',
content: 'describe this image',
createAt: 1,
parts: [
{
type: 'input_file',
filename: 'chart.png',
mimeType: 'image/png',
data: 'data:image/png;base64,QUFBQQ==',
},
],
} as ChatMessage],
{
...DEFAULT_INPUTS,
model: 'claude-opus-4-6',
protocol: 'claude',
},
DEFAULT_PARAMETER_ENABLED,
);
expect(payload.jsonBody).toEqual({
model: 'claude-opus-4-6',
stream: false,
max_tokens: 4096,
messages: [
{
role: 'user',
content: [
{ type: 'text', text: 'describe this image' },
{
type: 'image',
source: {
type: 'base64',
media_type: 'image/png',
data: 'QUFBQQ==',
},
},
],
},
],
temperature: 0.7,
});
});
it('builds Gemini conversation payloads with inlineData document parts for conversation files', () => {
const payload = buildApiPayload(
[{
src/web/pages/helpers/modelTesterSession.ts
+34 -1
View File
@@ -355,11 +355,22 @@ const toClaudeContentPart = (part: ConversationContentPart): Record<string, unkn
if (part.type !== 'input_file' || typeof part.data !== 'string' || !part.data.trim()) return null;
const parsed = parseDataUrl(part.data);
if (!parsed) return null;
const mimeType = part.mimeType || parsed.mimeType || 'application/octet-stream';
if (mimeType.toLowerCase().startsWith('image/')) {
return {
type: 'image',
source: {
type: 'base64',
media_type: mimeType,
data: parsed.data,
},
};
}
return {
type: 'document',
source: {
type: 'base64',
media_type: part.mimeType || parsed.mimeType || 'application/octet-stream',
media_type: mimeType,
data: parsed.data,
},
...(typeof part.filename === 'string' && part.filename.trim()
@@ -901,6 +912,28 @@ export const createConversationUserMessage = (
});
};
export const extractConversationUploadedFilesFromMessage = (
message: ChatMessage,
): ConversationUploadedFile[] => {
const parts = Array.isArray(message.parts) ? message.parts : [];
return parts.flatMap((part) => {
if (part.type !== 'input_file') return [];
const fileId = typeof part.fileId === 'string' ? part.fileId.trim() : '';
const data = typeof part.data === 'string' ? part.data.trim() : '';
if (!fileId && !data) return [];
return [{
...(fileId ? { fileId } : {}),
...(typeof part.filename === 'string' && part.filename.trim()
? { filename: part.filename.trim() }
: {}),
...(typeof part.mimeType === 'string' && part.mimeType.trim()
? { mimeType: part.mimeType.trim() }
: {}),
...(data ? { data } : {}),
}];
});
};
export const createLoadingAssistantMessage = (): ChatMessage =>
createMessage('assistant', '', {
status: MESSAGE_STATUS.LOADING,