import { useEffect, useMemo, useRef, useState, type CSSProperties } from 'react'; import { useLocation } from 'react-router-dom'; import { marked } from 'marked'; import { getSessionToken } from '../authSession.js'; import { useToast } from '../components/Toast.js'; import { filterModelTesterModelNames } from './helpers/modelTesterSession.js'; type AgentRole = 'user' | 'assistant' | 'system'; type AgentImage = { url: string; character?: string; angle?: string; }; type AgentScene = { id: number; description: string; prompt: string; url?: string; }; type AgentTts = { character: string; text: string; url?: string; }; type AgentAttachment = { id: string; name: string; mimeType: string; size: number; mediaKind?: 'image' | 'video' | 'audio' | 'document' | 'text' | 'file'; referenceLabel?: string; text?: string; dataUrl?: string; }; type AgentMessage = { id: string; role: AgentRole; content: string; reasoningContent?: string | null; attachments?: AgentAttachment[]; images?: AgentImage[]; scenes?: AgentScene[]; tts?: AgentTts[]; workflowJson?: unknown; }; type AgentStatus = { model?: string; mode?: string; keyName?: string; comfyui?: { status?: string; mode?: string; base_url?: string; baseUrl?: string; model_backend?: string; modelBackend?: string; }; }; type StreamAccumulator = { content: string; reasoningContent: string; images: AgentImage[]; scenes: AgentScene[]; tts: AgentTts[]; workflowJson: unknown; }; type StoredAgentSession = { activeConversationId?: string; conversations?: StoredAgentConversation[]; messages?: AgentMessage[]; selectedModel?: string; selectedImageModel?: string; }; type StoredAgentConversation = { id: string; title: string; updatedAt: number; messages: AgentMessage[]; selectedModel: string; selectedImageModel: string; }; type RunningConversation = { draft: StreamAccumulator; controller: AbortController; }; const WELCOME_MESSAGE: AgentMessage = { id: 'welcome', role: 'system', content: '你好，我是视频助手。把脚本、分镜或角色想法发给我；确认后我会分步生成角色四视图、场景图、TTS 和 ComfyUI 工作流。', }; const MAX_ATTACHMENT_COUNT = 8; const MAX_INLINE_IMAGE_BYTES = 12 * 1024 * 1024; const MAX_INLINE_TOTAL_BYTES = 36 * 1024 * 1024; const MAX_ATTACHMENT_BYTES = 64 * 1024 * 1024; const QUICK_PROMPTS = [ '帮我把这个创意拆成角色、场景和镜头节奏', '开始制作，先生成角色和场景，不要一次输出工作流', '根据已有角色和场景，继续生成 ComfyUI 工作流 JSON', ]; const STREAMING_REASONING_PLACEHOLDER = '正在等待模型返回真实思考过程，收到后会实时替换这里。'; const STREAMING_BODY_PLACEHOLDER = '模型正在组织正文和素材步骤...'; const SEEDANCE_PROMPT_TEMPLATES = [ { title: '文生视频', description: '从创意扩写成可直接投喂视频模型的完整提示词。', prompt: [ '请参考 Seedance 2.0 的视频提示词结构，把下面创意改写成可直接用于视频生成模型的提示词。', '', '创意主题：', '主体/人物：', '场景环境：', '核心动作：', '运镜语言：', '时间轴：', '0-3秒：', '3-6秒：', '6-10秒：', '音效/音乐：', '风格氛围：电影级质感，浅景深，光影自然，动作连贯。', '', '输出要求：只返回最终视频提示词；消除冲突指令；如果时长超过8秒，请保留分时段描述。', ].join('\n'), }, { title: '图生视频', description: '保持角色或首帧一致，补充动作、镜头和声音。', prompt: [ '基于 @图片1 生成 Seedance 2.0 图生视频提示词：', '', '@图片1 作为首帧/角色参考，保持主体身份、服装、五官、背景风格一致。', '动作变化：', '情绪变化：', '运镜：慢推 / 跟随 / 环绕 / 固定镜头（选择一种并写清楚）', '时长：6-10秒', '音效/音乐：', '风格：', '', '请输出一段自然语言提示词，并检查不要出现“固定镜头”和“环绕镜头”等冲突要求。', ].join('\n'), }, { title: '多素材引用', description: '给图片、视频、音频分配明确用途，避免引用模糊。', prompt: [ '请把以下素材组织成 Seedance 2.0 多模态视频提示词：', '', '素材引用：', '@图片1：作为人物/产品外观参考', '@图片2：作为场景/背景参考', '@视频1：参考运镜、动作节奏或转场效果', '@音频1：参考背景音乐、旁白音色或节拍', '', '视频目标：', '画面主体：', '动作编排：', '镜头运动：', '音画同步：', '生成时长：', '', '输出要求：每个 @ 引用都必须说明用途；不要只写“参考 @视频1”。', ].join('\n'), }, { title: '短剧分镜', description: '适合剧情、口播、短剧和带台词的视频。', prompt: [ '请写一段 15 秒短剧视频提示词，要求情绪递进清晰、台词自然、镜头可执行。', '', '角色设定：', '场景：', '冲突点：', '', '0-5秒：开场画面、角色表情、第一句台词', '6-10秒：冲突升级、动作、第二句台词', '11-15秒：反转/揭示/收尾，镜头定格', '', '音效：环境声、呼吸声、脚步声、音乐情绪', '风格：短剧质感，镜头紧凑，表情真实，节奏明确。', ].join('\n'), }, { title: '产品广告', description: '产品展示、材质特写、卖点节奏和品牌定格。', prompt: [ '请生成一段 15 秒产品广告视频提示词：', '', '@图片1 作为产品外观参考，保持 logo、颜色、材质和比例一致。', '0-3秒：产品动感入场，特写材质和 logo 细节。', '4-8秒：多角度展示产品结构、功能或核心卖点，加入扫光效果。', '9-12秒：产品进入使用场景，展示实际价值。', '13-15秒：主视觉定格，品牌文案出现，音乐推向高潮。', '', '运镜：微距特写、环绕、慢推。', '音效：产品交互声、轻微机械声或清脆提示音。', '风格：高级商业广告，干净背景，光影精致。', ].join('\n'), }, { title: '视频续写/编辑', description: '延长已有视频或定向修改局部内容。', prompt: [ '请基于 @视频1 写 Seedance 2.0 视频续写/编辑提示词：', '', '@视频1 作为原始视频参考，保留主体、场景、光影、节奏和镜头连续性。', '任务类型：向后延长 / 向前延长 / 局部编辑 / 角色替换 / 元素添加', '新增或修改内容：', '生成时长：', '', '时间轴：', '0-3秒：', '3-6秒：', '6-10秒：', '', '音效：延续原视频声音氛围，关键动作增加同步音效。', '输出要求：明确哪些内容保持不变，哪些内容需要改变。', ].join('\n'), }, ]; function createId(prefix: string) { return `${prefix}-${Date.now()}-${Math.random().toString(16).slice(2)}`; } function sessionStorageKey(audience: 'admin' | 'user') { return `boosapi.comfyuiAgent.${audience}.session`; } function buildConversationTitle(messages: AgentMessage[]) { const firstUser = messages.find((message) => message.role === 'user')?.content.trim(); if (!firstUser) return '新对话'; return firstUser.length > 24 ? `${firstUser.slice(0, 24)}...` : firstUser; } function createConversation( messages: AgentMessage[] = [WELCOME_MESSAGE], selectedModel = '', selectedImageModel = '', ): StoredAgentConversation { return { id: createId('conversation'), title: buildConversationTitle(messages), updatedAt: Date.now(), messages, selectedModel, selectedImageModel, }; } function isValidMessage(value: unknown): value is AgentMessage { if (!value || typeof value !== 'object') return false; const item = value as Partial; return ( typeof item.id === 'string' && (item.role === 'user' || item.role === 'assistant' || item.role === 'system') && typeof item.content === 'string' ); } function isValidAttachment(value: unknown): value is AgentAttachment { if (!value || typeof value !== 'object') return false; const item = value as Partial; return ( typeof item.id === 'string' && typeof item.name === 'string' && typeof item.mimeType === 'string' && typeof item.size === 'number' ); } function fileToDataUrl(file: File): Promise { return new Promise((resolve, reject) => { const reader = new FileReader(); reader.onload = () => resolve(String(reader.result || '')); reader.onerror = () => reject(reader.error || new Error('读取附件失败')); reader.readAsDataURL(file); }); } const TEXT_ATTACHMENT_EXTENSIONS = new Set([ 'txt', 'md', 'markdown', 'json', 'csv', 'srt', 'vtt', ]); const DOCUMENT_ATTACHMENT_EXTENSIONS = new Set([ 'pdf', 'doc', 'docx', 'xls', 'xlsx', 'ppt', 'pptx', ]); function fileExtension(name: string): string { return name.split('.').pop()?.trim().toLowerCase() || ''; } function formatBytes(bytes: number): string { if (!Number.isFinite(bytes) || bytes <= 0) return '0 B'; const units = ['B', 'KB', 'MB', 'GB']; let value = bytes; let unitIndex = 0; while (value >= 1024 && unitIndex < units.length - 1) { value /= 1024; unitIndex += 1; } return `${value.toFixed(value >= 10 || unitIndex === 0 ? 0 : 1)} ${units[unitIndex]}`; } function isTextAttachment(file: File, mimeType: string): boolean { if (mimeType.startsWith('text/')) return true; if (['application/json', 'application/x-ndjson', 'application/xml'].includes(mimeType)) { return true; } return TEXT_ATTACHMENT_EXTENSIONS.has(fileExtension(file.name)); } function isDocumentAttachment(file: File, mimeType: string): boolean { if (DOCUMENT_ATTACHMENT_EXTENSIONS.has(fileExtension(file.name))) return true; return [ 'application/pdf', 'application/msword', 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', 'application/vnd.ms-excel', 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', 'application/vnd.ms-powerpoint', 'application/vnd.openxmlformats-officedocument.presentationml.presentation', ].includes(mimeType); } function mediaKindForAttachment(name: string, mimeType: string): NonNullable { if (mimeType.startsWith('image/')) return 'image'; if (mimeType.startsWith('video/')) return 'video'; if (mimeType.startsWith('audio/')) return 'audio'; const extension = fileExtension(name); if ( DOCUMENT_ATTACHMENT_EXTENSIONS.has(extension) || [ 'application/pdf', 'application/msword', 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', 'application/vnd.ms-excel', 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', 'application/vnd.ms-powerpoint', 'application/vnd.openxmlformats-officedocument.presentationml.presentation', ].includes(mimeType) ) { return 'document'; } if ( mimeType.startsWith('text/') || ['application/json', 'application/x-ndjson', 'application/xml'].includes(mimeType) || TEXT_ATTACHMENT_EXTENSIONS.has(fileExtension(name)) ) { return 'text'; } return 'file'; } function referencePrefixForKind(kind: NonNullable): string { switch (kind) { case 'image': return '图片'; case 'video': return '视频'; case 'audio': return '音频'; case 'document': return '文档'; case 'text': return '文本'; default: return '素材'; } } function assignAttachmentReferences(items: AgentAttachment[]): AgentAttachment[] { const counters: Record = {}; return items.map((item) => { const mediaKind = item.mediaKind || mediaKindForAttachment(item.name, item.mimeType); const prefix = referencePrefixForKind(mediaKind); counters[prefix] = (counters[prefix] || 0) + 1; return { ...item, mediaKind, referenceLabel: `@${prefix}${counters[prefix]}`, }; }); } async function fileToAttachment(file: File): Promise { const mimeType = file.type || 'application/octet-stream'; const mediaKind = mediaKindForAttachment(file.name || 'untitled', mimeType); const base = { id: createId('attachment'), name: file.name || 'untitled', mimeType, size: file.size, mediaKind, }; if (mimeType.startsWith('image/')) { if (file.size > MAX_INLINE_IMAGE_BYTES) { return { ...base, text: [ `[图片素材: ${file.name}]`, `类型: ${mimeType}`, `大小: ${formatBytes(file.size)}`, `图片超过 ${formatBytes(MAX_INLINE_IMAGE_BYTES)}，已作为引用记录，未内联发送原图数据。`, '如需精确参考，请压缩图片或上传较小版本。', ].join('\n'), }; } return { ...base, dataUrl: await fileToDataUrl(file) }; } if (mimeType.startsWith('video/') || mimeType.startsWith('audio/')) { return { ...base, text: [ `[${mimeType.startsWith('video/') ? '视频' : '音频'}素材: ${file.name}]`, `类型: ${mimeType}`, `大小: ${formatBytes(file.size)}`, '当前版本会记录该素材引用和用途说明，不会把完整二进制内容塞进对话请求。', '请在输入框说明需要参考的镜头、节奏、音色或时间段；后续接入服务端解析后会自动提取媒体信息。', ].join('\n'), }; } if (isDocumentAttachment(file, mimeType) && !isTextAttachment(file, mimeType)) { return { ...base, text: [ `[文档素材: ${file.name}]`, `类型: ${mimeType}`, '当前版本已记录该文件，但还没有启用服务端 PDF/Office 正文和内嵌图片解析。', '请先把剧本文字复制到输入框，或上传 txt/md/csv/json 版本；后续解析器接入后会自动提取正文、表格和图片。', ].join('\n'), }; } if (!isTextAttachment(file, mimeType)) { return { ...base, text: `[附件: ${file.name}]\n类型: ${mimeType}\n当前版本暂不支持读取该二进制文件内容，请改用图片或文本类素材。`, }; } const text = await file.text(); return { ...base, text: text.slice(0, 120_000), }; } function readStoredSession(audience: 'admin' | 'user'): StoredAgentSession { try { const raw = localStorage.getItem(sessionStorageKey(audience)); if (!raw) return {}; const parsed = JSON.parse(raw) as StoredAgentSession; const conversations: StoredAgentConversation[] = Array.isArray(parsed.conversations) ? parsed.conversations .map((conversation): StoredAgentConversation | null => { const messages = Array.isArray(conversation?.messages) ? conversation.messages.filter(isValidMessage).map((message) => ({ ...message, reasoningContent: typeof message.reasoningContent === 'string' ? message.reasoningContent : undefined, attachments: Array.isArray(message.attachments) ? message.attachments.filter(isValidAttachment) : undefined, })) : []; if (!conversation?.id || !messages.length) return null; return { id: String(conversation.id), title: typeof conversation.title === 'string' && conversation.title.trim() ? conversation.title.trim() : buildConversationTitle(messages), updatedAt: typeof conversation.updatedAt === 'number' ? conversation.updatedAt : Date.now(), messages, selectedModel: typeof conversation.selectedModel === 'string' ? conversation.selectedModel : '', selectedImageModel: typeof conversation.selectedImageModel === 'string' ? conversation.selectedImageModel : '', }; }) .filter((conversation): conversation is StoredAgentConversation => !!conversation) : []; if (conversations.length) { const activeConversationId = conversations.some( (conversation) => conversation.id === parsed.activeConversationId, ) ? parsed.activeConversationId : conversations[0].id; return { conversations, activeConversationId }; } const messages = Array.isArray(parsed.messages) ? parsed.messages.filter(isValidMessage).map((message) => ({ ...message, reasoningContent: typeof message.reasoningContent === 'string' ? message.reasoningContent : undefined, attachments: Array.isArray(message.attachments) ? message.attachments.filter(isValidAttachment) : undefined, })) : undefined; return { messages: messages?.length ? messages : undefined, selectedModel: typeof parsed.selectedModel === 'string' ? parsed.selectedModel : undefined, selectedImageModel: typeof parsed.selectedImageModel === 'string' ? parsed.selectedImageModel : undefined, }; } catch { return {}; } } function persistStoredSession( audience: 'admin' | 'user', conversations: StoredAgentConversation[], activeConversationId: string, ) { try { localStorage.setItem( sessionStorageKey(audience), JSON.stringify({ conversations, activeConversationId }), ); } catch { // Ignore quota/private-mode failures; chat still works for the current page. } } function angleLabel(angle?: string) { if (angle === 'front') return '正面'; if (angle === 'back') return '背面'; if (angle === 'left') return '左侧'; if (angle === 'right') return '右侧'; return angle || '图像'; } function appendLine(current: string, line: string) { if (!line) return current; return current ? `${current}\n${line}` : line; } function escapeHtml(value: string): string { return value .replace(/&/g, '&') .replace(//g, '>') .replace(/"/g, '"') .replace(/'/g, '''); } function sanitizeUrl(raw: string | null | undefined, allowDataImage = false): string | null { const value = String(raw || '').trim(); if (!value) return null; if (value.startsWith('#') || value.startsWith('/')) return value; if (allowDataImage && /^data:image\/[a-z0-9.+-]+;base64,/i.test(value)) return value; if (/^https?:\/\//i.test(value)) return value; if (/^mailto:/i.test(value) || /^tel:/i.test(value)) return value; return null; } function sanitizeMarkdownHtml(html: string): string { if (typeof DOMParser !== 'function' || typeof Node === 'undefined') { return `

${escapeHtml(html)}

`; } const allowedTags = new Set([ 'a', 'blockquote', 'br', 'code', 'del', 'em', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'hr', 'img', 'li', 'ol', 'p', 'pre', 'strong', 'table', 'tbody', 'td', 'th', 'thead', 'tr', 'ul', ]); const droppedTags = new Set(['script', 'style', 'iframe', 'object', 'embed', 'link', 'meta']); const voidTags = new Set(['br', 'hr', 'img']); const parser = new DOMParser(); const doc = parser.parseFromString(`${html}`, 'text/html'); const cleanNode = (node: Node): string => { if (node.nodeType === Node.TEXT_NODE) return escapeHtml(node.textContent || ''); if (node.nodeType !== Node.ELEMENT_NODE) return ''; const element = node as Element; const tag = element.tagName.toLowerCase(); if (droppedTags.has(tag)) return ''; const children = Array.from(element.childNodes).map(cleanNode).join(''); if (!allowedTags.has(tag)) return children; const attrs: string[] = []; if (tag === 'a') { const href = sanitizeUrl(element.getAttribute('href')); if (href) { attrs.push(` href="${escapeHtml(href)}"`); attrs.push(' target="_blank" rel="noopener noreferrer"'); } } else if (tag === 'img') { const src = sanitizeUrl(element.getAttribute('src'), true); if (!src) return ''; attrs.push(` src="${escapeHtml(src)}"`); const alt = element.getAttribute('alt'); if (alt) attrs.push(` alt="${escapeHtml(alt)}"`); } else if (tag === 'code') { const className = String(element.getAttribute('class') || '').trim(); if (/^language-[a-z0-9_-]+$/i.test(className)) { attrs.push(` class="${escapeHtml(className)}"`); } } return voidTags.has(tag) ? `<${tag}${attrs.join('')}>` : `<${tag}${attrs.join('')}>${children}`; }; return Array.from(doc.body.childNodes).map(cleanNode).join(''); } function renderMarkdown(content: string): string { const raw = String(content || '').trim(); if (!raw) return ''; const rendered = String(marked.parse(raw, { gfm: true, breaks: true })); return sanitizeMarkdownHtml(rendered || `

${escapeHtml(raw)}

`); } function processThinkTags(content: string, reasoningContent = '') { if (!content || !//i.test(content)) { return { content, reasoningContent }; } const thoughts: string[] = []; const replyParts: string[] = []; let lastIndex = 0; const regex = /([\s\S]*?)<\/think>/gi; let match: RegExpExecArray | null; while ((match = regex.exec(content)) !== null) { replyParts.push(content.substring(lastIndex, match.index)); thoughts.push(match[1].trim()); lastIndex = match.index + match[0].length; } replyParts.push(content.substring(lastIndex)); let cleanContent = replyParts.join(''); const lastOpenThinkIndex = cleanContent.toLowerCase().lastIndexOf(''); if (lastOpenThinkIndex !== -1) { const fragmentAfterLastOpen = cleanContent.substring(lastOpenThinkIndex + ''.length); if (!/<\/think>/i.test(fragmentAfterLastOpen)) { thoughts.push(fragmentAfterLastOpen.trim()); cleanContent = cleanContent.substring(0, lastOpenThinkIndex); } } cleanContent = cleanContent.replace(/<\/?think>/gi, '').trim(); const thoughtsText = thoughts.filter(Boolean).join('\n\n---\n\n'); return { content: cleanContent, reasoningContent: reasoningContent && thoughtsText ? `${reasoningContent}\n\n---\n\n${thoughtsText}` : (reasoningContent || thoughtsText), }; } async function copyTextToClipboard(text: string) { if (navigator.clipboard?.writeText) { await navigator.clipboard.writeText(text); return; } const textarea = document.createElement('textarea'); textarea.value = text; textarea.style.position = 'fixed'; textarea.style.opacity = '0'; document.body.appendChild(textarea); textarea.focus(); textarea.select(); document.execCommand('copy'); document.body.removeChild(textarea); } function parseWorkflow(content: string) { const trimmed = content.trim(); if (!trimmed) return null; try { return JSON.parse(trimmed); } catch { return content; } } async function readSseStream( response: Response, onEvent: (event: any) => void, ) { const reader = response.body?.getReader(); if (!reader) throw new Error('浏览器无法读取流式响应'); const decoder = new TextDecoder(); let buffer = ''; while (true) { const { done, value } = await reader.read(); if (done) break; buffer += decoder.decode(value, { stream: true }); const lines = buffer.split('\n'); buffer = lines.pop() || ''; for (const rawLine of lines) { const line = rawLine.trim(); if (!line.startsWith('data:')) continue; const data = line.slice(5).trim(); if (!data || data === '[DONE]') continue; onEvent(JSON.parse(data)); } } } export default function ComfyUIAgent() { const toast = useToast(); const location = useLocation(); const sessionAudience = location.pathname.startsWith('/user/') ? 'user' : 'admin'; const initialStoredSession = useMemo( () => readStoredSession(sessionAudience), [sessionAudience], ); const initialConversations = useMemo(() => { if (initialStoredSession.conversations?.length) return initialStoredSession.conversations; return [ createConversation( initialStoredSession.messages || [WELCOME_MESSAGE], initialStoredSession.selectedModel || '', initialStoredSession.selectedImageModel || '', ), ]; }, [initialStoredSession]); const initialActiveConversation = initialConversations.find( (conversation) => conversation.id === initialStoredSession.activeConversationId, ) || initialConversations[0]; const [conversations, setConversations] = useState( () => initialConversations, ); const [activeConversationId, setActiveConversationId] = useState( () => initialActiveConversation.id, ); const [messages, setMessages] = useState( () => initialActiveConversation.messages, ); const [input, setInput] = useState(''); const [runningByConversation, setRunningByConversation] = useState< Record >({}); const [status, setStatus] = useState(null); const [modelOptions, setModelOptions] = useState([]); const [selectedModel, setSelectedModel] = useState( () => initialActiveConversation.selectedModel || '', ); const [imageModelOptions, setImageModelOptions] = useState([]); const [selectedImageModel, setSelectedImageModel] = useState( () => initialActiveConversation.selectedImageModel || '', ); const [modelSearch, setModelSearch] = useState(''); const [imageModelSearch, setImageModelSearch] = useState(''); const [conversationSidebarCollapsed, setConversationSidebarCollapsed] = useState(false); const [loadingModels, setLoadingModels] = useState(false); const [attachments, setAttachments] = useState([]); const [readingAttachments, setReadingAttachments] = useState(false); const chatEndRef = useRef(null); const fileInputRef = useRef(null); const activeConversationIdRef = useRef(activeConversationId); const messagesRef = useRef(messages); const conversationsRef = useRef(conversations); const streaming = runningByConversation[activeConversationId]?.draft || null; const loading = !!runningByConversation[activeConversationId]; const canSend = (input.trim().length > 0 || attachments.length > 0) && !loading && !readingAttachments; const latestWorkflow = useMemo(() => { for (let index = messages.length - 1; index >= 0; index -= 1) { if (messages[index].workflowJson) return messages[index].workflowJson; } return streaming?.workflowJson || null; }, [messages, streaming]); const filteredModelOptions = useMemo( () => { const filtered = filterModelTesterModelNames(modelOptions, modelSearch).slice(0, 80); if (selectedModel && !filtered.includes(selectedModel)) { return [selectedModel, ...filtered]; } return filtered; }, [modelOptions, modelSearch, selectedModel], ); const filteredImageModelOptions = useMemo( () => { const search = imageModelSearch.trim().toLowerCase(); const filtered = imageModelOptions .filter((model) => !search || model.toLowerCase().includes(search)) .slice(0, 80); if (selectedImageModel && !filtered.includes(selectedImageModel)) { return [selectedImageModel, ...filtered]; } return filtered; }, [imageModelOptions, imageModelSearch, selectedImageModel], ); useEffect(() => { chatEndRef.current?.scrollIntoView({ behavior: 'smooth', block: 'end' }); }, [messages, streaming?.content, streaming?.reasoningContent, streaming?.images.length, streaming?.scenes.length, streaming?.tts.length]); useEffect(() => { activeConversationIdRef.current = activeConversationId; }, [activeConversationId]); useEffect(() => { messagesRef.current = messages; }, [messages]); useEffect(() => { conversationsRef.current = conversations; }, [conversations]); useEffect(() => { const stored = readStoredSession(sessionAudience); const nextConversations = stored.conversations?.length ? stored.conversations : [ createConversation( stored.messages || [WELCOME_MESSAGE], stored.selectedModel || '', stored.selectedImageModel || '', ), ]; const nextActive = nextConversations.find( (conversation) => conversation.id === stored.activeConversationId, ) || nextConversations[0]; setConversations(nextConversations); setActiveConversationId(nextActive.id); setMessages(nextActive.messages); setSelectedModel(nextActive.selectedModel || ''); setSelectedImageModel(nextActive.selectedImageModel || ''); setModelSearch(''); setImageModelSearch(''); Object.values(runningByConversation).forEach((running) => running.controller.abort()); setRunningByConversation({}); }, [sessionAudience]); useEffect(() => { setConversations((current) => { const now = Date.now(); const next = current.length ? current.map((conversation) => conversation.id === activeConversationId ? { ...conversation, title: buildConversationTitle(messages), updatedAt: now, messages, selectedModel, selectedImageModel, } : conversation, ) : [createConversation(messages, selectedModel, selectedImageModel)]; const ordered = [...next].sort((a, b) => { if (a.id === activeConversationId) return -1; if (b.id === activeConversationId) return 1; return b.updatedAt - a.updatedAt; }); persistStoredSession(sessionAudience, ordered, activeConversationId); return ordered; }); }, [activeConversationId, messages, selectedModel, selectedImageModel, sessionAudience]); useEffect(() => { const token = getSessionToken(sessionAudience, localStorage); if (!token) return; void fetch('/api/comfyui-agent/status', { headers: { Authorization: `Bearer ${token}` }, }) .then((response) => (response.ok ? response.json() : null)) .then((payload) => setStatus(payload || null)) .catch(() => setStatus(null)); }, [sessionAudience]); useEffect(() => { const loadRouteModels = async () => { const token = getSessionToken(sessionAudience, localStorage); if (!token) return; setLoadingModels(true); try { const response = await fetch('/api/comfyui-agent/models', { headers: { Authorization: `Bearer ${token}` }, }); if (!response.ok) throw new Error(`HTTP ${response.status}`); const payload = await response.json(); const names = Array.isArray(payload?.models) ? payload.models .map((model: unknown) => (typeof model === 'string' ? model.trim() : '')) .filter(Boolean) : []; setModelOptions(names); setSelectedModel((current) => { if (current && names.includes(current)) return current; const configuredModel = String(payload?.defaultModel || status?.model || '').trim(); if (configuredModel && names.includes(configuredModel)) return configuredModel; return names[0] || configuredModel; }); const imageResponse = await fetch('/api/comfyui-agent/image-models', { headers: { Authorization: `Bearer ${token}` }, }); if (!imageResponse.ok) throw new Error(`HTTP ${imageResponse.status}`); const imagePayload = await imageResponse.json(); const imageNames = Array.isArray(imagePayload?.models) ? imagePayload.models .map((model: unknown) => (typeof model === 'string' ? model.trim() : '')) .filter(Boolean) : []; setImageModelOptions(imageNames); setSelectedImageModel((current) => { if (current && imageNames.includes(current)) return current; const configuredModel = String(imagePayload?.defaultModel || '').trim(); if (configuredModel && imageNames.includes(configuredModel)) return configuredModel; return imageNames[0] || configuredModel || 'gpt-image-2'; }); } catch { toast.error('加载模型路由失败'); } finally { setLoadingModels(false); } }; void loadRouteModels(); }, [sessionAudience, status?.model, toast]); const exportWorkflow = () => { const workflow = latestWorkflow || { name: 'BoosAPI Video Agent Project', version: 1, messages: messages .filter((message) => message.role !== 'system') .map((message) => ({ role: message.role, content: message.content, attachments: message.attachments || [], images: message.images || [], scenes: message.scenes || [], tts: message.tts || [], })), }; const blob = new Blob([JSON.stringify(workflow, null, 2)], { type: 'application/json', }); const url = URL.createObjectURL(blob); const link = document.createElement('a'); link.href = url; link.download = 'comfyui-video-agent-workflow.json'; link.click(); URL.revokeObjectURL(url); }; const applySeedanceTemplate = (template: string) => { setInput((current) => { const trimmed = current.trim(); if (!trimmed) return template; return `${trimmed}\n\n---\n${template}`; }); }; const handleStreamEvent = ( event: any, draft: StreamAccumulator, apply: (next: StreamAccumulator) => void, ) => { const next: StreamAccumulator = { content: draft.content, reasoningContent: draft.reasoningContent, images: [...draft.images], scenes: [...draft.scenes], tts: [...draft.tts], workflowJson: draft.workflowJson, }; if (event.type === 'text') { next.content += String(event.content || ''); } else if (event.type === 'reasoning') { next.reasoningContent += String(event.content || ''); } else if (event.type === 'character_prompts') { next.content = appendLine( next.content, `${event.character || '角色'} 的四视图提示词已生成。`, ); } else if (event.type === 'image') { next.images.push({ url: String(event.url || ''), character: event.character, angle: event.angle, }); next.content = appendLine( next.content, `已生成 ${event.character || '角色'} ${angleLabel(event.angle)}。`, ); } else if (event.type === 'scene') { next.scenes.push({ id: Number(event.id || 0), description: String(event.description || ''), prompt: String(event.prompt || ''), url: event.url ? String(event.url) : undefined, }); next.content = appendLine( next.content, `场景 ${event.id || ''} 已准备：${event.description || event.prompt || ''}`, ); } else if (event.type === 'tts') { next.tts.push({ character: String(event.character || ''), text: String(event.text || ''), url: event.url ? String(event.url) : undefined, }); next.content = appendLine( next.content, `已生成 ${event.character || '角色'} 的语音。`, ); } else if (event.type === 'comfyui_workflow') { next.workflowJson = parseWorkflow(String(event.content || '')); next.content = appendLine(next.content, 'ComfyUI 工作流 JSON 已就绪。'); } else if (event.type === 'error') { next.content = appendLine(next.content, `错误：${event.message || '生成失败'}`); } draft.content = next.content; draft.reasoningContent = next.reasoningContent; draft.images = next.images; draft.scenes = next.scenes; draft.tts = next.tts; draft.workflowJson = next.workflowJson; apply(next); }; const appendAssistantMessageToConversation = ( conversationId: string, assistantMessage: AgentMessage, ) => { setConversations((current) => { const now = Date.now(); const next = current.map((conversation) => ( conversation.id === conversationId ? { ...conversation, updatedAt: now, messages: [...conversation.messages, assistantMessage], } : conversation )); persistStoredSession(sessionAudience, next, activeConversationIdRef.current); return next; }); if (activeConversationIdRef.current === conversationId) { setMessages((current) => [...current, assistantMessage]); } }; const sendMessage = async () => { const content = input.trim(); if ((!content && attachments.length === 0) || loading || readingAttachments) return; const token = getSessionToken(sessionAudience, localStorage); if (!token) { toast.error('登录状态已过期，请重新登录'); return; } const userMessage: AgentMessage = { id: createId('user'), role: 'user', content, attachments: attachments.length ? attachments : undefined, }; const nextMessages = [...messages, userMessage]; const controller = new AbortController(); const conversationId = activeConversationId; const modelForRequest = selectedModel || status?.model; const imageModelForRequest = selectedImageModel || imageModelOptions[0] || 'gpt-image-2'; const draft: StreamAccumulator = { content: '', reasoningContent: '', images: [], scenes: [], tts: [], workflowJson: null, }; setMessages(nextMessages); setInput(''); setAttachments([]); setRunningByConversation((current) => ({ ...current, [conversationId]: { draft, controller }, })); try { const payloadMessages = nextMessages .filter((message) => message.role !== 'system') .map((message) => ({ role: message.role, content: message.content, reasoningContent: message.reasoningContent || '', attachments: message.attachments || [], })); const response = await fetch('/api/comfyui-agent/chat', { method: 'POST', headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${token}`, }, body: JSON.stringify({ messages: payloadMessages, model: modelForRequest, imageModel: imageModelForRequest, }), signal: controller.signal, }); if (!response.ok) { const text = await response.text(); throw new Error(text || `HTTP ${response.status}`); } await readSseStream(response, (event) => { handleStreamEvent(event, draft, (nextDraft) => { setRunningByConversation((current) => { const running = current[conversationId]; if (!running) return current; return { ...current, [conversationId]: { ...running, draft: nextDraft, }, }; }); }); }); const processed = processThinkTags(draft.content, draft.reasoningContent); appendAssistantMessageToConversation(conversationId, { id: createId('assistant'), role: 'assistant', content: processed.content || '生成完成，但没有返回文本。', reasoningContent: processed.reasoningContent || undefined, images: draft.images.length ? draft.images : undefined, scenes: draft.scenes.length ? draft.scenes : undefined, tts: draft.tts.length ? draft.tts : undefined, workflowJson: draft.workflowJson || undefined, }); } catch (error: any) { if (error?.name === 'AbortError') { if (draft.content) { const processed = processThinkTags(draft.content, draft.reasoningContent); appendAssistantMessageToConversation(conversationId, { id: createId('assistant'), role: 'assistant', content: appendLine(processed.content, '已停止生成。'), reasoningContent: processed.reasoningContent || undefined, images: draft.images.length ? draft.images : undefined, scenes: draft.scenes.length ? draft.scenes : undefined, tts: draft.tts.length ? draft.tts : undefined, workflowJson: draft.workflowJson || undefined, }); } } else { toast.error(error?.message || '视频助手请求失败'); const processed = processThinkTags(draft.content, draft.reasoningContent); appendAssistantMessageToConversation(conversationId, { id: createId('assistant'), role: 'assistant', content: appendLine(processed.content, `请求失败：${error?.message || '未知错误'}`), reasoningContent: processed.reasoningContent || undefined, images: draft.images.length ? draft.images : undefined, scenes: draft.scenes.length ? draft.scenes : undefined, tts: draft.tts.length ? draft.tts : undefined, workflowJson: draft.workflowJson || undefined, }); } } finally { setRunningByConversation((current) => { const next = { ...current }; delete next[conversationId]; return next; }); } }; const stopGeneration = () => { runningByConversation[activeConversationId]?.controller.abort(); }; const startNewConversation = () => { const conversation = createConversation( [WELCOME_MESSAGE], selectedModel || status?.model || '', selectedImageModel || imageModelOptions[0] || '', ); setConversations((current) => { const next = [conversation, ...current]; persistStoredSession(sessionAudience, next, conversation.id); return next; }); setActiveConversationId(conversation.id); setMessages(conversation.messages); setSelectedModel(conversation.selectedModel || ''); setSelectedImageModel(conversation.selectedImageModel || selectedImageModel || ''); setInput(''); setAttachments([]); setModelSearch(''); setImageModelSearch(''); }; const selectConversation = (conversation: StoredAgentConversation) => { if (conversation.id === activeConversationId) return; setActiveConversationId(conversation.id); setMessages(conversation.messages); setSelectedModel(conversation.selectedModel || ''); setSelectedImageModel(conversation.selectedImageModel || ''); setInput(''); setAttachments([]); setModelSearch(''); setImageModelSearch(''); }; const deleteConversation = (conversationId: string) => { runningByConversation[conversationId]?.controller.abort(); setConversations((current) => { const remaining = current.filter((conversation) => conversation.id !== conversationId); const next = remaining.length ? remaining : [createConversation([WELCOME_MESSAGE], selectedModel, selectedImageModel)]; const nextActive = conversationId === activeConversationId ? next[0] : next.find((conversation) => conversation.id === activeConversationId) || next[0]; setActiveConversationId(nextActive.id); setMessages(nextActive.messages); setSelectedModel(nextActive.selectedModel || ''); setSelectedImageModel(nextActive.selectedImageModel || ''); setInput(''); setAttachments([]); persistStoredSession(sessionAudience, next, nextActive.id); return next; }); setRunningByConversation((current) => { const next = { ...current }; delete next[conversationId]; return next; }); }; const handleFilesSelected = async (files: FileList | null) => { if (!files?.length) return; setReadingAttachments(true); try { const selectedFiles = Array.from(files).slice(0, MAX_ATTACHMENT_COUNT); const oversized = selectedFiles.filter((file) => file.size > MAX_ATTACHMENT_BYTES); if (oversized.length > 0) { toast.error(`附件过大：${oversized[0].name} 超过 ${formatBytes(MAX_ATTACHMENT_BYTES)}`); return; } const existingInlineBytes = attachments.reduce((sum, attachment) => ( sum + (attachment.dataUrl ? attachment.size : 0) ), 0); const nextInlineBytes = selectedFiles.reduce((sum, file) => ( sum + (file.type.startsWith('image/') && file.size <= MAX_INLINE_IMAGE_BYTES ? file.size : 0) ), 0); if (existingInlineBytes + nextInlineBytes > MAX_INLINE_TOTAL_BYTES) { toast.error(`图片附件总量超过 ${formatBytes(MAX_INLINE_TOTAL_BYTES)}，请减少数量或压缩后再上传`); return; } if (files.length > MAX_ATTACHMENT_COUNT || attachments.length + selectedFiles.length > MAX_ATTACHMENT_COUNT) { toast.info(`最多保留 ${MAX_ATTACHMENT_COUNT} 个附件，超出的部分已忽略。`); } const next = await Promise.all(selectedFiles.map(fileToAttachment)); const withReferences = assignAttachmentReferences([...attachments, ...next].slice(0, MAX_ATTACHMENT_COUNT)); setAttachments(withReferences); if (next.some((attachment) => isDocumentAttachment( new File([], attachment.name, { type: attachment.mimeType }), attachment.mimeType, ))) { toast.info('PDF/Office 已作为素材记录；完整正文和内嵌图片解析会在后端解析器接入后启用。'); } } catch (error: any) { toast.error(error?.message || '读取附件失败'); } finally { setReadingAttachments(false); if (fileInputRef.current) fileInputRef.current.value = ''; } }; const removeAttachment = (id: string) => { setAttachments((current) => assignAttachmentReferences(current.filter((attachment) => attachment.id !== id))); }; const insertAttachmentReference = (label?: string) => { if (!label) return; setInput((current) => { const trimmed = current.trimEnd(); return trimmed ? `${trimmed} ${label} ` : `${label} `; }); }; const copyMessage = async (message: AgentMessage | StreamAccumulator) => { const processed = processThinkTags(message.content || '', message.reasoningContent || ''); const parts = [ processed.reasoningContent ? `## 思考过程\n\n${processed.reasoningContent}` : '', processed.content, ].filter(Boolean); if (!parts.length) { toast.error('没有可复制的文本内容。'); return; } try { await copyTextToClipboard(parts.join('\n\n')); toast.success('已复制对话内容'); } catch { toast.error('复制失败，请手动复制。'); } }; const renderReasoning = (message: AgentMessage | StreamAccumulator, isStreaming = false) => { const processed = processThinkTags(message.content || '', message.reasoningContent || ''); const hasReasoning = processed.reasoningContent.trim().length > 0; if (!hasReasoning && !isStreaming) return null; return (
思考过程 {isStreaming ? 实时 : null} {hasReasoning ? (
) : (
{STREAMING_REASONING_PLACEHOLDER}
)}
); }; const renderMessageContent = (message: AgentMessage | StreamAccumulator) => { const processed = processThinkTags(message.content || '', message.reasoningContent || ''); return (
); }; const renderAttachments = (items?: AgentAttachment[], compact = false) => { if (!items?.length) return null; return (
{items.map((attachment) => (
{attachment.dataUrl ? ( attachment.mimeType.startsWith('video/') ? (
))}
); }; const renderAssets = (message: AgentMessage | StreamAccumulator) => ( <> {!!message.images?.length && (
{message.images.map((image, index) => (
{`${image.character
{image.character || '角色'} / {angleLabel(image.angle)}
))}
)} {!!message.scenes?.length && (
{message.scenes.map((scene) => (
场景 {scene.id || '-'} {scene.description || scene.prompt} {scene.url ? 查看图像 : null}
))}
)} {!!message.tts?.length && (
{message.tts.map((item, index) => (
{item.character || '语音'} {item.text} {item.url ?
))}
)} ); return (

视频助手

对话 {selectedModel || status?.model || '加载中'} · 生图 {selectedImageModel || 'gpt-image-2'} · 模式 {status?.mode || 'local'} · Key {status?.keyName || '当前用户'}
{QUICK_PROMPTS.map((prompt) => ( ))}
Seedance 2.0 提示词
支持 @图片 / @视频 / @音频 引用、分时段镜头和音画同步说明
{SEEDANCE_PROMPT_TEMPLATES.map((template) => ( ))}
{messages.map((message) => (
{message.role === 'user' ? '你' : message.role === 'system' ? '系统' : '助手'}
{renderReasoning(message)} {renderMessageContent(message)} {renderAttachments(message.attachments, message.role !== 'user')} {renderAssets(message)}
))} {streaming && (
助手
{renderReasoning(streaming, true)} {streaming.content ? renderMessageContent(streaming) : null} {streaming.content ? | : null} {!streaming.content && !streaming.images.length && !streaming.scenes.length && !streaming.tts.length ? (
{STREAMING_BODY_PLACEHOLDER}
) : null} {renderAssets(streaming)}
)}