LaPaoPao/BoosAPI
1
0
Fork 0
forked from boos/BoosAPI
Code Pull Requests 1 Activity

feat: add desktop agent workbench

Browse Source
This commit is contained in:
boos
2026-06-26 13:08:00 +08:00
parent bc99c678f9
commit 424aaa9b1f
28 changed files with 1247 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files
desktop/src/pages/AgentWorkbenchPage.tsx
+526
View File
@@ -0,0 +1,526 @@
import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
import { App, Button, Dropdown, Input, Select, Switch, Tag, theme } from 'antd';
import type { MenuProps } from 'antd';
import {
ArrowUp,
AtSign,
ChevronDown,
Clapperboard,
ImagePlus,
Mic2,
Music2,
Plus,
SlidersHorizontal,
Sparkles,
UserRound,
WandSparkles,
Wrench,
X,
} from 'lucide-react';
import { invoke } from '@/lib/invoke';
import { useConversationStore, useProviderStore, useSkillStore, useUIStore } from '@/stores';
import type { AttachmentInput, Model } from '@/types';
type CreationMode = 'agent' | 'image' | 'video' | 'music' | 'voice' | 'digital-human' | 'action';
interface CreationModeOption {
key: CreationMode;
label: string;
hint: string;
icon: React.ReactNode;
}
interface AttachmentItem {
id: string;
name: string;
type: string;
size: number;
data: string;
}
interface ChatModelOption {
value: string;
label: string;
providerId: string;
modelId: string;
}
const CREATION_MODES: CreationModeOption[] = [
{ key: 'agent', label: 'Agent 模式', hint: '拆解任务、读写文件、执行工具', icon: <WandSparkles size={16} /> },
{ key: 'image', label: '图片生成', hint: '即梦图片、参考图与风格控制', icon: <ImagePlus size={16} /> },
{ key: 'video', label: '视频生成', hint: '即梦视频、图生视频与分镜', icon: <Clapperboard size={16} /> },
{ key: 'music', label: '音乐生成', hint: '配乐、氛围与短片 BGM', icon: <Music2 size={16} /> },
{ key: 'voice', label: '配音生成', hint: 'TTS 音色、语速与试听', icon: <Mic2 size={16} /> },
{ key: 'digital-human', label: '数字人', hint: '口播、形象与视频工作流', icon: <UserRound size={16} /> },
{ key: 'action', label: '动作模仿', hint: '动作迁移与主体参考', icon: <Sparkles size={16} /> },
];
const JIMENG_IMAGE_MODELS = [
{ value: 'jimeng_t2i_v40', label: '即梦图片 4.0' },
{ value: 'custom', label: '自定义 req_key' },
];
const JIMENG_VIDEO_MODELS = [
{ value: 'jimeng_ti2v_v30_pro', label: '即梦图生视频 3.0 Pro' },
{ value: 'custom', label: '自定义 req_key' },
];
const JIANYING_SKILL_NAME = 'jianying-editor';
const JIANYING_SKILL_SOURCE = 'luoluoluo22/jianying-editor-skill';
const JIANYING_PROMPT_PREFIX = '请使用 jianying-editor Skill 自动创建剪映草稿：';
function formatFileSize(size: number): string {
if (size < 1024) return `${size} B`;
if (size < 1024 * 1024) return `${(size / 1024).toFixed(1)} KB`;
return `${(size / 1024 / 1024).toFixed(1)} MB`;
}
function modelOptionLabel(model: Model, providerName: string): string {
return `${model.name || model.model_id} · ${providerName}`;
}
function isUsableChatModel(model: Model): boolean {
return model.enabled && (model.model_type === 'Chat' || model.capabilities.includes('TextChat'));
}
function getAttachmentMimeType(fileName: string, mimeType?: string): string {
if (mimeType) return mimeType;
const extension = fileName.split('.').pop()?.toLowerCase();
if (extension === 'pdf') return 'application/pdf';
if (extension === 'txt') return 'text/plain';
if (extension === 'md') return 'text/markdown';
if (extension === 'json') return 'application/json';
return 'application/octet-stream';
}
async function fileToAttachmentItem(file: File): Promise<AttachmentItem> {
return new Promise((resolve, reject) => {
const reader = new FileReader();
reader.onerror = () => reject(reader.error ?? new Error(`读取文件失败：${file.name}`));
reader.onload = () => {
const base64 = (reader.result as string).split(',')[1] || '';
resolve({
id: `${file.name}-${file.size}-${file.lastModified}-${globalThis.crypto?.randomUUID?.() ?? Date.now()}`,
name: file.name,
type: getAttachmentMimeType(file.name, file.type),
size: file.size,
data: base64,
});
};
reader.readAsDataURL(file);
});
}
function toAttachmentInput(item: AttachmentItem): AttachmentInput {
return {
file_name: item.name,
file_type: item.type,
file_size: item.size,
data: item.data,
};
}
export function AgentWorkbenchPage() {
const { token } = theme.useToken();
const { message } = App.useApp();
const fileInputRef = useRef<HTMLInputElement>(null);
const providers = useProviderStore((s) => s.providers);
const fetchProviders = useProviderStore((s) => s.fetchProviders);
const createConversation = useConversationStore((s) => s.createConversation);
const updateConversation = useConversationStore((s) => s.updateConversation);
const sendAgentMessage = useConversationStore((s) => s.sendAgentMessage);
const skills = useSkillStore((s) => s.skills);
const loadSkills = useSkillStore((s) => s.loadSkills);
const installSkill = useSkillStore((s) => s.installSkill);
const toggleSkill = useSkillStore((s) => s.toggleSkill);
const setActivePage = useUIStore((s) => s.setActivePage);
const [creationMode, setCreationMode] = useState<CreationMode>('agent');
const [autoMode, setAutoMode] = useState(true);
const [prompt, setPrompt] = useState('');
const [agentModel, setAgentModel] = useState('custom');
const [customAgentModel, setCustomAgentModel] = useState('');
const [imageModel, setImageModel] = useState('jimeng_t2i_v40');
const [videoModel, setVideoModel] = useState('jimeng_ti2v_v30_pro');
const [customJimengModel, setCustomJimengModel] = useState('');
const [attachments, setAttachments] = useState<AttachmentItem[]>([]);
const [subjectOpen, setSubjectOpen] = useState(false);
const [skillOpen, setSkillOpen] = useState(false);
const [submitting, setSubmitting] = useState(false);
const [installingJianying, setInstallingJianying] = useState(false);
const [result, setResult] = useState('');
useEffect(() => {
if (providers.length === 0) {
void fetchProviders().catch((error) => message.error(String(error)));
}
}, [fetchProviders, message, providers.length]);
useEffect(() => {
void loadSkills();
}, [loadSkills]);
const chatModelOptions = useMemo<ChatModelOption[]>(() => {
const rows = providers
.filter((provider) => provider.enabled)
.flatMap((provider) =>
provider.models
.filter(isUsableChatModel)
.map((model) => ({
value: `${provider.id}::${model.model_id}`,
label: modelOptionLabel(model, provider.name),
providerId: provider.id,
modelId: model.model_id,
})),
);
const seen = new Set<string>();
return rows.filter((item) => {
if (seen.has(item.value)) return false;
seen.add(item.value);
return true;
});
}, [providers]);
useEffect(() => {
if (agentModel !== 'custom') return;
if (customAgentModel || chatModelOptions.length === 0) return;
setAgentModel(chatModelOptions[0].value);
}, [agentModel, chatModelOptions, customAgentModel]);
const currentMode = CREATION_MODES.find((item) => item.key === creationMode) ?? CREATION_MODES[0];
const selectedModel = creationMode === 'agent'
? (agentModel === 'custom' ? customAgentModel.trim() || '自定义模型' : agentModel)
: creationMode === 'image'
? (imageModel === 'custom' ? customJimengModel.trim() || '自定义 req_key' : imageModel)
: creationMode === 'video'
? (videoModel === 'custom' ? customJimengModel.trim() || '自定义 req_key' : videoModel)
: '待配置';
const modeItems: MenuProps['items'] = CREATION_MODES.map((item) => ({
key: item.key,
label: (
<div className="agent-workbench-mode-item">
<span>{item.icon}</span>
<div>
<strong>{item.label}</strong>
<small>{item.hint}</small>
</div>
{item.key === creationMode && <em>✓</em>}
</div>
),
}));
const selectedChatModel = useMemo(() => (
chatModelOptions.find((item) => item.value === agentModel) ?? null
), [agentModel, chatModelOptions]);
const jianyingSkill = useMemo(() => (
skills.find((item) => item.name === JIANYING_SKILL_NAME) ?? null
), [skills]);
const handleFilesSelected = useCallback((files: FileList | null) => {
if (!files || files.length === 0) return;
void Promise.all(Array.from(files).map(fileToAttachmentItem))
.then((next) => {
setAttachments((current) => [...current, ...next]);
})
.catch((error) => message.error(String(error)))
.finally(() => {
if (fileInputRef.current) fileInputRef.current.value = '';
});
}, [message]);
const removeAttachment = useCallback((id: string) => {
setAttachments((current) => current.filter((item) => item.id !== id));
}, []);
const applyJianyingSkill = useCallback(async () => {
if (installingJianying) return;
setInstallingJianying(true);
try {
if (!jianyingSkill) {
await installSkill(JIANYING_SKILL_SOURCE);
message.success('剪映 Skill 已安装');
} else if (!jianyingSkill.enabled) {
await toggleSkill(JIANYING_SKILL_NAME, true);
message.success('剪映 Skill 已启用');
}
setCreationMode('agent');
setPrompt((value) => {
const text = value.trim();
if (text.includes(JIANYING_SKILL_NAME) || text.includes('剪映')) return value;
return text ? `${JIANYING_PROMPT_PREFIX}${text}` : `${JIANYING_PROMPT_PREFIX}`;
});
setSkillOpen(false);
} catch (error) {
message.error(`剪映 Skill 集成失败：${String(error)}`);
} finally {
setInstallingJianying(false);
}
}, [installSkill, installingJianying, jianyingSkill, message, toggleSkill]);
const submit = useCallback(async () => {
const text = prompt.trim();
if (!text) {
message.warning('先输入想法、剧本或任务说明');
return;
}
if (submitting) return;
if (creationMode !== 'agent') {
setResult(`${currentMode.label} 已准备好。模型：${selectedModel}；参考素材：${attachments.length} 个。即梦/媒体生成链路下一步接入。`);
return;
}
const fallbackProvider = providers.find((provider) => provider.enabled);
const providerId = selectedChatModel?.providerId ?? fallbackProvider?.id;
const modelId = selectedChatModel?.modelId ?? customAgentModel.trim();
if (!providerId || !modelId) {
message.warning('先在供应商里启用一个对话模型，或填写自定义模型名');
return;
}
setSubmitting(true);
try {
const title = text.slice(0, 24) || 'Agent 创作';
const conversation = await createConversation(title, modelId, providerId);
await updateConversation(conversation.id, { mode: 'agent', provider_id: providerId, model_id: modelId });
await invoke('agent_update_session', {
conversationId: conversation.id,
permissionMode: autoMode ? 'accept_edits' : 'default',
});
await sendAgentMessage(text, attachments.map(toAttachmentInput));
setResult(`Agent 已开始执行。模型：${modelId}；参考素材：${attachments.length} 个。`);
setPrompt('');
setAttachments([]);
setActivePage('chat');
} catch (error) {
message.error(String(error));
setResult(`Agent 启动失败：${String(error)}`);
} finally {
setSubmitting(false);
}
}, [
attachments,
autoMode,
createConversation,
creationMode,
currentMode.label,
customAgentModel,
message,
prompt,
providers,
selectedChatModel,
selectedModel,
sendAgentMessage,
setActivePage,
submitting,
updateConversation,
]);
return (
<div className="agent-workbench-desktop" style={{ background: token.colorBgContainer }}>
<div className="agent-workbench-desktop-bg" />
<section className="agent-workbench-desktop-hero">
<h1>你好，想创作什么？</h1>
<div className="agent-workbench-desktop-composer">
<input
ref={fileInputRef}
className="agent-workbench-file-input"
type="file"
multiple
accept="image/*,video/*,audio/*,.pdf,.txt,.md,.json"
onChange={(event) => handleFilesSelected(event.target.files)}
/>
<button
className="agent-workbench-upload"
type="button"
onClick={() => fileInputRef.current?.click()}
aria-label="上传参考素材"
>
<Plus size={24} />
</button>
<Input.TextArea
className="agent-workbench-prompt"
value={prompt}
onChange={(event) => setPrompt(event.target.value)}
onPressEnter={(event) => {
if (event.metaKey || event.ctrlKey) {
event.preventDefault();
void submit();
}
}}
placeholder="输入想法、剧本或上传参考，支持 “/” 使用技能，@ 添加主体，和 Agent 一起创作"
autoSize={{ minRows: 4, maxRows: 8 }}
/>
{attachments.length > 0 && (
<div className="agent-workbench-attachments">
{attachments.map((item) => (
<Tag
key={item.id}
closeIcon={<X size={12} />}
onClose={(event) => {
event.preventDefault();
removeAttachment(item.id);
}}
>
{item.name} <span>{formatFileSize(item.size)}</span>
</Tag>
))}
</div>
)}
<div className="agent-workbench-toolbar">
<div className="agent-workbench-tool-left">
<Dropdown
trigger={['click']}
menu={{
items: modeItems,
selectable: true,
selectedKeys: [creationMode],
onClick: ({ key }) => setCreationMode(key as CreationMode),
}}
overlayClassName="agent-workbench-mode-menu"
>
<Button className="agent-workbench-pill active" icon={currentMode.icon}>
{currentMode.label} <ChevronDown size={14} />
</Button>
</Dropdown>
<Button className="agent-workbench-pill" icon={<SlidersHorizontal size={15} />} onClick={() => setAutoMode((value) => !value)}>
自动
<Switch size="small" checked={autoMode} onChange={setAutoMode} onClick={(_, event) => event.stopPropagation()} />
</Button>
{creationMode === 'agent' && (
<>
<Select
className="agent-workbench-model-select"
value={agentModel}
onChange={(value) => {
setAgentModel(value);
if (value !== 'custom') setCustomAgentModel('');
}}
options={[
...chatModelOptions,
{ value: 'custom', label: '自定义模型' },
]}
popupMatchSelectWidth={false}
/>
{agentModel === 'custom' && (
<Input
className="agent-workbench-model-input"
value={customAgentModel}
onChange={(event) => setCustomAgentModel(event.target.value)}
placeholder="输入 BoosAPI 模型名"
/>
)}
</>
)}
{creationMode === 'image' && (
<>
<Select
className="agent-workbench-model-select"
value={imageModel}
onChange={setImageModel}
options={JIMENG_IMAGE_MODELS}
popupMatchSelectWidth={false}
/>
{imageModel === 'custom' && (
<Input
className="agent-workbench-model-input"
value={customJimengModel}
onChange={(event) => setCustomJimengModel(event.target.value)}
placeholder="输入火山 req_key"
/>
)}
</>
)}
{creationMode === 'video' && (
<>
<Select
className="agent-workbench-model-select"
value={videoModel}
onChange={setVideoModel}
options={JIMENG_VIDEO_MODELS}
popupMatchSelectWidth={false}
/>
{videoModel === 'custom' && (
<Input
className="agent-workbench-model-input"
value={customJimengModel}
onChange={(event) => setCustomJimengModel(event.target.value)}
placeholder="输入火山 req_key"
/>
)}
</>
)}
<Dropdown
open={skillOpen}
onOpenChange={setSkillOpen}
trigger={['click']}
dropdownRender={() => (
<div className="agent-workbench-popover">
<Input placeholder="搜索技能" prefix={<Wrench size={15} />} />
<button type="button" onClick={() => void applyJianyingSkill()}>
<Clapperboard size={16} />
剪映自动剪辑
<small>
{installingJianying
? '正在安装/启用...'
: jianyingSkill?.enabled
? '已启用，生成草稿、字幕、配音、特效和导出'
: jianyingSkill
? '已安装，点击启用并写入提示'
: '一键安装 luoluoluo22/jianying-editor-skill'}
</small>
</button>
<button type="button"><Wrench size={16} /> 剧情短片 <small>自动生成故事大纲、分镜脚本并产出短片</small></button>
<button type="button"><Wrench size={16} /> 电商套图 <small>生成风格统一的商品全套视觉素材</small></button>
<button type="button"><Plus size={16} /> 用 Agent 创建技能</button>
</div>
)}
>
<Button className="agent-workbench-pill" icon={<Wrench size={15} />}>使用技能</Button>
</Dropdown>
<Dropdown
open={subjectOpen}
onOpenChange={setSubjectOpen}
trigger={['click']}
dropdownRender={() => (
<div className="agent-workbench-subject-popover">
<span>可能@的内容</span>
<div>
<AtSign size={36} />
<strong>你还没有创建过主体</strong>
<button
type="button"
onClick={() => {
setPrompt((value) => `${value}${value && !value.endsWith(' ') ? ' ' : ''}@主体 `);
setSubjectOpen(false);
}}
>
+ 创建主体
</button>
</div>
</div>
)}
>
<Button className="agent-workbench-pill square" icon={<AtSign size={16} />} />
</Dropdown>
</div>
<Button
className="agent-workbench-submit"
type="primary"
icon={<ArrowUp size={18} />}
loading={submitting}
onClick={() => void submit()}
/>
</div>
</div>
{result && (
<div className="agent-workbench-result">
<strong>{currentMode.label}</strong>
<p>{result}</p>
</div>
)}
</section>
</div>
);
}