Nvidia/BossAI
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve digital human template UI

Browse Source
This commit is contained in:
boos
2026-06-02 12:17:05 +08:00
parent fa5a2419d8
commit 056e9fb651
2 changed files with 462 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/web/pages/DigitalHuman.tsx
+267 -15
View File
@@ -1,10 +1,11 @@
import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react';
import { useLocation } from 'react-router-dom';
import { api, fetchSessionResponse, userApi, type DigitalHumanJob, type WorkflowSummary } from '../api.js';
import { api, fetchSessionResponse, userApi, type DigitalHumanJob, type WorkflowSummary, type WorkspaceAsset } from '../api.js';
import type { SessionAudience } from '../authSession.js';
type AudioRole = 'lip_sync' | 'voice_reference' | 'both';
type UploadPurpose = 'avatar' | 'audio';
type MappingKey = keyof typeof DEFAULT_MAPPING;
const DEFAULT_MAPPING = {
prompt: { nodeId: '', field: 'prompt' },
@@ -51,7 +52,7 @@ function graphMapping(workflow?: WorkflowSummary | null) {
|| DEFAULT_MAPPING;
}
function mappingValue(mapping: any, key: keyof typeof DEFAULT_MAPPING) {
function mappingValue(mapping: any, key: MappingKey) {
const value = mapping?.[key] || DEFAULT_MAPPING[key];
if (typeof value === 'string') return { nodeId: value, field: DEFAULT_MAPPING[key].field };
return {
@@ -64,6 +65,117 @@ function jobAssets(job: DigitalHumanJob) {
return Array.isArray(job.result?.assets) ? job.result.assets : [];
}
function templateHasMapping(workflow: WorkflowSummary) {
const mapping = graphMapping(workflow);
return ['prompt', 'script', 'avatar', 'referenceAudio', 'output'].every((key) => {
return Boolean(mappingValue(mapping, key as MappingKey).nodeId);
});
}
function workflowKindLabel(workflow: WorkflowSummary) {
if (workflow.isTemplate || workflow.visibility === 'template') return '官方模板';
if (workflow.visibility === 'public') return '共享模板';
return '私有工作流';
}
function sourcePrompt(graph: any) {
return graph?.prompt || graph?.workflow || graph?.comfyPrompt || graph || {};
}
function collectPromptNodes(graph: any) {
const prompt = sourcePrompt(graph);
const nodes: Array<{ id: string; classType: string; inputs: Record<string, any>; title: string }> = [];
if (prompt && typeof prompt === 'object' && !Array.isArray(prompt)) {
for (const [id, node] of Object.entries(prompt as Record<string, any>)) {
if (!node || typeof node !== 'object') continue;
const classType = String((node as any).class_type || (node as any).type || '');
const inputs = ((node as any).inputs && typeof (node as any).inputs === 'object') ? (node as any).inputs : {};
if (classType || Object.keys(inputs).length > 0) {
nodes.push({
id: String(id),
classType,
inputs,
title: String((node as any)._meta?.title || (node as any).title || classType || id),
});
}
}
}
const graphNodes = graph?.nodes;
if (Array.isArray(graphNodes)) {
for (const node of graphNodes) {
if (!node || typeof node !== 'object') continue;
const id = String(node.id || node.nodeId || '');
if (!id || nodes.some((item) => item.id === id)) continue;
const classType = String(node.type || node.class_type || '');
nodes.push({
id,
classType,
inputs: {},
title: String(node.title || node.label || classType || id),
});
}
}
return nodes;
}
function includesAny(value: string, words: string[]) {
const text = value.toLowerCase();
return words.some((word) => text.includes(word));
}
function findNode(
nodes: ReturnType<typeof collectPromptNodes>,
words: string[],
inputWords: string[] = [],
) {
return nodes.find((node) => {
const haystack = `${node.classType} ${node.title}`;
const inputNames = Object.keys(node.inputs || {}).join(' ');
return includesAny(haystack, words) || (inputWords.length > 0 && includesAny(inputNames, inputWords));
});
}
function inferMappingFromWorkflow(workflow: WorkflowSummary | null) {
if (!workflow) return DEFAULT_MAPPING;
const nodes = collectPromptNodes(workflow.graph);
const promptNode = findNode(nodes, ['prompt', 'positive', 'textgenerate', 'text generate', 'wan', 'ltx'], ['prompt']);
const scriptNode = findNode(nodes, ['script', 'text', 'tts', 'voice', 'audio'], ['text', 'script', 'caption']);
const avatarNode = findNode(nodes, ['loadimage', 'image', 'video', 'avatar', 'portrait', 'person'], ['image', 'video', 'avatar']);
const audioNode = findNode(nodes, ['audio', 'sound', 'voice', 'tts', 'load audio'], ['audio', 'voice']);
const outputNode = findNode(nodes, ['savevideo', 'save video', 'previewvideo', 'video combine', 'vhs', 'output'], ['video', 'filename_prefix'])
|| [...nodes].reverse().find((node) => includesAny(`${node.classType} ${node.title}`, ['save', 'preview', 'output']));
return {
prompt: { nodeId: promptNode?.id || '', field: promptNode?.inputs?.prompt !== undefined ? 'prompt' : 'text' },
script: { nodeId: scriptNode?.id || promptNode?.id || '', field: scriptNode?.inputs?.script !== undefined ? 'script' : 'text' },
avatar: { nodeId: avatarNode?.id || '', field: avatarNode?.inputs?.video !== undefined ? 'video' : 'image' },
referenceAudio: { nodeId: audioNode?.id || '', field: 'audio' },
voiceReference: { nodeId: audioNode?.id || '', field: 'audio' },
output: { nodeId: outputNode?.id || '', field: outputNode?.inputs?.filename_prefix !== undefined ? 'filename_prefix' : 'video' },
};
}
function assetFileUrl(asset: WorkspaceAsset, audience: SessionAudience) {
if (asset.remoteUrl) return asset.remoteUrl;
if (!asset.hasLocalFile) return '';
const prefix = audience === 'user' ? '/api/user' : '/api';
return `${prefix}/workspace/assets/${encodeURIComponent(asset.publicId)}/file`;
}
function assetName(asset: WorkspaceAsset) {
return asset.title || asset.fileName || asset.publicId;
}
function formatJobTime(value?: string | null) {
if (!value) return '';
const date = new Date(value);
if (Number.isNaN(date.getTime())) return '';
return date.toLocaleString('zh-CN', { hour12: false });
}
export default function DigitalHuman() {
const location = useLocation();
const audience: SessionAudience = location.pathname.startsWith('/user/') ? 'user' : 'admin';
@@ -72,6 +184,7 @@ export default function DigitalHuman() {
const audioInputRef = useRef<HTMLInputElement | null>(null);
const [workflows, setWorkflows] = useState<WorkflowSummary[]>([]);
const [jobs, setJobs] = useState<DigitalHumanJob[]>([]);
const [workspaceAssets, setWorkspaceAssets] = useState<WorkspaceAsset[]>([]);
const [templateId, setTemplateId] = useState('');
const [mapping, setMapping] = useState<any>(DEFAULT_MAPPING);
const [avatarUrl, setAvatarUrl] = useState('');
@@ -84,10 +197,19 @@ export default function DigitalHuman() {
const [subtitles, setSubtitles] = useState(true);
const [submitting, setSubmitting] = useState(false);
const [uploading, setUploading] = useState<UploadPurpose | ''>('');
const [assetPicker, setAssetPicker] = useState<UploadPurpose | ''>('');
const [loadingAssets, setLoadingAssets] = useState(false);
const [error, setError] = useState('');
const client = useMemo(() => (audience === 'user' ? userApi : api), [audience]);
const selectedWorkflow = workflows.find((item) => item.publicId === templateId) || null;
const sortedWorkflows = useMemo(() => {
return [...workflows].sort((a, b) => {
const score = (item: WorkflowSummary) => (templateHasMapping(item) ? 0 : 10)
+ (item.isTemplate || item.visibility === 'template' ? 0 : item.visibility === 'public' ? 2 : 4);
return score(a) - score(b) || a.name.localeCompare(b.name, 'zh-CN');
});
}, [workflows]);
const mapped = useMemo(() => ({
prompt: mappingValue(mapping, 'prompt'),
script: mappingValue(mapping, 'script'),
@@ -111,6 +233,12 @@ export default function DigitalHuman() {
&& !!productScript.trim()
&& missingMapping.length === 0
&& !submitting;
const selectedTemplateReady = selectedWorkflow ? templateHasMapping(selectedWorkflow) || missingMapping.length === 0 : false;
const pickerAssets = workspaceAssets.filter((asset) => {
if (assetPicker === 'audio') return asset.kind === 'audio';
if (assetPicker === 'avatar') return asset.kind === 'image' || asset.kind === 'video';
return false;
});
const loadData = useCallback(async () => {
const [workflowPayload, jobPayload] = await Promise.all([
@@ -175,6 +303,36 @@ export default function DigitalHuman() {
}));
};
const inferMapping = () => {
setMapping(inferMappingFromWorkflow(selectedWorkflow));
};
const openAssetPicker = async (purpose: UploadPurpose) => {
setAssetPicker(purpose);
setLoadingAssets(true);
setError('');
try {
const kind = purpose === 'audio' ? 'audio' : undefined;
const payload = await client.getWorkspaceAssets({ kind, pageSize: 60 });
setWorkspaceAssets(Array.isArray(payload.items) ? payload.items : []);
} catch (err: any) {
setError(err?.message || '工作空间素材加载失败');
} finally {
setLoadingAssets(false);
}
};
const chooseAsset = (asset: WorkspaceAsset) => {
const url = assetFileUrl(asset, audience);
if (!url) {
setError('这个素材没有可用文件地址');
return;
}
if (assetPicker === 'avatar') setAvatarUrl(url);
if (assetPicker === 'audio') setReferenceAudioUrl(url);
setAssetPicker('');
};
const submit = async () => {
if (!canSubmit) return;
setSubmitting(true);
@@ -243,14 +401,46 @@ export default function DigitalHuman() {
<div className="digital-human-layout">
<section className="digital-human-panel">
<h2>1. 选择工作流模板</h2>
<label className="digital-human-field">
<span>模板</span>
<div className="digital-human-section-head">
<div>
<h2>1. 选择工作流模板</h2>
<p>优先选择已配置数字人节点映射的模板；普通 Comfy 工作流也可以用自动猜测补齐。</p>
</div>
<button type="button" className="btn btn-ghost" onClick={inferMapping} disabled={!selectedWorkflow}>猜测映射</button>
</div>
<div className="digital-human-template-grid">
{sortedWorkflows.slice(0, 6).map((workflow) => {
const ready = templateHasMapping(workflow);
const active = workflow.publicId === templateId;
return (
<button
key={workflow.publicId}
type="button"
className={`digital-human-template-card${active ? ' is-active' : ''}`}
onClick={() => setTemplateId(workflow.publicId)}
>
<span className="digital-human-template-top">
<strong>{workflow.name}</strong>
<em>{ready ? '可直接生成' : '需映射'}</em>
</span>
<span className="digital-human-template-desc">
{workflow.description || 'Comfy 工作流模板，可注入人物素材、文案和参考音频。'}
</span>
<span className="digital-human-template-tags">
<i>{workflowKindLabel(workflow)}</i>
<i>{workflow.visibility}</i>
</span>
</button>
);
})}
</div>
<label className="digital-human-field digital-human-template-select">
<span>全部模板</span>
<select value={templateId} onChange={(event) => setTemplateId(event.target.value)}>
<option value="">请选择模板</option>
{workflows.map((workflow) => (
{sortedWorkflows.map((workflow) => (
<option key={workflow.publicId} value={workflow.publicId}>
{workflow.name} · {workflow.visibility}
{workflow.name} · {workflowKindLabel(workflow)}
</option>
))}
</select>
@@ -259,6 +449,7 @@ export default function DigitalHuman() {
<div className="digital-human-template-meta">
<strong>{selectedWorkflow.name}</strong>
<span>{selectedWorkflow.description || '这个模板需要配置节点映射后才能注入人物、文案和音频。'}</span>
<span>{selectedTemplateReady ? '节点映射已准备好。' : '节点映射不完整，可点击“猜测映射”后再微调。'}</span>
</div>
)}
@@ -267,27 +458,39 @@ export default function DigitalHuman() {
<div className="digital-human-upload-card">
<strong>人物图片 / 视频</strong>
<span>{avatarUrl ? '已准备素材' : '支持本地上传或粘贴 URL'}</span>
{avatarUrl && (
avatarUrl.match(/\.(mp4|webm|mov|m4v)(\?|$)/i)
? <video className="digital-human-upload-preview" src={avatarUrl} muted controls preload="metadata" />
: <img className="digital-human-upload-preview" src={avatarUrl} alt="" loading="lazy" />
)}
<input ref={avatarInputRef} type="file" hidden accept="image/*,video/*" onChange={(event) => {
const file = event.target.files?.[0];
if (file) void uploadFile(file, 'avatar');
event.currentTarget.value = '';
}} />
<button type="button" className="btn btn-ghost" onClick={() => avatarInputRef.current?.click()} disabled={uploading === 'avatar'}>
{uploading === 'avatar' ? '上传中...' : '上传人物素材'}
</button>
<div className="digital-human-upload-actions">
<button type="button" className="btn btn-ghost" onClick={() => avatarInputRef.current?.click()} disabled={uploading === 'avatar'}>
{uploading === 'avatar' ? '上传中...' : '上传人物素材'}
</button>
<button type="button" className="btn btn-ghost" onClick={() => void openAssetPicker('avatar')}>工作空间</button>
</div>
<input value={avatarUrl} onChange={(event) => setAvatarUrl(event.target.value)} placeholder="或粘贴图片/视频 URL" />
</div>
<div className="digital-human-upload-card">
<strong>参考音频</strong>
<span>{referenceAudioUrl ? '已准备音频' : '用于口型驱动或音色参考'}</span>
{referenceAudioUrl && <audio className="digital-human-audio-preview" src={referenceAudioUrl} controls />}
<input ref={audioInputRef} type="file" hidden accept="audio/*" onChange={(event) => {
const file = event.target.files?.[0];
if (file) void uploadFile(file, 'audio');
event.currentTarget.value = '';
}} />
<button type="button" className="btn btn-ghost" onClick={() => audioInputRef.current?.click()} disabled={uploading === 'audio'}>
{uploading === 'audio' ? '上传中...' : '上传参考音频'}
</button>
<div className="digital-human-upload-actions">
<button type="button" className="btn btn-ghost" onClick={() => audioInputRef.current?.click()} disabled={uploading === 'audio'}>
{uploading === 'audio' ? '上传中...' : '上传参考音频'}
</button>
<button type="button" className="btn btn-ghost" onClick={() => void openAssetPicker('audio')}>工作空间</button>
</div>
<input value={referenceAudioUrl} onChange={(event) => setReferenceAudioUrl(event.target.value)} placeholder="或粘贴音频 URL" />
</div>
</div>
@@ -333,7 +536,13 @@ export default function DigitalHuman() {
</section>
<section className="digital-human-panel">
<h2>节点映射</h2>
<div className="digital-human-section-head">
<div>
<h2>节点映射</h2>
<p>把页面输入写入对应 Comfy 节点。字段名可按模板节点输入口微调。</p>
</div>
<button type="button" className="btn btn-ghost" onClick={inferMapping} disabled={!selectedWorkflow}>重新猜测</button>
</div>
<div className="digital-human-mapping-grid">
{([
['prompt', '提示词节点'],
@@ -376,10 +585,14 @@ export default function DigitalHuman() {
<div className="digital-human-job-head">
<div>
<strong>{job.title || '数字人讲解任务'}</strong>
<span>{job.message || statusLabel(job.status)}</span>
<span>{job.message || statusLabel(job.status)}{formatJobTime(job.createdAt) ? ` · ${formatJobTime(job.createdAt)}` : ''}</span>
</div>
<em>{statusLabel(job.status)}</em>
</div>
<div className="digital-human-job-meta">
<span>进度 {progress}%</span>
{job.result?.prompt_id && <span>Prompt {job.result.prompt_id}</span>}
</div>
<div className="digital-human-progress"><span style={{ width: `${progress}%` }} /></div>
{job.error && <div className="digital-human-job-error">{job.error}</div>}
{jobAssets(job).length > 0 && (
@@ -401,6 +614,45 @@ export default function DigitalHuman() {
</div>
</section>
</div>
{assetPicker && (
<div className="modal-overlay" onClick={() => setAssetPicker('')}>
<div className="digital-human-picker modal-card" onClick={(event) => event.stopPropagation()}>
<div className="modal-header">
<div>
<h3>{assetPicker === 'audio' ? '选择参考音频' : '选择人物素材'}</h3>
<p>从工作空间选取已有素材，只有最终产出仍会归档到工作空间。</p>
</div>
<button type="button" className="btn btn-ghost" onClick={() => setAssetPicker('')}>关闭</button>
</div>
{loadingAssets ? (
<div className="empty-state" style={{ minHeight: 180 }}>
<div className="empty-state-title">正在加载素材</div>
</div>
) : pickerAssets.length === 0 ? (
<div className="empty-state" style={{ minHeight: 180 }}>
<div className="empty-state-title">没有可选素材</div>
<div className="empty-state-desc">可以先上传本地文件，或去工作空间确认素材类型。</div>
</div>
) : (
<div className="digital-human-picker-grid">
{pickerAssets.map((asset) => {
const preview = assetFileUrl(asset, audience);
return (
<button key={asset.publicId} type="button" className="digital-human-picker-item" onClick={() => chooseAsset(asset)}>
{asset.kind === 'image' && preview ? <img src={preview} alt="" loading="lazy" /> : null}
{asset.kind === 'video' && preview ? <video src={preview} muted preload="metadata" /> : null}
{asset.kind === 'audio' ? <span className="digital-human-picker-audio">音频</span> : null}
<strong>{assetName(asset)}</strong>
<span>{asset.kind} · {asset.source || 'workspace'}</span>
</button>
);
})}
</div>
)}
</div>
</div>
)}
</div>
);
}