// Avatar / Persona studio (DoD): persistent avatars as workspace asset type `avatar` // avatars//avatar.json + ref.webp + voice.wav · reused by Chat, Image (Kontext), agents. import { STUDIOS, onStop } from './widgets.js?v=50'; import { run, result, MODULES, esc, hf, stTxt, url, badge, setS, keep, LS, FLUX, KONTEXT, TTS, CHAT, signedIn } from './real.js?v=136'; export const MODA = 'multimodalart/MoDA-fast-talking-head', W2L = 'pragnakalp/Wav2lip-ZeroGPU'; const LANGS = ['German', 'English', 'French', 'Spanish', 'Auto']; const slug = (n) => (n || 'avatar').toLowerCase().normalize('NFKD').replace(/[^\w]+/g, '-').replace(/^-|-$/g, '').slice(0, 40) || 'avatar'; const blank = () => ({ type: 'studius.avatar', v: 2, id: '', name: '', system: '', backstory: '', look: '', seed: Math.floor(Math.random() * 1e6), voice: { description: 'Warm, friendly, energetic voice', language: 'German', sample_text: '' }, ref: null, voiceRef: null, updated: 0 }); const bucket = () => signedIn() && window.__ws?.bucket; const wsFile = (p) => '/api/ws/file?bucket=' + encodeURIComponent(window.__ws.bucket) + '&path=' + encodeURIComponent(p); async function upload(path, blob, studio = 'avatar') { const f = new FormData(); f.append('bucket', window.__ws.bucket); f.append('path', path); f.append('studio', studio); f.append('file', blob, path.split('/').pop()); const r = await fetch('/api/ws/upload', { method: 'POST', body: f, credentials: 'include' }); if (!r.ok) throw new Error((await r.json().catch(() => ({}))).detail || 'Upload ' + r.status); return r.json(); } const toBlob = async (u) => u instanceof Blob ? u : (await fetch(u, { credentials: 'include' })).blob(); const dataURL = (b) => new Promise(r => { const f = new FileReader(); f.onload = () => r(f.result); f.readAsDataURL(b); }); /** Avatar store: workspace bucket when connected, else localStorage (data URLs). */ export const Avatars = { local() { try { return JSON.parse(LS('studius-avatars')) || {}; } catch (e) { return {}; } }, async list() { const out = { ...this.local() }; if (bucket()) { try { const r = await (await fetch('/api/ws/list?prefix=avatars/&bucket=' + encodeURIComponent(window.__ws.bucket), { credentials: 'include' })).json(); await Promise.all((r.files || []).filter(f => f.path.endsWith('/avatar.json')).map(async f => { try { const a = await (await fetch(wsFile(f.path), { credentials: 'include' })).json(); out[a.id] = { ...a, _ws: true }; } catch (e) {} })); } catch (e) {} } return Object.values(out).sort((a, b) => (b.updated || 0) - (a.updated || 0)); }, async save(av, refBlob, voiceBlob) { av.id ||= slug(av.name); av.updated = Date.now(); const base = `avatars/${av.id}/`; if (bucket()) { if (refBlob) { await upload(base + 'ref.webp', refBlob); av.ref = base + 'ref.webp'; } if (voiceBlob) { await upload(base + 'voice.wav', voiceBlob); av.voiceRef = base + 'voice.wav'; } const { _ws, _refUrl, _voiceUrl, ...clean } = av; await upload(base + 'avatar.json', new Blob([JSON.stringify(clean, null, 1)], { type: 'application/json' })); } else { if (refBlob) av.ref = await dataURL(refBlob); if (voiceBlob && voiceBlob.size < 1.5e6) av.voiceRef = await dataURL(voiceBlob); } const loc = this.local(); const { _refUrl, _voiceUrl, ...keepA } = av; loc[av.id] = bucket() ? { ...keepA, _ws: true, ref: null, voiceRef: null, _hasRef: !!av.ref, _hasVoice: !!av.voiceRef } : keepA; try { LS('studius-avatars', JSON.stringify(loc)); } catch (e) {} window.dispatchEvent(new CustomEvent('studius-avatars')); return av; }, refUrl: (av) => !av?.ref ? null : av.ref.startsWith('data:') || av.ref.startsWith('/api/') ? av.ref : wsFile(av.ref), voiceUrl: (av) => !av?.voiceRef ? null : av.voiceRef.startsWith('data:') || av.voiceRef.startsWith('/api/') ? av.voiceRef : wsFile(av.voiceRef), persona: (av) => `Du bist ${av.name}. ${av.system}${av.backstory ? ' Hintergrund: ' + av.backstory : ''} Antworte in der Rolle von ${av.name}, kurz und natürlich.`, async speak(av, text, opt = {}) { const lang = av.voice?.language || 'Auto', vu = this.voiceUrl(av); const d = vu && av.voice?.sample_text ? await run(TTS, '/generate_voice_clone', { ref_audio: await toBlob(vu), ref_text: av.voice.sample_text, target_text: text, language: lang, use_xvector_only: false, model_size: '1.7B' }, { timeout: 180000, ...opt }) : await run(TTS, '/generate_voice_design', { text, language: lang, voice_description: av.voice?.description || 'neutral' }, { timeout: 180000, ...opt }); return url(d[0]); }, /** Talking head: ref image + voice (TTS text or audio) → lip-sync MP4. MoDA (ZeroGPU, ~20 s) → fallback Wav2Lip (CPU, no GPU quota). */ async talk(av, { text, audio } = {}, opt = {}) { const ru = this.refUrl(av); if (!ru) throw new Error(av.name + ' hat kein Referenzbild'); const st = opt.onStatus || (() => {}); if (!audio) { st({ msg: 'Stimme (TTS)…' }); audio = await this.speak(av, text, opt); } const img = await toBlob(ru), au = await toBlob(audio); let d, model = MODA, err = ''; const still = async () => { st({ msg: 'Standbild-Video (ffmpeg, ohne GPU)…' }); const f = new FormData(); f.append('image', img, 'avatar.' + ((img.type || '').split('/')[1] || 'png')); f.append('audio', au, 'voice.wav'); f.append('name', (av.id || 'avatar') + '-talk'); const r = await fetch('/api/stillvideo', { method: 'POST', body: f, credentials: 'include' }); if (!r.ok) throw new Error('Standbild-Video: ' + ((await r.json().catch(() => ({}))).detail || r.status)); return [{ video: (await r.json()).url }]; }; try { st({ msg: 'Lip-Sync (MoDA)…' }); d = await run(MODA, '/generate_motion', { source_image_path: img, driving_audio_path: au, emotion_name: 'None', cfg_scale: 1.2 }, { timeout: 240000, noAnonOn: /no face/i, ...opt }); } catch (e) { err = e.message; // no human face (e.g. mascot) → lip-sync impossible; Wav2Lip would silently return its demo face, so use the still-image video instead if (/no face/i.test(err)) { model = 'ffmpeg still+voice (kein Gesicht erkannt)'; d = await still(); } else { try { model = W2L; st({ msg: 'MoDA nicht verfügbar → Wav2Lip (CPU)…' }); d = await run(W2L, '/run_infrence', { input_image: img, input_audio: au }, { timeout: 420000, ...opt }); } catch (e2) { err += ' | ' + e2.message; model = 'ffmpeg still+voice'; d = await still(); } } } const v = d[0]?.video ?? d[0]; return { url: url(v), audio, model, fallback: err }; }, async imagine(av, prompt, opt = {}) { const ru = this.refUrl(av); if (!ru) throw new Error('Avatar hat noch kein Referenzbild'); const d = await run(KONTEXT, '/infer', { input_image: await toBlob(ru), prompt: `${prompt}. Keep the exact same character (${av.look}).`, seed: av.seed || 7, randomize_seed: false, guidance_scale: 2.5, steps: 20 }, { timeout: 240000, ...opt }); return url(d[0]); }, }; window.__avatars = Avatars; /* ---------- Avatar Studio ---------- */ STUDIOS.push({ id: 'avatar', cat: 'Bild', icon: '🧑', name: 'Avatar Studio', like: 'Persona-Studio', real: 'API', accepts: ['image', 'avatar', 'audio'], build(el) { let av = blank(), refBlob = null, voiceBlob = null; el.innerHTML = `
REAL
Look
Stimme
Sprechendes Video (Talking Head)
Look: ${hf(FLUX)} + ${hf(KONTEXT)} · Stimme: ${hf(TTS)} · Lip-Sync: ${hf(MODA)} (Fallback ${hf(W2L)}) · gespeichert als avatars/<id>/ im Workspace
`; keep(el); const q = (s) => el.querySelector(s); const fill = () => { q('.n').value = av.name; q('.sy').value = av.system; q('.bs').value = av.backstory; q('.lk').value = av.look; q('.vd').value = av.voice.description; q('.lang').value = av.voice.language || 'German'; q('.vt').value = av.voice.sample_text || `Hallo, ich bin ${av.name || 'dein Avatar'}. Schön, dich zu sehen!`; showRef(); showVoice(); }; const read = () => { av.name = q('.n').value.trim(); av.system = q('.sy').value.trim(); av.backstory = q('.bs').value.trim(); av.look = q('.lk').value.trim(); av.voice = { ...av.voice, description: q('.vd').value.trim(), language: q('.lang').value, sample_text: voiceBlob ? q('.vt').value.trim() : av.voice.sample_text }; }; const showRef = (u) => { const r = u || Avatars.refUrl(av); q('.refbox').innerHTML = r ? `Referenzbild` : 'Noch kein Referenzbild – erzeugen oder hochladen.'; }; const showVoice = (u) => { const v = u || Avatars.voiceUrl(av); q('.voicebox').innerHTML = v ? `${voiceBlob ? 'Neue Stimme – mit Speichern fixieren' : 'Fixierte Stimme (Klon-Referenz)'}` : ''; }; const refreshPick = async () => { const list = await Avatars.list(); if (!el.isConnected) return; const p = q('.pick'), cur = av.id; p.innerHTML = '' + list.map(a => ``).join(''); p.value = cur || ''; p._list = list; }; q('.mktalk').onclick = async (e) => { read(); const t = q('.tt').value.trim() || q('.vt').value.trim(); if (!t) return setS(el, '⚠ Bitte Text eingeben', true); busy(e.target, true); try { const r = await Avatars.talk(av, { text: t.slice(0, 300) }, { el, onStatus: m => setS(el, 'Talking Head: ' + stTxt(m)) }); q('.talkout').innerHTML = ``; window.__talk = { url: r.url, model: r.model, fallback: r.fallback }; result(q('.talkout'), { kind: 'video', mime: 'video/mp4', url: r.url, name: `${av.id || slug(av.name)}-talk-${Date.now()}.mp4`, studio: 'avatar', prompt: t, avatar: av.id, model: r.model }); setS(el, '✔ Sprechendes Video · ' + r.model.split('/')[1] + ' → senden an Video Edit'); badge(el, 'REAL'); } catch (er) { setS(el, '⚠ ' + er.message, true); window.__talk = { err: er.message }; } finally { busy(e.target, false); } }; q('.pick').onchange = (e) => { const a = e.target._list?.find(x => x.id === e.target.value); av = a ? structuredClone(a) : blank(); refBlob = voiceBlob = null; fill(); setS(el, a ? 'Geladen: ' + a.name : 'Neuer Avatar'); }; q('.del-local').onclick = () => { av = blank(); refBlob = voiceBlob = null; q('.pick').value = ''; fill(); }; const busy = (b, on) => { b.disabled = on; }; q('.mkref').onclick = async (e) => { read(); if (!av.look) return setS(el, '⚠ Bitte Look beschreiben', true); busy(e.target, true); try { const d = await run(FLUX, '/infer', { prompt: `character reference portrait, centered, plain background, ${av.look}`, seed: av.seed, randomize_seed: false, width: 512, height: 512, num_inference_steps: 4 }, { el, onStatus: m => setS(el, 'Referenz: ' + stTxt(m)) }); const u = url(d[0]); refBlob = await toBlob(u); showRef(u); setS(el, '✔ Referenzbild (Seed ' + av.seed + ')'); badge(el, 'REAL'); } catch (er) { setS(el, '⚠ ' + er.message, true); } finally { busy(e.target, false); } }; q('.upref').onchange = (e) => { const f = e.target.files[0]; if (!f) return; refBlob = f; showRef(URL.createObjectURL(f)); setS(el, 'Eigenes Referenzbild – speichern nicht vergessen'); }; q('.imag').onclick = async (e) => { read(); const sc = q('.scene').value.trim() || 'waving hello'; busy(e.target, true); try { const tmp = { ...av, ref: refBlob ? URL.createObjectURL(refBlob) : av.ref }; if (refBlob) tmp.ref = 'data:'; const ru = refBlob ? refBlob : await toBlob(Avatars.refUrl(av)); const d = await run(KONTEXT, '/infer', { input_image: ru, prompt: `${sc}. Keep the exact same character (${av.look}).`, seed: av.seed, randomize_seed: false, guidance_scale: 2.5, steps: 20 }, { el, timeout: 240000, onStatus: m => setS(el, 'Im Look: ' + stTxt(m)) }); const u = url(d[0]); q('.imgout').innerHTML = `Avatar in Szene`; window.__avatarImg = u; result(q('.imgout'), { kind: 'image', mime: 'image/webp', url: u, name: `${av.id || slug(av.name)}-${Date.now()}.webp`, studio: 'avatar', prompt: sc, avatar: av.id }); setS(el, '✔ Bild im Avatar-Look'); } catch (er) { setS(el, '⚠ ' + er.message, true); } finally { busy(e.target, false); } }; q('.mkvoice').onclick = async (e) => { read(); const txt = q('.vt').value.trim() || `Hallo, ich bin ${av.name}.`; busy(e.target, true); try { const d = await run(TTS, '/generate_voice_design', { text: txt, language: q('.lang').value, voice_description: av.voice.description || 'friendly voice' }, { el, timeout: 180000, onStatus: m => setS(el, 'Stimme: ' + stTxt(m)) }); const u = url(d[0]); voiceBlob = await toBlob(u); av.voice.sample_text = txt; showVoice(u); window.__voiceUrl = u; setS(el, '✔ Stimme entworfen – „Speichern“ fixiert sie als Klon-Referenz'); } catch (er) { setS(el, '⚠ ' + er.message, true); } finally { busy(e.target, false); } }; q('.sv').onclick = async (e) => { read(); if (!av.name) return setS(el, '⚠ Name fehlt', true); busy(e.target, true); setS(el, bucket() ? 'Speichere im Workspace…' : 'Speichere lokal…'); try { av.id ||= slug(av.name); await Avatars.save(av, refBlob, voiceBlob); refBlob = voiceBlob = null; window.__avatarSaved = { id: av.id, ws: !!bucket() }; setS(el, '✔ ' + (bucket() ? `Workspace: avatars/${av.id}/` : 'lokal gespeichert (für Workspace: anmelden + Bucket verbinden)')); await refreshPick(); } catch (er) { setS(el, '⚠ ' + er.message, true); } finally { busy(e.target, false); } }; q('.talk').onclick = () => { read(); if (!av.name) return setS(el, '⚠ Erst speichern', true); pendingTalk = av.id; window.__studius.open(window.__studius.STUDIOS.findIndex(s => s.id === 'chat')); }; fill(); refreshPick(); } }); const AV = STUDIOS.find(s => s.id === 'avatar'); AV.accept = async (el, a) => { if (a.kind === 'image') { const b = await toBlob(a.url || a.blob); const inp = el.querySelector('.upref'); const dt = new DataTransfer(); dt.items.add(new File([b], a.name || 'ref.webp', { type: b.type })); inp.files = dt.files; inp.dispatchEvent(new Event('change')); return true; } if (a.kind === 'avatar') { const j = JSON.parse(a.text ?? await (await toBlob(a.url)).text()); const p = el.querySelector('.pick'); const o = document.createElement('option'); o.value = j.id; o.textContent = j.name; p.append(o); p._list = [...(p._list || []), j]; p.value = j.id; p.dispatchEvent(new Event('change')); return true; } }; /* ---------- reuse: Chat (talk to avatar) + Image (generate in avatar look) ---------- */ let pendingTalk = null; async function avatarSelect(host, cls, label) { const list = await Avatars.list(); if (!host.isConnected) return null; const row = document.createElement('div'); row.className = 'row avrow'; row.innerHTML = ``; host.prepend(row); const sel = row.querySelector('select'); sel._list = list; return sel; } const chosen = (sel) => sel?._list?.find(a => a.id === sel.value) || null; window.addEventListener('studius-opened', async (e) => { const { id, el } = e.detail; if (id === 'chat') { const sel = await avatarSelect(el, 'avsel', 'Avatar'); if (!sel) return; if (pendingTalk) { sel.value = pendingTalk; pendingTalk = null; } const greet = () => { const a = chosen(sel); const p = el.querySelector('.chat p.a'); if (p) p.textContent = a ? `◆ ${a.name}: Hallo! Frag mich etwas.` : '◆ STUDIUS-Chat (Llama 3.2 3B). Frag mich etwas.'; }; sel.onchange = greet; greet(); el.addEventListener('chat-reply', (ev) => { const a = chosen(sel); if (!a) return; const b = document.createElement('button'); b.className = 'btn alt speak'; b.textContent = '🔊 ' + a.name; b.setAttribute('aria-label', 'Antwort vorlesen'); b.onclick = async () => { b.disabled = true; b.textContent = '🔊 …'; try { const u = await Avatars.speak(a, ev.detail.text.slice(0, 400), { el }); const au = document.createElement('audio'); au.controls = true; au.autoplay = true; au.src = u; b.replaceWith(au); window.__spoke = u; } catch (er) { b.textContent = '⚠ ' + er.message.slice(0, 60); } }; ev.detail.p.after(b); }); } if (id === 'image') { const sel = await avatarSelect(el, 'avsel', 'Im Look von'); if (!sel) return; } }); window.__persona = (el) => { const a = chosen(el.querySelector('.avsel')); return a ? Avatars.persona(a) : null; }; window.__avatarRef = async (el) => { const a = chosen(el.querySelector('.avsel')); if (!a) return null; const ru = Avatars.refUrl(a); if (!ru) throw new Error(a.name + ' hat kein Referenzbild'); return { blob: await toBlob(ru), look: a.look, seed: a.seed || 7 }; }; /* ---------- Module API: avatar ---------- */ const findAv = async (name) => { const l = await Avatars.list(); const n = (name || '').toLowerCase(); const a = l.find(x => x.id === n || x.name?.toLowerCase() === n) || l.find(x => n && x.name?.toLowerCase().includes(n)) || l[0]; if (!a) throw new Error('Kein Avatar vorhanden – im Avatar Studio anlegen'); return a; }; MODULES.avatar = { id: 'avatar', name: 'Avatar', backend: KONTEXT + ' + ' + TTS + ' + ' + MODA, inputs: [{ name: 'avatar', type: 'string' }, { name: 'prompt', type: 'text' }], outputs: [{ kind: 'image' }, { kind: 'audio' }, { kind: 'video' }, { kind: 'text' }], actions: { imagine: { desc: 'Erzeugt ein Bild des Avatars in einer Szene (konsistenter Look)', params: { avatar: 'string', prompt: 'string' }, run: async ({ avatar, prompt }, on = {}) => { const a = await findAv(avatar); const u = await Avatars.imagine(a, prompt, { onStatus: on.status }); return { kind: 'image', url: u, name: `${a.id}-${Date.now()}.webp`, studio: 'avatar', prompt, avatar: a.id }; } }, speak: { desc: 'Spricht einen Text mit der Stimme des Avatars', params: { avatar: 'string', prompt: 'string' }, run: async ({ avatar, prompt }, on = {}) => { const a = await findAv(avatar); const u = await Avatars.speak(a, prompt, { onStatus: on.status }); return { kind: 'audio', url: u, name: `${a.id}-voice-${Date.now()}.wav`, studio: 'avatar', prompt, avatar: a.id }; } }, talk: { desc: 'Sprechendes Video (Talking Head): Avatar-Bild + Avatar-Stimme sprechen den Text lippensynchron', params: { avatar: 'string', prompt: 'string' }, run: async ({ avatar, prompt }, on = {}) => { const a = await findAv(avatar); const r = await Avatars.talk(a, { text: String(prompt || '').slice(0, 300) }, { onStatus: on.status }); return { kind: 'video', mime: 'video/mp4', url: r.url, name: `${a.id}-talk-${Date.now()}.mp4`, studio: 'avatar', prompt, avatar: a.id, model: r.model }; } }, reply: { desc: 'Antwortet in der Rolle (Persona) des Avatars', params: { avatar: 'string', prompt: 'string' }, run: async ({ avatar, prompt }, on = {}) => { const a = await findAv(avatar); return MODULES.chat.actions.reply.run({ prompt: `System: ${Avatars.persona(a)}\nUser: ${prompt}\nAssistant:` }, on); } }, } };