studius / avatar.js
2Btainment's picture
P9.3: Space-Paket Direkt-Fallback zu SpaceForge bei HF-Limit, v136
9ceefcc verified
Raw History Blame Contribute Delete
21.3 kB
// Avatar / Persona studio (DoD): persistent avatars as workspace asset type `avatar`
// avatars/<slug>/avatar.json + ref.webp + voice.wav · reused by Chat, Image (Kontext), agents.
import { STUDIOS, onStop } from './widgets.js?v=50';
import { run, result, MODULES, esc, hf, stTxt, url, badge, setS, keep, LS, FLUX, KONTEXT, TTS, CHAT, signedIn } from './real.js?v=136';
export const MODA = 'multimodalart/MoDA-fast-talking-head', W2L = 'pragnakalp/Wav2lip-ZeroGPU';
const LANGS = ['German', 'English', 'French', 'Spanish', 'Auto'];
const slug = (n) => (n || 'avatar').toLowerCase().normalize('NFKD').replace(/[^\w]+/g, '-').replace(/^-|-$/g, '').slice(0, 40) || 'avatar';
const blank = () => ({ type: 'studius.avatar', v: 2, id: '', name: '', system: '', backstory: '', look: '', seed: Math.floor(Math.random() * 1e6),
voice: { description: 'Warm, friendly, energetic voice', language: 'German', sample_text: '' }, ref: null, voiceRef: null, updated: 0 });
const bucket = () => signedIn() && window.__ws?.bucket;
const wsFile = (p) => '/api/ws/file?bucket=' + encodeURIComponent(window.__ws.bucket) + '&path=' + encodeURIComponent(p);
async function upload(path, blob, studio = 'avatar') { const f = new FormData(); f.append('bucket', window.__ws.bucket); f.append('path', path); f.append('studio', studio); f.append('file', blob, path.split('/').pop());
const r = await fetch('/api/ws/upload', { method: 'POST', body: f, credentials: 'include' }); if (!r.ok) throw new Error((await r.json().catch(() => ({}))).detail || 'Upload ' + r.status); return r.json(); }
const toBlob = async (u) => u instanceof Blob ? u : (await fetch(u, { credentials: 'include' })).blob();
const dataURL = (b) => new Promise(r => { const f = new FileReader(); f.onload = () => r(f.result); f.readAsDataURL(b); });
/** Avatar store: workspace bucket when connected, else localStorage (data URLs). */
export const Avatars = {
local() { try { return JSON.parse(LS('studius-avatars')) || {}; } catch (e) { return {}; } },
async list() {
const out = { ...this.local() };
if (bucket()) { try { const r = await (await fetch('/api/ws/list?prefix=avatars/&bucket=' + encodeURIComponent(window.__ws.bucket), { credentials: 'include' })).json();
await Promise.all((r.files || []).filter(f => f.path.endsWith('/avatar.json')).map(async f => { try { const a = await (await fetch(wsFile(f.path), { credentials: 'include' })).json(); out[a.id] = { ...a, _ws: true }; } catch (e) {} })); } catch (e) {} }
return Object.values(out).sort((a, b) => (b.updated || 0) - (a.updated || 0));
},
async save(av, refBlob, voiceBlob) {
av.id ||= slug(av.name); av.updated = Date.now(); const base = `avatars/${av.id}/`;
if (bucket()) {
if (refBlob) { await upload(base + 'ref.webp', refBlob); av.ref = base + 'ref.webp'; }
if (voiceBlob) { await upload(base + 'voice.wav', voiceBlob); av.voiceRef = base + 'voice.wav'; }
const { _ws, _refUrl, _voiceUrl, ...clean } = av; await upload(base + 'avatar.json', new Blob([JSON.stringify(clean, null, 1)], { type: 'application/json' }));
} else {
if (refBlob) av.ref = await dataURL(refBlob); if (voiceBlob && voiceBlob.size < 1.5e6) av.voiceRef = await dataURL(voiceBlob);
}
const loc = this.local(); const { _refUrl, _voiceUrl, ...keepA } = av; loc[av.id] = bucket() ? { ...keepA, _ws: true, ref: null, voiceRef: null, _hasRef: !!av.ref, _hasVoice: !!av.voiceRef } : keepA;
try { LS('studius-avatars', JSON.stringify(loc)); } catch (e) {} window.dispatchEvent(new CustomEvent('studius-avatars')); return av;
},
refUrl: (av) => !av?.ref ? null : av.ref.startsWith('data:') || av.ref.startsWith('/api/') ? av.ref : wsFile(av.ref),
voiceUrl: (av) => !av?.voiceRef ? null : av.voiceRef.startsWith('data:') || av.voiceRef.startsWith('/api/') ? av.voiceRef : wsFile(av.voiceRef),
persona: (av) => `Du bist ${av.name}. ${av.system}${av.backstory ? ' Hintergrund: ' + av.backstory : ''} Antworte in der Rolle von ${av.name}, kurz und natürlich.`,
async speak(av, text, opt = {}) {
const lang = av.voice?.language || 'Auto', vu = this.voiceUrl(av);
const d = vu && av.voice?.sample_text
? await run(TTS, '/generate_voice_clone', { ref_audio: await toBlob(vu), ref_text: av.voice.sample_text, target_text: text, language: lang, use_xvector_only: false, model_size: '1.7B' }, { timeout: 180000, ...opt })
: await run(TTS, '/generate_voice_design', { text, language: lang, voice_description: av.voice?.description || 'neutral' }, { timeout: 180000, ...opt });
return url(d[0]);
},
/** Talking head: ref image + voice (TTS text or audio) → lip-sync MP4. MoDA (ZeroGPU, ~20 s) → fallback Wav2Lip (CPU, no GPU quota). */
async talk(av, { text, audio } = {}, opt = {}) {
const ru = this.refUrl(av); if (!ru) throw new Error(av.name + ' hat kein Referenzbild');
const st = opt.onStatus || (() => {}); if (!audio) { st({ msg: 'Stimme (TTS)…' }); audio = await this.speak(av, text, opt); }
const img = await toBlob(ru), au = await toBlob(audio); let d, model = MODA, err = '';
const still = async () => { st({ msg: 'Standbild-Video (ffmpeg, ohne GPU)…' }); const f = new FormData(); f.append('image', img, 'avatar.' + ((img.type || '').split('/')[1] || 'png')); f.append('audio', au, 'voice.wav'); f.append('name', (av.id || 'avatar') + '-talk');
const r = await fetch('/api/stillvideo', { method: 'POST', body: f, credentials: 'include' }); if (!r.ok) throw new Error('Standbild-Video: ' + ((await r.json().catch(() => ({}))).detail || r.status)); return [{ video: (await r.json()).url }]; };
try { st({ msg: 'Lip-Sync (MoDA)…' }); d = await run(MODA, '/generate_motion', { source_image_path: img, driving_audio_path: au, emotion_name: 'None', cfg_scale: 1.2 }, { timeout: 240000, noAnonOn: /no face/i, ...opt }); }
catch (e) { err = e.message;
// no human face (e.g. mascot) → lip-sync impossible; Wav2Lip would silently return its demo face, so use the still-image video instead
if (/no face/i.test(err)) { model = 'ffmpeg still+voice (kein Gesicht erkannt)'; d = await still(); }
else { try { model = W2L; st({ msg: 'MoDA nicht verfügbar → Wav2Lip (CPU)…' }); d = await run(W2L, '/run_infrence', { input_image: img, input_audio: au }, { timeout: 420000, ...opt }); }
catch (e2) { err += ' | ' + e2.message; model = 'ffmpeg still+voice'; d = await still(); } } }
const v = d[0]?.video ?? d[0]; return { url: url(v), audio, model, fallback: err };
},
async imagine(av, prompt, opt = {}) {
const ru = this.refUrl(av); if (!ru) throw new Error('Avatar hat noch kein Referenzbild');
const d = await run(KONTEXT, '/infer', { input_image: await toBlob(ru), prompt: `${prompt}. Keep the exact same character (${av.look}).`, seed: av.seed || 7, randomize_seed: false, guidance_scale: 2.5, steps: 20 }, { timeout: 240000, ...opt });
return url(d[0]);
},
};
window.__avatars = Avatars;
/* ---------- Avatar Studio ---------- */
STUDIOS.push({ id: 'avatar', cat: 'Bild', icon: '🧑', name: 'Avatar Studio', like: 'Persona-Studio', real: 'API', accepts: ['image', 'avatar', 'audio'], build(el) {
let av = blank(), refBlob = null, voiceBlob = null;
el.innerHTML = `<div class="row" style="margin:0 0 6px"><span class="badge real">REAL</span><span class="stat"></span></div>
<div class="row"><select class="pick" aria-label="Avatar wählen"><option value="">+ Neuer Avatar</option></select><button class="btn alt del-local" title="Formular leeren">Neu</button></div>
<label class="fl">Name<input class="n" aria-label="Name" placeholder="z.B. Jenni Bee"></label>
<label class="fl">Persönlichkeit / System-Prompt<textarea class="sy" rows="2" aria-label="System-Prompt" placeholder="Wie spricht und denkt der Avatar?"></textarea></label>
<label class="fl">Backstory<textarea class="bs" rows="2" aria-label="Backstory"></textarea></label>
<fieldset class="grp"><legend>Look</legend><textarea class="lk" rows="2" aria-label="Look-Beschreibung" placeholder="z.B. golden bee mascot, big friendly eyes, black-gold hex style"></textarea>
<div class="row"><button class="btn alt mkref">✨ Referenzbild erzeugen</button><label class="btn alt">⬆ Eigenes Bild<input type="file" accept="image/*" hidden class="upref"></label></div>
<div class="refbox"></div><div class="row"><input class="scene" aria-label="Szene im Avatar-Look" placeholder="Szene: winkt am Strand"><button class="btn alt imag">Im Look erzeugen</button></div><div class="imgout"></div></fieldset>
<fieldset class="grp"><legend>Stimme</legend><input class="vd" aria-label="Stimmbeschreibung" placeholder="Stimme beschreiben"><div class="row"><select class="lang" aria-label="Sprache">${LANGS.map(l => `<option>${l}</option>`).join('')}</select><button class="btn alt mkvoice">🔊 Stimme entwerfen</button></div>
<input class="vt" aria-label="Probesatz" placeholder="Probesatz"><div class="voicebox"></div></fieldset>
<fieldset class="grp"><legend>Sprechendes Video (Talking Head)</legend><textarea class="tt" rows="2" aria-label="Text für das sprechende Video" placeholder="Was soll der Avatar sagen? (kurz, ~1–2 Sätze)"></textarea>
<div class="row"><button class="btn alt mktalk">🎬 Sprechendes Video</button></div><div class="talkout"></div></fieldset>
<div class="row"><button class="btn sv">💾 Avatar speichern</button><button class="btn alt talk">💬 Mit Avatar chatten</button></div>
<div class="credit">Look: ${hf(FLUX)} + ${hf(KONTEXT)} · Stimme: ${hf(TTS)} · Lip-Sync: ${hf(MODA)} (Fallback ${hf(W2L)}) · gespeichert als <code>avatars/&lt;id&gt;/</code> im Workspace</div>`;
keep(el); const q = (s) => el.querySelector(s);
const fill = () => { q('.n').value = av.name; q('.sy').value = av.system; q('.bs').value = av.backstory; q('.lk').value = av.look; q('.vd').value = av.voice.description; q('.lang').value = av.voice.language || 'German'; q('.vt').value = av.voice.sample_text || `Hallo, ich bin ${av.name || 'dein Avatar'}. Schön, dich zu sehen!`; showRef(); showVoice(); };
const read = () => { av.name = q('.n').value.trim(); av.system = q('.sy').value.trim(); av.backstory = q('.bs').value.trim(); av.look = q('.lk').value.trim(); av.voice = { ...av.voice, description: q('.vd').value.trim(), language: q('.lang').value, sample_text: voiceBlob ? q('.vt').value.trim() : av.voice.sample_text }; };
const showRef = (u) => { const r = u || Avatars.refUrl(av); q('.refbox').innerHTML = r ? `<img alt="Referenzbild" src="${esc(r)}" class="avref">` : '<small class="mut">Noch kein Referenzbild – erzeugen oder hochladen.</small>'; };
const showVoice = (u) => { const v = u || Avatars.voiceUrl(av); q('.voicebox').innerHTML = v ? `<audio controls src="${esc(v)}"></audio><small class="mut">${voiceBlob ? 'Neue Stimme – mit Speichern fixieren' : 'Fixierte Stimme (Klon-Referenz)'}</small>` : ''; };
const refreshPick = async () => { const list = await Avatars.list(); if (!el.isConnected) return; const p = q('.pick'), cur = av.id; p.innerHTML = '<option value="">+ Neuer Avatar</option>' + list.map(a => `<option value="${esc(a.id)}">${esc(a.name || a.id)}${a._ws ? ' ☁' : ''}</option>`).join(''); p.value = cur || ''; p._list = list; };
q('.mktalk').onclick = async (e) => { read(); const t = q('.tt').value.trim() || q('.vt').value.trim(); if (!t) return setS(el, '⚠ Bitte Text eingeben', true); busy(e.target, true);
try { const r = await Avatars.talk(av, { text: t.slice(0, 300) }, { el, onStatus: m => setS(el, 'Talking Head: ' + stTxt(m)) });
q('.talkout').innerHTML = `<video controls playsinline src="${esc(r.url)}" class="avout"></video>`; window.__talk = { url: r.url, model: r.model, fallback: r.fallback };
result(q('.talkout'), { kind: 'video', mime: 'video/mp4', url: r.url, name: `${av.id || slug(av.name)}-talk-${Date.now()}.mp4`, studio: 'avatar', prompt: t, avatar: av.id, model: r.model }); setS(el, '✔ Sprechendes Video · ' + r.model.split('/')[1] + ' → senden an Video Edit'); badge(el, 'REAL'); }
catch (er) { setS(el, '⚠ ' + er.message, true); window.__talk = { err: er.message }; } finally { busy(e.target, false); } };
q('.pick').onchange = (e) => { const a = e.target._list?.find(x => x.id === e.target.value); av = a ? structuredClone(a) : blank(); refBlob = voiceBlob = null; fill(); setS(el, a ? 'Geladen: ' + a.name : 'Neuer Avatar'); };
q('.del-local').onclick = () => { av = blank(); refBlob = voiceBlob = null; q('.pick').value = ''; fill(); };
const busy = (b, on) => { b.disabled = on; };
q('.mkref').onclick = async (e) => { read(); if (!av.look) return setS(el, '⚠ Bitte Look beschreiben', true); busy(e.target, true);
try { const d = await run(FLUX, '/infer', { prompt: `character reference portrait, centered, plain background, ${av.look}`, seed: av.seed, randomize_seed: false, width: 512, height: 512, num_inference_steps: 4 }, { el, onStatus: m => setS(el, 'Referenz: ' + stTxt(m)) });
const u = url(d[0]); refBlob = await toBlob(u); showRef(u); setS(el, '✔ Referenzbild (Seed ' + av.seed + ')'); badge(el, 'REAL'); } catch (er) { setS(el, '⚠ ' + er.message, true); } finally { busy(e.target, false); } };
q('.upref').onchange = (e) => { const f = e.target.files[0]; if (!f) return; refBlob = f; showRef(URL.createObjectURL(f)); setS(el, 'Eigenes Referenzbild – speichern nicht vergessen'); };
q('.imag').onclick = async (e) => { read(); const sc = q('.scene').value.trim() || 'waving hello'; busy(e.target, true);
try { const tmp = { ...av, ref: refBlob ? URL.createObjectURL(refBlob) : av.ref }; if (refBlob) tmp.ref = 'data:'; const ru = refBlob ? refBlob : await toBlob(Avatars.refUrl(av));
const d = await run(KONTEXT, '/infer', { input_image: ru, prompt: `${sc}. Keep the exact same character (${av.look}).`, seed: av.seed, randomize_seed: false, guidance_scale: 2.5, steps: 20 }, { el, timeout: 240000, onStatus: m => setS(el, 'Im Look: ' + stTxt(m)) });
const u = url(d[0]); q('.imgout').innerHTML = `<img alt="Avatar in Szene" src="${esc(u)}" class="avout">`; window.__avatarImg = u; result(q('.imgout'), { kind: 'image', mime: 'image/webp', url: u, name: `${av.id || slug(av.name)}-${Date.now()}.webp`, studio: 'avatar', prompt: sc, avatar: av.id }); setS(el, '✔ Bild im Avatar-Look'); }
catch (er) { setS(el, '⚠ ' + er.message, true); } finally { busy(e.target, false); } };
q('.mkvoice').onclick = async (e) => { read(); const txt = q('.vt').value.trim() || `Hallo, ich bin ${av.name}.`; busy(e.target, true);
try { const d = await run(TTS, '/generate_voice_design', { text: txt, language: q('.lang').value, voice_description: av.voice.description || 'friendly voice' }, { el, timeout: 180000, onStatus: m => setS(el, 'Stimme: ' + stTxt(m)) });
const u = url(d[0]); voiceBlob = await toBlob(u); av.voice.sample_text = txt; showVoice(u); window.__voiceUrl = u; setS(el, '✔ Stimme entworfen – „Speichern“ fixiert sie als Klon-Referenz'); } catch (er) { setS(el, '⚠ ' + er.message, true); } finally { busy(e.target, false); } };
q('.sv').onclick = async (e) => { read(); if (!av.name) return setS(el, '⚠ Name fehlt', true); busy(e.target, true); setS(el, bucket() ? 'Speichere im Workspace…' : 'Speichere lokal…');
try { av.id ||= slug(av.name); await Avatars.save(av, refBlob, voiceBlob); refBlob = voiceBlob = null; window.__avatarSaved = { id: av.id, ws: !!bucket() }; setS(el, '✔ ' + (bucket() ? `Workspace: avatars/${av.id}/` : 'lokal gespeichert (für Workspace: anmelden + Bucket verbinden)')); await refreshPick(); }
catch (er) { setS(el, '⚠ ' + er.message, true); } finally { busy(e.target, false); } };
q('.talk').onclick = () => { read(); if (!av.name) return setS(el, '⚠ Erst speichern', true); pendingTalk = av.id; window.__studius.open(window.__studius.STUDIOS.findIndex(s => s.id === 'chat')); };
fill(); refreshPick();
} });
const AV = STUDIOS.find(s => s.id === 'avatar');
AV.accept = async (el, a) => { if (a.kind === 'image') { const b = await toBlob(a.url || a.blob); const inp = el.querySelector('.upref'); const dt = new DataTransfer(); dt.items.add(new File([b], a.name || 'ref.webp', { type: b.type })); inp.files = dt.files; inp.dispatchEvent(new Event('change')); return true; }
if (a.kind === 'avatar') { const j = JSON.parse(a.text ?? await (await toBlob(a.url)).text()); const p = el.querySelector('.pick'); const o = document.createElement('option'); o.value = j.id; o.textContent = j.name; p.append(o); p._list = [...(p._list || []), j]; p.value = j.id; p.dispatchEvent(new Event('change')); return true; } };
/* ---------- reuse: Chat (talk to avatar) + Image (generate in avatar look) ---------- */
let pendingTalk = null;
async function avatarSelect(host, cls, label) {
const list = await Avatars.list(); if (!host.isConnected) return null;
const row = document.createElement('div'); row.className = 'row avrow'; row.innerHTML = `<label class="fl inline">${label}<select class="${cls}" aria-label="${label}"><option value="">— keiner —</option>${list.map(a => `<option value="${esc(a.id)}">${esc(a.name)}</option>`).join('')}</select></label>`;
host.prepend(row); const sel = row.querySelector('select'); sel._list = list; return sel;
}
const chosen = (sel) => sel?._list?.find(a => a.id === sel.value) || null;
window.addEventListener('studius-opened', async (e) => {
const { id, el } = e.detail;
if (id === 'chat') { const sel = await avatarSelect(el, 'avsel', 'Avatar'); if (!sel) return;
if (pendingTalk) { sel.value = pendingTalk; pendingTalk = null; }
const greet = () => { const a = chosen(sel); const p = el.querySelector('.chat p.a'); if (p) p.textContent = a ? `◆ ${a.name}: Hallo! Frag mich etwas.` : '◆ STUDIUS-Chat (Llama 3.2 3B). Frag mich etwas.'; };
sel.onchange = greet; greet();
el.addEventListener('chat-reply', (ev) => { const a = chosen(sel); if (!a) return; const b = document.createElement('button'); b.className = 'btn alt speak'; b.textContent = '🔊 ' + a.name; b.setAttribute('aria-label', 'Antwort vorlesen');
b.onclick = async () => { b.disabled = true; b.textContent = '🔊 …'; try { const u = await Avatars.speak(a, ev.detail.text.slice(0, 400), { el }); const au = document.createElement('audio'); au.controls = true; au.autoplay = true; au.src = u; b.replaceWith(au); window.__spoke = u; } catch (er) { b.textContent = '⚠ ' + er.message.slice(0, 60); } };
ev.detail.p.after(b); });
}
if (id === 'image') { const sel = await avatarSelect(el, 'avsel', 'Im Look von'); if (!sel) return; }
});
window.__persona = (el) => { const a = chosen(el.querySelector('.avsel')); return a ? Avatars.persona(a) : null; };
window.__avatarRef = async (el) => { const a = chosen(el.querySelector('.avsel')); if (!a) return null; const ru = Avatars.refUrl(a); if (!ru) throw new Error(a.name + ' hat kein Referenzbild'); return { blob: await toBlob(ru), look: a.look, seed: a.seed || 7 }; };
/* ---------- Module API: avatar ---------- */
const findAv = async (name) => { const l = await Avatars.list(); const n = (name || '').toLowerCase(); const a = l.find(x => x.id === n || x.name?.toLowerCase() === n) || l.find(x => n && x.name?.toLowerCase().includes(n)) || l[0]; if (!a) throw new Error('Kein Avatar vorhanden – im Avatar Studio anlegen'); return a; };
MODULES.avatar = { id: 'avatar', name: 'Avatar', backend: KONTEXT + ' + ' + TTS + ' + ' + MODA, inputs: [{ name: 'avatar', type: 'string' }, { name: 'prompt', type: 'text' }], outputs: [{ kind: 'image' }, { kind: 'audio' }, { kind: 'video' }, { kind: 'text' }],
actions: {
imagine: { desc: 'Erzeugt ein Bild des Avatars in einer Szene (konsistenter Look)', params: { avatar: 'string', prompt: 'string' }, run: async ({ avatar, prompt }, on = {}) => { const a = await findAv(avatar); const u = await Avatars.imagine(a, prompt, { onStatus: on.status }); return { kind: 'image', url: u, name: `${a.id}-${Date.now()}.webp`, studio: 'avatar', prompt, avatar: a.id }; } },
speak: { desc: 'Spricht einen Text mit der Stimme des Avatars', params: { avatar: 'string', prompt: 'string' }, run: async ({ avatar, prompt }, on = {}) => { const a = await findAv(avatar); const u = await Avatars.speak(a, prompt, { onStatus: on.status }); return { kind: 'audio', url: u, name: `${a.id}-voice-${Date.now()}.wav`, studio: 'avatar', prompt, avatar: a.id }; } },
talk: { desc: 'Sprechendes Video (Talking Head): Avatar-Bild + Avatar-Stimme sprechen den Text lippensynchron', params: { avatar: 'string', prompt: 'string' }, run: async ({ avatar, prompt }, on = {}) => { const a = await findAv(avatar); const r = await Avatars.talk(a, { text: String(prompt || '').slice(0, 300) }, { onStatus: on.status }); return { kind: 'video', mime: 'video/mp4', url: r.url, name: `${a.id}-talk-${Date.now()}.mp4`, studio: 'avatar', prompt, avatar: a.id, model: r.model }; } },
reply: { desc: 'Antwortet in der Rolle (Persona) des Avatars', params: { avatar: 'string', prompt: 'string' }, run: async ({ avatar, prompt }, on = {}) => { const a = await findAv(avatar); return MODULES.chat.actions.reply.run({ prompt: `System: ${Avatars.persona(a)}\nUser: ${prompt}\nAssistant:` }, on); } },
} };