Spaces:
Paused
Paused
P4.4: talking-head avatar (MoDA lip-sync, Wav2Lip/ffmpeg fallbacks), upload filenames
Browse files- app.js +6 -6
- avatar.js +27 -3
- index.html +2 -2
- master.js +1 -1
- real.js +2 -2
- server/infer.py +2 -0
- server/main.py +20 -0
- video.js +2 -2
- workspace.js +3 -3
app.js
CHANGED
|
@@ -1,11 +1,11 @@
|
|
| 1 |
import * as THREE from './vendor/three.module.min.js';
|
| 2 |
import { STUDIOS, stopLoops } from './widgets.js?v=50';
|
| 3 |
-
import './real.js?v=
|
| 4 |
-
import './avatar.js?v=
|
| 5 |
-
import './master.js?v=
|
| 6 |
-
import './video.js?v=
|
| 7 |
-
import './workspace.js?v=
|
| 8 |
-
import { icon } from './icons.js?v=
|
| 9 |
import { RoomEnvironment } from './vendor/addons/environments/RoomEnvironment.js';
|
| 10 |
import { EffectComposer } from './vendor/addons/postprocessing/EffectComposer.js';
|
| 11 |
import { RenderPass } from './vendor/addons/postprocessing/RenderPass.js';
|
|
|
|
| 1 |
import * as THREE from './vendor/three.module.min.js';
|
| 2 |
import { STUDIOS, stopLoops } from './widgets.js?v=50';
|
| 3 |
+
import './real.js?v=93';
|
| 4 |
+
import './avatar.js?v=93';
|
| 5 |
+
import './master.js?v=93';
|
| 6 |
+
import './video.js?v=93';
|
| 7 |
+
import './workspace.js?v=93';
|
| 8 |
+
import { icon } from './icons.js?v=93';
|
| 9 |
import { RoomEnvironment } from './vendor/addons/environments/RoomEnvironment.js';
|
| 10 |
import { EffectComposer } from './vendor/addons/postprocessing/EffectComposer.js';
|
| 11 |
import { RenderPass } from './vendor/addons/postprocessing/RenderPass.js';
|
avatar.js
CHANGED
|
@@ -1,8 +1,9 @@
|
|
| 1 |
// Avatar / Persona studio (DoD): persistent avatars as workspace asset type `avatar`
|
| 2 |
// avatars/<slug>/avatar.json + ref.webp + voice.wav · reused by Chat, Image (Kontext), agents.
|
| 3 |
import { STUDIOS, onStop } from './widgets.js?v=50';
|
| 4 |
-
import { run, result, MODULES, esc, hf, stTxt, url, badge, setS, keep, LS, FLUX, KONTEXT, TTS, CHAT, signedIn } from './real.js?v=
|
| 5 |
|
|
|
|
| 6 |
const LANGS = ['German', 'English', 'French', 'Spanish', 'Auto'];
|
| 7 |
const slug = (n) => (n || 'avatar').toLowerCase().normalize('NFKD').replace(/[^\w]+/g, '-').replace(/^-|-$/g, '').slice(0, 40) || 'avatar';
|
| 8 |
const blank = () => ({ type: 'studius.avatar', v: 2, id: '', name: '', system: '', backstory: '', look: '', seed: Math.floor(Math.random() * 1e6),
|
|
@@ -45,6 +46,21 @@ export const Avatars = {
|
|
| 45 |
: await run(TTS, '/generate_voice_design', { text, language: lang, voice_description: av.voice?.description || 'neutral' }, { timeout: 180000, ...opt });
|
| 46 |
return url(d[0]);
|
| 47 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 48 |
async imagine(av, prompt, opt = {}) {
|
| 49 |
const ru = this.refUrl(av); if (!ru) throw new Error('Avatar hat noch kein Referenzbild');
|
| 50 |
const d = await run(KONTEXT, '/infer', { input_image: await toBlob(ru), prompt: `${prompt}. Keep the exact same character (${av.look}).`, seed: av.seed || 7, randomize_seed: false, guidance_scale: 2.5, steps: 20 }, { timeout: 240000, ...opt });
|
|
@@ -66,14 +82,21 @@ STUDIOS.push({ id: 'avatar', cat: 'Bild', icon: '🧑', name: 'Avatar Studio', l
|
|
| 66 |
<div class="refbox"></div><div class="row"><input class="scene" aria-label="Szene im Avatar-Look" placeholder="Szene: winkt am Strand"><button class="btn alt imag">Im Look erzeugen</button></div><div class="imgout"></div></fieldset>
|
| 67 |
<fieldset class="grp"><legend>Stimme</legend><input class="vd" aria-label="Stimmbeschreibung" placeholder="Stimme beschreiben"><div class="row"><select class="lang" aria-label="Sprache">${LANGS.map(l => `<option>${l}</option>`).join('')}</select><button class="btn alt mkvoice">🔊 Stimme entwerfen</button></div>
|
| 68 |
<input class="vt" aria-label="Probesatz" placeholder="Probesatz"><div class="voicebox"></div></fieldset>
|
|
|
|
|
|
|
| 69 |
<div class="row"><button class="btn sv">💾 Avatar speichern</button><button class="btn alt talk">💬 Mit Avatar chatten</button></div>
|
| 70 |
-
<div class="credit">Look: ${hf(FLUX)} + ${hf(KONTEXT)} · Stimme: ${hf(TTS)} · gespeichert als <code>avatars/<id>/</code> im Workspace</div>`;
|
| 71 |
keep(el); const q = (s) => el.querySelector(s);
|
| 72 |
const fill = () => { q('.n').value = av.name; q('.sy').value = av.system; q('.bs').value = av.backstory; q('.lk').value = av.look; q('.vd').value = av.voice.description; q('.lang').value = av.voice.language || 'German'; q('.vt').value = av.voice.sample_text || `Hallo, ich bin ${av.name || 'dein Avatar'}. Schön, dich zu sehen!`; showRef(); showVoice(); };
|
| 73 |
const read = () => { av.name = q('.n').value.trim(); av.system = q('.sy').value.trim(); av.backstory = q('.bs').value.trim(); av.look = q('.lk').value.trim(); av.voice = { ...av.voice, description: q('.vd').value.trim(), language: q('.lang').value, sample_text: voiceBlob ? q('.vt').value.trim() : av.voice.sample_text }; };
|
| 74 |
const showRef = (u) => { const r = u || Avatars.refUrl(av); q('.refbox').innerHTML = r ? `<img alt="Referenzbild" src="${esc(r)}" class="avref">` : '<small class="mut">Noch kein Referenzbild – erzeugen oder hochladen.</small>'; };
|
| 75 |
const showVoice = (u) => { const v = u || Avatars.voiceUrl(av); q('.voicebox').innerHTML = v ? `<audio controls src="${esc(v)}"></audio><small class="mut">${voiceBlob ? 'Neue Stimme – mit Speichern fixieren' : 'Fixierte Stimme (Klon-Referenz)'}</small>` : ''; };
|
| 76 |
const refreshPick = async () => { const list = await Avatars.list(); if (!el.isConnected) return; const p = q('.pick'), cur = av.id; p.innerHTML = '<option value="">+ Neuer Avatar</option>' + list.map(a => `<option value="${esc(a.id)}">${esc(a.name || a.id)}${a._ws ? ' ☁' : ''}</option>`).join(''); p.value = cur || ''; p._list = list; };
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 77 |
q('.pick').onchange = (e) => { const a = e.target._list?.find(x => x.id === e.target.value); av = a ? structuredClone(a) : blank(); refBlob = voiceBlob = null; fill(); setS(el, a ? 'Geladen: ' + a.name : 'Neuer Avatar'); };
|
| 78 |
q('.del-local').onclick = () => { av = blank(); refBlob = voiceBlob = null; q('.pick').value = ''; fill(); };
|
| 79 |
const busy = (b, on) => { b.disabled = on; };
|
|
@@ -124,9 +147,10 @@ window.__avatarRef = async (el) => { const a = chosen(el.querySelector('.avsel')
|
|
| 124 |
|
| 125 |
/* ---------- Module API: avatar ---------- */
|
| 126 |
const findAv = async (name) => { const l = await Avatars.list(); const n = (name || '').toLowerCase(); const a = l.find(x => x.id === n || x.name?.toLowerCase() === n) || l.find(x => n && x.name?.toLowerCase().includes(n)) || l[0]; if (!a) throw new Error('Kein Avatar vorhanden – im Avatar Studio anlegen'); return a; };
|
| 127 |
-
MODULES.avatar = { id: 'avatar', name: 'Avatar', backend: KONTEXT + ' + ' + TTS, inputs: [{ name: 'avatar', type: 'string' }, { name: 'prompt', type: 'text' }], outputs: [{ kind: 'image' }, { kind: 'audio' }, { kind: 'text' }],
|
| 128 |
actions: {
|
| 129 |
imagine: { desc: 'Erzeugt ein Bild des Avatars in einer Szene (konsistenter Look)', params: { avatar: 'string', prompt: 'string' }, run: async ({ avatar, prompt }, on = {}) => { const a = await findAv(avatar); const u = await Avatars.imagine(a, prompt, { onStatus: on.status }); return { kind: 'image', url: u, name: `${a.id}-${Date.now()}.webp`, studio: 'avatar', prompt, avatar: a.id }; } },
|
| 130 |
speak: { desc: 'Spricht einen Text mit der Stimme des Avatars', params: { avatar: 'string', prompt: 'string' }, run: async ({ avatar, prompt }, on = {}) => { const a = await findAv(avatar); const u = await Avatars.speak(a, prompt, { onStatus: on.status }); return { kind: 'audio', url: u, name: `${a.id}-voice-${Date.now()}.wav`, studio: 'avatar', prompt, avatar: a.id }; } },
|
|
|
|
| 131 |
reply: { desc: 'Antwortet in der Rolle (Persona) des Avatars', params: { avatar: 'string', prompt: 'string' }, run: async ({ avatar, prompt }, on = {}) => { const a = await findAv(avatar); return MODULES.chat.actions.reply.run({ prompt: `System: ${Avatars.persona(a)}\nUser: ${prompt}\nAssistant:` }, on); } },
|
| 132 |
} };
|
|
|
|
| 1 |
// Avatar / Persona studio (DoD): persistent avatars as workspace asset type `avatar`
|
| 2 |
// avatars/<slug>/avatar.json + ref.webp + voice.wav · reused by Chat, Image (Kontext), agents.
|
| 3 |
import { STUDIOS, onStop } from './widgets.js?v=50';
|
| 4 |
+
import { run, result, MODULES, esc, hf, stTxt, url, badge, setS, keep, LS, FLUX, KONTEXT, TTS, CHAT, signedIn } from './real.js?v=93';
|
| 5 |
|
| 6 |
+
export const MODA = 'multimodalart/MoDA-fast-talking-head', W2L = 'pragnakalp/Wav2lip-ZeroGPU';
|
| 7 |
const LANGS = ['German', 'English', 'French', 'Spanish', 'Auto'];
|
| 8 |
const slug = (n) => (n || 'avatar').toLowerCase().normalize('NFKD').replace(/[^\w]+/g, '-').replace(/^-|-$/g, '').slice(0, 40) || 'avatar';
|
| 9 |
const blank = () => ({ type: 'studius.avatar', v: 2, id: '', name: '', system: '', backstory: '', look: '', seed: Math.floor(Math.random() * 1e6),
|
|
|
|
| 46 |
: await run(TTS, '/generate_voice_design', { text, language: lang, voice_description: av.voice?.description || 'neutral' }, { timeout: 180000, ...opt });
|
| 47 |
return url(d[0]);
|
| 48 |
},
|
| 49 |
+
/** Talking head: ref image + voice (TTS text or audio) → lip-sync MP4. MoDA (ZeroGPU, ~20 s) → fallback Wav2Lip (CPU, no GPU quota). */
|
| 50 |
+
async talk(av, { text, audio } = {}, opt = {}) {
|
| 51 |
+
const ru = this.refUrl(av); if (!ru) throw new Error(av.name + ' hat kein Referenzbild');
|
| 52 |
+
const st = opt.onStatus || (() => {}); if (!audio) { st({ msg: 'Stimme (TTS)…' }); audio = await this.speak(av, text, opt); }
|
| 53 |
+
const img = await toBlob(ru), au = await toBlob(audio); let d, model = MODA, err = '';
|
| 54 |
+
const still = async () => { st({ msg: 'Standbild-Video (ffmpeg, ohne GPU)…' }); const f = new FormData(); f.append('image', img, 'avatar.' + ((img.type || '').split('/')[1] || 'png')); f.append('audio', au, 'voice.wav'); f.append('name', (av.id || 'avatar') + '-talk');
|
| 55 |
+
const r = await fetch('/api/stillvideo', { method: 'POST', body: f, credentials: 'include' }); if (!r.ok) throw new Error('Standbild-Video: ' + ((await r.json().catch(() => ({}))).detail || r.status)); return [{ video: (await r.json()).url }]; };
|
| 56 |
+
try { st({ msg: 'Lip-Sync (MoDA)…' }); d = await run(MODA, '/generate_motion', { source_image_path: img, driving_audio_path: au, emotion_name: 'None', cfg_scale: 1.2 }, { timeout: 240000, noAnonOn: /no face/i, ...opt }); }
|
| 57 |
+
catch (e) { err = e.message;
|
| 58 |
+
// no human face (e.g. mascot) → lip-sync impossible; Wav2Lip would silently return its demo face, so use the still-image video instead
|
| 59 |
+
if (/no face/i.test(err)) { model = 'ffmpeg still+voice (kein Gesicht erkannt)'; d = await still(); }
|
| 60 |
+
else { try { model = W2L; st({ msg: 'MoDA nicht verfügbar → Wav2Lip (CPU)…' }); d = await run(W2L, '/run_infrence', { input_image: img, input_audio: au }, { timeout: 420000, ...opt }); }
|
| 61 |
+
catch (e2) { err += ' | ' + e2.message; model = 'ffmpeg still+voice'; d = await still(); } } }
|
| 62 |
+
const v = d[0]?.video ?? d[0]; return { url: url(v), audio, model, fallback: err };
|
| 63 |
+
},
|
| 64 |
async imagine(av, prompt, opt = {}) {
|
| 65 |
const ru = this.refUrl(av); if (!ru) throw new Error('Avatar hat noch kein Referenzbild');
|
| 66 |
const d = await run(KONTEXT, '/infer', { input_image: await toBlob(ru), prompt: `${prompt}. Keep the exact same character (${av.look}).`, seed: av.seed || 7, randomize_seed: false, guidance_scale: 2.5, steps: 20 }, { timeout: 240000, ...opt });
|
|
|
|
| 82 |
<div class="refbox"></div><div class="row"><input class="scene" aria-label="Szene im Avatar-Look" placeholder="Szene: winkt am Strand"><button class="btn alt imag">Im Look erzeugen</button></div><div class="imgout"></div></fieldset>
|
| 83 |
<fieldset class="grp"><legend>Stimme</legend><input class="vd" aria-label="Stimmbeschreibung" placeholder="Stimme beschreiben"><div class="row"><select class="lang" aria-label="Sprache">${LANGS.map(l => `<option>${l}</option>`).join('')}</select><button class="btn alt mkvoice">🔊 Stimme entwerfen</button></div>
|
| 84 |
<input class="vt" aria-label="Probesatz" placeholder="Probesatz"><div class="voicebox"></div></fieldset>
|
| 85 |
+
<fieldset class="grp"><legend>Sprechendes Video (Talking Head)</legend><textarea class="tt" rows="2" aria-label="Text für das sprechende Video" placeholder="Was soll der Avatar sagen? (kurz, ~1–2 Sätze)"></textarea>
|
| 86 |
+
<div class="row"><button class="btn alt mktalk">🎬 Sprechendes Video</button></div><div class="talkout"></div></fieldset>
|
| 87 |
<div class="row"><button class="btn sv">💾 Avatar speichern</button><button class="btn alt talk">💬 Mit Avatar chatten</button></div>
|
| 88 |
+
<div class="credit">Look: ${hf(FLUX)} + ${hf(KONTEXT)} · Stimme: ${hf(TTS)} · Lip-Sync: ${hf(MODA)} (Fallback ${hf(W2L)}) · gespeichert als <code>avatars/<id>/</code> im Workspace</div>`;
|
| 89 |
keep(el); const q = (s) => el.querySelector(s);
|
| 90 |
const fill = () => { q('.n').value = av.name; q('.sy').value = av.system; q('.bs').value = av.backstory; q('.lk').value = av.look; q('.vd').value = av.voice.description; q('.lang').value = av.voice.language || 'German'; q('.vt').value = av.voice.sample_text || `Hallo, ich bin ${av.name || 'dein Avatar'}. Schön, dich zu sehen!`; showRef(); showVoice(); };
|
| 91 |
const read = () => { av.name = q('.n').value.trim(); av.system = q('.sy').value.trim(); av.backstory = q('.bs').value.trim(); av.look = q('.lk').value.trim(); av.voice = { ...av.voice, description: q('.vd').value.trim(), language: q('.lang').value, sample_text: voiceBlob ? q('.vt').value.trim() : av.voice.sample_text }; };
|
| 92 |
const showRef = (u) => { const r = u || Avatars.refUrl(av); q('.refbox').innerHTML = r ? `<img alt="Referenzbild" src="${esc(r)}" class="avref">` : '<small class="mut">Noch kein Referenzbild – erzeugen oder hochladen.</small>'; };
|
| 93 |
const showVoice = (u) => { const v = u || Avatars.voiceUrl(av); q('.voicebox').innerHTML = v ? `<audio controls src="${esc(v)}"></audio><small class="mut">${voiceBlob ? 'Neue Stimme – mit Speichern fixieren' : 'Fixierte Stimme (Klon-Referenz)'}</small>` : ''; };
|
| 94 |
const refreshPick = async () => { const list = await Avatars.list(); if (!el.isConnected) return; const p = q('.pick'), cur = av.id; p.innerHTML = '<option value="">+ Neuer Avatar</option>' + list.map(a => `<option value="${esc(a.id)}">${esc(a.name || a.id)}${a._ws ? ' ☁' : ''}</option>`).join(''); p.value = cur || ''; p._list = list; };
|
| 95 |
+
q('.mktalk').onclick = async (e) => { read(); const t = q('.tt').value.trim() || q('.vt').value.trim(); if (!t) return setS(el, '⚠ Bitte Text eingeben', true); busy(e.target, true);
|
| 96 |
+
try { const r = await Avatars.talk(av, { text: t.slice(0, 300) }, { el, onStatus: m => setS(el, 'Talking Head: ' + stTxt(m)) });
|
| 97 |
+
q('.talkout').innerHTML = `<video controls playsinline src="${esc(r.url)}" class="avout"></video>`; window.__talk = { url: r.url, model: r.model, fallback: r.fallback };
|
| 98 |
+
result(q('.talkout'), { kind: 'video', mime: 'video/mp4', url: r.url, name: `${av.id || slug(av.name)}-talk-${Date.now()}.mp4`, studio: 'avatar', prompt: t, avatar: av.id, model: r.model }); setS(el, '✔ Sprechendes Video · ' + r.model.split('/')[1] + ' → senden an Video Edit'); badge(el, 'REAL'); }
|
| 99 |
+
catch (er) { setS(el, '⚠ ' + er.message, true); window.__talk = { err: er.message }; } finally { busy(e.target, false); } };
|
| 100 |
q('.pick').onchange = (e) => { const a = e.target._list?.find(x => x.id === e.target.value); av = a ? structuredClone(a) : blank(); refBlob = voiceBlob = null; fill(); setS(el, a ? 'Geladen: ' + a.name : 'Neuer Avatar'); };
|
| 101 |
q('.del-local').onclick = () => { av = blank(); refBlob = voiceBlob = null; q('.pick').value = ''; fill(); };
|
| 102 |
const busy = (b, on) => { b.disabled = on; };
|
|
|
|
| 147 |
|
| 148 |
/* ---------- Module API: avatar ---------- */
|
| 149 |
const findAv = async (name) => { const l = await Avatars.list(); const n = (name || '').toLowerCase(); const a = l.find(x => x.id === n || x.name?.toLowerCase() === n) || l.find(x => n && x.name?.toLowerCase().includes(n)) || l[0]; if (!a) throw new Error('Kein Avatar vorhanden – im Avatar Studio anlegen'); return a; };
|
| 150 |
+
MODULES.avatar = { id: 'avatar', name: 'Avatar', backend: KONTEXT + ' + ' + TTS + ' + ' + MODA, inputs: [{ name: 'avatar', type: 'string' }, { name: 'prompt', type: 'text' }], outputs: [{ kind: 'image' }, { kind: 'audio' }, { kind: 'video' }, { kind: 'text' }],
|
| 151 |
actions: {
|
| 152 |
imagine: { desc: 'Erzeugt ein Bild des Avatars in einer Szene (konsistenter Look)', params: { avatar: 'string', prompt: 'string' }, run: async ({ avatar, prompt }, on = {}) => { const a = await findAv(avatar); const u = await Avatars.imagine(a, prompt, { onStatus: on.status }); return { kind: 'image', url: u, name: `${a.id}-${Date.now()}.webp`, studio: 'avatar', prompt, avatar: a.id }; } },
|
| 153 |
speak: { desc: 'Spricht einen Text mit der Stimme des Avatars', params: { avatar: 'string', prompt: 'string' }, run: async ({ avatar, prompt }, on = {}) => { const a = await findAv(avatar); const u = await Avatars.speak(a, prompt, { onStatus: on.status }); return { kind: 'audio', url: u, name: `${a.id}-voice-${Date.now()}.wav`, studio: 'avatar', prompt, avatar: a.id }; } },
|
| 154 |
+
talk: { desc: 'Sprechendes Video (Talking Head): Avatar-Bild + Avatar-Stimme sprechen den Text lippensynchron', params: { avatar: 'string', prompt: 'string' }, run: async ({ avatar, prompt }, on = {}) => { const a = await findAv(avatar); const r = await Avatars.talk(a, { text: String(prompt || '').slice(0, 300) }, { onStatus: on.status }); return { kind: 'video', mime: 'video/mp4', url: r.url, name: `${a.id}-talk-${Date.now()}.mp4`, studio: 'avatar', prompt, avatar: a.id, model: r.model }; } },
|
| 155 |
reply: { desc: 'Antwortet in der Rolle (Persona) des Avatars', params: { avatar: 'string', prompt: 'string' }, run: async ({ avatar, prompt }, on = {}) => { const a = await findAv(avatar); return MODULES.chat.actions.reply.run({ prompt: `System: ${Avatars.persona(a)}\nUser: ${prompt}\nAssistant:` }, on); } },
|
| 156 |
} };
|
index.html
CHANGED
|
@@ -8,7 +8,7 @@
|
|
| 8 |
<title>STUDIUS · DAS Studio — Einswarm</title>
|
| 9 |
<link rel="icon" href="assets/einswarm-mascot.webp">
|
| 10 |
<link rel="preload" href="fonts/outfit-latin-wght.woff2" as="font" type="font/woff2" crossorigin>
|
| 11 |
-
<link rel="stylesheet" href="style.css?v=
|
| 12 |
</head>
|
| 13 |
<body>
|
| 14 |
<canvas id="gl"></canvas>
|
|
@@ -49,6 +49,6 @@
|
|
| 49 |
|
| 50 |
<a id="bmc" href="https://www.buymeacoffee.com/2btainment" target="_blank" rel="noopener noreferrer">Buy me a coffee</a>
|
| 51 |
<noscript><p style="color:#f2c94c;padding:2em">STUDIUS braucht JavaScript.</p></noscript>
|
| 52 |
-
<script type="module" src="app.js?v=
|
| 53 |
</body>
|
| 54 |
</html>
|
|
|
|
| 8 |
<title>STUDIUS · DAS Studio — Einswarm</title>
|
| 9 |
<link rel="icon" href="assets/einswarm-mascot.webp">
|
| 10 |
<link rel="preload" href="fonts/outfit-latin-wght.woff2" as="font" type="font/woff2" crossorigin>
|
| 11 |
+
<link rel="stylesheet" href="style.css?v=93">
|
| 12 |
</head>
|
| 13 |
<body>
|
| 14 |
<canvas id="gl"></canvas>
|
|
|
|
| 49 |
|
| 50 |
<a id="bmc" href="https://www.buymeacoffee.com/2btainment" target="_blank" rel="noopener noreferrer">Buy me a coffee</a>
|
| 51 |
<noscript><p style="color:#f2c94c;padding:2em">STUDIUS braucht JavaScript.</p></noscript>
|
| 52 |
+
<script type="module" src="app.js?v=93"></script>
|
| 53 |
</body>
|
| 54 |
</html>
|
master.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
| 1 |
// Mastering studio (DoD): client-side WebAudio chain EQ → compressor → limiter, loudness normalisation to target LUFS
|
| 2 |
// (ITU-R BS.1770-4 K-weighting + gating, measured offline), live meter, A/B, WAV/MP3 export, save to workspace.
|
| 3 |
import { STUDIOS, onStop, loop, cv } from './widgets.js?v=50';
|
| 4 |
-
import { result, MODULES, esc, setS, keep } from './real.js?v=
|
| 5 |
|
| 6 |
/* ---------- BS.1770 loudness ---------- */
|
| 7 |
function biquad(x, b0, b1, b2, a0, a1, a2) { const y = new Float32Array(x.length); b0 /= a0; b1 /= a0; b2 /= a0; a1 /= a0; a2 /= a0; let x1 = 0, x2 = 0, y1 = 0, y2 = 0;
|
|
|
|
| 1 |
// Mastering studio (DoD): client-side WebAudio chain EQ → compressor → limiter, loudness normalisation to target LUFS
|
| 2 |
// (ITU-R BS.1770-4 K-weighting + gating, measured offline), live meter, A/B, WAV/MP3 export, save to workspace.
|
| 3 |
import { STUDIOS, onStop, loop, cv } from './widgets.js?v=50';
|
| 4 |
+
import { result, MODULES, esc, setS, keep } from './real.js?v=93';
|
| 5 |
|
| 6 |
/* ---------- BS.1770 loudness ---------- */
|
| 7 |
function biquad(x, b0, b1, b2, a0, a1, a2) { const y = new Float32Array(x.length); b0 /= a0; b1 /= a0; b2 /= a0; a1 /= a0; a2 /= a0; let x1 = 0, x2 = 0, y1 = 0, y2 = 0;
|
real.js
CHANGED
|
@@ -29,7 +29,7 @@ async function runAnon(space, ep, data, { onStatus = () => {}, onData = () => {}
|
|
| 29 |
async function runUser(space, ep, data, { onStatus = () => {}, onData = () => {}, timeout = 150000, el } = {}) {
|
| 30 |
const t0 = performance.now(), f = new FormData(), args = {}; let n = 0;
|
| 31 |
for (const [k, v] of Object.entries(data)) {
|
| 32 |
-
if (v instanceof Blob) { const id = 'f' + n++; f.append(id, v, v.name || 'input'); args[k] = { __file__: id }; }
|
| 33 |
else if (typeof v === 'string' && v.startsWith('/api/tmp/') && k !== 'prompt') args[k] = { __tmp__: v };
|
| 34 |
else args[k] = v;
|
| 35 |
}
|
|
@@ -59,7 +59,7 @@ export async function run(space, ep, data, opt = {}) { LASTM = space; let why =
|
|
| 59 |
if (signedIn()) {
|
| 60 |
const q = window.__quota; if (q?.ok && q.remaining < 60) notice(opt.el, `⚡ Nur noch <b>${Math.round(q.remaining)} s</b> deiner ZeroGPU-Quota${q.resetsAt ? ' · Reset ' + fmtReset(q.resetsAt) : ''} – evtl. Fallback.`);
|
| 61 |
try { const d = await runUser(space, ep, data, opt); opt.el && via(opt.el, 'user'); refreshQuota(); return d; }
|
| 62 |
-
catch (e) { if (e.message === 'abgebrochen' || (opt.el && !opt.el.isConnected)) throw e; console.warn('user-quota run failed, trying anonymous:', e.message); why = e.message;
|
| 63 |
const q2 = await refreshQuota();
|
| 64 |
notice(opt.el, isQuota(why) ? `⚠ <b>Deine ZeroGPU-Quota ist aufgebraucht</b>${(m => m ? ` (${m[1]} s angefragt, ${m[2]} s übrig)` : '')(why.match(/\((\d+)s requested vs\. (\d+)s left\)/))}${q2?.ok && q2.resetsAt ? ' · Reset ' + fmtReset(q2.resetsAt) : ''}. Fallback: <b>anonyme Quota</b> – kann langsamer sein oder ebenfalls scheitern. <a href="https://huggingface.co/pricing" target="_blank" rel="noopener">Mehr Quota (PRO)</a>` : `⚠ Lauf mit deinem Konto fehlgeschlagen (${esc(why.slice(0, 120))}). Fallback: anonyme Quota.`);
|
| 65 |
opt.onStatus?.({ stage: 'pending', message: 'Fallback: anonyme Quota' }); }
|
|
|
|
| 29 |
async function runUser(space, ep, data, { onStatus = () => {}, onData = () => {}, timeout = 150000, el } = {}) {
|
| 30 |
const t0 = performance.now(), f = new FormData(), args = {}; let n = 0;
|
| 31 |
for (const [k, v] of Object.entries(data)) {
|
| 32 |
+
if (v instanceof Blob) { const id = 'f' + n++; f.append(id, v, v.name || ('input.' + ({ 'image/jpeg': 'jpg', 'image/png': 'png', 'image/webp': 'webp', 'audio/wav': 'wav', 'audio/x-wav': 'wav', 'audio/wave': 'wav', 'audio/mpeg': 'mp3', 'audio/ogg': 'ogg', 'audio/flac': 'flac', 'video/mp4': 'mp4', 'video/webm': 'webm' }[(v.type || '').split(';')[0]] || 'bin'))); args[k] = { __file__: id }; }
|
| 33 |
else if (typeof v === 'string' && v.startsWith('/api/tmp/') && k !== 'prompt') args[k] = { __tmp__: v };
|
| 34 |
else args[k] = v;
|
| 35 |
}
|
|
|
|
| 59 |
if (signedIn()) {
|
| 60 |
const q = window.__quota; if (q?.ok && q.remaining < 60) notice(opt.el, `⚡ Nur noch <b>${Math.round(q.remaining)} s</b> deiner ZeroGPU-Quota${q.resetsAt ? ' · Reset ' + fmtReset(q.resetsAt) : ''} – evtl. Fallback.`);
|
| 61 |
try { const d = await runUser(space, ep, data, opt); opt.el && via(opt.el, 'user'); refreshQuota(); return d; }
|
| 62 |
+
catch (e) { if (e.message === 'abgebrochen' || (opt.el && !opt.el.isConnected)) throw e; if (opt.noAnonOn?.test(e.message)) throw e; console.warn('user-quota run failed, trying anonymous:', e.message); why = e.message;
|
| 63 |
const q2 = await refreshQuota();
|
| 64 |
notice(opt.el, isQuota(why) ? `⚠ <b>Deine ZeroGPU-Quota ist aufgebraucht</b>${(m => m ? ` (${m[1]} s angefragt, ${m[2]} s übrig)` : '')(why.match(/\((\d+)s requested vs\. (\d+)s left\)/))}${q2?.ok && q2.resetsAt ? ' · Reset ' + fmtReset(q2.resetsAt) : ''}. Fallback: <b>anonyme Quota</b> – kann langsamer sein oder ebenfalls scheitern. <a href="https://huggingface.co/pricing" target="_blank" rel="noopener">Mehr Quota (PRO)</a>` : `⚠ Lauf mit deinem Konto fehlgeschlagen (${esc(why.slice(0, 120))}). Fallback: anonyme Quota.`);
|
| 65 |
opt.onStatus?.({ stage: 'pending', message: 'Fallback: anonyme Quota' }); }
|
server/infer.py
CHANGED
|
@@ -16,6 +16,8 @@ ALLOW = { # space -> allowed endpoints (Module API backends)
|
|
| 16 |
"declare-lab/TangoFlux": {"/predict"},
|
| 17 |
"abidlabs/music-separation": {"/inference"},
|
| 18 |
"Lightricks/ltx-video-distilled": {"/text_to_video"},
|
|
|
|
|
|
|
| 19 |
"Qwen/Qwen3-TTS": {"/generate_voice_design", "/generate_voice_clone"},
|
| 20 |
}
|
| 21 |
PATH_ARGS = {("Lightricks/ltx-video-distilled", "/text_to_video"): {"input_image_filepath", "input_video_filepath"}} # str-typed filepath args: upload to the Space, pass its server path
|
|
|
|
| 16 |
"declare-lab/TangoFlux": {"/predict"},
|
| 17 |
"abidlabs/music-separation": {"/inference"},
|
| 18 |
"Lightricks/ltx-video-distilled": {"/text_to_video"},
|
| 19 |
+
"multimodalart/MoDA-fast-talking-head": {"/generate_motion"},
|
| 20 |
+
"pragnakalp/Wav2lip-ZeroGPU": {"/run_infrence"},
|
| 21 |
"Qwen/Qwen3-TTS": {"/generate_voice_design", "/generate_voice_clone"},
|
| 22 |
}
|
| 23 |
PATH_ARGS = {("Lightricks/ltx-video-distilled", "/text_to_video"): {"input_image_filepath", "input_video_filepath"}} # str-typed filepath args: upload to the Space, pass its server path
|
server/main.py
CHANGED
|
@@ -311,6 +311,26 @@ async def transcode(req: Request, file: UploadFile = File(...), name: str = Form
|
|
| 311 |
shutil.rmtree(d, ignore_errors=True)
|
| 312 |
return res
|
| 313 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 314 |
@app.get("/api/quota")
|
| 315 |
async def quota(req: Request):
|
| 316 |
"""User's own ZeroGPU quota (GPU-seconds) from the Hub. Token stays server-side."""
|
|
|
|
| 311 |
shutil.rmtree(d, ignore_errors=True)
|
| 312 |
return res
|
| 313 |
|
| 314 |
+
@app.post("/api/stillvideo")
|
| 315 |
+
async def stillvideo(req: Request, image: UploadFile = File(...), audio: UploadFile = File(...), name: str = Form("talk")):
|
| 316 |
+
"""Talking-head fallback without GPU: still avatar image + voice -> gentle zoom H.264/AAC MP4 (faststart)."""
|
| 317 |
+
import asyncio, shutil, subprocess
|
| 318 |
+
from .infer import tmp_put
|
| 319 |
+
need(req)
|
| 320 |
+
if not shutil.which("ffmpeg"): raise HTTPException(503, "ffmpeg not installed")
|
| 321 |
+
d = tempfile.mkdtemp(); im, au = f"{d}/in.img", f"{d}/in.aud"; stem = re.sub(r"[^\w.-]+", "-", Path(name).stem)[:80] or "talk"; out = f"{d}/{stem}.mp4"
|
| 322 |
+
for f, pth in ((image, im), (audio, au)):
|
| 323 |
+
b = await f.read(50 * 2**20 + 1)
|
| 324 |
+
if len(b) > 50 * 2**20: shutil.rmtree(d, ignore_errors=True); raise HTTPException(413, "max 50 MB")
|
| 325 |
+
Path(pth).write_bytes(b)
|
| 326 |
+
vf = "scale=1024:1024:force_original_aspect_ratio=increase,crop=1024:1024,zoompan=z='min(1.0+0.0008*on,1.15)':x='iw/2-(iw/zoom/2)':y='ih/2-(ih/zoom/2)':d=1:s=720x720:fps=30,format=yuv420p"
|
| 327 |
+
cmd = ["ffmpeg", "-hide_banner", "-loglevel", "error", "-y", "-loop", "1", "-i", im, "-i", au, "-vf", vf, "-c:v", "libx264", "-profile:v", "high", "-level", "4.1", "-preset", "veryfast", "-crf", "21", "-pix_fmt", "yuv420p",
|
| 328 |
+
"-c:a", "aac", "-b:a", "160k", "-ar", "48000", "-ac", "2", "-shortest", "-t", "120", "-movflags", "+faststart", out]
|
| 329 |
+
p = await asyncio.to_thread(subprocess.run, cmd, capture_output=True, text=True, timeout=300)
|
| 330 |
+
if p.returncode != 0 or not os.path.exists(out): shutil.rmtree(d, ignore_errors=True); raise HTTPException(422, "ffmpeg: " + (p.stderr or "failed")[-300:])
|
| 331 |
+
u = tmp_put(out); shutil.rmtree(d, ignore_errors=True)
|
| 332 |
+
return {"url": u}
|
| 333 |
+
|
| 334 |
@app.get("/api/quota")
|
| 335 |
async def quota(req: Request):
|
| 336 |
"""User's own ZeroGPU quota (GPU-seconds) from the Hub. Token stays server-side."""
|
video.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
| 1 |
// STUDIUS P3.4: Video Gen (LTX-Video, text→video + image→video), Video Edit (trim, soundtrack, title, fades → MP4/WebM export)
|
| 2 |
// and Stems (Demucs split → synced mixer with gain/mute/solo → WAV mixdown). All results go on the asset bus (save / send).
|
| 3 |
-
import { run, result, MODULES, esc, hf, stTxt, url, badge, timer, setS, S, keep } from './real.js?v=
|
| 4 |
-
import { toWav } from './master.js?v=
|
| 5 |
import { onStop } from './widgets.js?v=50';
|
| 6 |
|
| 7 |
export const LTX = 'Lightricks/ltx-video-distilled';
|
|
|
|
| 1 |
// STUDIUS P3.4: Video Gen (LTX-Video, text→video + image→video), Video Edit (trim, soundtrack, title, fades → MP4/WebM export)
|
| 2 |
// and Stems (Demucs split → synced mixer with gain/mute/solo → WAV mixdown). All results go on the asset bus (save / send).
|
| 3 |
+
import { run, result, MODULES, esc, hf, stTxt, url, badge, timer, setS, S, keep } from './real.js?v=93';
|
| 4 |
+
import { toWav } from './master.js?v=93';
|
| 5 |
import { onStop } from './widgets.js?v=50';
|
| 6 |
|
| 7 |
export const LTX = 'Lightricks/ltx-video-distilled';
|
workspace.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
// STUDIUS empire backbone (frontend): HF sign-in, Workspace (user's HF Bucket), asset bus hand-off, bee agent dock.
|
| 2 |
-
import { MODULES, refreshQuota } from './real.js?v=
|
| 3 |
-
import { Avatars } from './avatar.js?v=
|
| 4 |
const $ = (s, r = document) => r.querySelector(s);
|
| 5 |
const esc = (t) => String(t ?? '').replace(/[&<>"']/g, c => ({ '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' }[c]));
|
| 6 |
const ST = () => window.__studius.STUDIOS;
|
|
@@ -148,7 +148,7 @@ const TEMPLATES = [
|
|
| 148 |
{ id: 'shop-team', name: 'Shop/Marketing-Team', icon: '🛍', task: 'Bewirb ein goldenes Bienen-T-Shirt: Produkttext, Produktfoto und kurzer Werbeclip',
|
| 149 |
members: [['produkttexter', 'Produkttexter', 'Schreibt Produktbeschreibung (3 Sätze) und Slogan', ['chat']], ['produktfoto', 'Produktfotograf', 'Erzeugt ein Produktfoto im Studio-Look', ['image']], ['werbespot', 'Werbespot-Regie', 'Erzeugt aus dem Produktfoto-Prompt einen kurzen Clip', ['video']], ['archivar', 'Archivar', 'Speichert fertige Ergebnisse im Workspace', ['workspace']]] },
|
| 150 |
{ id: 'avatar-team', name: 'Avatar-Content-Team', icon: '👤', task: 'Der Avatar stellt sich in einem kurzen Clip vor',
|
| 151 |
-
members: [['autor', 'Avatar-Autor', 'Schreibt in der Persona des Avatars einen Vorstellungstext (2 Sätze)', ['avatar']], ['fotograf', 'Avatar-Fotograf', 'Erzeugt ein Bild des Avatars in einer passenden Szene', ['avatar']], ['stimme', 'Avatar-
|
| 152 |
];
|
| 153 |
async function fromTemplate(tid) { const T = TEMPLATES.find(t => t.id === tid); if (!T) return; const av = avList[0]?.id || '';
|
| 154 |
const made = []; for (const [id, name, role, modules] of T.members) { const aid = (id === 'archivar' ? '' : T.id.split('-')[0] + '-') + id;
|
|
|
|
| 1 |
// STUDIUS empire backbone (frontend): HF sign-in, Workspace (user's HF Bucket), asset bus hand-off, bee agent dock.
|
| 2 |
+
import { MODULES, refreshQuota } from './real.js?v=93';
|
| 3 |
+
import { Avatars } from './avatar.js?v=93';
|
| 4 |
const $ = (s, r = document) => r.querySelector(s);
|
| 5 |
const esc = (t) => String(t ?? '').replace(/[&<>"']/g, c => ({ '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' }[c]));
|
| 6 |
const ST = () => window.__studius.STUDIOS;
|
|
|
|
| 148 |
{ id: 'shop-team', name: 'Shop/Marketing-Team', icon: '🛍', task: 'Bewirb ein goldenes Bienen-T-Shirt: Produkttext, Produktfoto und kurzer Werbeclip',
|
| 149 |
members: [['produkttexter', 'Produkttexter', 'Schreibt Produktbeschreibung (3 Sätze) und Slogan', ['chat']], ['produktfoto', 'Produktfotograf', 'Erzeugt ein Produktfoto im Studio-Look', ['image']], ['werbespot', 'Werbespot-Regie', 'Erzeugt aus dem Produktfoto-Prompt einen kurzen Clip', ['video']], ['archivar', 'Archivar', 'Speichert fertige Ergebnisse im Workspace', ['workspace']]] },
|
| 150 |
{ id: 'avatar-team', name: 'Avatar-Content-Team', icon: '👤', task: 'Der Avatar stellt sich in einem kurzen Clip vor',
|
| 151 |
+
members: [['autor', 'Avatar-Autor', 'Schreibt in der Persona des Avatars einen Vorstellungstext (2 Sätze)', ['avatar']], ['fotograf', 'Avatar-Fotograf', 'Erzeugt ein Bild des Avatars in einer passenden Szene', ['avatar']], ['stimme', 'Avatar-Sprecher', 'Macht aus dem Text ein sprechendes Video des Avatars (avatar.talk, Lip-Sync)', ['avatar']], ['archivar', 'Archivar', 'Speichert fertige Ergebnisse im Workspace', ['workspace']]] },
|
| 152 |
];
|
| 153 |
async function fromTemplate(tid) { const T = TEMPLATES.find(t => t.id === tid); if (!T) return; const av = avList[0]?.id || '';
|
| 154 |
const made = []; for (const [id, name, role, modules] of T.members) { const aid = (id === 'archivar' ? '' : T.id.split('-')[0] + '-') + id;
|