2Btainment commited on
Commit
61657d1
·
verified ·
1 Parent(s): d00d60c

P4.4: talking-head avatar (MoDA lip-sync, Wav2Lip/ffmpeg fallbacks), upload filenames

Browse files
Files changed (9) hide show
  1. app.js +6 -6
  2. avatar.js +27 -3
  3. index.html +2 -2
  4. master.js +1 -1
  5. real.js +2 -2
  6. server/infer.py +2 -0
  7. server/main.py +20 -0
  8. video.js +2 -2
  9. workspace.js +3 -3
app.js CHANGED
@@ -1,11 +1,11 @@
1
  import * as THREE from './vendor/three.module.min.js';
2
  import { STUDIOS, stopLoops } from './widgets.js?v=50';
3
- import './real.js?v=89';
4
- import './avatar.js?v=89';
5
- import './master.js?v=89';
6
- import './video.js?v=89';
7
- import './workspace.js?v=89';
8
- import { icon } from './icons.js?v=89';
9
  import { RoomEnvironment } from './vendor/addons/environments/RoomEnvironment.js';
10
  import { EffectComposer } from './vendor/addons/postprocessing/EffectComposer.js';
11
  import { RenderPass } from './vendor/addons/postprocessing/RenderPass.js';
 
1
  import * as THREE from './vendor/three.module.min.js';
2
  import { STUDIOS, stopLoops } from './widgets.js?v=50';
3
+ import './real.js?v=93';
4
+ import './avatar.js?v=93';
5
+ import './master.js?v=93';
6
+ import './video.js?v=93';
7
+ import './workspace.js?v=93';
8
+ import { icon } from './icons.js?v=93';
9
  import { RoomEnvironment } from './vendor/addons/environments/RoomEnvironment.js';
10
  import { EffectComposer } from './vendor/addons/postprocessing/EffectComposer.js';
11
  import { RenderPass } from './vendor/addons/postprocessing/RenderPass.js';
avatar.js CHANGED
@@ -1,8 +1,9 @@
1
  // Avatar / Persona studio (DoD): persistent avatars as workspace asset type `avatar`
2
  // avatars/<slug>/avatar.json + ref.webp + voice.wav · reused by Chat, Image (Kontext), agents.
3
  import { STUDIOS, onStop } from './widgets.js?v=50';
4
- import { run, result, MODULES, esc, hf, stTxt, url, badge, setS, keep, LS, FLUX, KONTEXT, TTS, CHAT, signedIn } from './real.js?v=89';
5
 
 
6
  const LANGS = ['German', 'English', 'French', 'Spanish', 'Auto'];
7
  const slug = (n) => (n || 'avatar').toLowerCase().normalize('NFKD').replace(/[^\w]+/g, '-').replace(/^-|-$/g, '').slice(0, 40) || 'avatar';
8
  const blank = () => ({ type: 'studius.avatar', v: 2, id: '', name: '', system: '', backstory: '', look: '', seed: Math.floor(Math.random() * 1e6),
@@ -45,6 +46,21 @@ export const Avatars = {
45
  : await run(TTS, '/generate_voice_design', { text, language: lang, voice_description: av.voice?.description || 'neutral' }, { timeout: 180000, ...opt });
46
  return url(d[0]);
47
  },
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
48
  async imagine(av, prompt, opt = {}) {
49
  const ru = this.refUrl(av); if (!ru) throw new Error('Avatar hat noch kein Referenzbild');
50
  const d = await run(KONTEXT, '/infer', { input_image: await toBlob(ru), prompt: `${prompt}. Keep the exact same character (${av.look}).`, seed: av.seed || 7, randomize_seed: false, guidance_scale: 2.5, steps: 20 }, { timeout: 240000, ...opt });
@@ -66,14 +82,21 @@ STUDIOS.push({ id: 'avatar', cat: 'Bild', icon: '🧑', name: 'Avatar Studio', l
66
  <div class="refbox"></div><div class="row"><input class="scene" aria-label="Szene im Avatar-Look" placeholder="Szene: winkt am Strand"><button class="btn alt imag">Im Look erzeugen</button></div><div class="imgout"></div></fieldset>
67
  <fieldset class="grp"><legend>Stimme</legend><input class="vd" aria-label="Stimmbeschreibung" placeholder="Stimme beschreiben"><div class="row"><select class="lang" aria-label="Sprache">${LANGS.map(l => `<option>${l}</option>`).join('')}</select><button class="btn alt mkvoice">🔊 Stimme entwerfen</button></div>
68
  <input class="vt" aria-label="Probesatz" placeholder="Probesatz"><div class="voicebox"></div></fieldset>
 
 
69
  <div class="row"><button class="btn sv">💾 Avatar speichern</button><button class="btn alt talk">💬 Mit Avatar chatten</button></div>
70
- <div class="credit">Look: ${hf(FLUX)} + ${hf(KONTEXT)} · Stimme: ${hf(TTS)} · gespeichert als <code>avatars/&lt;id&gt;/</code> im Workspace</div>`;
71
  keep(el); const q = (s) => el.querySelector(s);
72
  const fill = () => { q('.n').value = av.name; q('.sy').value = av.system; q('.bs').value = av.backstory; q('.lk').value = av.look; q('.vd').value = av.voice.description; q('.lang').value = av.voice.language || 'German'; q('.vt').value = av.voice.sample_text || `Hallo, ich bin ${av.name || 'dein Avatar'}. Schön, dich zu sehen!`; showRef(); showVoice(); };
73
  const read = () => { av.name = q('.n').value.trim(); av.system = q('.sy').value.trim(); av.backstory = q('.bs').value.trim(); av.look = q('.lk').value.trim(); av.voice = { ...av.voice, description: q('.vd').value.trim(), language: q('.lang').value, sample_text: voiceBlob ? q('.vt').value.trim() : av.voice.sample_text }; };
74
  const showRef = (u) => { const r = u || Avatars.refUrl(av); q('.refbox').innerHTML = r ? `<img alt="Referenzbild" src="${esc(r)}" class="avref">` : '<small class="mut">Noch kein Referenzbild – erzeugen oder hochladen.</small>'; };
75
  const showVoice = (u) => { const v = u || Avatars.voiceUrl(av); q('.voicebox').innerHTML = v ? `<audio controls src="${esc(v)}"></audio><small class="mut">${voiceBlob ? 'Neue Stimme – mit Speichern fixieren' : 'Fixierte Stimme (Klon-Referenz)'}</small>` : ''; };
76
  const refreshPick = async () => { const list = await Avatars.list(); if (!el.isConnected) return; const p = q('.pick'), cur = av.id; p.innerHTML = '<option value="">+ Neuer Avatar</option>' + list.map(a => `<option value="${esc(a.id)}">${esc(a.name || a.id)}${a._ws ? ' ☁' : ''}</option>`).join(''); p.value = cur || ''; p._list = list; };
 
 
 
 
 
77
  q('.pick').onchange = (e) => { const a = e.target._list?.find(x => x.id === e.target.value); av = a ? structuredClone(a) : blank(); refBlob = voiceBlob = null; fill(); setS(el, a ? 'Geladen: ' + a.name : 'Neuer Avatar'); };
78
  q('.del-local').onclick = () => { av = blank(); refBlob = voiceBlob = null; q('.pick').value = ''; fill(); };
79
  const busy = (b, on) => { b.disabled = on; };
@@ -124,9 +147,10 @@ window.__avatarRef = async (el) => { const a = chosen(el.querySelector('.avsel')
124
 
125
  /* ---------- Module API: avatar ---------- */
126
  const findAv = async (name) => { const l = await Avatars.list(); const n = (name || '').toLowerCase(); const a = l.find(x => x.id === n || x.name?.toLowerCase() === n) || l.find(x => n && x.name?.toLowerCase().includes(n)) || l[0]; if (!a) throw new Error('Kein Avatar vorhanden – im Avatar Studio anlegen'); return a; };
127
- MODULES.avatar = { id: 'avatar', name: 'Avatar', backend: KONTEXT + ' + ' + TTS, inputs: [{ name: 'avatar', type: 'string' }, { name: 'prompt', type: 'text' }], outputs: [{ kind: 'image' }, { kind: 'audio' }, { kind: 'text' }],
128
  actions: {
129
  imagine: { desc: 'Erzeugt ein Bild des Avatars in einer Szene (konsistenter Look)', params: { avatar: 'string', prompt: 'string' }, run: async ({ avatar, prompt }, on = {}) => { const a = await findAv(avatar); const u = await Avatars.imagine(a, prompt, { onStatus: on.status }); return { kind: 'image', url: u, name: `${a.id}-${Date.now()}.webp`, studio: 'avatar', prompt, avatar: a.id }; } },
130
  speak: { desc: 'Spricht einen Text mit der Stimme des Avatars', params: { avatar: 'string', prompt: 'string' }, run: async ({ avatar, prompt }, on = {}) => { const a = await findAv(avatar); const u = await Avatars.speak(a, prompt, { onStatus: on.status }); return { kind: 'audio', url: u, name: `${a.id}-voice-${Date.now()}.wav`, studio: 'avatar', prompt, avatar: a.id }; } },
 
131
  reply: { desc: 'Antwortet in der Rolle (Persona) des Avatars', params: { avatar: 'string', prompt: 'string' }, run: async ({ avatar, prompt }, on = {}) => { const a = await findAv(avatar); return MODULES.chat.actions.reply.run({ prompt: `System: ${Avatars.persona(a)}\nUser: ${prompt}\nAssistant:` }, on); } },
132
  } };
 
1
  // Avatar / Persona studio (DoD): persistent avatars as workspace asset type `avatar`
2
  // avatars/<slug>/avatar.json + ref.webp + voice.wav · reused by Chat, Image (Kontext), agents.
3
  import { STUDIOS, onStop } from './widgets.js?v=50';
4
+ import { run, result, MODULES, esc, hf, stTxt, url, badge, setS, keep, LS, FLUX, KONTEXT, TTS, CHAT, signedIn } from './real.js?v=93';
5
 
6
+ export const MODA = 'multimodalart/MoDA-fast-talking-head', W2L = 'pragnakalp/Wav2lip-ZeroGPU';
7
  const LANGS = ['German', 'English', 'French', 'Spanish', 'Auto'];
8
  const slug = (n) => (n || 'avatar').toLowerCase().normalize('NFKD').replace(/[^\w]+/g, '-').replace(/^-|-$/g, '').slice(0, 40) || 'avatar';
9
  const blank = () => ({ type: 'studius.avatar', v: 2, id: '', name: '', system: '', backstory: '', look: '', seed: Math.floor(Math.random() * 1e6),
 
46
  : await run(TTS, '/generate_voice_design', { text, language: lang, voice_description: av.voice?.description || 'neutral' }, { timeout: 180000, ...opt });
47
  return url(d[0]);
48
  },
49
+ /** Talking head: ref image + voice (TTS text or audio) → lip-sync MP4. MoDA (ZeroGPU, ~20 s) → fallback Wav2Lip (CPU, no GPU quota). */
50
+ async talk(av, { text, audio } = {}, opt = {}) {
51
+ const ru = this.refUrl(av); if (!ru) throw new Error(av.name + ' hat kein Referenzbild');
52
+ const st = opt.onStatus || (() => {}); if (!audio) { st({ msg: 'Stimme (TTS)…' }); audio = await this.speak(av, text, opt); }
53
+ const img = await toBlob(ru), au = await toBlob(audio); let d, model = MODA, err = '';
54
+ const still = async () => { st({ msg: 'Standbild-Video (ffmpeg, ohne GPU)…' }); const f = new FormData(); f.append('image', img, 'avatar.' + ((img.type || '').split('/')[1] || 'png')); f.append('audio', au, 'voice.wav'); f.append('name', (av.id || 'avatar') + '-talk');
55
+ const r = await fetch('/api/stillvideo', { method: 'POST', body: f, credentials: 'include' }); if (!r.ok) throw new Error('Standbild-Video: ' + ((await r.json().catch(() => ({}))).detail || r.status)); return [{ video: (await r.json()).url }]; };
56
+ try { st({ msg: 'Lip-Sync (MoDA)…' }); d = await run(MODA, '/generate_motion', { source_image_path: img, driving_audio_path: au, emotion_name: 'None', cfg_scale: 1.2 }, { timeout: 240000, noAnonOn: /no face/i, ...opt }); }
57
+ catch (e) { err = e.message;
58
+ // no human face (e.g. mascot) → lip-sync impossible; Wav2Lip would silently return its demo face, so use the still-image video instead
59
+ if (/no face/i.test(err)) { model = 'ffmpeg still+voice (kein Gesicht erkannt)'; d = await still(); }
60
+ else { try { model = W2L; st({ msg: 'MoDA nicht verfügbar → Wav2Lip (CPU)…' }); d = await run(W2L, '/run_infrence', { input_image: img, input_audio: au }, { timeout: 420000, ...opt }); }
61
+ catch (e2) { err += ' | ' + e2.message; model = 'ffmpeg still+voice'; d = await still(); } } }
62
+ const v = d[0]?.video ?? d[0]; return { url: url(v), audio, model, fallback: err };
63
+ },
64
  async imagine(av, prompt, opt = {}) {
65
  const ru = this.refUrl(av); if (!ru) throw new Error('Avatar hat noch kein Referenzbild');
66
  const d = await run(KONTEXT, '/infer', { input_image: await toBlob(ru), prompt: `${prompt}. Keep the exact same character (${av.look}).`, seed: av.seed || 7, randomize_seed: false, guidance_scale: 2.5, steps: 20 }, { timeout: 240000, ...opt });
 
82
  <div class="refbox"></div><div class="row"><input class="scene" aria-label="Szene im Avatar-Look" placeholder="Szene: winkt am Strand"><button class="btn alt imag">Im Look erzeugen</button></div><div class="imgout"></div></fieldset>
83
  <fieldset class="grp"><legend>Stimme</legend><input class="vd" aria-label="Stimmbeschreibung" placeholder="Stimme beschreiben"><div class="row"><select class="lang" aria-label="Sprache">${LANGS.map(l => `<option>${l}</option>`).join('')}</select><button class="btn alt mkvoice">🔊 Stimme entwerfen</button></div>
84
  <input class="vt" aria-label="Probesatz" placeholder="Probesatz"><div class="voicebox"></div></fieldset>
85
+ <fieldset class="grp"><legend>Sprechendes Video (Talking Head)</legend><textarea class="tt" rows="2" aria-label="Text für das sprechende Video" placeholder="Was soll der Avatar sagen? (kurz, ~1–2 Sätze)"></textarea>
86
+ <div class="row"><button class="btn alt mktalk">🎬 Sprechendes Video</button></div><div class="talkout"></div></fieldset>
87
  <div class="row"><button class="btn sv">💾 Avatar speichern</button><button class="btn alt talk">💬 Mit Avatar chatten</button></div>
88
+ <div class="credit">Look: ${hf(FLUX)} + ${hf(KONTEXT)} · Stimme: ${hf(TTS)} · Lip-Sync: ${hf(MODA)} (Fallback ${hf(W2L)}) · gespeichert als <code>avatars/&lt;id&gt;/</code> im Workspace</div>`;
89
  keep(el); const q = (s) => el.querySelector(s);
90
  const fill = () => { q('.n').value = av.name; q('.sy').value = av.system; q('.bs').value = av.backstory; q('.lk').value = av.look; q('.vd').value = av.voice.description; q('.lang').value = av.voice.language || 'German'; q('.vt').value = av.voice.sample_text || `Hallo, ich bin ${av.name || 'dein Avatar'}. Schön, dich zu sehen!`; showRef(); showVoice(); };
91
  const read = () => { av.name = q('.n').value.trim(); av.system = q('.sy').value.trim(); av.backstory = q('.bs').value.trim(); av.look = q('.lk').value.trim(); av.voice = { ...av.voice, description: q('.vd').value.trim(), language: q('.lang').value, sample_text: voiceBlob ? q('.vt').value.trim() : av.voice.sample_text }; };
92
  const showRef = (u) => { const r = u || Avatars.refUrl(av); q('.refbox').innerHTML = r ? `<img alt="Referenzbild" src="${esc(r)}" class="avref">` : '<small class="mut">Noch kein Referenzbild – erzeugen oder hochladen.</small>'; };
93
  const showVoice = (u) => { const v = u || Avatars.voiceUrl(av); q('.voicebox').innerHTML = v ? `<audio controls src="${esc(v)}"></audio><small class="mut">${voiceBlob ? 'Neue Stimme – mit Speichern fixieren' : 'Fixierte Stimme (Klon-Referenz)'}</small>` : ''; };
94
  const refreshPick = async () => { const list = await Avatars.list(); if (!el.isConnected) return; const p = q('.pick'), cur = av.id; p.innerHTML = '<option value="">+ Neuer Avatar</option>' + list.map(a => `<option value="${esc(a.id)}">${esc(a.name || a.id)}${a._ws ? ' ☁' : ''}</option>`).join(''); p.value = cur || ''; p._list = list; };
95
+ q('.mktalk').onclick = async (e) => { read(); const t = q('.tt').value.trim() || q('.vt').value.trim(); if (!t) return setS(el, '⚠ Bitte Text eingeben', true); busy(e.target, true);
96
+ try { const r = await Avatars.talk(av, { text: t.slice(0, 300) }, { el, onStatus: m => setS(el, 'Talking Head: ' + stTxt(m)) });
97
+ q('.talkout').innerHTML = `<video controls playsinline src="${esc(r.url)}" class="avout"></video>`; window.__talk = { url: r.url, model: r.model, fallback: r.fallback };
98
+ result(q('.talkout'), { kind: 'video', mime: 'video/mp4', url: r.url, name: `${av.id || slug(av.name)}-talk-${Date.now()}.mp4`, studio: 'avatar', prompt: t, avatar: av.id, model: r.model }); setS(el, '✔ Sprechendes Video · ' + r.model.split('/')[1] + ' → senden an Video Edit'); badge(el, 'REAL'); }
99
+ catch (er) { setS(el, '⚠ ' + er.message, true); window.__talk = { err: er.message }; } finally { busy(e.target, false); } };
100
  q('.pick').onchange = (e) => { const a = e.target._list?.find(x => x.id === e.target.value); av = a ? structuredClone(a) : blank(); refBlob = voiceBlob = null; fill(); setS(el, a ? 'Geladen: ' + a.name : 'Neuer Avatar'); };
101
  q('.del-local').onclick = () => { av = blank(); refBlob = voiceBlob = null; q('.pick').value = ''; fill(); };
102
  const busy = (b, on) => { b.disabled = on; };
 
147
 
148
  /* ---------- Module API: avatar ---------- */
149
  const findAv = async (name) => { const l = await Avatars.list(); const n = (name || '').toLowerCase(); const a = l.find(x => x.id === n || x.name?.toLowerCase() === n) || l.find(x => n && x.name?.toLowerCase().includes(n)) || l[0]; if (!a) throw new Error('Kein Avatar vorhanden – im Avatar Studio anlegen'); return a; };
150
+ MODULES.avatar = { id: 'avatar', name: 'Avatar', backend: KONTEXT + ' + ' + TTS + ' + ' + MODA, inputs: [{ name: 'avatar', type: 'string' }, { name: 'prompt', type: 'text' }], outputs: [{ kind: 'image' }, { kind: 'audio' }, { kind: 'video' }, { kind: 'text' }],
151
  actions: {
152
  imagine: { desc: 'Erzeugt ein Bild des Avatars in einer Szene (konsistenter Look)', params: { avatar: 'string', prompt: 'string' }, run: async ({ avatar, prompt }, on = {}) => { const a = await findAv(avatar); const u = await Avatars.imagine(a, prompt, { onStatus: on.status }); return { kind: 'image', url: u, name: `${a.id}-${Date.now()}.webp`, studio: 'avatar', prompt, avatar: a.id }; } },
153
  speak: { desc: 'Spricht einen Text mit der Stimme des Avatars', params: { avatar: 'string', prompt: 'string' }, run: async ({ avatar, prompt }, on = {}) => { const a = await findAv(avatar); const u = await Avatars.speak(a, prompt, { onStatus: on.status }); return { kind: 'audio', url: u, name: `${a.id}-voice-${Date.now()}.wav`, studio: 'avatar', prompt, avatar: a.id }; } },
154
+ talk: { desc: 'Sprechendes Video (Talking Head): Avatar-Bild + Avatar-Stimme sprechen den Text lippensynchron', params: { avatar: 'string', prompt: 'string' }, run: async ({ avatar, prompt }, on = {}) => { const a = await findAv(avatar); const r = await Avatars.talk(a, { text: String(prompt || '').slice(0, 300) }, { onStatus: on.status }); return { kind: 'video', mime: 'video/mp4', url: r.url, name: `${a.id}-talk-${Date.now()}.mp4`, studio: 'avatar', prompt, avatar: a.id, model: r.model }; } },
155
  reply: { desc: 'Antwortet in der Rolle (Persona) des Avatars', params: { avatar: 'string', prompt: 'string' }, run: async ({ avatar, prompt }, on = {}) => { const a = await findAv(avatar); return MODULES.chat.actions.reply.run({ prompt: `System: ${Avatars.persona(a)}\nUser: ${prompt}\nAssistant:` }, on); } },
156
  } };
index.html CHANGED
@@ -8,7 +8,7 @@
8
  <title>STUDIUS · DAS Studio — Einswarm</title>
9
  <link rel="icon" href="assets/einswarm-mascot.webp">
10
  <link rel="preload" href="fonts/outfit-latin-wght.woff2" as="font" type="font/woff2" crossorigin>
11
- <link rel="stylesheet" href="style.css?v=89">
12
  </head>
13
  <body>
14
  <canvas id="gl"></canvas>
@@ -49,6 +49,6 @@
49
 
50
  <a id="bmc" href="https://www.buymeacoffee.com/2btainment" target="_blank" rel="noopener noreferrer">Buy me a coffee</a>
51
  <noscript><p style="color:#f2c94c;padding:2em">STUDIUS braucht JavaScript.</p></noscript>
52
- <script type="module" src="app.js?v=89"></script>
53
  </body>
54
  </html>
 
8
  <title>STUDIUS · DAS Studio — Einswarm</title>
9
  <link rel="icon" href="assets/einswarm-mascot.webp">
10
  <link rel="preload" href="fonts/outfit-latin-wght.woff2" as="font" type="font/woff2" crossorigin>
11
+ <link rel="stylesheet" href="style.css?v=93">
12
  </head>
13
  <body>
14
  <canvas id="gl"></canvas>
 
49
 
50
  <a id="bmc" href="https://www.buymeacoffee.com/2btainment" target="_blank" rel="noopener noreferrer">Buy me a coffee</a>
51
  <noscript><p style="color:#f2c94c;padding:2em">STUDIUS braucht JavaScript.</p></noscript>
52
+ <script type="module" src="app.js?v=93"></script>
53
  </body>
54
  </html>
master.js CHANGED
@@ -1,7 +1,7 @@
1
  // Mastering studio (DoD): client-side WebAudio chain EQ → compressor → limiter, loudness normalisation to target LUFS
2
  // (ITU-R BS.1770-4 K-weighting + gating, measured offline), live meter, A/B, WAV/MP3 export, save to workspace.
3
  import { STUDIOS, onStop, loop, cv } from './widgets.js?v=50';
4
- import { result, MODULES, esc, setS, keep } from './real.js?v=89';
5
 
6
  /* ---------- BS.1770 loudness ---------- */
7
  function biquad(x, b0, b1, b2, a0, a1, a2) { const y = new Float32Array(x.length); b0 /= a0; b1 /= a0; b2 /= a0; a1 /= a0; a2 /= a0; let x1 = 0, x2 = 0, y1 = 0, y2 = 0;
 
1
  // Mastering studio (DoD): client-side WebAudio chain EQ → compressor → limiter, loudness normalisation to target LUFS
2
  // (ITU-R BS.1770-4 K-weighting + gating, measured offline), live meter, A/B, WAV/MP3 export, save to workspace.
3
  import { STUDIOS, onStop, loop, cv } from './widgets.js?v=50';
4
+ import { result, MODULES, esc, setS, keep } from './real.js?v=93';
5
 
6
  /* ---------- BS.1770 loudness ---------- */
7
  function biquad(x, b0, b1, b2, a0, a1, a2) { const y = new Float32Array(x.length); b0 /= a0; b1 /= a0; b2 /= a0; a1 /= a0; a2 /= a0; let x1 = 0, x2 = 0, y1 = 0, y2 = 0;
real.js CHANGED
@@ -29,7 +29,7 @@ async function runAnon(space, ep, data, { onStatus = () => {}, onData = () => {}
29
  async function runUser(space, ep, data, { onStatus = () => {}, onData = () => {}, timeout = 150000, el } = {}) {
30
  const t0 = performance.now(), f = new FormData(), args = {}; let n = 0;
31
  for (const [k, v] of Object.entries(data)) {
32
- if (v instanceof Blob) { const id = 'f' + n++; f.append(id, v, v.name || 'input'); args[k] = { __file__: id }; }
33
  else if (typeof v === 'string' && v.startsWith('/api/tmp/') && k !== 'prompt') args[k] = { __tmp__: v };
34
  else args[k] = v;
35
  }
@@ -59,7 +59,7 @@ export async function run(space, ep, data, opt = {}) { LASTM = space; let why =
59
  if (signedIn()) {
60
  const q = window.__quota; if (q?.ok && q.remaining < 60) notice(opt.el, `⚡ Nur noch <b>${Math.round(q.remaining)} s</b> deiner ZeroGPU-Quota${q.resetsAt ? ' · Reset ' + fmtReset(q.resetsAt) : ''} – evtl. Fallback.`);
61
  try { const d = await runUser(space, ep, data, opt); opt.el && via(opt.el, 'user'); refreshQuota(); return d; }
62
- catch (e) { if (e.message === 'abgebrochen' || (opt.el && !opt.el.isConnected)) throw e; console.warn('user-quota run failed, trying anonymous:', e.message); why = e.message;
63
  const q2 = await refreshQuota();
64
  notice(opt.el, isQuota(why) ? `⚠ <b>Deine ZeroGPU-Quota ist aufgebraucht</b>${(m => m ? ` (${m[1]} s angefragt, ${m[2]} s übrig)` : '')(why.match(/\((\d+)s requested vs\. (\d+)s left\)/))}${q2?.ok && q2.resetsAt ? ' · Reset ' + fmtReset(q2.resetsAt) : ''}. Fallback: <b>anonyme Quota</b> – kann langsamer sein oder ebenfalls scheitern. <a href="https://huggingface.co/pricing" target="_blank" rel="noopener">Mehr Quota (PRO)</a>` : `⚠ Lauf mit deinem Konto fehlgeschlagen (${esc(why.slice(0, 120))}). Fallback: anonyme Quota.`);
65
  opt.onStatus?.({ stage: 'pending', message: 'Fallback: anonyme Quota' }); }
 
29
  async function runUser(space, ep, data, { onStatus = () => {}, onData = () => {}, timeout = 150000, el } = {}) {
30
  const t0 = performance.now(), f = new FormData(), args = {}; let n = 0;
31
  for (const [k, v] of Object.entries(data)) {
32
+ if (v instanceof Blob) { const id = 'f' + n++; f.append(id, v, v.name || ('input.' + ({ 'image/jpeg': 'jpg', 'image/png': 'png', 'image/webp': 'webp', 'audio/wav': 'wav', 'audio/x-wav': 'wav', 'audio/wave': 'wav', 'audio/mpeg': 'mp3', 'audio/ogg': 'ogg', 'audio/flac': 'flac', 'video/mp4': 'mp4', 'video/webm': 'webm' }[(v.type || '').split(';')[0]] || 'bin'))); args[k] = { __file__: id }; }
33
  else if (typeof v === 'string' && v.startsWith('/api/tmp/') && k !== 'prompt') args[k] = { __tmp__: v };
34
  else args[k] = v;
35
  }
 
59
  if (signedIn()) {
60
  const q = window.__quota; if (q?.ok && q.remaining < 60) notice(opt.el, `⚡ Nur noch <b>${Math.round(q.remaining)} s</b> deiner ZeroGPU-Quota${q.resetsAt ? ' · Reset ' + fmtReset(q.resetsAt) : ''} – evtl. Fallback.`);
61
  try { const d = await runUser(space, ep, data, opt); opt.el && via(opt.el, 'user'); refreshQuota(); return d; }
62
+ catch (e) { if (e.message === 'abgebrochen' || (opt.el && !opt.el.isConnected)) throw e; if (opt.noAnonOn?.test(e.message)) throw e; console.warn('user-quota run failed, trying anonymous:', e.message); why = e.message;
63
  const q2 = await refreshQuota();
64
  notice(opt.el, isQuota(why) ? `⚠ <b>Deine ZeroGPU-Quota ist aufgebraucht</b>${(m => m ? ` (${m[1]} s angefragt, ${m[2]} s übrig)` : '')(why.match(/\((\d+)s requested vs\. (\d+)s left\)/))}${q2?.ok && q2.resetsAt ? ' · Reset ' + fmtReset(q2.resetsAt) : ''}. Fallback: <b>anonyme Quota</b> – kann langsamer sein oder ebenfalls scheitern. <a href="https://huggingface.co/pricing" target="_blank" rel="noopener">Mehr Quota (PRO)</a>` : `⚠ Lauf mit deinem Konto fehlgeschlagen (${esc(why.slice(0, 120))}). Fallback: anonyme Quota.`);
65
  opt.onStatus?.({ stage: 'pending', message: 'Fallback: anonyme Quota' }); }
server/infer.py CHANGED
@@ -16,6 +16,8 @@ ALLOW = { # space -> allowed endpoints (Module API backends)
16
  "declare-lab/TangoFlux": {"/predict"},
17
  "abidlabs/music-separation": {"/inference"},
18
  "Lightricks/ltx-video-distilled": {"/text_to_video"},
 
 
19
  "Qwen/Qwen3-TTS": {"/generate_voice_design", "/generate_voice_clone"},
20
  }
21
  PATH_ARGS = {("Lightricks/ltx-video-distilled", "/text_to_video"): {"input_image_filepath", "input_video_filepath"}} # str-typed filepath args: upload to the Space, pass its server path
 
16
  "declare-lab/TangoFlux": {"/predict"},
17
  "abidlabs/music-separation": {"/inference"},
18
  "Lightricks/ltx-video-distilled": {"/text_to_video"},
19
+ "multimodalart/MoDA-fast-talking-head": {"/generate_motion"},
20
+ "pragnakalp/Wav2lip-ZeroGPU": {"/run_infrence"},
21
  "Qwen/Qwen3-TTS": {"/generate_voice_design", "/generate_voice_clone"},
22
  }
23
  PATH_ARGS = {("Lightricks/ltx-video-distilled", "/text_to_video"): {"input_image_filepath", "input_video_filepath"}} # str-typed filepath args: upload to the Space, pass its server path
server/main.py CHANGED
@@ -311,6 +311,26 @@ async def transcode(req: Request, file: UploadFile = File(...), name: str = Form
311
  shutil.rmtree(d, ignore_errors=True)
312
  return res
313
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
314
  @app.get("/api/quota")
315
  async def quota(req: Request):
316
  """User's own ZeroGPU quota (GPU-seconds) from the Hub. Token stays server-side."""
 
311
  shutil.rmtree(d, ignore_errors=True)
312
  return res
313
 
314
+ @app.post("/api/stillvideo")
315
+ async def stillvideo(req: Request, image: UploadFile = File(...), audio: UploadFile = File(...), name: str = Form("talk")):
316
+ """Talking-head fallback without GPU: still avatar image + voice -> gentle zoom H.264/AAC MP4 (faststart)."""
317
+ import asyncio, shutil, subprocess
318
+ from .infer import tmp_put
319
+ need(req)
320
+ if not shutil.which("ffmpeg"): raise HTTPException(503, "ffmpeg not installed")
321
+ d = tempfile.mkdtemp(); im, au = f"{d}/in.img", f"{d}/in.aud"; stem = re.sub(r"[^\w.-]+", "-", Path(name).stem)[:80] or "talk"; out = f"{d}/{stem}.mp4"
322
+ for f, pth in ((image, im), (audio, au)):
323
+ b = await f.read(50 * 2**20 + 1)
324
+ if len(b) > 50 * 2**20: shutil.rmtree(d, ignore_errors=True); raise HTTPException(413, "max 50 MB")
325
+ Path(pth).write_bytes(b)
326
+ vf = "scale=1024:1024:force_original_aspect_ratio=increase,crop=1024:1024,zoompan=z='min(1.0+0.0008*on,1.15)':x='iw/2-(iw/zoom/2)':y='ih/2-(ih/zoom/2)':d=1:s=720x720:fps=30,format=yuv420p"
327
+ cmd = ["ffmpeg", "-hide_banner", "-loglevel", "error", "-y", "-loop", "1", "-i", im, "-i", au, "-vf", vf, "-c:v", "libx264", "-profile:v", "high", "-level", "4.1", "-preset", "veryfast", "-crf", "21", "-pix_fmt", "yuv420p",
328
+ "-c:a", "aac", "-b:a", "160k", "-ar", "48000", "-ac", "2", "-shortest", "-t", "120", "-movflags", "+faststart", out]
329
+ p = await asyncio.to_thread(subprocess.run, cmd, capture_output=True, text=True, timeout=300)
330
+ if p.returncode != 0 or not os.path.exists(out): shutil.rmtree(d, ignore_errors=True); raise HTTPException(422, "ffmpeg: " + (p.stderr or "failed")[-300:])
331
+ u = tmp_put(out); shutil.rmtree(d, ignore_errors=True)
332
+ return {"url": u}
333
+
334
  @app.get("/api/quota")
335
  async def quota(req: Request):
336
  """User's own ZeroGPU quota (GPU-seconds) from the Hub. Token stays server-side."""
video.js CHANGED
@@ -1,7 +1,7 @@
1
  // STUDIUS P3.4: Video Gen (LTX-Video, text→video + image→video), Video Edit (trim, soundtrack, title, fades → MP4/WebM export)
2
  // and Stems (Demucs split → synced mixer with gain/mute/solo → WAV mixdown). All results go on the asset bus (save / send).
3
- import { run, result, MODULES, esc, hf, stTxt, url, badge, timer, setS, S, keep } from './real.js?v=89';
4
- import { toWav } from './master.js?v=89';
5
  import { onStop } from './widgets.js?v=50';
6
 
7
  export const LTX = 'Lightricks/ltx-video-distilled';
 
1
  // STUDIUS P3.4: Video Gen (LTX-Video, text→video + image→video), Video Edit (trim, soundtrack, title, fades → MP4/WebM export)
2
  // and Stems (Demucs split → synced mixer with gain/mute/solo → WAV mixdown). All results go on the asset bus (save / send).
3
+ import { run, result, MODULES, esc, hf, stTxt, url, badge, timer, setS, S, keep } from './real.js?v=93';
4
+ import { toWav } from './master.js?v=93';
5
  import { onStop } from './widgets.js?v=50';
6
 
7
  export const LTX = 'Lightricks/ltx-video-distilled';
workspace.js CHANGED
@@ -1,6 +1,6 @@
1
  // STUDIUS empire backbone (frontend): HF sign-in, Workspace (user's HF Bucket), asset bus hand-off, bee agent dock.
2
- import { MODULES, refreshQuota } from './real.js?v=89';
3
- import { Avatars } from './avatar.js?v=89';
4
  const $ = (s, r = document) => r.querySelector(s);
5
  const esc = (t) => String(t ?? '').replace(/[&<>"']/g, c => ({ '&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;', "'": '&#39;' }[c]));
6
  const ST = () => window.__studius.STUDIOS;
@@ -148,7 +148,7 @@ const TEMPLATES = [
148
  { id: 'shop-team', name: 'Shop/Marketing-Team', icon: '🛍', task: 'Bewirb ein goldenes Bienen-T-Shirt: Produkttext, Produktfoto und kurzer Werbeclip',
149
  members: [['produkttexter', 'Produkttexter', 'Schreibt Produktbeschreibung (3 Sätze) und Slogan', ['chat']], ['produktfoto', 'Produktfotograf', 'Erzeugt ein Produktfoto im Studio-Look', ['image']], ['werbespot', 'Werbespot-Regie', 'Erzeugt aus dem Produktfoto-Prompt einen kurzen Clip', ['video']], ['archivar', 'Archivar', 'Speichert fertige Ergebnisse im Workspace', ['workspace']]] },
150
  { id: 'avatar-team', name: 'Avatar-Content-Team', icon: '👤', task: 'Der Avatar stellt sich in einem kurzen Clip vor',
151
- members: [['autor', 'Avatar-Autor', 'Schreibt in der Persona des Avatars einen Vorstellungstext (2 Sätze)', ['avatar']], ['fotograf', 'Avatar-Fotograf', 'Erzeugt ein Bild des Avatars in einer passenden Szene', ['avatar']], ['stimme', 'Avatar-Stimme', 'Spricht den Text mit der Stimme des Avatars', ['avatar']], ['archivar', 'Archivar', 'Speichert fertige Ergebnisse im Workspace', ['workspace']]] },
152
  ];
153
  async function fromTemplate(tid) { const T = TEMPLATES.find(t => t.id === tid); if (!T) return; const av = avList[0]?.id || '';
154
  const made = []; for (const [id, name, role, modules] of T.members) { const aid = (id === 'archivar' ? '' : T.id.split('-')[0] + '-') + id;
 
1
  // STUDIUS empire backbone (frontend): HF sign-in, Workspace (user's HF Bucket), asset bus hand-off, bee agent dock.
2
+ import { MODULES, refreshQuota } from './real.js?v=93';
3
+ import { Avatars } from './avatar.js?v=93';
4
  const $ = (s, r = document) => r.querySelector(s);
5
  const esc = (t) => String(t ?? '').replace(/[&<>"']/g, c => ({ '&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;', "'": '&#39;' }[c]));
6
  const ST = () => window.__studius.STUDIOS;
 
148
  { id: 'shop-team', name: 'Shop/Marketing-Team', icon: '🛍', task: 'Bewirb ein goldenes Bienen-T-Shirt: Produkttext, Produktfoto und kurzer Werbeclip',
149
  members: [['produkttexter', 'Produkttexter', 'Schreibt Produktbeschreibung (3 Sätze) und Slogan', ['chat']], ['produktfoto', 'Produktfotograf', 'Erzeugt ein Produktfoto im Studio-Look', ['image']], ['werbespot', 'Werbespot-Regie', 'Erzeugt aus dem Produktfoto-Prompt einen kurzen Clip', ['video']], ['archivar', 'Archivar', 'Speichert fertige Ergebnisse im Workspace', ['workspace']]] },
150
  { id: 'avatar-team', name: 'Avatar-Content-Team', icon: '👤', task: 'Der Avatar stellt sich in einem kurzen Clip vor',
151
+ members: [['autor', 'Avatar-Autor', 'Schreibt in der Persona des Avatars einen Vorstellungstext (2 Sätze)', ['avatar']], ['fotograf', 'Avatar-Fotograf', 'Erzeugt ein Bild des Avatars in einer passenden Szene', ['avatar']], ['stimme', 'Avatar-Sprecher', 'Macht aus dem Text ein sprechendes Video des Avatars (avatar.talk, Lip-Sync)', ['avatar']], ['archivar', 'Archivar', 'Speichert fertige Ergebnisse im Workspace', ['workspace']]] },
152
  ];
153
  async function fromTemplate(tid) { const T = TEMPLATES.find(t => t.id === tid); if (!T) return; const av = avList[0]?.id || '';
154
  const made = []; for (const [id, name, role, modules] of T.members) { const aid = (id === 'archivar' ? '' : T.id.split('-')[0] + '-') + id;