// STUDIUS P3.4: Video Gen (LTX-Video, text→video + image→video), Video Edit (trim, soundtrack, title, fades → MP4/WebM export) // and Stems (Demucs split → synced mixer with gain/mute/solo → WAV mixdown). All results go on the asset bus (save / send). import { run, result, MODULES, esc, hf, stTxt, url, badge, timer, setS, S, keep } from './real.js?v=136'; import { toWav } from './master.js?v=136'; import { onStop } from './widgets.js?v=50'; export const LTX = 'Lightricks/ltx-video-distilled'; const toBlob = async (u) => (await fetch(u, { credentials: 'include' })).blob(); const fmt = (s) => String(Math.floor(s / 60)).padStart(2, '0') + ':' + (s % 60).toFixed(1).padStart(4, '0'); /* ---------- VIDEO GEN ---------- */ export async function genVideo({ prompt, image, seconds = 2, size = '704x512' }, on = {}) { const [w, h] = size.split('x').map(Number); const d = await run(LTX, '/text_to_video', { prompt, negative_prompt: 'worst quality, inconsistent motion, blurry, jittery, distorted', input_image_filepath: image || null, input_video_filepath: null, height_ui: h, width_ui: w, mode: image ? 'image-to-video' : 'text-to-video', duration_ui: Math.max(1, Math.min(6, +seconds || 2)), ui_frames_to_use: 9, seed_ui: 42, randomize_seed: true, ui_guidance_scale: 1, improve_texture_flag: true }, { el: on.el, timeout: 240000, onStatus: on.status }); const v = d[0]?.video ?? d[0]; return { kind: 'video', mime: 'video/mp4', url: url(v), name: `ltx-${d[1] ?? Date.now()}.mp4`, studio: 'videogen', prompt, seed: d[1] }; } { const s = S('videogen'); s.real = 'API'; s.accepts = ['image']; s.build = (el) => { el.innerHTML = `
REALText → Video oder Bild → Video
kein Startbild → Text-zu-Video
powered by ${hf(LTX)} (LTX-Video distilled) · Ergebnis → Video Edit für Schnitt, Musik & Titel
`; keep(el); let img = null; const q = (x) => el.querySelector(x); const setImg = (f) => { img = f; GEN.img = f; q('.imgn').textContent = f ? '🖼 ' + (f.name || 'Startbild') + ' → Bild-zu-Video' : 'kein Startbild → Text-zu-Video'; q('.clr').hidden = !f; }; q('.img').onchange = () => setImg(q('.img').files[0] || null); q('.clr').onclick = () => setImg(null); el.__setImage = setImg; if (GEN.img) setImg(GEN.img); q('.go').onclick = async () => { const go = q('.go'); go.disabled = true; const tm = timer(el, '.stat'); setS(el, 'Starte…'); try { const a = await genVideo({ prompt: q('.pr').value.trim(), image: img, seconds: q('.dur').value, size: q('.sz').value }, { el, status: (m) => setS(el, stTxt(m)) }); q('.out').innerHTML = ``; const dl = q('.dl'); dl.href = a.url; dl.download = a.name; dl.hidden = false; window.__videoGen = { url: a.url, seed: a.seed, mode: img ? 'i2v' : 't2v' }; result(q('.out'), a); setS(el, `✔ ${img ? 'Bild' : 'Text'}-zu-Video · Seed ${a.seed}`); badge(el, 'REAL'); } catch (e) { if (!el.isConnected) return; setS(el, '⚠ ' + e.message, true); window.__videoGen = { err: e.message }; } finally { clearInterval(tm); go.disabled = false; } }; }; s.accept = async (el, a) => { const b = a.blob || await toBlob(a.url); el.__setImage?.(new File([b], a.name || 'start.webp', { type: b.type || 'image/webp' })); return true; }; } /* server-side H.264/AAC MP4 (faststart) – plays on iPhone/Instagram/TikTok */ export async function toH264(blob, name, onP = () => {}) { if (!window.__ws?.me?.signedIn) return null; onP('MP4 (H.264) wird auf dem Server erzeugt…'); const f = new FormData(); f.append('file', blob, name); f.append('name', name); const r = await fetch('/api/transcode', { method: 'POST', body: f, credentials: 'include' }); const j = await r.json().catch(() => ({})); if (!r.ok) throw new Error(j.detail || 'Transcode HTTP ' + r.status); return j; } /* ---------- VIDEO EDIT: trim, soundtrack, title, fades → export ---------- */ const recMime = () => (window.__ws?.me?.signedIn ? ['video/webm;codecs=vp8,opus', 'video/webm', 'video/webm;codecs=vp9,opus', 'video/mp4'] : ['video/mp4;codecs=avc1.42E01E,mp4a.40.2', 'video/webm;codecs=vp9,opus', 'video/webm;codecs=vp8,opus', 'video/webm', 'video/mp4']).find(m => window.MediaRecorder?.isTypeSupported?.(m)) || 'video/webm'; export async function renderEdit({ video, audio, a = 0, b = null, title = '', pos = 'bottom', fade = 0.5, music = 0.8, orig = 0.3 }, onP = () => {}) { const v = document.createElement('video'); v.src = video; v.crossOrigin = 'anonymous'; v.playsInline = true; v.preload = 'auto'; await new Promise((ok, no) => { v.onloadedmetadata = ok; v.onerror = () => no(new Error('Video nicht ladbar')); }); const end = Math.min(b ?? v.duration, v.duration), start = Math.max(0, Math.min(a, end - 0.2)), dur = end - start; const W = v.videoWidth || 704, H = v.videoHeight || 512, c = document.createElement('canvas'); c.width = W; c.height = H; const x = c.getContext('2d'); const ac = new AudioContext(), dst = ac.createMediaStreamDestination(); const og = ac.createGain(); og.gain.value = orig; try { ac.createMediaElementSource(v).connect(og).connect(dst); } catch (e) {} let ab = null; if (audio) { const buf = await ac.decodeAudioData(await (await toBlob(audio)).arrayBuffer()); ab = ac.createBufferSource(); ab.buffer = buf; const mg = ac.createGain(); ab.connect(mg).connect(dst); mg.gain.setValueAtTime(0, ac.currentTime); mg.gain.linearRampToValueAtTime(music, ac.currentTime + Math.max(.05, fade)); mg.gain.setValueAtTime(music, ac.currentTime + Math.max(.06, dur - fade)); mg.gain.linearRampToValueAtTime(0, ac.currentTime + dur); } const stream = new MediaStream([...c.captureStream(30).getVideoTracks(), ...dst.stream.getAudioTracks()]); const mime = recMime(), rec = new MediaRecorder(stream, { mimeType: mime, videoBitsPerSecond: 6e6 }), parts = []; rec.ondataavailable = (e) => e.data.size && parts.push(e.data); v.currentTime = start; await new Promise(ok => v.onseeked = ok); v.muted = false; v.volume = 1; const draw = () => { const t = v.currentTime - start; x.drawImage(v, 0, 0, W, H); const al = Math.min(1, t / Math.max(.01, fade), (dur - t) / Math.max(.01, fade)); if (fade > 0 && al < 1) { x.fillStyle = `rgba(0,0,0,${1 - Math.max(0, al)})`; x.fillRect(0, 0, W, H); } if (title) { const fs = Math.round(H / 12); x.font = `700 ${fs}px Outfit, sans-serif`; x.textAlign = 'center'; const y = pos === 'top' ? fs * 1.6 : pos === 'center' ? H / 2 : H - fs * 1.1; x.lineWidth = fs / 6; x.strokeStyle = 'rgba(0,0,0,.75)'; x.strokeText(title, W / 2, y); x.fillStyle = '#f2c94c'; x.fillText(title, W / 2, y); } onP(Math.min(1, t / dur)); }; const dbg = window.__recDbg = { mime, chunks: 0, err: null }; rec.onerror = (e) => dbg.err = String(e.error || e); const od = rec.ondataavailable; rec.ondataavailable = (e) => { dbg.chunks++; od(e); }; const done = new Promise(ok => rec.onstop = ok); rec.start(250); ab?.start(); await v.play(); await new Promise(ok => { const tick = () => { draw(); if (v.currentTime >= end - 0.03 || v.ended) return ok(); requestAnimationFrame(tick); }; tick(); }); draw(); await new Promise(r => setTimeout(r, 400)); v.pause(); ab?.stop(); dbg.frames = dbg.chunks; dbg.vt = v.currentTime; dbg.state = rec.state; if (rec.state !== 'inactive') { rec.requestData(); rec.stop(); } await done; ac.close(); const type = (rec.mimeType || mime).split(';')[0], blob = new Blob(parts, { type }); if (!blob.size) throw new Error('Aufnahme leer – Video-Codec im Browser nicht abspielbar?'); return { blob, type, codec: rec.mimeType || mime, ext: type.includes('mp4') ? 'mp4' : 'webm', dur, w: W, h: H }; } const EDIT = { v: null, vn: '', a: null, an: '' }, GEN = { img: null }, STM = { src: null, name: '' }; // survive studio re-open (hand-offs) { const s = S('videoedit'); s.real = 'LOKAL'; s.accepts = ['video', 'audio', 'image']; s.build = (el) => { el.innerHTML = `
LOKALVideo laden – oder aus Video Gen / Workspace senden
keine Musik

Noch kein Video.

Schnitt, Musik, Titel & Fades im Browser (Canvas + WebAudio → MediaRecorder, ${esc(recMime().split(';')[0])})
`; keep(el); const q = (x) => el.querySelector(x); let vu = null, au = null, vname = 'clip'; const setVideo = (u, name) => { vu = u; EDIT.v = u; EDIT.vn = name; vname = (name || 'clip').replace(/\.[^.]+$/, ''); q('.vw').innerHTML = ``; q('.ctl').hidden = false; const v = q('video'); v.onloadedmetadata = () => { sync(); window.__video = { dur: v.duration, w: v.videoWidth, h: v.videoHeight }; setS(el, `${name || 'Video'} · ${v.videoWidth}×${v.videoHeight} · ${v.duration.toFixed(1)} s`); }; v.ontimeupdate = () => { const [a, b] = T(); q('.tc').textContent = fmt(v.currentTime); if (v.currentTime >= b) { v.pause(); q('.pp').textContent = '▶ Vorschau'; } }; }; const setAudio = (u, name) => { au = u; EDIT.a = u; EDIT.an = name; q('.an').textContent = '🎵 ' + (name || 'Musik'); }; const T = () => { const d = q('video')?.duration || 0; return [q('.ti').value / 1000 * d, q('.to').value / 1000 * d]; }; const sync = () => { if (+q('.ti').value > +q('.to').value - 20) q('.ti').value = +q('.to').value - 20; const v = q('video'); if (!v) return; const [a, b] = T(); v.currentTime = a; setS(el, `Schnitt ${fmt(a)} – ${fmt(b)} (${(b - a).toFixed(1)} s)`); }; q('.ti').oninput = sync; q('.to').oninput = sync; q('.vf').onchange = () => { const f = q('.vf').files[0]; if (!f) return; setVideo(URL.createObjectURL(f), f.name); }; q('.af').onchange = () => { const f = q('.af').files[0]; if (!f) return; setAudio(URL.createObjectURL(f), f.name); }; q('.pp').onclick = () => { const v = q('video'); if (!v) return; if (v.paused) { v.currentTime = T()[0]; v.play(); q('.pp').textContent = '⏸ Stop'; } else { v.pause(); q('.pp').textContent = '▶ Vorschau'; } }; q('.go').onclick = async () => { if (!vu) return; const go = q('.go'); go.disabled = true; q('video')?.pause(); const [a, b] = T(); try { const r = await renderEdit({ video: vu, audio: au, a, b, title: q('.tt').value.trim(), pos: q('.tp').value, fade: +q('.fd').value, music: q('.mv').value / 100, orig: q('.ov').value / 100 }, p => setS(el, `Rendere… ${Math.round(p * 100)} %`)); const u = URL.createObjectURL(r.blob); onStop(() => URL.revokeObjectURL(u)); const stem = `${vname}-edit-${Date.now()}`, raw = `${stem}.${r.ext === 'mp4' ? 'rec.mp4' : 'webm'}`; let h = null, herr = ''; try { h = await toH264(r.blob, raw, m => setS(el, m)); } catch (e) { herr = e.message; } const fu = h ? h.url : u, fname = h ? stem + '.mp4' : raw, vs = h?.probe?.streams?.find(x => x.codec_type === 'video'); q('.out').innerHTML = `
${h ? `⬇ MP4 H.264/AAC (${(h.size / 1e6).toFixed(1)} MB)` : ''}⬇ Roh-${r.ext.toUpperCase()}
${h ? 'iPhone · Instagram · TikTok-tauglich (faststart)' : `${herr ? '⚠ H.264: ' + esc(herr) : 'Für H.264-MP4 anmelden'} – Roh-Export im Browser-Codec.`}`; window.__videoEdit = { size: h?.size || r.blob.size, type: h ? 'video/mp4' : r.type, codec: h ? `${vs?.codec_name} ${vs?.profile} ${vs?.pix_fmt}` : r.codec, h264: !!h, url: fu, dur: r.dur, music: !!au, title: q('.tt').value, err: herr || undefined }; result(q('.out'), h ? { kind: 'video', mime: 'video/mp4', url: h.url, name: fname, studio: 'videoedit', model: 'ffmpeg libx264+aac (server)', prompt: q('.tt').value } : { kind: 'video', mime: r.type, blob: r.blob, url: u, name: raw, studio: 'videoedit', prompt: q('.tt').value }); setS(el, `✔ Export ${h ? 'MP4 H.264' : r.ext.toUpperCase()} · ${r.dur.toFixed(1)} s${au ? ' · mit Musik' : ''}${q('.tt').value ? ' · mit Titel' : ''}`); badge(el, 'LOKAL'); } catch (e) { setS(el, '⚠ ' + e.message, true); window.__videoEdit = { err: e.message }; } finally { go.disabled = false; } }; el.__setVideo = setVideo; el.__setAudio = setAudio; if (EDIT.v) setVideo(EDIT.v, EDIT.vn); if (EDIT.a) setAudio(EDIT.a, EDIT.an); }; s.accept = async (el, a) => { const u = a.blob ? URL.createObjectURL(a.blob) : a.url; if (a.kind === 'video') el.__setVideo(u, a.name); else if (a.kind === 'audio') el.__setAudio(u, a.name); else { const vg = S('videogen'); setS(el, '🖼 Bilder bitte über Video Gen (Bild → Video) animieren.'); } return true; }; } /* ---------- STEMS: Demucs split → synced mixer → mixdown ---------- */ const SEP = 'abidlabs/music-separation'; { const s = S('stems'); s.real = 'API'; s.accepts = ['audio']; s.build = (el) => { el.innerHTML = `
REALSong laden – oder aus Music / Workspace senden
kein Audio
powered by ${hf(SEP)} (Demucs) · Mixer & Mixdown lokal (WebAudio) · Mix → Mastering
`; keep(el); const q = (x) => el.querySelector(x); let src = null, srcName = 'song'; const setSrc = (blob, name) => { src = blob; STM.src = blob; STM.name = name; srcName = (name || 'song').replace(/\.[^.]+$/, ''); q('.an').textContent = '🎵 ' + (name || 'Audio') + ' · ' + (blob.size / 1e6).toFixed(1) + ' MB'; q('.go').disabled = false; }; q('.af').onchange = () => { const f = q('.af').files[0]; if (f) setSrc(f, f.name); }; el.__setSrc = setSrc; if (STM.src) setSrc(STM.src, STM.name); q('.go').onclick = async () => { if (!src) return; if (src.size > 25e6) return setS(el, '⚠ Datei > 25 MB', true); const go = q('.go'); go.disabled = true; const tm = timer(el, '.stat'); setS(el, 'Lade hoch…'); try { const d = await run(SEP, '/inference', { audio: new File([src], srcName + '.wav', { type: src.type || 'audio/wav' }) }, { el, timeout: 240000, onStatus: m => setS(el, stTxt(m)) }); const names = d.length === 2 ? ['Vocals', 'Instrumental'] : ['Vocals', 'Drums', 'Bass', 'Other'].slice(0, d.length); const urls = d.map(url); await mixer(el, names.map((n, i) => ({ name: n, url: urls[i] })), srcName); window.__stems = urls; setS(el, `✔ ${urls.length} Stems · Mixer bereit`); badge(el, 'REAL'); } catch (e) { if (!el.isConnected) return; setS(el, '⚠ ' + e.message, true); window.__stems = { err: e.message }; } finally { clearInterval(tm); go.disabled = false; } }; }; s.accept = async (el, a) => { const b = a.blob || await toBlob(a.url); el.__setSrc(b, a.name); return true; }; } async function mixer(el, stems, base) { const ac = new AudioContext(); onStop(() => ac.close()); const box = el.querySelector('.mix'); const bufs = await Promise.all(stems.map(async s => ac.decodeAudioData(await (await toBlob(s.url)).arrayBuffer()))); const st = stems.map((s, i) => ({ ...s, buf: bufs[i], gain: 1, mute: false, solo: false })); box.innerHTML = `
${st.map((s, i) => `
${esc(s.name)}
`).join('')}
00:00.0
`; st.forEach((s, i) => result(box.querySelectorAll('.stem .sv')[i], { kind: 'audio', url: s.url, name: `${base}-${s.name.toLowerCase()}.wav`, studio: 'stems', prompt: s.name })); const eff = (s) => (s.mute || (st.some(x => x.solo) && !s.solo)) ? 0 : s.gain; let nodes = null, t0 = 0, raf = 0; const stop = () => { nodes?.forEach(n => { try { n.src.stop(); } catch (e) {} }); nodes = null; cancelAnimationFrame(raf); box.querySelector('.pl').textContent = '▶ Mix abspielen'; }; const play = () => { stop(); t0 = ac.currentTime; nodes = st.map(s => { const src = ac.createBufferSource(); src.buffer = s.buf; const g = ac.createGain(); g.gain.value = eff(s); src.connect(g).connect(ac.destination); src.start(); return { src, g }; }); nodes[0].src.onended = stop; box.querySelector('.pl').textContent = '⏹ Stop'; const tick = () => { box.querySelector('.tc').textContent = fmt(ac.currentTime - t0); raf = requestAnimationFrame(tick); }; tick(); }; const upd = () => { nodes?.forEach((n, i) => n.g.gain.setTargetAtTime(eff(st[i]), ac.currentTime, .02)); box.querySelectorAll('.stem').forEach((r, i) => { r.classList.toggle('off', eff(st[i]) === 0); r.querySelector('.m').setAttribute('aria-pressed', st[i].mute); r.querySelector('.so').setAttribute('aria-pressed', st[i].solo); }); window.__mix = st.map(s => ({ name: s.name, gain: eff(s) })); }; box.querySelector('.stems').oninput = (e) => { const r = e.target.closest('.stem'); if (r && e.target.classList.contains('g')) { st[r.dataset.i].gain = e.target.value / 100; upd(); } }; box.querySelector('.stems').onclick = (e) => { const r = e.target.closest('.stem'); if (!r) return; const s = st[r.dataset.i]; if (e.target.classList.contains('m')) s.mute = !s.mute; else if (e.target.classList.contains('so')) s.solo = !s.solo; else return; upd(); }; box.querySelector('.pl').onclick = () => nodes ? stop() : play(); box.querySelector('.go2').onclick = async () => { const n = Math.max(...bufs.map(b => b.length)), sr = bufs[0].sampleRate, oc = new OfflineAudioContext(2, n, sr); st.forEach(s => { const src = oc.createBufferSource(); src.buffer = s.buf; const g = oc.createGain(); g.gain.value = eff(s); src.connect(g).connect(oc.destination); src.start(); }); const out = await oc.startRendering(); let pk = 0; for (let c = 0; c < out.numberOfChannels; c++) { const d = out.getChannelData(c); for (let i = 0; i < d.length; i++) pk = Math.max(pk, Math.abs(d[i])); } if (pk > .99) for (let c = 0; c < out.numberOfChannels; c++) { const d = out.getChannelData(c); for (let i = 0; i < d.length; i++) d[i] *= .99 / pk; } const blob = toWav(out), u = URL.createObjectURL(blob); onStop(() => URL.revokeObjectURL(u)); const name = `${base}-mix-${Date.now()}.wav`, tag = st.map(s => s.name + ':' + Math.round(eff(s) * 100)).join(' '); const mo = box.querySelector('.mo'); mo.innerHTML = `
⬇ Mix WAV
`; window.__mixdown = { size: blob.size, tag, limited: pk > .99 }; result(mo, { kind: 'audio', mime: 'audio/wav', blob, url: u, name, studio: 'stems', prompt: 'Mix ' + tag }); setS(el, '✔ Mixdown · ' + tag); }; upd(); } /* ---------- Module API ---------- */ MODULES.video = { id: 'video', name: 'Video', backend: LTX + ' + lokaler Schnitt', inputs: [{ name: 'prompt', type: 'text' }, { name: 'image', type: 'image-url' }], outputs: [{ kind: 'video' }], actions: { generate: { desc: 'Erzeugt ein 2–5 s Video aus einem Prompt (optional Startbild-url eines vorherigen Bildes)', params: { prompt: 'string', seconds: 'number' }, run: async ({ prompt, seconds = 2, image }, on = {}) => genVideo({ prompt, seconds, image: image ? await toBlob(image) : null }, { status: on.status }) }, score: { desc: 'Legt Musik (audio url) unter ein Video (video url), optional mit Titel, und exportiert es', params: { video: 'string', audio: 'string', title: 'string' }, run: async ({ video, audio, title = '' }) => { const r = await renderEdit({ video, audio, title }); const n = `scored-${Date.now()}`; const h = await toH264(r.blob, n + '.webm').catch(() => null); if (h) return { kind: 'video', mime: 'video/mp4', url: h.url, name: n + '.mp4', studio: 'videoedit', prompt: title, text: `Video mit Musik (${r.dur.toFixed(1)} s, H.264)` }; const u = URL.createObjectURL(r.blob); return { kind: 'video', mime: r.type, blob: r.blob, url: u, name: `${n}.${r.ext}`, studio: 'videoedit', prompt: title, text: `Video mit Musik (${r.dur.toFixed(1)} s)` }; } } } }; MODULES.stems = { id: 'stems', name: 'Stems', backend: SEP, inputs: [{ name: 'source', type: 'audio-url' }], outputs: [{ kind: 'audio' }], actions: { split: { desc: 'Trennt ein Audio (url) in Vocals + Instrumental; liefert das Instrumental', params: { source: 'string' }, run: async ({ source }, on = {}) => { const d = await run(SEP, '/inference', { audio: new File([await toBlob(source)], 'in.wav', { type: 'audio/wav' }) }, { onStatus: on.status, timeout: 240000 }); const u = d.map(url); return { kind: 'audio', url: u[1] || u[0], stems: u, name: `instrumental-${Date.now()}.wav`, studio: 'stems', text: `${u.length} Stems getrennt` }; } } } };