Spaces:
Running on Zero
Running on Zero
Two tabs (Image, one-button Video), sharper 1152x768 clips, better narration planner
Browse files
README.md
CHANGED
|
@@ -12,11 +12,15 @@ short_description: FLUX image, LTX video, MusicGen music, Kokoro+Qwen narration
|
|
| 12 |
|
| 13 |
# Graphics Studio
|
| 14 |
|
| 15 |
-
|
| 16 |
-
2. **LTX-Video** (Lightricks) animates that still, or any uploaded image, into a 2 to 8 second clip at 768×512, 10 fps.
|
| 17 |
-
3. **MusicGen medium** (facebook) writes a soundtrack for the clip from a text description.
|
| 18 |
-
4. **Qwen2.5 1.5B** plans the narration (what to say and when), and **Kokoro** speaks it over the music.
|
| 19 |
|
| 20 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 21 |
|
| 22 |
Runs on ZeroGPU: set the Space hardware to **ZeroGPU** in Settings.
|
|
|
|
| 12 |
|
| 13 |
# Graphics Studio
|
| 14 |
|
| 15 |
+
**Image tab:** FLUX.1-schnell (black-forest-labs) turns a prompt into a 1152×768 still in 4 steps.
|
|
|
|
|
|
|
|
|
|
| 16 |
|
| 17 |
+
**Video tab:** one button makes a whole clip from a scene and a motion description.
|
| 18 |
+
|
| 19 |
+
1. **FLUX.1-schnell** draws the scene (or you supply a start image).
|
| 20 |
+
2. **LTX-Video 0.9.8 13B distilled** (Lightricks) animates it: 2 to 8 seconds, 1152×768, 10 fps.
|
| 21 |
+
3. **MusicGen medium** (facebook) adds a soundtrack if you describe one.
|
| 22 |
+
4. **Qwen2.5 1.5B** plans the narration (what to say and when) and **Kokoro** speaks it, if narration is ticked.
|
| 23 |
+
|
| 24 |
+
FLUX.1-schnell, LTX-Video, Qwen and Kokoro are openly licensed; MusicGen medium is CC-BY-NC 4.0 (non-commercial).
|
| 25 |
|
| 26 |
Runs on ZeroGPU: set the Space hardware to **ZeroGPU** in Settings.
|
app.py
CHANGED
|
@@ -1,51 +1,40 @@
|
|
| 1 |
-
"""
|
| 2 |
|
| 3 |
-
|
| 4 |
-
|
| 5 |
-
|
| 6 |
-
|
| 7 |
-
Kokoro speaks it, and it is mixed over the music.
|
| 8 |
"""
|
| 9 |
import os
|
| 10 |
|
| 11 |
|
| 12 |
def _pick_model_store():
|
| 13 |
-
"""
|
| 14 |
-
|
| 15 |
-
|
| 16 |
-
|
| 17 |
-
|
| 18 |
-
|
| 19 |
-
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
|
| 23 |
-
os.remove(probe)
|
| 24 |
-
return os.path.join(path, "huggingface")
|
| 25 |
-
except OSError:
|
| 26 |
-
continue
|
| 27 |
-
return None
|
| 28 |
|
| 29 |
|
| 30 |
_store = _pick_model_store()
|
| 31 |
if _store:
|
| 32 |
os.environ["HF_HOME"] = _store # Must be set before the Hugging Face libraries are imported
|
| 33 |
-
print("MODEL STORE:", _store or "none found, using the default temporary cache",
|
| 34 |
-
"| mounts seen:", [m for m in ("/data", "/bucket", "/storage", "/mnt") if os.path.isdir(m)], flush=True)
|
| 35 |
|
| 36 |
-
import json
|
| 37 |
import random
|
| 38 |
import re
|
| 39 |
import subprocess
|
| 40 |
|
| 41 |
import gradio as gr
|
| 42 |
-
import imageio
|
| 43 |
import imageio_ffmpeg
|
| 44 |
import numpy as np
|
| 45 |
import scipy.io.wavfile
|
| 46 |
import spaces
|
| 47 |
import torch
|
| 48 |
-
from PIL import Image
|
| 49 |
from diffusers import FluxPipeline, LTXConditionPipeline
|
| 50 |
from diffusers.pipelines.ltx.pipeline_ltx_condition import LTXVideoCondition
|
| 51 |
from diffusers.utils import export_to_video
|
|
@@ -54,17 +43,17 @@ from transformers import AutoModelForCausalLM, AutoProcessor, AutoTokenizer, Mus
|
|
| 54 |
|
| 55 |
IMAGE_MODEL = "black-forest-labs/FLUX.1-schnell"
|
| 56 |
VIDEO_MODEL = "Lightricks/LTX-Video-0.9.8-13B-distilled"
|
| 57 |
-
MUSIC_MODEL = "facebook/musicgen-medium"
|
| 58 |
PLANNER_MODEL = "Qwen/Qwen2.5-1.5B-Instruct" # Writes the narration and its timing
|
| 59 |
VOICE = "af_heart" # Kokoro voice
|
| 60 |
VOICE_RATE = 24000
|
| 61 |
WORDS_PER_SECOND = 2.5
|
| 62 |
|
| 63 |
IMAGE_WIDTH, IMAGE_HEIGHT = 1152, 768 # 3:2, multiples of 16 for FLUX
|
| 64 |
-
VIDEO_WIDTH, VIDEO_HEIGHT =
|
|
|
|
| 65 |
FPS = 10
|
| 66 |
TRIM_FRAMES = 8 # Frames cut from the start of every clip, where the noise is
|
| 67 |
-
FADE_FRAMES = 0 # Loop cross-fade switched off (set to 20 for a 2-second fade)
|
| 68 |
MAX_SEED = 2**31 - 1
|
| 69 |
|
| 70 |
NEGATIVE_PROMPT = ("worst quality, inconsistent motion, blurry, blurry motion, motion blur, "
|
|
@@ -72,15 +61,14 @@ NEGATIVE_PROMPT = ("worst quality, inconsistent motion, blurry, blurry motion, m
|
|
| 72 |
NEGATIVE_SLOW = ", slow motion, slow-motion video" # Left out when the motion text asks for slow motion
|
| 73 |
# The distilled model runs without guidance, so the negative prompt alone has no effect on it.
|
| 74 |
# These words are added to every prompt to get the same result from the positive side.
|
| 75 |
-
STYLE_SUFFIX = "Sharp, crisp
|
| 76 |
SPEED_SUFFIX = ", moving at natural real-time speed" # Also left out when slow motion is asked for
|
| 77 |
|
| 78 |
-
|
| 79 |
# Loaded once at start-up. ZeroGPU attaches the GPU only while a @spaces.GPU function runs.
|
|
|
|
|
|
|
| 80 |
image_pipe = FluxPipeline.from_pretrained(IMAGE_MODEL, torch_dtype=torch.bfloat16).to("cuda")
|
| 81 |
-
# FLUX
|
| 82 |
-
# slice, so both steps ask for the whole card (size='xlarge'; it counts double against the quota).
|
| 83 |
-
# Both models use the same T5-XXL text encoder (9.5 GB), so LTX-Video borrows FLUX's copy.
|
| 84 |
video_pipe = LTXConditionPipeline.from_pretrained(
|
| 85 |
VIDEO_MODEL, text_encoder=image_pipe.text_encoder_2, torch_dtype=torch.bfloat16).to("cuda")
|
| 86 |
video_pipe.vae.enable_tiling() # Decode the video in tiles to keep memory low
|
|
@@ -95,12 +83,10 @@ def pick_seed(seed, randomize):
|
|
| 95 |
return random.randint(0, MAX_SEED) if randomize else int(seed)
|
| 96 |
|
| 97 |
|
| 98 |
-
|
| 99 |
-
|
| 100 |
-
|
| 101 |
-
|
| 102 |
-
seed = pick_seed(seed, randomize_seed)
|
| 103 |
-
image = image_pipe(
|
| 104 |
prompt=prompt,
|
| 105 |
width=IMAGE_WIDTH,
|
| 106 |
height=IMAGE_HEIGHT,
|
|
@@ -109,50 +95,26 @@ def make_image(prompt, seed, randomize_seed):
|
|
| 109 |
max_sequence_length=256,
|
| 110 |
generator=torch.Generator("cpu").manual_seed(seed),
|
| 111 |
).images[0]
|
| 112 |
-
return image, seed
|
| 113 |
|
| 114 |
|
| 115 |
-
def
|
| 116 |
-
"""
|
| 117 |
-
and those closing frames are then dropped. The last frame shown flows into the first."""
|
| 118 |
-
n = min(FADE_FRAMES, len(frames) // 3)
|
| 119 |
-
if n < 2:
|
| 120 |
-
return frames
|
| 121 |
-
body = frames[:len(frames) - n]
|
| 122 |
-
tail = frames[len(frames) - n:]
|
| 123 |
-
for i in range(n):
|
| 124 |
-
body[i] = Image.blend(tail[i], body[i], (i + 1) / (n + 1))
|
| 125 |
-
return body
|
| 126 |
-
|
| 127 |
-
|
| 128 |
-
def video_duration(image, scene_prompt, motion_prompt, seconds, seed, randomize_seed):
|
| 129 |
-
"""Ask ZeroGPU for only as much GPU time as this clip needs."""
|
| 130 |
-
return int(20 + seconds * 3)
|
| 131 |
-
|
| 132 |
-
|
| 133 |
-
@spaces.GPU(duration=video_duration, size='xlarge')
|
| 134 |
-
def make_video(image, scene_prompt, motion_prompt, seconds, seed, randomize_seed):
|
| 135 |
-
if image is None:
|
| 136 |
-
raise gr.Error("Make or upload an image first.")
|
| 137 |
-
seed = pick_seed(seed, randomize_seed)
|
| 138 |
# Generate the clip plus the frames that get cut off. LTX-Video wants 8n+1 frames.
|
| 139 |
-
num_frames = -(-(int(seconds * FPS) + TRIM_FRAMES
|
| 140 |
-
|
| 141 |
-
motion = motion_prompt.strip() or "gentle cinematic camera motion, natural movement"
|
| 142 |
-
# The motion comes first so the model gives it the most weight
|
| 143 |
# Only the phrase itself counts: "slow motion", "slow-motion", "slow mo". "Slowly" does not.
|
| 144 |
wants_slow = re.search(r"slow[\s-]?mo", motion.lower()) is not None
|
| 145 |
style = STYLE_SUFFIX + ("" if wants_slow else SPEED_SUFFIX) + "."
|
| 146 |
-
|
| 147 |
-
prompt = f"{motion}. {
|
| 148 |
start = LTXVideoCondition(image=image.convert("RGB").resize((VIDEO_WIDTH, VIDEO_HEIGHT)), frame_index=0)
|
| 149 |
frames = video_pipe(
|
| 150 |
conditions=[start],
|
| 151 |
prompt=prompt,
|
|
|
|
| 152 |
image_cond_noise_scale=0.0,
|
| 153 |
decode_timestep=0.05,
|
| 154 |
decode_noise_scale=0.025,
|
| 155 |
-
negative_prompt=negative,
|
| 156 |
width=VIDEO_WIDTH,
|
| 157 |
height=VIDEO_HEIGHT,
|
| 158 |
num_frames=num_frames,
|
|
@@ -162,40 +124,15 @@ def make_video(image, scene_prompt, motion_prompt, seconds, seed, randomize_seed
|
|
| 162 |
guidance_rescale=0.7,
|
| 163 |
generator=torch.Generator("cpu").manual_seed(seed),
|
| 164 |
).frames[0]
|
| 165 |
-
|
| 166 |
-
|
| 167 |
-
|
| 168 |
-
|
| 169 |
-
|
| 170 |
-
def clip_seconds(video):
|
| 171 |
-
try:
|
| 172 |
-
return float(imageio.get_reader(video).get_meta_data()["duration"])
|
| 173 |
-
except Exception:
|
| 174 |
-
return 6.0
|
| 175 |
-
|
| 176 |
-
|
| 177 |
-
def mix(video, music_wav, voice_wav):
|
| 178 |
-
"""Put whichever sound tracks exist onto the silent clip. With both, the music plays
|
| 179 |
-
quietly under the voice. Called after every sound step, so the order never matters."""
|
| 180 |
-
ffmpeg = [imageio_ffmpeg.get_ffmpeg_exe(), "-y", "-i", video]
|
| 181 |
-
if music_wav and voice_wav:
|
| 182 |
-
ffmpeg += ["-i", voice_wav, "-i", music_wav, "-filter_complex",
|
| 183 |
-
"[1:a]aresample=44100,volume=1.6[v];[2:a]aresample=44100,volume=0.35[m];"
|
| 184 |
-
"[v][m]amix=inputs=2:duration=longest:normalize=0[a]", "-map", "0:v:0", "-map", "[a]"]
|
| 185 |
-
else:
|
| 186 |
-
ffmpeg += ["-i", music_wav or voice_wav, "-map", "0:v:0", "-map", "1:a:0"]
|
| 187 |
-
out_path = f"/tmp/mixed_{random.randint(0, MAX_SEED)}.mp4"
|
| 188 |
-
subprocess.run(ffmpeg + ["-c:v", "copy", "-c:a", "aac", "-shortest", out_path], check=True, capture_output=True)
|
| 189 |
-
return out_path
|
| 190 |
-
|
| 191 |
-
|
| 192 |
-
def music_duration(video, music_prompt):
|
| 193 |
-
return int(15 + clip_seconds(video) * 3) if video else 15
|
| 194 |
|
| 195 |
|
| 196 |
-
|
| 197 |
-
|
| 198 |
-
seconds = min(max(
|
| 199 |
inputs = music_processor(text=[music_prompt], padding=True, return_tensors="pt").to("cuda")
|
| 200 |
audio = music_model.generate(**inputs, do_sample=True, guidance_scale=3.0,
|
| 201 |
max_new_tokens=int(seconds * 50) + 8) # MusicGen makes 50 tokens a second
|
|
@@ -209,47 +146,40 @@ def compose_music(video, music_prompt):
|
|
| 209 |
return wav_path
|
| 210 |
|
| 211 |
|
| 212 |
-
def add_music(video, music_prompt, voice_wav):
|
| 213 |
-
if not video:
|
| 214 |
-
raise gr.Error("Animate an image first.")
|
| 215 |
-
if not music_prompt.strip():
|
| 216 |
-
raise gr.Error("Describe the music first.")
|
| 217 |
-
music_wav = compose_music(video, music_prompt)
|
| 218 |
-
return mix(video, music_wav, voice_wav), music_wav
|
| 219 |
-
|
| 220 |
-
|
| 221 |
-
@spaces.GPU(duration=20, size='xlarge')
|
| 222 |
def plan_narration(scene, motion, seconds):
|
| 223 |
"""The planning model writes 1 to 3 short lines and picks the second each one starts."""
|
| 224 |
-
budget = int(seconds * WORDS_PER_SECOND)
|
| 225 |
-
system = ("You
|
| 226 |
-
|
| 227 |
-
|
| 228 |
-
|
| 229 |
-
"
|
| 230 |
-
"
|
| 231 |
-
|
|
|
|
|
|
|
|
|
|
| 232 |
text = planner_tokenizer.apply_chat_template(
|
| 233 |
[{"role": "system", "content": system}, {"role": "user", "content": user}],
|
| 234 |
tokenize=False, add_generation_prompt=True)
|
| 235 |
inputs = planner_tokenizer([text], return_tensors="pt").to("cuda")
|
| 236 |
-
out = planner_model.generate(**inputs, max_new_tokens=
|
| 237 |
return planner_tokenizer.decode(out[0][inputs.input_ids.shape[1]:], skip_special_tokens=True)
|
| 238 |
|
| 239 |
|
| 240 |
-
|
| 241 |
-
|
|
|
|
|
|
|
|
|
|
| 242 |
plan = []
|
| 243 |
-
|
| 244 |
-
|
| 245 |
-
|
| 246 |
-
|
| 247 |
-
plan.append((max(0.0, float(item["start"])), line))
|
| 248 |
-
except Exception:
|
| 249 |
-
plan = []
|
| 250 |
if not plan:
|
| 251 |
-
words = (
|
| 252 |
-
plan = [(0.5, " ".join(words))]
|
| 253 |
return sorted(plan)[:3]
|
| 254 |
|
| 255 |
|
|
@@ -258,16 +188,11 @@ def speak(text):
|
|
| 258 |
return np.concatenate(chunks) if chunks else np.zeros(0, dtype=np.float32)
|
| 259 |
|
| 260 |
|
| 261 |
-
def
|
| 262 |
-
|
| 263 |
-
raise gr.Error("Animate an image first.")
|
| 264 |
-
seconds = clip_seconds(video)
|
| 265 |
-
reply = plan_narration(scene, motion, seconds)
|
| 266 |
-
plan = parse_plan(reply, scene, seconds)
|
| 267 |
-
|
| 268 |
track = np.zeros(int(seconds * VOICE_RATE), dtype=np.float32)
|
| 269 |
cursor, script = 0.0, []
|
| 270 |
-
for start, line in
|
| 271 |
voice = speak(line)
|
| 272 |
start = max(start, cursor) # Never talk over the previous line
|
| 273 |
begin = int(start * VOICE_RATE)
|
|
@@ -279,60 +204,116 @@ def add_narration(video, music_wav, scene, motion):
|
|
| 279 |
cursor = start + len(voice) / VOICE_RATE + 0.3
|
| 280 |
script.append(f"{start:4.1f} s {line}")
|
| 281 |
if not script:
|
| 282 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 283 |
|
| 284 |
-
|
| 285 |
-
|
| 286 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 287 |
|
| 288 |
|
| 289 |
with gr.Blocks(title="Graphics Studio") as demo:
|
| 290 |
-
|
| 291 |
-
|
| 292 |
-
|
| 293 |
-
gr.
|
| 294 |
-
|
| 295 |
-
|
| 296 |
-
|
| 297 |
-
|
| 298 |
-
|
| 299 |
-
|
| 300 |
-
|
| 301 |
-
|
| 302 |
-
|
| 303 |
-
|
| 304 |
-
|
| 305 |
-
|
| 306 |
-
|
| 307 |
-
|
| 308 |
-
|
| 309 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 310 |
seconds = gr.Slider(2, 8, value=5, step=1, label="Length (seconds)")
|
| 311 |
-
|
| 312 |
-
|
| 313 |
-
|
| 314 |
-
|
| 315 |
-
|
| 316 |
-
|
| 317 |
-
|
| 318 |
-
|
| 319 |
-
|
| 320 |
-
narration_script = gr.Textbox(label="Narration plan (start time and line)", lines=3, interactive=False)
|
| 321 |
-
|
| 322 |
-
gr.Examples(
|
| 323 |
-
examples=[
|
| 324 |
-
["A breathtaking aurora over a frozen lake at night, snowy mountains, ultra detailed"],
|
| 325 |
-
["A lone lighthouse on a cliff during a thunderstorm, giant waves, cinematic lighting"],
|
| 326 |
-
["A floating island with waterfalls at sunrise, golden clouds, fantasy landscape"],
|
| 327 |
-
],
|
| 328 |
-
inputs=[image_prompt],
|
| 329 |
-
)
|
| 330 |
|
| 331 |
image_button.click(make_image, [image_prompt, image_seed, image_random], [image_out, image_seed])
|
| 332 |
-
video_button.click(make_video,
|
| 333 |
-
[
|
| 334 |
-
|
| 335 |
-
|
| 336 |
-
[video_out, narration_script, voice_track])
|
| 337 |
|
| 338 |
demo.queue().launch()
|
|
|
|
| 1 |
+
"""Graphics Studio on ZeroGPU.
|
| 2 |
|
| 3 |
+
Image tab: FLUX.1-schnell turns a prompt into a still image.
|
| 4 |
+
Video tab: one button makes a whole clip. FLUX draws the scene, LTX-Video (13B) animates it,
|
| 5 |
+
MusicGen medium adds a soundtrack if you describe one, and if narration is ticked a small
|
| 6 |
+
planning model (Qwen2.5 1.5B) writes the lines and their timing and Kokoro speaks them.
|
|
|
|
| 7 |
"""
|
| 8 |
import os
|
| 9 |
|
| 10 |
|
| 11 |
def _pick_model_store():
|
| 12 |
+
"""Off unless MODEL_STORE is set. Tried /data (the attached bucket) on 2026-10-04: it reads at
|
| 13 |
+
about 8 MB/s, so loading 70 GB of models from it takes hours. The local disk downloads at 1 GB/s."""
|
| 14 |
+
path = os.environ.get("MODEL_STORE")
|
| 15 |
+
if not path:
|
| 16 |
+
return None
|
| 17 |
+
try:
|
| 18 |
+
os.makedirs(os.path.join(path, "huggingface"), exist_ok=True)
|
| 19 |
+
return os.path.join(path, "huggingface")
|
| 20 |
+
except OSError:
|
| 21 |
+
return None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 22 |
|
| 23 |
|
| 24 |
_store = _pick_model_store()
|
| 25 |
if _store:
|
| 26 |
os.environ["HF_HOME"] = _store # Must be set before the Hugging Face libraries are imported
|
|
|
|
|
|
|
| 27 |
|
|
|
|
| 28 |
import random
|
| 29 |
import re
|
| 30 |
import subprocess
|
| 31 |
|
| 32 |
import gradio as gr
|
|
|
|
| 33 |
import imageio_ffmpeg
|
| 34 |
import numpy as np
|
| 35 |
import scipy.io.wavfile
|
| 36 |
import spaces
|
| 37 |
import torch
|
|
|
|
| 38 |
from diffusers import FluxPipeline, LTXConditionPipeline
|
| 39 |
from diffusers.pipelines.ltx.pipeline_ltx_condition import LTXVideoCondition
|
| 40 |
from diffusers.utils import export_to_video
|
|
|
|
| 43 |
|
| 44 |
IMAGE_MODEL = "black-forest-labs/FLUX.1-schnell"
|
| 45 |
VIDEO_MODEL = "Lightricks/LTX-Video-0.9.8-13B-distilled"
|
| 46 |
+
MUSIC_MODEL = "facebook/musicgen-medium" # Non-commercial licence (CC-BY-NC 4.0)
|
| 47 |
PLANNER_MODEL = "Qwen/Qwen2.5-1.5B-Instruct" # Writes the narration and its timing
|
| 48 |
VOICE = "af_heart" # Kokoro voice
|
| 49 |
VOICE_RATE = 24000
|
| 50 |
WORDS_PER_SECOND = 2.5
|
| 51 |
|
| 52 |
IMAGE_WIDTH, IMAGE_HEIGHT = 1152, 768 # 3:2, multiples of 16 for FLUX
|
| 53 |
+
VIDEO_WIDTH, VIDEO_HEIGHT = 1152, 768 # Same size as the image, so nothing is shrunk (multiples of 32)
|
| 54 |
+
VIDEO_QUALITY = 9 # 0 to 10. Low values make the saved file blurry
|
| 55 |
FPS = 10
|
| 56 |
TRIM_FRAMES = 8 # Frames cut from the start of every clip, where the noise is
|
|
|
|
| 57 |
MAX_SEED = 2**31 - 1
|
| 58 |
|
| 59 |
NEGATIVE_PROMPT = ("worst quality, inconsistent motion, blurry, blurry motion, motion blur, "
|
|
|
|
| 61 |
NEGATIVE_SLOW = ", slow motion, slow-motion video" # Left out when the motion text asks for slow motion
|
| 62 |
# The distilled model runs without guidance, so the negative prompt alone has no effect on it.
|
| 63 |
# These words are added to every prompt to get the same result from the positive side.
|
| 64 |
+
STYLE_SUFFIX = "Sharp, crisp, highly detailed, in focus, no blur"
|
| 65 |
SPEED_SUFFIX = ", moving at natural real-time speed" # Also left out when slow motion is asked for
|
| 66 |
|
|
|
|
| 67 |
# Loaded once at start-up. ZeroGPU attaches the GPU only while a @spaces.GPU function runs.
|
| 68 |
+
# Everything together is about 70 GB, more than the default 47 GB ZeroGPU slice, so the GPU steps
|
| 69 |
+
# ask for the whole card (size='xlarge'; it counts double against the quota).
|
| 70 |
image_pipe = FluxPipeline.from_pretrained(IMAGE_MODEL, torch_dtype=torch.bfloat16).to("cuda")
|
| 71 |
+
# FLUX and LTX-Video use the same T5-XXL text encoder (9.5 GB), so LTX-Video borrows FLUX's copy.
|
|
|
|
|
|
|
| 72 |
video_pipe = LTXConditionPipeline.from_pretrained(
|
| 73 |
VIDEO_MODEL, text_encoder=image_pipe.text_encoder_2, torch_dtype=torch.bfloat16).to("cuda")
|
| 74 |
video_pipe.vae.enable_tiling() # Decode the video in tiles to keep memory low
|
|
|
|
| 83 |
return random.randint(0, MAX_SEED) if randomize else int(seed)
|
| 84 |
|
| 85 |
|
| 86 |
+
# ---------------------------------------------------------------- The four model steps (run on the GPU)
|
| 87 |
+
|
| 88 |
+
def draw_image(prompt, seed):
|
| 89 |
+
return image_pipe(
|
|
|
|
|
|
|
| 90 |
prompt=prompt,
|
| 91 |
width=IMAGE_WIDTH,
|
| 92 |
height=IMAGE_HEIGHT,
|
|
|
|
| 95 |
max_sequence_length=256,
|
| 96 |
generator=torch.Generator("cpu").manual_seed(seed),
|
| 97 |
).images[0]
|
|
|
|
| 98 |
|
| 99 |
|
| 100 |
+
def animate(image, scene, motion, seconds, seed):
|
| 101 |
+
"""Returns the path of a silent clip and its length in seconds."""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 102 |
# Generate the clip plus the frames that get cut off. LTX-Video wants 8n+1 frames.
|
| 103 |
+
num_frames = -(-(int(seconds * FPS) + TRIM_FRAMES - 1) // 8) * 8 + 1
|
| 104 |
+
motion = motion.strip() or "gentle cinematic camera motion, natural movement"
|
|
|
|
|
|
|
| 105 |
# Only the phrase itself counts: "slow motion", "slow-motion", "slow mo". "Slowly" does not.
|
| 106 |
wants_slow = re.search(r"slow[\s-]?mo", motion.lower()) is not None
|
| 107 |
style = STYLE_SUFFIX + ("" if wants_slow else SPEED_SUFFIX) + "."
|
| 108 |
+
# The motion comes first so the model gives it the most weight
|
| 109 |
+
prompt = f"{motion}. {scene.strip()}. {style}" if scene.strip() else f"{motion}. {style}"
|
| 110 |
start = LTXVideoCondition(image=image.convert("RGB").resize((VIDEO_WIDTH, VIDEO_HEIGHT)), frame_index=0)
|
| 111 |
frames = video_pipe(
|
| 112 |
conditions=[start],
|
| 113 |
prompt=prompt,
|
| 114 |
+
negative_prompt=NEGATIVE_PROMPT + ("" if wants_slow else NEGATIVE_SLOW),
|
| 115 |
image_cond_noise_scale=0.0,
|
| 116 |
decode_timestep=0.05,
|
| 117 |
decode_noise_scale=0.025,
|
|
|
|
| 118 |
width=VIDEO_WIDTH,
|
| 119 |
height=VIDEO_HEIGHT,
|
| 120 |
num_frames=num_frames,
|
|
|
|
| 124 |
guidance_rescale=0.7,
|
| 125 |
generator=torch.Generator("cpu").manual_seed(seed),
|
| 126 |
).frames[0]
|
| 127 |
+
frames = frames[TRIM_FRAMES:]
|
| 128 |
+
path = f"/tmp/video_{seed}_{random.randint(0, 99999)}.mp4"
|
| 129 |
+
export_to_video(frames, path, fps=FPS, quality=VIDEO_QUALITY)
|
| 130 |
+
return path, len(frames) / FPS
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 131 |
|
| 132 |
|
| 133 |
+
def compose_music(music_prompt, seconds):
|
| 134 |
+
"""Returns the path of a WAV file as long as the clip."""
|
| 135 |
+
seconds = min(max(seconds, 2.0), 28.0)
|
| 136 |
inputs = music_processor(text=[music_prompt], padding=True, return_tensors="pt").to("cuda")
|
| 137 |
audio = music_model.generate(**inputs, do_sample=True, guidance_scale=3.0,
|
| 138 |
max_new_tokens=int(seconds * 50) + 8) # MusicGen makes 50 tokens a second
|
|
|
|
| 146 |
return wav_path
|
| 147 |
|
| 148 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 149 |
def plan_narration(scene, motion, seconds):
|
| 150 |
"""The planning model writes 1 to 3 short lines and picks the second each one starts."""
|
| 151 |
+
budget = max(4, int((seconds - 1) * WORDS_PER_SECOND))
|
| 152 |
+
system = ("You are the narrator of a short film clip. You are given what the clip shows. Write what the "
|
| 153 |
+
"narrator SAYS over it: a thought, a feeling or a tiny story about the scene, like a nature "
|
| 154 |
+
"documentary. Never repeat or list the description itself, and never mention cameras, prompts or "
|
| 155 |
+
"video. Reply with 1 to 3 lines and nothing else. Each line is: start second | sentence.\n"
|
| 156 |
+
f"The clip lasts {seconds:.0f} seconds. Use at most {budget} words in total, so every line is short. "
|
| 157 |
+
"The first line starts at 0.5 or later, and later lines start after the earlier ones have been spoken.\n\n"
|
| 158 |
+
"Example for a clip of a lighthouse in a storm, 6 seconds:\n"
|
| 159 |
+
"0.5 | Some lights refuse to go out.\n"
|
| 160 |
+
"3.5 | Even when the sea says otherwise.")
|
| 161 |
+
user = f"The clip shows: {scene or 'a quiet scene'}\nWhat moves: {motion or 'gentle movement'}"
|
| 162 |
text = planner_tokenizer.apply_chat_template(
|
| 163 |
[{"role": "system", "content": system}, {"role": "user", "content": user}],
|
| 164 |
tokenize=False, add_generation_prompt=True)
|
| 165 |
inputs = planner_tokenizer([text], return_tensors="pt").to("cuda")
|
| 166 |
+
out = planner_model.generate(**inputs, max_new_tokens=120, do_sample=True, temperature=0.8, top_p=0.9)
|
| 167 |
return planner_tokenizer.decode(out[0][inputs.input_ids.shape[1]:], skip_special_tokens=True)
|
| 168 |
|
| 169 |
|
| 170 |
+
# ---------------------------------------------------------------- Narration and mixing (CPU only)
|
| 171 |
+
|
| 172 |
+
def parse_plan(reply, seconds):
|
| 173 |
+
"""Turn the planner's reply into [(start, text)]. If it ignored the format, its own words are
|
| 174 |
+
still used as one line; the scene description is never read out."""
|
| 175 |
plan = []
|
| 176 |
+
for row in reply.splitlines():
|
| 177 |
+
match = re.match(r"\s*[-*]?\s*(\d+(?:\.\d+)?)\s*(?:s|sec|seconds)?\s*[|:–-]\s*(.+)", row)
|
| 178 |
+
if match and match.group(2).strip():
|
| 179 |
+
plan.append((float(match.group(1)), match.group(2).strip().strip('"')))
|
|
|
|
|
|
|
|
|
|
| 180 |
if not plan:
|
| 181 |
+
words = re.sub(r"[\[\]{}\"|]", " ", reply).split()[:int(seconds * WORDS_PER_SECOND)]
|
| 182 |
+
plan = [(0.5, " ".join(words) or "A quiet moment, captured in motion.")]
|
| 183 |
return sorted(plan)[:3]
|
| 184 |
|
| 185 |
|
|
|
|
| 188 |
return np.concatenate(chunks) if chunks else np.zeros(0, dtype=np.float32)
|
| 189 |
|
| 190 |
|
| 191 |
+
def narrate(reply, seconds):
|
| 192 |
+
"""Speak the plan into one WAV as long as the clip. Returns (wav path or None, the plan as text)."""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 193 |
track = np.zeros(int(seconds * VOICE_RATE), dtype=np.float32)
|
| 194 |
cursor, script = 0.0, []
|
| 195 |
+
for start, line in parse_plan(reply, seconds):
|
| 196 |
voice = speak(line)
|
| 197 |
start = max(start, cursor) # Never talk over the previous line
|
| 198 |
begin = int(start * VOICE_RATE)
|
|
|
|
| 204 |
cursor = start + len(voice) / VOICE_RATE + 0.3
|
| 205 |
script.append(f"{start:4.1f} s {line}")
|
| 206 |
if not script:
|
| 207 |
+
return None, "The clip is too short for narration."
|
| 208 |
+
path = f"/tmp/voice_{random.randint(0, MAX_SEED)}.wav"
|
| 209 |
+
scipy.io.wavfile.write(path, VOICE_RATE, (np.clip(track, -1, 1) * 32767).astype(np.int16))
|
| 210 |
+
return path, "\n".join(script)
|
| 211 |
+
|
| 212 |
+
|
| 213 |
+
def mix(video, music_wav, voice_wav):
|
| 214 |
+
"""Put whichever sound tracks exist onto the silent clip. With both, the music plays quietly under the voice."""
|
| 215 |
+
if not music_wav and not voice_wav:
|
| 216 |
+
return video
|
| 217 |
+
ffmpeg = [imageio_ffmpeg.get_ffmpeg_exe(), "-y", "-i", video]
|
| 218 |
+
if music_wav and voice_wav:
|
| 219 |
+
ffmpeg += ["-i", voice_wav, "-i", music_wav, "-filter_complex",
|
| 220 |
+
"[1:a]aresample=44100,volume=1.6[v];[2:a]aresample=44100,volume=0.35[m];"
|
| 221 |
+
"[v][m]amix=inputs=2:duration=longest:normalize=0[a]", "-map", "0:v:0", "-map", "[a]"]
|
| 222 |
+
else:
|
| 223 |
+
ffmpeg += ["-i", music_wav or voice_wav, "-map", "0:v:0", "-map", "1:a:0"]
|
| 224 |
+
out_path = f"/tmp/mixed_{random.randint(0, MAX_SEED)}.mp4"
|
| 225 |
+
subprocess.run(ffmpeg + ["-c:v", "copy", "-c:a", "aac", "-shortest", out_path], check=True, capture_output=True)
|
| 226 |
+
return out_path
|
| 227 |
|
| 228 |
+
|
| 229 |
+
# ---------------------------------------------------------------- The two buttons
|
| 230 |
+
|
| 231 |
+
@spaces.GPU(duration=10, size='xlarge')
|
| 232 |
+
def make_image(prompt, seed, randomize_seed):
|
| 233 |
+
if not prompt.strip():
|
| 234 |
+
raise gr.Error("Describe the image first.")
|
| 235 |
+
seed = pick_seed(seed, randomize_seed)
|
| 236 |
+
return draw_image(prompt, seed), seed
|
| 237 |
+
|
| 238 |
+
|
| 239 |
+
def studio_duration(image, scene, motion, music_prompt, want_narration, seconds, seed):
|
| 240 |
+
"""Ask ZeroGPU for only as much GPU time as this clip needs."""
|
| 241 |
+
total = 20 + seconds * 5 # Animation
|
| 242 |
+
if image is None:
|
| 243 |
+
total += 8 # FLUX draws the scene first
|
| 244 |
+
if music_prompt.strip():
|
| 245 |
+
total += 8 + seconds * 2.5
|
| 246 |
+
if want_narration:
|
| 247 |
+
total += 8
|
| 248 |
+
return int(total)
|
| 249 |
+
|
| 250 |
+
|
| 251 |
+
@spaces.GPU(duration=studio_duration, size='xlarge')
|
| 252 |
+
def run_models(image, scene, motion, music_prompt, want_narration, seconds, seed):
|
| 253 |
+
"""Everything that needs the GPU, in one go, so the models are moved onto it only once."""
|
| 254 |
+
if image is None:
|
| 255 |
+
image = draw_image(scene, seed)
|
| 256 |
+
video, length = animate(image, scene, motion, seconds, seed)
|
| 257 |
+
music_wav = compose_music(music_prompt, length) if music_prompt.strip() else None
|
| 258 |
+
reply = plan_narration(scene, motion, length) if want_narration else None
|
| 259 |
+
return video, length, music_wav, reply
|
| 260 |
+
|
| 261 |
+
|
| 262 |
+
def make_video(image, scene, motion, music_prompt, want_narration, seconds, seed, randomize_seed):
|
| 263 |
+
if image is None and not scene.strip():
|
| 264 |
+
raise gr.Error("Describe the scene, or add a start image.")
|
| 265 |
+
seed = pick_seed(seed, randomize_seed)
|
| 266 |
+
video, length, music_wav, reply = run_models(image, scene, motion, music_prompt, want_narration, seconds, seed)
|
| 267 |
+
voice_wav, script = narrate(reply, length) if reply is not None else (None, "")
|
| 268 |
+
return mix(video, music_wav, voice_wav), script, seed
|
| 269 |
|
| 270 |
|
| 271 |
with gr.Blocks(title="Graphics Studio") as demo:
|
| 272 |
+
gr.Markdown("# Graphics Studio\n**FLUX.1-schnell** images, **LTX-Video** animation, **MusicGen** music, "
|
| 273 |
+
"**Qwen** + **Kokoro** narration.")
|
| 274 |
+
|
| 275 |
+
with gr.Tab("Image"):
|
| 276 |
+
image_prompt = gr.Textbox(label="Describe the image", lines=3,
|
| 277 |
+
placeholder="A breathtaking aurora over a frozen lake at night, ultra detailed")
|
| 278 |
+
with gr.Row():
|
| 279 |
+
image_seed = gr.Slider(0, MAX_SEED, value=0, step=1, label="Seed")
|
| 280 |
+
image_random = gr.Checkbox(value=True, label="Random seed")
|
| 281 |
+
image_button = gr.Button("Generate image", variant="primary")
|
| 282 |
+
image_out = gr.Image(label="Image", type="pil")
|
| 283 |
+
gr.Examples(
|
| 284 |
+
examples=[
|
| 285 |
+
["A breathtaking aurora over a frozen lake at night, snowy mountains, ultra detailed"],
|
| 286 |
+
["A lone lighthouse on a cliff during a thunderstorm, giant waves, cinematic lighting"],
|
| 287 |
+
["A floating island with waterfalls at sunrise, golden clouds, fantasy landscape"],
|
| 288 |
+
],
|
| 289 |
+
inputs=[image_prompt],
|
| 290 |
+
)
|
| 291 |
+
|
| 292 |
+
with gr.Tab("Video"):
|
| 293 |
+
with gr.Row():
|
| 294 |
+
with gr.Column():
|
| 295 |
+
scene_prompt = gr.Textbox(label="Scene", lines=3,
|
| 296 |
+
placeholder="A lone lighthouse on a cliff during a thunderstorm, giant waves")
|
| 297 |
+
motion_prompt = gr.Textbox(label="Motion", lines=2,
|
| 298 |
+
placeholder="Waves crash against the rocks, the light beam sweeps to the right")
|
| 299 |
+
music_prompt = gr.Textbox(label="Music (optional, leave empty for none)", lines=2,
|
| 300 |
+
placeholder="Tense orchestral strings with distant thunder")
|
| 301 |
+
want_narration = gr.Checkbox(value=False, label="Add narration")
|
| 302 |
seconds = gr.Slider(2, 8, value=5, step=1, label="Length (seconds)")
|
| 303 |
+
with gr.Accordion("More options", open=False):
|
| 304 |
+
start_image = gr.Image(label="Start image (optional; otherwise FLUX draws the scene)", type="pil")
|
| 305 |
+
with gr.Row():
|
| 306 |
+
video_seed = gr.Slider(0, MAX_SEED, value=0, step=1, label="Seed")
|
| 307 |
+
video_random = gr.Checkbox(value=True, label="Random seed")
|
| 308 |
+
video_button = gr.Button("Make video", variant="primary")
|
| 309 |
+
with gr.Column():
|
| 310 |
+
video_out = gr.Video(label="Video", autoplay=False, loop=True)
|
| 311 |
+
narration_script = gr.Textbox(label="Narration (start time and line)", lines=3, interactive=False)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 312 |
|
| 313 |
image_button.click(make_image, [image_prompt, image_seed, image_random], [image_out, image_seed])
|
| 314 |
+
video_button.click(make_video,
|
| 315 |
+
[start_image, scene_prompt, motion_prompt, music_prompt, want_narration, seconds,
|
| 316 |
+
video_seed, video_random],
|
| 317 |
+
[video_out, narration_script, video_seed])
|
|
|
|
| 318 |
|
| 319 |
demo.queue().launch()
|