vrishv commited on
Commit
aff8ac3
·
verified ·
1 Parent(s): f6e9e12

Two tabs (Image, one-button Video), sharper 1152x768 clips, better narration planner

Browse files
Files changed (2) hide show
  1. README.md +9 -5
  2. app.py +170 -189
README.md CHANGED
@@ -12,11 +12,15 @@ short_description: FLUX image, LTX video, MusicGen music, Kokoro+Qwen narration
12
 
13
  # Graphics Studio
14
 
15
- 1. **FLUX.1-schnell** (black-forest-labs) turns a prompt into a 1152×768 still in 4 steps.
16
- 2. **LTX-Video** (Lightricks) animates that still, or any uploaded image, into a 2 to 8 second clip at 768×512, 10 fps.
17
- 3. **MusicGen medium** (facebook) writes a soundtrack for the clip from a text description.
18
- 4. **Qwen2.5 1.5B** plans the narration (what to say and when), and **Kokoro** speaks it over the music.
19
 
20
- FLUX.1-schnell and LTX-Video are openly licensed; MusicGen medium is CC-BY-NC 4.0 (non-commercial).
 
 
 
 
 
 
 
21
 
22
  Runs on ZeroGPU: set the Space hardware to **ZeroGPU** in Settings.
 
12
 
13
  # Graphics Studio
14
 
15
+ **Image tab:** FLUX.1-schnell (black-forest-labs) turns a prompt into a 1152×768 still in 4 steps.
 
 
 
16
 
17
+ **Video tab:** one button makes a whole clip from a scene and a motion description.
18
+
19
+ 1. **FLUX.1-schnell** draws the scene (or you supply a start image).
20
+ 2. **LTX-Video 0.9.8 13B distilled** (Lightricks) animates it: 2 to 8 seconds, 1152×768, 10 fps.
21
+ 3. **MusicGen medium** (facebook) adds a soundtrack if you describe one.
22
+ 4. **Qwen2.5 1.5B** plans the narration (what to say and when) and **Kokoro** speaks it, if narration is ticked.
23
+
24
+ FLUX.1-schnell, LTX-Video, Qwen and Kokoro are openly licensed; MusicGen medium is CC-BY-NC 4.0 (non-commercial).
25
 
26
  Runs on ZeroGPU: set the Space hardware to **ZeroGPU** in Settings.
app.py CHANGED
@@ -1,51 +1,40 @@
1
- """Image to video on ZeroGPU.
2
 
3
- Step 1: FLUX.1-schnell turns a prompt into a still image (4 steps).
4
- Step 2: LTX-Video animates that image, or any image you upload, into a short clip.
5
- Step 3: MusicGen medium writes a soundtrack and attaches it to the clip.
6
- Step 4: a small planning model (Qwen2.5 1.5B) writes narration and decides when each line starts;
7
- Kokoro speaks it, and it is mixed over the music.
8
  """
9
  import os
10
 
11
 
12
  def _pick_model_store():
13
- """Keep downloaded models on the Space's attached storage, so restarts don't download them again.
14
- Uses MODEL_STORE if that variable is set, else the first writable mount it finds."""
15
- # Off unless MODEL_STORE is set. Tried /data (the attached bucket) on 2026-10-04: it reads at about
16
- # 8 MB/s, so loading 70 GB of models from it takes hours. The local disk downloads at about 1 GB/s.
17
- candidates = [os.environ.get("MODEL_STORE")]
18
- for path in [c for c in candidates if c]:
19
- try:
20
- os.makedirs(os.path.join(path, "huggingface"), exist_ok=True)
21
- probe = os.path.join(path, "huggingface", ".write_test")
22
- open(probe, "w").close()
23
- os.remove(probe)
24
- return os.path.join(path, "huggingface")
25
- except OSError:
26
- continue
27
- return None
28
 
29
 
30
  _store = _pick_model_store()
31
  if _store:
32
  os.environ["HF_HOME"] = _store # Must be set before the Hugging Face libraries are imported
33
- print("MODEL STORE:", _store or "none found, using the default temporary cache",
34
- "| mounts seen:", [m for m in ("/data", "/bucket", "/storage", "/mnt") if os.path.isdir(m)], flush=True)
35
 
36
- import json
37
  import random
38
  import re
39
  import subprocess
40
 
41
  import gradio as gr
42
- import imageio
43
  import imageio_ffmpeg
44
  import numpy as np
45
  import scipy.io.wavfile
46
  import spaces
47
  import torch
48
- from PIL import Image
49
  from diffusers import FluxPipeline, LTXConditionPipeline
50
  from diffusers.pipelines.ltx.pipeline_ltx_condition import LTXVideoCondition
51
  from diffusers.utils import export_to_video
@@ -54,17 +43,17 @@ from transformers import AutoModelForCausalLM, AutoProcessor, AutoTokenizer, Mus
54
 
55
  IMAGE_MODEL = "black-forest-labs/FLUX.1-schnell"
56
  VIDEO_MODEL = "Lightricks/LTX-Video-0.9.8-13B-distilled"
57
- MUSIC_MODEL = "facebook/musicgen-medium" # Non-commercial licence (CC-BY-NC 4.0)
58
  PLANNER_MODEL = "Qwen/Qwen2.5-1.5B-Instruct" # Writes the narration and its timing
59
  VOICE = "af_heart" # Kokoro voice
60
  VOICE_RATE = 24000
61
  WORDS_PER_SECOND = 2.5
62
 
63
  IMAGE_WIDTH, IMAGE_HEIGHT = 1152, 768 # 3:2, multiples of 16 for FLUX
64
- VIDEO_WIDTH, VIDEO_HEIGHT = 768, 512 # 3:2, multiples of 32 for LTX-Video
 
65
  FPS = 10
66
  TRIM_FRAMES = 8 # Frames cut from the start of every clip, where the noise is
67
- FADE_FRAMES = 0 # Loop cross-fade switched off (set to 20 for a 2-second fade)
68
  MAX_SEED = 2**31 - 1
69
 
70
  NEGATIVE_PROMPT = ("worst quality, inconsistent motion, blurry, blurry motion, motion blur, "
@@ -72,15 +61,14 @@ NEGATIVE_PROMPT = ("worst quality, inconsistent motion, blurry, blurry motion, m
72
  NEGATIVE_SLOW = ", slow motion, slow-motion video" # Left out when the motion text asks for slow motion
73
  # The distilled model runs without guidance, so the negative prompt alone has no effect on it.
74
  # These words are added to every prompt to get the same result from the positive side.
75
- STYLE_SUFFIX = "Sharp, crisp motion with no blur"
76
  SPEED_SUFFIX = ", moving at natural real-time speed" # Also left out when slow motion is asked for
77
 
78
-
79
  # Loaded once at start-up. ZeroGPU attaches the GPU only while a @spaces.GPU function runs.
 
 
80
  image_pipe = FluxPipeline.from_pretrained(IMAGE_MODEL, torch_dtype=torch.bfloat16).to("cuda")
81
- # FLUX (12B) and LTX-Video (13B) together need about 60 GB, more than the default 47 GB ZeroGPU
82
- # slice, so both steps ask for the whole card (size='xlarge'; it counts double against the quota).
83
- # Both models use the same T5-XXL text encoder (9.5 GB), so LTX-Video borrows FLUX's copy.
84
  video_pipe = LTXConditionPipeline.from_pretrained(
85
  VIDEO_MODEL, text_encoder=image_pipe.text_encoder_2, torch_dtype=torch.bfloat16).to("cuda")
86
  video_pipe.vae.enable_tiling() # Decode the video in tiles to keep memory low
@@ -95,12 +83,10 @@ def pick_seed(seed, randomize):
95
  return random.randint(0, MAX_SEED) if randomize else int(seed)
96
 
97
 
98
- @spaces.GPU(duration=10, size='xlarge')
99
- def make_image(prompt, seed, randomize_seed):
100
- if not prompt.strip():
101
- raise gr.Error("Describe the image first.")
102
- seed = pick_seed(seed, randomize_seed)
103
- image = image_pipe(
104
  prompt=prompt,
105
  width=IMAGE_WIDTH,
106
  height=IMAGE_HEIGHT,
@@ -109,50 +95,26 @@ def make_image(prompt, seed, randomize_seed):
109
  max_sequence_length=256,
110
  generator=torch.Generator("cpu").manual_seed(seed),
111
  ).images[0]
112
- return image, seed
113
 
114
 
115
- def seamless_loop(frames):
116
- """Make the clip loop smoothly: its opening frames fade in from its closing frames,
117
- and those closing frames are then dropped. The last frame shown flows into the first."""
118
- n = min(FADE_FRAMES, len(frames) // 3)
119
- if n < 2:
120
- return frames
121
- body = frames[:len(frames) - n]
122
- tail = frames[len(frames) - n:]
123
- for i in range(n):
124
- body[i] = Image.blend(tail[i], body[i], (i + 1) / (n + 1))
125
- return body
126
-
127
-
128
- def video_duration(image, scene_prompt, motion_prompt, seconds, seed, randomize_seed):
129
- """Ask ZeroGPU for only as much GPU time as this clip needs."""
130
- return int(20 + seconds * 3)
131
-
132
-
133
- @spaces.GPU(duration=video_duration, size='xlarge')
134
- def make_video(image, scene_prompt, motion_prompt, seconds, seed, randomize_seed):
135
- if image is None:
136
- raise gr.Error("Make or upload an image first.")
137
- seed = pick_seed(seed, randomize_seed)
138
  # Generate the clip plus the frames that get cut off. LTX-Video wants 8n+1 frames.
139
- num_frames = -(-(int(seconds * FPS) + TRIM_FRAMES + FADE_FRAMES - 1) // 8) * 8 + 1
140
- # LTX-Video needs the scene AND the motion described, or it tends to hold the picture still
141
- motion = motion_prompt.strip() or "gentle cinematic camera motion, natural movement"
142
- # The motion comes first so the model gives it the most weight
143
  # Only the phrase itself counts: "slow motion", "slow-motion", "slow mo". "Slowly" does not.
144
  wants_slow = re.search(r"slow[\s-]?mo", motion.lower()) is not None
145
  style = STYLE_SUFFIX + ("" if wants_slow else SPEED_SUFFIX) + "."
146
- negative = NEGATIVE_PROMPT + ("" if wants_slow else NEGATIVE_SLOW)
147
- prompt = f"{motion}. {scene_prompt.strip()}. {style}" if scene_prompt.strip() else f"{motion}. {style}"
148
  start = LTXVideoCondition(image=image.convert("RGB").resize((VIDEO_WIDTH, VIDEO_HEIGHT)), frame_index=0)
149
  frames = video_pipe(
150
  conditions=[start],
151
  prompt=prompt,
 
152
  image_cond_noise_scale=0.0,
153
  decode_timestep=0.05,
154
  decode_noise_scale=0.025,
155
- negative_prompt=negative,
156
  width=VIDEO_WIDTH,
157
  height=VIDEO_HEIGHT,
158
  num_frames=num_frames,
@@ -162,40 +124,15 @@ def make_video(image, scene_prompt, motion_prompt, seconds, seed, randomize_seed
162
  guidance_rescale=0.7,
163
  generator=torch.Generator("cpu").manual_seed(seed),
164
  ).frames[0]
165
- path = f"/tmp/video_{seed}.mp4"
166
- export_to_video(seamless_loop(frames[TRIM_FRAMES:]), path, fps=FPS)
167
- return path, seed, path, None, None, ""
168
-
169
-
170
- def clip_seconds(video):
171
- try:
172
- return float(imageio.get_reader(video).get_meta_data()["duration"])
173
- except Exception:
174
- return 6.0
175
-
176
-
177
- def mix(video, music_wav, voice_wav):
178
- """Put whichever sound tracks exist onto the silent clip. With both, the music plays
179
- quietly under the voice. Called after every sound step, so the order never matters."""
180
- ffmpeg = [imageio_ffmpeg.get_ffmpeg_exe(), "-y", "-i", video]
181
- if music_wav and voice_wav:
182
- ffmpeg += ["-i", voice_wav, "-i", music_wav, "-filter_complex",
183
- "[1:a]aresample=44100,volume=1.6[v];[2:a]aresample=44100,volume=0.35[m];"
184
- "[v][m]amix=inputs=2:duration=longest:normalize=0[a]", "-map", "0:v:0", "-map", "[a]"]
185
- else:
186
- ffmpeg += ["-i", music_wav or voice_wav, "-map", "0:v:0", "-map", "1:a:0"]
187
- out_path = f"/tmp/mixed_{random.randint(0, MAX_SEED)}.mp4"
188
- subprocess.run(ffmpeg + ["-c:v", "copy", "-c:a", "aac", "-shortest", out_path], check=True, capture_output=True)
189
- return out_path
190
-
191
-
192
- def music_duration(video, music_prompt):
193
- return int(15 + clip_seconds(video) * 3) if video else 15
194
 
195
 
196
- @spaces.GPU(duration=music_duration, size='xlarge')
197
- def compose_music(video, music_prompt):
198
- seconds = min(max(clip_seconds(video), 2.0), 28.0)
199
  inputs = music_processor(text=[music_prompt], padding=True, return_tensors="pt").to("cuda")
200
  audio = music_model.generate(**inputs, do_sample=True, guidance_scale=3.0,
201
  max_new_tokens=int(seconds * 50) + 8) # MusicGen makes 50 tokens a second
@@ -209,47 +146,40 @@ def compose_music(video, music_prompt):
209
  return wav_path
210
 
211
 
212
- def add_music(video, music_prompt, voice_wav):
213
- if not video:
214
- raise gr.Error("Animate an image first.")
215
- if not music_prompt.strip():
216
- raise gr.Error("Describe the music first.")
217
- music_wav = compose_music(video, music_prompt)
218
- return mix(video, music_wav, voice_wav), music_wav
219
-
220
-
221
- @spaces.GPU(duration=20, size='xlarge')
222
  def plan_narration(scene, motion, seconds):
223
  """The planning model writes 1 to 3 short lines and picks the second each one starts."""
224
- budget = int(seconds * WORDS_PER_SECOND)
225
- system = ("You write narration for a short silent video clip. Reply with ONLY a JSON list, no other text. "
226
- 'Each item is {"start": seconds from the beginning, "text": one short spoken sentence}. '
227
- f"The clip is {seconds:.1f} seconds long. Speech takes one second per {WORDS_PER_SECOND} words, "
228
- f"so use at most {budget} words in total. Use 1 to 3 items. Lines must not overlap, and the last "
229
- "one must finish before the clip ends. Start the first line after a short pause. "
230
- "Leave silence where nothing needs saying.")
231
- user = f"Scene: {scene or 'unknown'}\nMotion: {motion or 'gentle movement'}\nNarrate the scene with a sense of wonder."
 
 
 
232
  text = planner_tokenizer.apply_chat_template(
233
  [{"role": "system", "content": system}, {"role": "user", "content": user}],
234
  tokenize=False, add_generation_prompt=True)
235
  inputs = planner_tokenizer([text], return_tensors="pt").to("cuda")
236
- out = planner_model.generate(**inputs, max_new_tokens=200, do_sample=False)
237
  return planner_tokenizer.decode(out[0][inputs.input_ids.shape[1]:], skip_special_tokens=True)
238
 
239
 
240
- def parse_plan(reply, fallback, seconds):
241
- """Turn the planner's reply into [(start, text)], or one safe line if it isn't valid JSON."""
 
 
 
242
  plan = []
243
- try:
244
- for item in json.loads(re.search(r"\[.*\]", reply, re.S).group(0)):
245
- line = str(item["text"]).strip()
246
- if line:
247
- plan.append((max(0.0, float(item["start"])), line))
248
- except Exception:
249
- plan = []
250
  if not plan:
251
- words = (fallback or "A quiet moment, captured in motion.").split()[:int(seconds * WORDS_PER_SECOND)]
252
- plan = [(0.5, " ".join(words))]
253
  return sorted(plan)[:3]
254
 
255
 
@@ -258,16 +188,11 @@ def speak(text):
258
  return np.concatenate(chunks) if chunks else np.zeros(0, dtype=np.float32)
259
 
260
 
261
- def add_narration(video, music_wav, scene, motion):
262
- if not video:
263
- raise gr.Error("Animate an image first.")
264
- seconds = clip_seconds(video)
265
- reply = plan_narration(scene, motion, seconds)
266
- plan = parse_plan(reply, scene, seconds)
267
-
268
  track = np.zeros(int(seconds * VOICE_RATE), dtype=np.float32)
269
  cursor, script = 0.0, []
270
- for start, line in plan:
271
  voice = speak(line)
272
  start = max(start, cursor) # Never talk over the previous line
273
  begin = int(start * VOICE_RATE)
@@ -279,60 +204,116 @@ def add_narration(video, music_wav, scene, motion):
279
  cursor = start + len(voice) / VOICE_RATE + 0.3
280
  script.append(f"{start:4.1f} s {line}")
281
  if not script:
282
- raise gr.Error("The clip is too short for any narration.")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
283
 
284
- voice_path = f"/tmp/voice_{random.randint(0, MAX_SEED)}.wav"
285
- scipy.io.wavfile.write(voice_path, VOICE_RATE, (np.clip(track, -1, 1) * 32767).astype(np.int16))
286
- return mix(video, music_wav, voice_path), "\n".join(script), voice_path
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
287
 
288
 
289
  with gr.Blocks(title="Graphics Studio") as demo:
290
- silent_clip = gr.State(None) # The clip without sound, as Animate made it
291
- music_track = gr.State(None) # The soundtrack file, once Add music has run
292
- voice_track = gr.State(None) # The narration file, once Add narration has run
293
- gr.Markdown("# Graphics Studio\nMake a still with **FLUX.1-schnell**, bring it to life with **LTX-Video**, "
294
- "score it with **MusicGen**, then narrate it with **Qwen** and **Kokoro**.")
295
-
296
- with gr.Row():
297
- with gr.Column():
298
- image_prompt = gr.Textbox(label="1. Describe the image", lines=3,
299
- placeholder="A breathtaking aurora over a frozen lake at night, ultra detailed")
300
- with gr.Row():
301
- image_seed = gr.Slider(0, MAX_SEED, value=0, step=1, label="Seed")
302
- image_random = gr.Checkbox(value=True, label="Random seed")
303
- image_button = gr.Button("Generate image", variant="primary")
304
- image_out = gr.Image(label="Image (or upload your own)", type="pil")
305
-
306
- with gr.Column():
307
- motion_prompt = gr.Textbox(label="2. Describe the motion", lines=3,
308
- placeholder="The aurora ripples slowly, stars twinkle, the camera drifts forward")
309
- with gr.Row():
 
 
 
 
 
 
 
 
 
 
310
  seconds = gr.Slider(2, 8, value=5, step=1, label="Length (seconds)")
311
- with gr.Row():
312
- video_seed = gr.Slider(0, MAX_SEED, value=0, step=1, label="Seed")
313
- video_random = gr.Checkbox(value=True, label="Random seed")
314
- video_button = gr.Button("Animate", variant="primary")
315
- video_out = gr.Video(label="Video", autoplay=False, loop=True)
316
- music_prompt = gr.Textbox(label="3. Describe the music", lines=2,
317
- placeholder="Calm ambient synth pads, slow and dreamy, night sky mood")
318
- music_button = gr.Button("Add music (optional)", variant="primary")
319
- narration_button = gr.Button("4. Add narration (optional)", variant="primary")
320
- narration_script = gr.Textbox(label="Narration plan (start time and line)", lines=3, interactive=False)
321
-
322
- gr.Examples(
323
- examples=[
324
- ["A breathtaking aurora over a frozen lake at night, snowy mountains, ultra detailed"],
325
- ["A lone lighthouse on a cliff during a thunderstorm, giant waves, cinematic lighting"],
326
- ["A floating island with waterfalls at sunrise, golden clouds, fantasy landscape"],
327
- ],
328
- inputs=[image_prompt],
329
- )
330
 
331
  image_button.click(make_image, [image_prompt, image_seed, image_random], [image_out, image_seed])
332
- video_button.click(make_video, [image_out, image_prompt, motion_prompt, seconds, video_seed, video_random],
333
- [video_out, video_seed, silent_clip, music_track, voice_track, narration_script])
334
- music_button.click(add_music, [silent_clip, music_prompt, voice_track], [video_out, music_track])
335
- narration_button.click(add_narration, [silent_clip, music_track, image_prompt, motion_prompt],
336
- [video_out, narration_script, voice_track])
337
 
338
  demo.queue().launch()
 
1
+ """Graphics Studio on ZeroGPU.
2
 
3
+ Image tab: FLUX.1-schnell turns a prompt into a still image.
4
+ Video tab: one button makes a whole clip. FLUX draws the scene, LTX-Video (13B) animates it,
5
+ MusicGen medium adds a soundtrack if you describe one, and if narration is ticked a small
6
+ planning model (Qwen2.5 1.5B) writes the lines and their timing and Kokoro speaks them.
 
7
  """
8
  import os
9
 
10
 
11
  def _pick_model_store():
12
+ """Off unless MODEL_STORE is set. Tried /data (the attached bucket) on 2026-10-04: it reads at
13
+ about 8 MB/s, so loading 70 GB of models from it takes hours. The local disk downloads at 1 GB/s."""
14
+ path = os.environ.get("MODEL_STORE")
15
+ if not path:
16
+ return None
17
+ try:
18
+ os.makedirs(os.path.join(path, "huggingface"), exist_ok=True)
19
+ return os.path.join(path, "huggingface")
20
+ except OSError:
21
+ return None
 
 
 
 
 
22
 
23
 
24
  _store = _pick_model_store()
25
  if _store:
26
  os.environ["HF_HOME"] = _store # Must be set before the Hugging Face libraries are imported
 
 
27
 
 
28
  import random
29
  import re
30
  import subprocess
31
 
32
  import gradio as gr
 
33
  import imageio_ffmpeg
34
  import numpy as np
35
  import scipy.io.wavfile
36
  import spaces
37
  import torch
 
38
  from diffusers import FluxPipeline, LTXConditionPipeline
39
  from diffusers.pipelines.ltx.pipeline_ltx_condition import LTXVideoCondition
40
  from diffusers.utils import export_to_video
 
43
 
44
  IMAGE_MODEL = "black-forest-labs/FLUX.1-schnell"
45
  VIDEO_MODEL = "Lightricks/LTX-Video-0.9.8-13B-distilled"
46
+ MUSIC_MODEL = "facebook/musicgen-medium" # Non-commercial licence (CC-BY-NC 4.0)
47
  PLANNER_MODEL = "Qwen/Qwen2.5-1.5B-Instruct" # Writes the narration and its timing
48
  VOICE = "af_heart" # Kokoro voice
49
  VOICE_RATE = 24000
50
  WORDS_PER_SECOND = 2.5
51
 
52
  IMAGE_WIDTH, IMAGE_HEIGHT = 1152, 768 # 3:2, multiples of 16 for FLUX
53
+ VIDEO_WIDTH, VIDEO_HEIGHT = 1152, 768 # Same size as the image, so nothing is shrunk (multiples of 32)
54
+ VIDEO_QUALITY = 9 # 0 to 10. Low values make the saved file blurry
55
  FPS = 10
56
  TRIM_FRAMES = 8 # Frames cut from the start of every clip, where the noise is
 
57
  MAX_SEED = 2**31 - 1
58
 
59
  NEGATIVE_PROMPT = ("worst quality, inconsistent motion, blurry, blurry motion, motion blur, "
 
61
  NEGATIVE_SLOW = ", slow motion, slow-motion video" # Left out when the motion text asks for slow motion
62
  # The distilled model runs without guidance, so the negative prompt alone has no effect on it.
63
  # These words are added to every prompt to get the same result from the positive side.
64
+ STYLE_SUFFIX = "Sharp, crisp, highly detailed, in focus, no blur"
65
  SPEED_SUFFIX = ", moving at natural real-time speed" # Also left out when slow motion is asked for
66
 
 
67
  # Loaded once at start-up. ZeroGPU attaches the GPU only while a @spaces.GPU function runs.
68
+ # Everything together is about 70 GB, more than the default 47 GB ZeroGPU slice, so the GPU steps
69
+ # ask for the whole card (size='xlarge'; it counts double against the quota).
70
  image_pipe = FluxPipeline.from_pretrained(IMAGE_MODEL, torch_dtype=torch.bfloat16).to("cuda")
71
+ # FLUX and LTX-Video use the same T5-XXL text encoder (9.5 GB), so LTX-Video borrows FLUX's copy.
 
 
72
  video_pipe = LTXConditionPipeline.from_pretrained(
73
  VIDEO_MODEL, text_encoder=image_pipe.text_encoder_2, torch_dtype=torch.bfloat16).to("cuda")
74
  video_pipe.vae.enable_tiling() # Decode the video in tiles to keep memory low
 
83
  return random.randint(0, MAX_SEED) if randomize else int(seed)
84
 
85
 
86
+ # ---------------------------------------------------------------- The four model steps (run on the GPU)
87
+
88
+ def draw_image(prompt, seed):
89
+ return image_pipe(
 
 
90
  prompt=prompt,
91
  width=IMAGE_WIDTH,
92
  height=IMAGE_HEIGHT,
 
95
  max_sequence_length=256,
96
  generator=torch.Generator("cpu").manual_seed(seed),
97
  ).images[0]
 
98
 
99
 
100
+ def animate(image, scene, motion, seconds, seed):
101
+ """Returns the path of a silent clip and its length in seconds."""
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
102
  # Generate the clip plus the frames that get cut off. LTX-Video wants 8n+1 frames.
103
+ num_frames = -(-(int(seconds * FPS) + TRIM_FRAMES - 1) // 8) * 8 + 1
104
+ motion = motion.strip() or "gentle cinematic camera motion, natural movement"
 
 
105
  # Only the phrase itself counts: "slow motion", "slow-motion", "slow mo". "Slowly" does not.
106
  wants_slow = re.search(r"slow[\s-]?mo", motion.lower()) is not None
107
  style = STYLE_SUFFIX + ("" if wants_slow else SPEED_SUFFIX) + "."
108
+ # The motion comes first so the model gives it the most weight
109
+ prompt = f"{motion}. {scene.strip()}. {style}" if scene.strip() else f"{motion}. {style}"
110
  start = LTXVideoCondition(image=image.convert("RGB").resize((VIDEO_WIDTH, VIDEO_HEIGHT)), frame_index=0)
111
  frames = video_pipe(
112
  conditions=[start],
113
  prompt=prompt,
114
+ negative_prompt=NEGATIVE_PROMPT + ("" if wants_slow else NEGATIVE_SLOW),
115
  image_cond_noise_scale=0.0,
116
  decode_timestep=0.05,
117
  decode_noise_scale=0.025,
 
118
  width=VIDEO_WIDTH,
119
  height=VIDEO_HEIGHT,
120
  num_frames=num_frames,
 
124
  guidance_rescale=0.7,
125
  generator=torch.Generator("cpu").manual_seed(seed),
126
  ).frames[0]
127
+ frames = frames[TRIM_FRAMES:]
128
+ path = f"/tmp/video_{seed}_{random.randint(0, 99999)}.mp4"
129
+ export_to_video(frames, path, fps=FPS, quality=VIDEO_QUALITY)
130
+ return path, len(frames) / FPS
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
131
 
132
 
133
+ def compose_music(music_prompt, seconds):
134
+ """Returns the path of a WAV file as long as the clip."""
135
+ seconds = min(max(seconds, 2.0), 28.0)
136
  inputs = music_processor(text=[music_prompt], padding=True, return_tensors="pt").to("cuda")
137
  audio = music_model.generate(**inputs, do_sample=True, guidance_scale=3.0,
138
  max_new_tokens=int(seconds * 50) + 8) # MusicGen makes 50 tokens a second
 
146
  return wav_path
147
 
148
 
 
 
 
 
 
 
 
 
 
 
149
  def plan_narration(scene, motion, seconds):
150
  """The planning model writes 1 to 3 short lines and picks the second each one starts."""
151
+ budget = max(4, int((seconds - 1) * WORDS_PER_SECOND))
152
+ system = ("You are the narrator of a short film clip. You are given what the clip shows. Write what the "
153
+ "narrator SAYS over it: a thought, a feeling or a tiny story about the scene, like a nature "
154
+ "documentary. Never repeat or list the description itself, and never mention cameras, prompts or "
155
+ "video. Reply with 1 to 3 lines and nothing else. Each line is: start second | sentence.\n"
156
+ f"The clip lasts {seconds:.0f} seconds. Use at most {budget} words in total, so every line is short. "
157
+ "The first line starts at 0.5 or later, and later lines start after the earlier ones have been spoken.\n\n"
158
+ "Example for a clip of a lighthouse in a storm, 6 seconds:\n"
159
+ "0.5 | Some lights refuse to go out.\n"
160
+ "3.5 | Even when the sea says otherwise.")
161
+ user = f"The clip shows: {scene or 'a quiet scene'}\nWhat moves: {motion or 'gentle movement'}"
162
  text = planner_tokenizer.apply_chat_template(
163
  [{"role": "system", "content": system}, {"role": "user", "content": user}],
164
  tokenize=False, add_generation_prompt=True)
165
  inputs = planner_tokenizer([text], return_tensors="pt").to("cuda")
166
+ out = planner_model.generate(**inputs, max_new_tokens=120, do_sample=True, temperature=0.8, top_p=0.9)
167
  return planner_tokenizer.decode(out[0][inputs.input_ids.shape[1]:], skip_special_tokens=True)
168
 
169
 
170
+ # ---------------------------------------------------------------- Narration and mixing (CPU only)
171
+
172
+ def parse_plan(reply, seconds):
173
+ """Turn the planner's reply into [(start, text)]. If it ignored the format, its own words are
174
+ still used as one line; the scene description is never read out."""
175
  plan = []
176
+ for row in reply.splitlines():
177
+ match = re.match(r"\s*[-*]?\s*(\d+(?:\.\d+)?)\s*(?:s|sec|seconds)?\s*[|:–-]\s*(.+)", row)
178
+ if match and match.group(2).strip():
179
+ plan.append((float(match.group(1)), match.group(2).strip().strip('"')))
 
 
 
180
  if not plan:
181
+ words = re.sub(r"[\[\]{}\"|]", " ", reply).split()[:int(seconds * WORDS_PER_SECOND)]
182
+ plan = [(0.5, " ".join(words) or "A quiet moment, captured in motion.")]
183
  return sorted(plan)[:3]
184
 
185
 
 
188
  return np.concatenate(chunks) if chunks else np.zeros(0, dtype=np.float32)
189
 
190
 
191
+ def narrate(reply, seconds):
192
+ """Speak the plan into one WAV as long as the clip. Returns (wav path or None, the plan as text)."""
 
 
 
 
 
193
  track = np.zeros(int(seconds * VOICE_RATE), dtype=np.float32)
194
  cursor, script = 0.0, []
195
+ for start, line in parse_plan(reply, seconds):
196
  voice = speak(line)
197
  start = max(start, cursor) # Never talk over the previous line
198
  begin = int(start * VOICE_RATE)
 
204
  cursor = start + len(voice) / VOICE_RATE + 0.3
205
  script.append(f"{start:4.1f} s {line}")
206
  if not script:
207
+ return None, "The clip is too short for narration."
208
+ path = f"/tmp/voice_{random.randint(0, MAX_SEED)}.wav"
209
+ scipy.io.wavfile.write(path, VOICE_RATE, (np.clip(track, -1, 1) * 32767).astype(np.int16))
210
+ return path, "\n".join(script)
211
+
212
+
213
+ def mix(video, music_wav, voice_wav):
214
+ """Put whichever sound tracks exist onto the silent clip. With both, the music plays quietly under the voice."""
215
+ if not music_wav and not voice_wav:
216
+ return video
217
+ ffmpeg = [imageio_ffmpeg.get_ffmpeg_exe(), "-y", "-i", video]
218
+ if music_wav and voice_wav:
219
+ ffmpeg += ["-i", voice_wav, "-i", music_wav, "-filter_complex",
220
+ "[1:a]aresample=44100,volume=1.6[v];[2:a]aresample=44100,volume=0.35[m];"
221
+ "[v][m]amix=inputs=2:duration=longest:normalize=0[a]", "-map", "0:v:0", "-map", "[a]"]
222
+ else:
223
+ ffmpeg += ["-i", music_wav or voice_wav, "-map", "0:v:0", "-map", "1:a:0"]
224
+ out_path = f"/tmp/mixed_{random.randint(0, MAX_SEED)}.mp4"
225
+ subprocess.run(ffmpeg + ["-c:v", "copy", "-c:a", "aac", "-shortest", out_path], check=True, capture_output=True)
226
+ return out_path
227
 
228
+
229
+ # ---------------------------------------------------------------- The two buttons
230
+
231
+ @spaces.GPU(duration=10, size='xlarge')
232
+ def make_image(prompt, seed, randomize_seed):
233
+ if not prompt.strip():
234
+ raise gr.Error("Describe the image first.")
235
+ seed = pick_seed(seed, randomize_seed)
236
+ return draw_image(prompt, seed), seed
237
+
238
+
239
+ def studio_duration(image, scene, motion, music_prompt, want_narration, seconds, seed):
240
+ """Ask ZeroGPU for only as much GPU time as this clip needs."""
241
+ total = 20 + seconds * 5 # Animation
242
+ if image is None:
243
+ total += 8 # FLUX draws the scene first
244
+ if music_prompt.strip():
245
+ total += 8 + seconds * 2.5
246
+ if want_narration:
247
+ total += 8
248
+ return int(total)
249
+
250
+
251
+ @spaces.GPU(duration=studio_duration, size='xlarge')
252
+ def run_models(image, scene, motion, music_prompt, want_narration, seconds, seed):
253
+ """Everything that needs the GPU, in one go, so the models are moved onto it only once."""
254
+ if image is None:
255
+ image = draw_image(scene, seed)
256
+ video, length = animate(image, scene, motion, seconds, seed)
257
+ music_wav = compose_music(music_prompt, length) if music_prompt.strip() else None
258
+ reply = plan_narration(scene, motion, length) if want_narration else None
259
+ return video, length, music_wav, reply
260
+
261
+
262
+ def make_video(image, scene, motion, music_prompt, want_narration, seconds, seed, randomize_seed):
263
+ if image is None and not scene.strip():
264
+ raise gr.Error("Describe the scene, or add a start image.")
265
+ seed = pick_seed(seed, randomize_seed)
266
+ video, length, music_wav, reply = run_models(image, scene, motion, music_prompt, want_narration, seconds, seed)
267
+ voice_wav, script = narrate(reply, length) if reply is not None else (None, "")
268
+ return mix(video, music_wav, voice_wav), script, seed
269
 
270
 
271
  with gr.Blocks(title="Graphics Studio") as demo:
272
+ gr.Markdown("# Graphics Studio\n**FLUX.1-schnell** images, **LTX-Video** animation, **MusicGen** music, "
273
+ "**Qwen** + **Kokoro** narration.")
274
+
275
+ with gr.Tab("Image"):
276
+ image_prompt = gr.Textbox(label="Describe the image", lines=3,
277
+ placeholder="A breathtaking aurora over a frozen lake at night, ultra detailed")
278
+ with gr.Row():
279
+ image_seed = gr.Slider(0, MAX_SEED, value=0, step=1, label="Seed")
280
+ image_random = gr.Checkbox(value=True, label="Random seed")
281
+ image_button = gr.Button("Generate image", variant="primary")
282
+ image_out = gr.Image(label="Image", type="pil")
283
+ gr.Examples(
284
+ examples=[
285
+ ["A breathtaking aurora over a frozen lake at night, snowy mountains, ultra detailed"],
286
+ ["A lone lighthouse on a cliff during a thunderstorm, giant waves, cinematic lighting"],
287
+ ["A floating island with waterfalls at sunrise, golden clouds, fantasy landscape"],
288
+ ],
289
+ inputs=[image_prompt],
290
+ )
291
+
292
+ with gr.Tab("Video"):
293
+ with gr.Row():
294
+ with gr.Column():
295
+ scene_prompt = gr.Textbox(label="Scene", lines=3,
296
+ placeholder="A lone lighthouse on a cliff during a thunderstorm, giant waves")
297
+ motion_prompt = gr.Textbox(label="Motion", lines=2,
298
+ placeholder="Waves crash against the rocks, the light beam sweeps to the right")
299
+ music_prompt = gr.Textbox(label="Music (optional, leave empty for none)", lines=2,
300
+ placeholder="Tense orchestral strings with distant thunder")
301
+ want_narration = gr.Checkbox(value=False, label="Add narration")
302
  seconds = gr.Slider(2, 8, value=5, step=1, label="Length (seconds)")
303
+ with gr.Accordion("More options", open=False):
304
+ start_image = gr.Image(label="Start image (optional; otherwise FLUX draws the scene)", type="pil")
305
+ with gr.Row():
306
+ video_seed = gr.Slider(0, MAX_SEED, value=0, step=1, label="Seed")
307
+ video_random = gr.Checkbox(value=True, label="Random seed")
308
+ video_button = gr.Button("Make video", variant="primary")
309
+ with gr.Column():
310
+ video_out = gr.Video(label="Video", autoplay=False, loop=True)
311
+ narration_script = gr.Textbox(label="Narration (start time and line)", lines=3, interactive=False)
 
 
 
 
 
 
 
 
 
 
312
 
313
  image_button.click(make_image, [image_prompt, image_seed, image_random], [image_out, image_seed])
314
+ video_button.click(make_video,
315
+ [start_image, scene_prompt, motion_prompt, music_prompt, want_narration, seconds,
316
+ video_seed, video_random],
317
+ [video_out, narration_script, video_seed])
 
318
 
319
  demo.queue().launch()