akhaliq's picture
akhaliq HF Staff
Torch-tensor audio for encode_video mux
bc8f5d7 verified
Raw History Blame Contribute Delete
16.6 kB
from __future__ import annotations
import json
import mimetypes
import os
import random
import time
import traceback
from pathlib import Path
import spaces # ZeroGPU: must be imported before torch/gradio touch CUDA
import gradio as gr
WEIGHTS = "VideoRebirthAI/hyperflow-h3-flash"
OUT_DIR = Path("/tmp/hyperflow-h3-flash")
OUT_DIR.mkdir(parents=True, exist_ok=True)
GPU_SIZE = os.environ.get("HF_GPU_SIZE", "xlarge")
OFFLOAD = os.environ.get("HF_OFFLOAD", "stream")
TE_PRECISION = os.environ.get("HF_TE_PRECISION", "fp8")
FPS = 24
MIN_FRAMES, MAX_FRAMES = 124, 362 # 5.2 s to 15 s at 24 fps, rounds up to 17n+5
MAX_KEYFRAME_PIXELS = 768 * 1344
STATUS: dict[str, str | None] = {"error": None, "loaded_in": None}
PIPE = None
OFFLOAD_DONE = False
TASK_LABELS = {
"t2va": "text → video + audio",
"i2va": "first frame → video + audio",
"fl2va": "first + last frame → video + audio",
}
def align_num_frames(num_frames: int) -> int:
while num_frames % 17 != 5:
num_frames += 1
return max(MIN_FRAMES, min(MAX_FRAMES, num_frames))
def canvas(aspect: str) -> tuple[int, int]:
ratio_w, ratio_h = (int(part) for part in aspect.split(":"))
ratio = ratio_w / ratio_h
width, height = (
(768.0 * ratio, 768.0) if ratio >= 1 else (768.0, 768.0 / ratio)
)
if width * height > MAX_KEYFRAME_PIXELS:
scale = (MAX_KEYFRAME_PIXELS / (width * height)) ** 0.5
width, height = width * scale, height * scale
m = 32
return max(m, round(height / m) * m), max(m, round(width / m) * m)
def gpu_seconds(task: str, steps: int, height: int, width: int, num_frames: int) -> int:
"""Duration budget for the @spaces.GPU call, fitted to the card's 1×H200 numbers:
36 s end-to-end at 5.2 s / 1344×768, 229 s at 15 s. Rows = packed video + audio latent
tokens, computed per hyperflow_h3_flash.pipeline.request_tokens (inlined — not re-exported
by the installed package's __init__)."""
frames = align_num_frames(int(num_frames))
latent_frames = (frames - 5) // 17 * 5 + 2
patches = (int(height) // 32) * (int(width) // 32)
rows = latent_frames * patches + round(frames / FPS * 40) * 2 # video + audio latents
keyframes = {"i2va": 1, "fl2va": 2}.get(task, 0)
rows += keyframes * patches
denoise = int(steps) * 2.47e-9 * rows**2
decode = 15 + 15 * (int(height) * int(width) * frames) / (1344 * 768 * 124)
return int(min(900, max(70, denoise + decode + 48)))
def snapshot_transformer(subfolder: str = "fl2va_int8_convrot") -> str:
from huggingface_hub import snapshot_download
started = time.time()
path = snapshot_download(WEIGHTS, allow_patterns=[f"{subfolder}/*", "*.json"])
print(f"[boot] {WEIGHTS}/{subfolder} in {time.time() - started:.0f}s", flush=True)
return path
def _shim_cudart() -> None:
"""ZeroGPU replaces torch.cuda.cudart() with a shim that lacks cudaHostRegister/Unregister, which
hyperflow_h3_flash.offload uses to page-lock its host arenas. Register no-ops: the arenas stay pageable
(slower H2D streaming, same results) instead of crashing at load."""
try:
import spaces.zero.torch.cudart as zc
except Exception:
return
for name in ("cudaHostRegister", "cudaHostUnregister"):
if not hasattr(zc, name):
setattr(zc, name, lambda *args, **kwargs: 0)
print(f"[boot] shimmed spaces.zero.torch.cudart.{name}", flush=True)
def load() -> None:
global PIPE
try:
import torch
_shim_cudart()
from hyperflow_h3_flash import load_pipeline
path = snapshot_transformer()
started = time.time()
# CPU-only boot: ZeroGPU workers may boot with no GPU attached (build-time Stream() fails otherwise).
# resolve_offload returns "none" on CPU, so components load to host RAM; the CUDA-side
# apply_offload(...) runs inside the first @spaces.GPU call (see _run).
PIPE = load_pipeline(
"t2va",
weights=path,
device="cpu",
text_encoder_precision=TE_PRECISION,
)
STATUS["loaded_in"] = f"{time.time() - started:.0f}s"
print(f"[boot] pipeline ready on CPU in {STATUS['loaded_in']}s (te={TE_PRECISION}; {OFFLOAD} applied on first GPU call)", flush=True)
except Exception:
STATUS["error"] = traceback.format_exc(limit=6)
traceback.print_exc()
def _progress_hook(total: int, progress) -> object:
calls = {"n": 0}
def hook(module, args, kwargs):
calls["n"] += 1
progress(calls["n"] / total, desc=f"Step {calls['n']} of {total}")
return PIPE.transformer.register_forward_pre_hook(hook, with_kwargs=True)
def _duration(*args, **kwargs):
task, _first, _last, height, width, num_frames, _seed, steps = args[:8]
return gpu_seconds(task, int(steps), int(height), int(width), int(num_frames))
@spaces.GPU(duration=_duration, size=GPU_SIZE)
def _run(task, first_image, last_image, height, width, num_frames, seed, steps, prompt, progress=None):
global OFFLOAD_DONE
import numpy as np
import torch
from diffusers.utils.export_utils import encode_video
if not OFFLOAD_DONE:
from hyperflow_h3_flash.offload import apply_offload, model_components
started = time.perf_counter()
apply_offload(model_components(PIPE), torch.device("cuda"), OFFLOAD)
OFFLOAD_DONE = True
print(f"[gpu] {OFFLOAD} offload hooks applied in {time.perf_counter() - started:.1f}s", flush=True)
prepared = {}
if task in ("i2va", "fl2va"):
from PIL import Image
prepared["image"] = Image.open(first_image).convert("RGB")
if task == "fl2va":
from PIL import Image
prepared["last_image"] = Image.open(last_image).convert("RGB")
hook = _progress_hook(int(steps), progress)
torch.cuda.synchronize()
torch.cuda.reset_peak_memory_stats()
started = time.perf_counter()
try:
with torch.inference_mode():
out = PIPE(
prompt=prompt,
height=int(height),
width=int(width),
num_frames=align_num_frames(int(num_frames)),
num_inference_steps=int(steps),
generator=torch.Generator().manual_seed(int(seed)),
output=["videos", "audio", "sampling_rate"],
**prepared,
)
torch.cuda.synchronize()
finally:
hook.remove()
total = time.perf_counter() - started
frames = out["videos"][0]
if isinstance(frames, (list, tuple)):
frames = np.stack([np.asarray(f) for f in frames])
if hasattr(frames, "detach"):
frames = frames.float().clamp(0, 1).cpu().numpy()
if frames.dtype != np.uint8 and frames.max() <= 1.5:
frames = (np.clip(frames, 0.0, 1.0) * 255.0).round().astype(np.uint8)
wave = out["audio"][0]
if isinstance(wave, (list, tuple)):
wave = np.stack([np.asarray(w) for w in wave])
if hasattr(wave, "detach"):
wave = wave.float().cpu().numpy()
wave = np.asarray(wave, dtype=np.float32)
if wave.ndim == 1:
wave = wave[None, :]
wave_t = torch.from_numpy(wave)
peak_gib = torch.cuda.max_memory_allocated() / 2**30
path = OUT_DIR / f"hyperflow-{int(time.time() * 1000)}.mp4"
enc_started = time.perf_counter()
encode_video(
frames,
fps=FPS,
output_path=str(path),
audio=wave_t,
audio_sample_rate=int(out["sampling_rate"]),
)
timings = {"pipeline": total, "encode": time.perf_counter() - enc_started, "peak_gib": peak_gib}
torch.cuda.reset_peak_memory_stats()
print(f"[gen] {task} {width}x{height}x{num_frames} seed={seed} {timings}", flush=True)
return str(path), timings
def _duration(task, first_image, last_image, height, width, num_frames, seed, steps, prompt, progress=None):
return gpu_seconds(task, int(steps), int(height), int(width), int(num_frames))
def resolve_task(mode, first_image, last_image):
if mode == "Frames":
if first_image and last_image:
return "fl2va"
if first_image:
return "i2va"
return "t2va"
def classify(path: str) -> str:
kind = (mimetypes.guess_type(path)[0] or "").split("/")[0]
if kind == "image":
return kind
raise gr.Error("Keyframes must be images.")
def generate(prompt, mode, first_image, last_image, aspect, seconds, steps, seed, randomize, progress=gr.Progress()):
if STATUS["error"]:
raise gr.Error("The model failed to load — check the Space logs.\n\n```\n" + STATUS["error"][-900:] + "\n```")
if not prompt or not prompt.strip():
raise gr.Error("HyperFlow H3 Flash always takes a prompt. Describe subject, action, setting, camera and sound.")
if randomize:
seed = random.randint(0, 2**31 - 1)
task = resolve_task(mode, first_image, last_image)
if mode == "Frames" and not first_image:
raise gr.Error("Add a first frame to animate from — a last frame alone isn't supported in this demo.")
if task == "t2va":
first_image, last_image = None, None
height, width = canvas(aspect)
num_frames = align_num_frames(int(seconds) * FPS)
desc = TASK_LABELS[task]
progress(0.0, desc=desc)
path, timings = _run(task, first_image, last_image, height, width, num_frames, int(seed), int(steps), prompt.strip(), progress=progress)
stages = " · ".join(
f"{key} {value:.2f}{' GiB' if key == 'peak_gib' else 's'}" for key, value in timings.items()
)
report = (
f"**{desc}** · `{width}×{height}` · {num_frames} frames ({num_frames / FPS:.2f}s) · {steps} steps · seed `{seed}` · "
f"loaded in {STATUS['loaded_in']}\n\n<span class='pk-dim'>{stages}</span>"
)
return path, report, int(seed)
def upright(image_path):
if not image_path:
return image_path
from PIL import Image, ImageOps
with Image.open(image_path) as image:
if image.getexif().get(0x0112, 1) == 1:
return image_path
fixed = ImageOps.exif_transpose(image).convert("RGB")
out = OUT_DIR / f"upright-{int(time.time() * 1000)}.png"
fixed.save(out)
return str(out)
THEME = gr.themes.Base(
primary_hue=gr.themes.colors.violet,
neutral_hue=gr.themes.colors.stone,
radius_size=gr.themes.sizes.radius_lg,
font=[gr.themes.GoogleFont("Inter Tight"), "ui-sans-serif", "system-ui", "sans-serif"],
font_mono=[gr.themes.GoogleFont("JetBrains Mono"), "ui-monospace", "monospace"],
).set(
body_background_fill="#faf8f4",
body_background_fill_dark="#0a0a0a",
background_fill_primary="#ffffff",
background_fill_primary_dark="#141312",
background_fill_secondary="#f3efe9",
background_fill_secondary_dark="#1b1a18",
block_background_fill="#ffffff",
block_background_fill_dark="#141312",
block_border_color="#ebe6da",
block_border_color_dark="#2a2826",
border_color_primary="#e2dacf",
border_color_primary_dark="#2a2826",
block_shadow="none",
block_label_text_color="#6b6560",
input_background_fill="#fcfaf7",
input_background_fill_dark="#1b1a18",
button_primary_background_fill="#0a0a0a",
button_primary_background_fill_hover="#2b2724",
button_primary_text_color="#faf8f4",
button_primary_background_fill_dark="#faf8f4",
button_primary_background_fill_hover_dark="#e2dacf",
button_primary_text_color_dark="#0a0a0a",
button_large_radius="999px",
slider_color="#806eca",
slider_color_dark="#cfc3ff",
checkbox_background_color_selected="#806eca",
)
CSS = """
.gradio-container{max-width:1280px !important;margin:0 auto}
#hf-head{display:flex;align-items:center;gap:18px;padding:18px 4px 6px}
#hf-head h1{font-weight:700;font-size:30px;letter-spacing:-.02em;margin:0;line-height:1.05}
#hf-head p{margin:4px 0 0;color:#6b6560;font-size:14px}
.dark #hf-head p{color:#a8a29e}
#hf-head .hf-chip{margin-left:auto;font-size:12px;padding:6px 12px;border-radius:999px;background:#cfc3ff33;color:#5b4aa8;border:1px solid #cfc3ff80;white-space:nowrap}
.dark #hf-head .hf-chip{color:#cfc3ff}
#hf-head .hf-chip a{color:inherit}
#hf-prompt textarea{font-size:15px;line-height:1.5}
#hf-go{min-height:52px;font-size:16px;font-weight:600;letter-spacing:.01em}
.pk-dim{color:#8a847d;font-size:12.5px}
#hf-report{min-height:72px}
#hf-mode .wrap{gap:6px}
.pk-note{padding:10px 14px 12px !important;font-size:12.5px;line-height:1.5;color:#8a847d}
.pk-note p{margin:0 !important}
.pk-frames{gap:1px}
#hf-secs{overflow:visible}
footer{display:none !important}
"""
HEADER_HTML = """
<div id="hf-head">
<div>
<h1>HyperFlow H3 Flash</h1>
<p>Five- to fifteen-second video with a synchronized stereo soundtrack · text and endpoint keyframes · 4 / 6 / 8 steps · 768p</p>
</div>
<span class="hf-chip"><a href="https://huggingface.co/VideoRebirthAI/hyperflow-h3-flash" target="_blank">VideoRebirthAI/hyperflow-h3-flash</a></span>
</div>
"""
EXAMPLES = [
["A chef slides a steaming bowl of ramen across a wooden counter; rain taps on the window.", "Text", None, None, "16:9", 5, "6"],
["A red fox trots through a snowy forest at dawn, breath steaming; wind through the pines.", "Text", None, None, "16:9", 5, "6"],
["A dancer spins in place under warm stage light; skirt swishes and cymbals crash.", "Text", None, None, "9:16", 5, "6"],
]
def build() -> gr.Blocks:
if STATUS["error"] is None and PIPE is None:
load()
with gr.Blocks(title="HyperFlow H3 Flash") as demo:
gr.HTML(HEADER_HTML)
if STATUS["error"]:
gr.Textbox(value=STATUS["error"], label="Loading failed — full traceback", lines=12, interactive=False)
with gr.Row(equal_height=False):
with gr.Column(scale=11):
mode = gr.Radio(["Text", "Frames"], value="Text", label="Condition on", elem_id="hf-mode")
with gr.Group(visible=False) as frames_group:
with gr.Row(elem_classes="pk-frames"):
first_image = gr.Image(label="First frame", type="filepath", height=220)
last_image = gr.Image(label="Last frame (optional)", type="filepath", height=220)
gr.Markdown(
"Keyframes are center-cropped onto the canvas. A single frame animates from it; with both, "
"the clip moves from the first to the last.",
elem_classes="pk-note",
)
prompt = gr.Textbox(
label="Prompt",
lines=3,
placeholder="Subject, action, setting, camera movement, and sound in one prompt. No negative prompt (CFG = 1).",
elem_id="hf-prompt",
)
with gr.Row():
aspect = gr.Dropdown(["16:9", "9:16", "1:1", "4:3", "3:4", "21:9"], value="16:9", label="Aspect", scale=2)
seconds = gr.Slider(5, 15, value=5.2, step=0.2, label="Seconds", scale=2, elem_id="hf-secs")
steps = gr.Radio(["4", "6", "8"], value="6", label="Steps", scale=2)
with gr.Row():
run = gr.Button("Generate", variant="primary", size="lg", elem_id="hf-go")
seed = gr.Number(42, precision=0, label="Seed", scale=1)
randomize = gr.Checkbox(True, label="Randomize", scale=1)
with gr.Column(scale=10):
video = gr.Video(label="Video + soundtrack", autoplay=True, height=460)
report = gr.Markdown(elem_id="hf-report")
gr.Markdown(
"<span class='pk-dim'>Prompt in English or Chinese; describe what you see and what you hear. "
"Steps: 4 = speed, 6 = balanced, 8 = quality. Longer clips and more steps need more GPU quota.</span>",
)
gr.Examples(EXAMPLES, [prompt, mode, first_image, last_image, aspect, seconds, steps])
def show(selected):
return gr.update(visible=selected == "Frames")
first_image.upload(upright, first_image, first_image)
last_image.upload(upright, last_image, last_image)
mode.change(show, mode, frames_group)
run.click(
generate,
[prompt, mode, first_image, last_image, aspect, seconds, steps, seed, randomize],
[video, report, seed],
api_name="generate",
)
return demo
demo = build()
if __name__ == "__main__":
demo.launch(show_error=True, max_threads=64)