Streaming CWM demo

.gitattributes

@@ -33,3 +33,9 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+examples/ex03_anchor.png filter=lfs diff=lfs merge=lfs -text
+examples/ex06_anchor.png filter=lfs diff=lfs merge=lfs -text
+examples/ex08_anchor.png filter=lfs diff=lfs merge=lfs -text
+examples/ex29_anchor.png filter=lfs diff=lfs merge=lfs -text
+examples/ex31_anchor.png filter=lfs diff=lfs merge=lfs -text
+examples/ex36_anchor.png filter=lfs diff=lfs merge=lfs -text

README.md

@@ -4,10 +4,32 @@ emoji: 🌍
 colorFrom: blue
 colorTo: red
 sdk: gradio
-sdk_version: 5.50.0
+sdk_version: 6.30.0
 app_file: app.py
 short_description: Causal chunk-by-chunk world-model video streaming
 python_version: "3.12"
 startup_duration_timeout: 1h
+models:
+  - Luffuly/streaming-code-world-model
+  - Comfy-Org/MiniMax-H3
 ---
-(prep stage)
+
+# Streaming Code World Model
+
+Demo of [Streaming Code World Model](https://github.com/TingtingLiao/streaming-code-world-model)
+([checkpoints](https://huggingface.co/Luffuly/streaming-code-world-model)): causal, chunk-by-chunk video generation
+from a live control proxy, built on [Code World Model](https://github.com/buaacyw/code-world-model)
+(MiniMax-H3 Ref2VA + CWM LoRA).
+
+Sampling follows `rtcwm.eval.open_world generate --steps 4 --rule x0 --shift 5 --kv-reuse --fp8` with
+`streaming_dmd_4step.safetensors`. Adaptations for ZeroGPU (RTX PRO 6000 Blackwell): FlashAttention-3 / flex-attention
+replaced by PyTorch SDPA over the same block-causal key prefixes, FP8 block linears (the authors' `--fp8`), and the
+per-block AdaLN projections precomputed for the sampler's fixed set of timesteps.
+
+Scenes are released Code World Model inference examples
+([NTU-yiwen/code-world-model-inference-examples-40](https://huggingface.co/datasets/NTU-yiwen/code-world-model-inference-examples-40)):
+their start frame, the "Appearance and scene" caption of the first window and the Depth + Semantic-ID proxy, pre-encoded
+(Qwen3-VL-32B states, VAE latents) exactly as `rtcwm.eval.open_world prepare` does.
+
+Vendored code: `musubi_tuner/` (kohya-ss/musubi-tuner @ c4dc6b8 + the CWM `minimax-h3-retake34` and headless patches,
+Apache-2.0), `cwm_h3_inference/` (buaacyw/code-world-model, Apache-2.0), `rtcwm/` (TingtingLiao/streaming-code-world-model).

app.py

@@ -1,124 +1,250 @@
-"""TEMPORARY prep stage: encode the conditioning caches of a few released CWM examples, exactly as
-`python -m rtcwm.eval.open_world prepare` does (Qwen3-VL-32B states of the static caption, Picture 1 latent,
-proxy (Depth + Semantic-ID) latents, silent audio latent). Replaced by the real demo afterwards."""
+"""Streaming Code World Model -- causal, chunk-by-chunk world-model video on ZeroGPU.
+
+Reference implementation: https://github.com/TingtingLiao/streaming-code-world-model (rtcwm), sampling exactly as
+`python -m rtcwm.eval.open_world generate --steps 4 --rule x0 --shift 5 --kv-reuse --fp8` with the
+`streaming_dmd_4step` LoRA on the released MiniMax-H3 Ref2VA model. Hardware adaptations (RTX PRO 6000, sm_120):
+  * FlashAttention-3 / flex_attention -> torch SDPA over the same key prefixes (rtcwm.model.attention.dense_df);
+  * the blocks' qkv / out / fc1 / fc2 in FP8 exactly as rtcwm.model.fp8 (the authors' --fp8 option);
+  * each block's AdaLN projection only ever sees the time embedding of a handful of fixed times (condition rows
+    1.0 / 0.999 and the 4 sampling steps), so its outputs are precomputed once at startup instead of keeping the
+    26 GB of AdaLN weights on the GPU.
+"""
 import os
 import sys
 
-sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
+APP_DIR = os.path.dirname(os.path.abspath(__file__))
+sys.path.insert(0, APP_DIR)
+
+import spaces  # noqa: E402  (before torch)
+
+import json  # noqa: E402
+import random  # noqa: E402
+import tempfile  # noqa: E402
+import time  # noqa: E402
+from pathlib import Path  # noqa: E402
+
+import av  # noqa: E402
+import gradio as gr  # noqa: E402
+import numpy as np  # noqa: E402
+import torch  # noqa: E402
+from huggingface_hub import hf_hub_download  # noqa: E402
+from PIL import Image  # noqa: E402
+from safetensors import safe_open  # noqa: E402
+from safetensors.torch import load_file  # noqa: E402
+
+from musubi_tuner.minimax_h3.model import load_h3_transformer  # noqa: E402
+from musubi_tuner.minimax_h3.video_vae import load_video_vae  # noqa: E402
+from rtcwm.model import sampler as rt_sampler  # noqa: E402
+from rtcwm.model.layout import CHUNK_LATENTS, CLIP_FRAMES, TARGET_LATENTS  # noqa: E402
+from rtcwm.model.packing import audio_frames  # noqa: E402
 
-import spaces
-import io
-import json
-import shutil
-import tempfile
-import zipfile
-from pathlib import Path
+H3_REPO, H3_REV = "Comfy-Org/MiniMax-H3", "0543966fbdce5ba05709a8f2031c94bdba629b4a"
+LORA_REPO, LORA_FILE = "Luffuly/streaming-code-world-model", "streaming_dmd_4step.safetensors"
+STEPS, SHIFT, RULE = 4, 5.0, "x0"        # the released 4-step DMD sampling (model card)
+MAX_CHUNKS = 24
+FPS = 24
+EX_DIR = Path(APP_DIR) / "examples"
 
-import numpy as np
-import torch
-import gradio as gr
-from huggingface_hub import hf_hub_download, snapshot_download
-from PIL import Image
-from safetensors.torch import save_file
+# --------------------------------------------------------------------------------------------- examples
+SCENES = json.loads((EX_DIR / "scenes.json").read_text())         # name -> {id, chunks, caption}
+SILENCE = load_file(str(EX_DIR / "silence_audio.safetensors"))["audio"]
+PALETTE = np.array([[0, 0, 0], [135, 206, 235], [30, 90, 200], [150, 120, 80], [90, 90, 90], [60, 160, 60],
+                    [200, 120, 80], [220, 200, 60], [230, 40, 40], [200, 80, 200], [40, 220, 220], [250, 250, 250]],
+                   dtype=np.uint8)                                   # rtcwm.eval.open_world.PALETTE
 
-H3_REPO, H3_REV = "Comfy-Org/MiniMax-H3", "0543966fbdce5ba05709a8f2031c94bdba629b4a"
-QWEN_REPO, QWEN_REV = "Qwen/Qwen3-VL-32B-Instruct", "0cfaf48183f594c314753d30a4c4974bc75f3ccb"
-EX_REPO, EX_REV = "NTU-yiwen/code-world-model-inference-examples-40", "f8307bf50cb19bed191682734fdf444738160f91"
-EXAMPLES = [3, 6, 8, 29, 31, 36]
-CHUNKS = 24
-CLIP_FRAMES = 17
-
-models = Path(snapshot_download(H3_REPO, revision=H3_REV, local_dir="/tmp/h3", allow_patterns=[
-    "text_encoders/qwen3vl_32b_minimax_h3_bf16.safetensors", "vae/minimax_h3_video_vae_fp16.safetensors",
-    "vae/minimax_h3_audio_vae_fp32.safetensors"]))
-proc_dir = Path(snapshot_download(QWEN_REPO, revision=QWEN_REV, local_dir="/tmp/qwen_proc", allow_patterns=[
-    "chat_template.json", "config.json", "generation_config.json", "merges.txt", "preprocessor_config.json",
-    "tokenizer_config.json", "tokenizer.json", "video_preprocessor_config.json", "vocab.json"]))
-src = Path(snapshot_download(EX_REPO, repo_type="dataset", revision=EX_REV, allow_patterns=[
-    "manifest.json", "semantic_id_palette.json"] + [f"{i}/*" for i in EXAMPLES] + [f"{i}/**" for i in EXAMPLES]))
-
-import materialize_hf_examples as mat
-
-REL = Path("/tmp/release")
-for i in EXAMPLES:
-    mat.materialize_example(src, REL, i, require_config=False, frame_limit=(CHUNKS + 1) * CLIP_FRAMES)
-print("materialized", flush=True)
-
-from musubi_tuner.minimax_h3.audio_vae import load_audio_vae
-from musubi_tuner.minimax_h3.text_encoder import load_h3_processor, load_h3_text_encoder
-from musubi_tuner.minimax_h3.video_vae import encode_video_condition, load_video_vae
-
-vae = load_video_vae(models / "vae/minimax_h3_video_vae_fp16.safetensors", device="cpu", dtype=torch.float16).eval().requires_grad_(False)
-vae.to("cuda")
-avae = load_audio_vae(models / "vae/minimax_h3_audio_vae_fp32.safetensors", device="cpu", dtype=torch.float32).eval()
-avae.to("cuda")
-proc = load_h3_processor(str(proc_dir))
-qwen = load_h3_text_encoder(str(models / "text_encoders/qwen3vl_32b_minimax_h3_bf16.safetensors"),
-                            processor_path=str(proc_dir), device="cpu", dtype=torch.bfloat16).eval()
-qwen.to("cuda")
-print("models loaded", flush=True)
-
-
-def _prepare_pixels(frames):  # musubi_tuner.minimax_h3_generate_video._prepare_pixels
-    if frames.dtype == torch.uint8:
-        frames = frames.float().div(127.5).sub(1.0)
-    else:
-        frames = frames.float().mul(2.0).sub(1.0)
-    return frames.permute(3, 0, 1, 2).unsqueeze(0).contiguous()
-
-
-@spaces.GPU(duration=600, size="xlarge")
-def prep() -> str:
-    """Encode the example caches and return a zip."""
-    from cwm_h3_inference.cache import _load_anchor
-    from cwm_h3_inference.config import _canonical_caption
-    from cwm_h3_inference.constants import TARGET_HEIGHT, TARGET_WIDTH, WINDOW_AUDIO_SAMPLES
-    from cwm_h3_inference.duv import load_duv_frame
-    from rtcwm.eval.open_world import static_caption
-    from rtcwm.train.preencode import static_text_states
 
+def _scene_tensors(name: str) -> dict:
+    with safe_open(str(EX_DIR / f"ex{SCENES[name]['id']:02d}.safetensors"), "pt") as f:
+        return {k: f.get_tensor(k) for k in f.keys()}
+
+
+SCENE_DATA = {name: _scene_tensors(name) for name in SCENES}
+
+
+def load_window(name: str, j: int) -> dict:
+    """rtcwm.train.dataset.load_window(..., start=0, with_target=False) on the precomputed caches."""
+    d = SCENE_DATA[name]
+    duv = d["duv"][:, CHUNK_LATENTS * j:CHUNK_LATENTS * j + TARGET_LATENTS]
+    assert duv.shape[1] == TARGET_LATENTS, (name, j, duv.shape)
+    ref = torch.nn.functional.pad(duv.unsqueeze(0), (0, 1, 0, 0), value=0.0)
+    audio = SILENCE[..., :audio_frames(TARGET_LATENTS)].unsqueeze(0).clone()
+    return dict(audio=audio, anchor=d["anchor"].unsqueeze(0), ref=ref, hidden=d["hidden"].unsqueeze(0),
+                tags=d["tags"].unsqueeze(0), stream_start=j == 0)
+
+
+# --------------------------------------------------------------------------------------------- load
+from streaming_runtime import prepare_transformer  # noqa: E402
+
+dit = hf_hub_download(H3_REPO, "diffusion_models/minimax_h3_ref2va_bf16.safetensors", revision=H3_REV, local_dir="/tmp/h3")
+lora_path = hf_hub_download(LORA_REPO, LORA_FILE)
+transformer = load_h3_transformer(dit, device="cpu", dtype=torch.bfloat16, attn_mode="torch")
+transformer = prepare_transformer(transformer, load_file(lora_path)).to("cuda")
+vae_path = hf_hub_download(H3_REPO, "vae/minimax_h3_video_vae_fp16.safetensors", revision=H3_REV, local_dir="/tmp/h3")
+vae = load_video_vae(vae_path, device="cpu", dtype=torch.float16).eval().requires_grad_(False).to("cuda")
+print("models ready", flush=True)
+
+
+# --------------------------------------------------------------------------------------------- helpers
+def _write_mp4(frames: np.ndarray, crf: int = 18) -> str:
+    path = tempfile.NamedTemporaryFile(suffix=".mp4", delete=False).name
+    with av.open(path, "w") as c:
+        s = c.add_stream("libx264", rate=FPS)
+        s.width, s.height, s.pix_fmt = frames.shape[2], frames.shape[1], "yuv420p"
+        s.options = {"crf": str(crf)}
+        for f in frames:
+            for p in s.encode(av.VideoFrame.from_ndarray(np.ascontiguousarray(f), format="rgb24")):
+                c.mux(p)
+        for p in s.encode():
+            c.mux(p)
+    return path
+
+
+def _to_uint8(px: torch.Tensor) -> np.ndarray:
+    """[3,F,H,W] in [-1,1] -> [F,H,W,3] uint8 (rtcwm.eval.video.decode)."""
+    return ((px.float().clamp(-1, 1) + 1) * 127.5).round().byte().permute(1, 2, 3, 0).cpu().numpy()
+
+
+_PROXY_CACHE = {}
+
+
+def proxy_video(name: str, chunks: int) -> str:
+    """The Semantic-ID half of the proxy the stream follows, coloured as in rtcwm.eval.open_world.proxy_frames."""
+    key = (name, int(chunks))
+    if key not in _PROXY_CACHE:
+        sem = np.load(EX_DIR / f"ex{SCENES[name]['id']:02d}_semantic.npz")["semantic_id"][: int(chunks) * CLIP_FRAMES]
+        frames = PALETTE[np.clip(sem, 0, 11)]
+        frames = np.stack([np.asarray(Image.fromarray(f).resize((672, 384), Image.NEAREST)) for f in frames])
+        _PROXY_CACHE[key] = _write_mp4(frames, crf=23)
+    return _PROXY_CACHE[key]
+
+
+def _duration(scene, chunks=8, seed=0, *args, **kwargs):
+    return int(40 + 14 * int(chunks))
+
+
+# --------------------------------------------------------------------------------------------- inference
+@spaces.GPU(duration=_duration, size="xlarge")
+def generate(scene: str, chunks: int = 8, seed: int = 0):
+    """Stream a world-model video chunk by chunk from a released Code World Model scene.
+
+    Each 17-frame chunk (0.71 s at 24 fps) is generated causally from the start image, its caption, the
+    Depth + Semantic-ID proxy up to that chunk and the previous chunks (KV cache, 6-chunk attention span),
+    with the 4-step DMD streaming model.
+
+    Args:
+        scene: one of the released example scenes.
+        chunks: number of 0.71 s chunks to stream (1-24).
+        seed: random seed.
+
+    Returns:
+        (latest-chunk preview frame, final mp4 path) -- yielded progressively.
+    """
+    chunks = max(1, min(MAX_CHUNKS, int(chunks)))
     dev = torch.device("cuda")
-    out = Path(tempfile.mkdtemp())
+    gen = torch.Generator().manual_seed(int(seed))
+    t_start = time.time()
+    torch.cuda.reset_peak_memory_stats()
+    events = []
+
+    def on_chunk(k, secs):
+        events.append((k, secs))
+        print(f"{scene}: chunk {k + 1}/{chunks} {secs:.1f}s", flush=True)
+
+    # stream the chunks (rtcwm.model.sampler.generate_stream), previewing each finished chunk
+    import queue
+    import threading
+    q = queue.Queue()
+    result = {}
+
+    def preview_chunk(k, secs):
+        on_chunk(k, secs)
+        q.put(k)
+
+    def worker():
+        try:
+            with torch.inference_mode():
+                result["lat"] = rt_sampler.generate_stream(
+                    transformer, lambda j: load_window(scene, j), chunks, STEPS, gen, dev, RULE, SHIFT,
+                    history_noise=0.0, kv_reuse=True, on_chunk=preview_chunk)
+        except Exception as e:  # noqa: BLE001
+            result["err"] = e
+        q.put(None)
+
+    th = threading.Thread(target=worker, daemon=True)
+    th.start()
+    while True:
+        k = q.get()
+        if k is None:
+            break
+        yield gr.update(label=f"Streaming… chunk {k + 1}/{chunks} ({(k + 1) * CLIP_FRAMES / FPS:.1f} s generated)"), None
+    th.join()
+    if "err" in result:
+        raise result["err"]
+    lat = result["lat"]
+    gen_s = time.time() - t_start
     with torch.inference_mode():
-        a = avae.encode(torch.zeros(1, 2, WINDOW_AUDIO_SAMPLES, device=dev)).float().cpu()[0]
-    save_file({"audio": a.contiguous()}, str(out / "silence_audio.safetensors"))
-    log = []
-    for i in EXAMPLES:
-        ex = REL / "examples" / f"example_{i:02d}"
-        cond = ex / "conditions"
-        prompt = (src / str(i) / "prompts" / "000.txt").read_text()
-        k = min(CHUNKS, len(list(cond.glob("*.semantic_id.png"))) // CLIP_FRAMES)
-        duv = np.stack([load_duv_frame(cond, f, for_qwen=False) for f in range(k * CLIP_FRAMES)])
-        duv = np.concatenate([duv, np.repeat(duv[-1:], CLIP_FRAMES, 0)], 0)
-        with torch.inference_mode():
-            ref = encode_video_condition(vae, _prepare_pixels(torch.from_numpy(duv)).to(dev, torch.float16))[0]
-        ref = ref.float().cpu()[:, :5 * k].contiguous()
-        picture1 = _load_anchor(ex / "first_frame.png", size=(TARGET_WIDTH, TARGET_HEIGHT))
-        png = out / f"ex{i:02d}_anchor.png"
-        Image.fromarray(picture1).save(png)
-        with torch.inference_mode():
-            lat = encode_video_condition(vae, _prepare_pixels(torch.from_numpy(picture1).unsqueeze(0)).to(dev, torch.float16))
-        caption = _canonical_caption(static_caption(prompt), f"ow_example_{i:02d}/static_c00")
-        hidden, tags = static_text_states(proc, qwen, picture1, png, cond, 0, caption)
-        sem = np.stack([np.asarray(Image.open(cond / f"{f:06d}.semantic_id.png")) for f in range(k * CLIP_FRAMES)])
-        np.savez_compressed(out / f"ex{i:02d}_semantic.npz", semantic_id=sem.astype(np.uint8))
-        save_file({"duv": ref, "anchor": lat[0].float().cpu().contiguous(),
-                   "hidden": hidden.to("cpu", torch.bfloat16).contiguous(), "tags": tags.to("cpu", torch.int64).contiguous()},
-                  str(out / f"ex{i:02d}.safetensors"), metadata={"chunks": str(k), "caption": caption})
-        log.append(dict(id=i, chunks=k, duv=list(ref.shape), anchor=list(lat.shape), hidden=list(hidden.shape)))
-        print(log[-1], flush=True)
-    (out / "log.json").write_text(json.dumps(log))
-    z = tempfile.NamedTemporaryFile(suffix=".zip", delete=False).name
-    with zipfile.ZipFile(z, "w", zipfile.ZIP_STORED) as zf:
-        for p in out.iterdir():
-            zf.write(p, p.name)
-    return z
+        px = vae.decode(lat.to(dev, torch.float16))[0]
+    frames = _to_uint8(px)
+    path = _write_mp4(frames)
+    peak = torch.cuda.max_memory_allocated() / 2**30
+    print(f"{scene}: {chunks} chunks, {len(frames)} frames, stream {gen_s:.1f}s, total {time.time() - t_start:.1f}s, "
+          f"peak {peak:.1f} GiB", flush=True)
+    yield gr.update(label=f"Done: {chunks} chunks ({len(frames) / FPS:.1f} s) in {time.time() - t_start:.0f} s",
+                    value=Image.fromarray(frames[-1]).resize((672, 384))), path
 
 
-with gr.Blocks() as demo:
-    gr.Markdown("Preparing example caches (temporary).")
-    b = gr.Button("prep")
-    f = gr.File()
-    b.click(prep, None, f, api_name="prep")
+# --------------------------------------------------------------------------------------------- UI
+def show_scene(scene: str, chunks: int = 8):
+    """Preview a scene's start image, caption and proxy."""
+    s = SCENES[scene]
+    return str(EX_DIR / f"ex{s['id']:02d}_anchor.png"), s["caption"], proxy_video(scene, chunks)
 
-demo.launch()
+
+CSS = """
+#col-container { max-width: 1280px; margin: 0 auto; }
+.dark .gradio-container { color: var(--body-text-color); }
+"""
+NAMES = list(SCENES)
+
+with gr.Blocks() as demo:
+    with gr.Column(elem_id="col-container"):
+        gr.Markdown(
+            "# Streaming Code World Model\n"
+            "Causal, chunk-by-chunk video generation from a live control proxy: each 0.71 s chunk is generated from "
+            "the start image, its caption, the Depth + Semantic-ID proxy **so far** and the previous chunks — never "
+            "the future. 4-step DMD streaming LoRA on MiniMax-H3 Ref2VA + "
+            "[Code World Model](https://github.com/buaacyw/code-world-model).  \n"
+            "[GitHub](https://github.com/TingtingLiao/streaming-code-world-model) · "
+            "[Checkpoints](https://huggingface.co/Luffuly/streaming-code-world-model)"
+        )
+        with gr.Row():
+            with gr.Column(scale=1):
+                scene = gr.Dropdown(NAMES, value=NAMES[0], label="Scene (released CWM example)")
+                chunks = gr.Slider(1, MAX_CHUNKS, value=8, step=1, label="Chunks to stream (0.71 s each)")
+                run = gr.Button("Stream", variant="primary")
+                anchor = gr.Image(label="Start image (Picture 1)", interactive=False, height=240)
+                proxy = gr.Video(label="Control proxy (Semantic-ID view)", interactive=False, autoplay=True, loop=True,
+                                 height=240)
+                with gr.Accordion("Advanced", open=False):
+                    seed = gr.Number(value=0, precision=0, label="Seed")
+                    randomize = gr.Checkbox(value=False, label="Randomize seed")
+                    caption = gr.Textbox(label="Static caption (pre-encoded by Qwen3-VL-32B)", lines=6,
+                                         interactive=False)
+            with gr.Column(scale=2):
+                status = gr.Image(label="Stream status", interactive=False, height=200)
+                video = gr.Video(label="Generated stream (1344×768, 24 fps)", autoplay=True)
+
+        def _pick_seed(s, r):
+            return random.randint(0, 2**31 - 1) if r else int(s)
+
+        scene.change(show_scene, [scene, chunks], [anchor, caption, proxy], api_name=False)
+        chunks.release(show_scene, [scene, chunks], [anchor, caption, proxy], api_name=False)
+        run.click(_pick_seed, [seed, randomize], seed, api_name=False).then(
+            generate, [scene, chunks, seed], [status, video], api_name="generate")
+        demo.load(show_scene, [scene, chunks], [anchor, caption, proxy], api_name=False)
+
+        gr.Examples(examples=[[n, 8] for n in NAMES], inputs=[scene, chunks], outputs=[status, video], fn=generate,
+                    cache_examples=True, cache_mode="lazy", label="Scenes")
+
+demo.queue().launch(theme=gr.themes.Citrus(), css=CSS, mcp_server=True)

examples/ex03.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4785caba442438656cd34056a7da119d2966878d67745868c61edcd86d21ea98
+size 32910376

examples/ex03_semantic.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:10d31cb12bf7c4b20df70213e8f6a17bab0127741b1e049326229ec1d18844e5
+size 707731

examples/ex06.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:352418f73ffa28f6b08c418ba61a029c3c27878f2e32563d06539d12a9c0239b
+size 32438672

examples/ex06_semantic.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a03e31f973b6582cbab5236eb83d5ee13232b954751fa81ab4fdeb27f699fd93
+size 340520

examples/ex08.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8cff117933fd681f8ae9ca002d15423c24c223a214ea9273d1bcce29b680436d
+size 31997816

examples/ex08_semantic.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1e803072d468f6b652253bf0bbb65c24071883f11a84261cbd3e7f1dbc315029
+size 947066

examples/ex29.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e8841affedb78a159c92822cf487781b96ffd81455d97dc5d9812bdc2a9b2f90
+size 32725840

examples/ex29_semantic.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ce08de120367af8cd161e880ee37789c1b8ab5c807e1dbf20f12cec3297ca7fc
+size 675227

examples/ex31.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c8fce85a06bfa466a187baa680ef268e5fec2b067c3b5033660b530697f675a7
+size 32028552

examples/ex31_semantic.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5c1eb24ccd1df760a4d6b439ed35821b1f7d81889cb0988a95a16f555fd9e55a
+size 591886

examples/ex36.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e76cbb7cb2b884ca2e19eb4d0912a3a937358d0b778f5e09267e1bbaefa4fb6
+size 34253600

examples/ex36_semantic.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:14aef84569137ffa9ee0365cc71a868519d95bde238eee94ab718e7cc3f560ef
+size 902838

examples/scenes.json

@@ -0,0 +1,32 @@
+{
+ "Clockwork courier \u00b7 retro-futurist promenade": {
+  "id": 3,
+  "chunks": 24,
+  "caption": "Appearance and scene:\n\nThe protagonist is a friendly nonhuman clockwork exposition courier with a compact, round-bodied mechanical build. Its warm ivory enamel torso is egg-shaped and edged by fine soft-charcoal seams, tiny rivets, circular coral fittings, and small brass mechanisms. A clear bubble helm surrounds its powder-blue head, and pointed coral-and-ivory receiver fins project from either side. Two coiled brass winding keys rise from its upper back. Its short articulated arms and legs have cream enamel shells, exposed brass pivot rings, mitten-like mechanical hands, and rounded coral-soled feet; these are stable identity features. A broad coral utility sash wraps diagonally around its waist and supports a square coral satchel with a brass clasp. At the beginning, it leans forward in a running stride near the lower center-left, facing away and toward screen-right, with one arm reaching forward, the other lowered, one foot planted behind, and the opposite foot lifted. Its long deep-teal printed shadow falls ahead-right across the butter-yellow roadway.\n\nThe directly approached vehicle later in view is the coral open-cockpit roadster, a compact atomic-age touring car rendered in the same layered print language. It has a low pointed nose, paired cream headlamp circles, smooth coral fenders, powder-blue windshield glass, a deep-teal open cabin, silver-gray wheel rims with coral hubs, and a tapering rear deck with restrained fin forms. When it first becomes visible ahead-right, its nose faces screen-left along the roadside while its rear remains screen-right beside the solar-topiary boundary. It stays at rideable road scale, and its solid simplified surfaces retain coarse stipple, slight ink misregistration, and flat charcoal undershadows.\n\nA powder-blue finned sedan briefly approaches from farther up the roadway. It has a rounded cream roof, broad powder-blue side panels, coral hub details, and a compact soft-charcoal windshield, all simplified into screen-printed shapes. No other moving identity is visible at the opening.\n\nThe setting is an optimistic retro-futurist exposition district printed on warm fibrous paper. A sweeping cream ceramic pavilion with a thick coral rim and four vertical coral oval windows fills the left side. Beyond it stands a dense deep-teal modular solar-topiary wall patterned with powder-blue panel grids, circular recesses, and clipped geometric foliage. A powder-blue pavilion and a tall cypress-shaped antenna sculpture rise behind the wall, while slender coral signal poles punctuate the fairground. Terraced pale-pink mooncrete with round teal shrubs and angular agave forms borders the road opening on the right. The broad butter-yellow road curves uphill through the exhibits, edged by powder-blue curbs and flat coral accents under a clear stippled blue sky. Halftone dots, coarse three-ink grain, imperfect color overlap, and flat layered shadows remain visible across every surface. The trailing camera begins low and wide behind the courier, preserving its lower center-left placement and the open roadway ahead, then follows forward and turns gradually toward the approaching roadster."
+ },
+ "Olivewood automaton \u00b7 Mediterranean cliffside": {
+  "id": 6,
+  "chunks": 24,
+  "caption": "Appearance and scene:\n\nThe protagonist is an olivewood courier automaton with a compact, athletic humanoid build. Its small cream-colored ceramic sun-mask head bears a crisp cobalt sun emblem and rests above a short dark-bronze neck. A ribbed, amphora-shaped olivewood torso narrows into a banded bronze waist, with curved shoulder plates, jointed wooden upper limbs, dark-bronze elbows, articulated hands, wooden shins, and round bronze knees. The ceramic head, amphora-shaped torso, articulated dark-bronze joints, and carefully fitted wooden limbs are stable identity features. Loose indigo cloth trousers gather below the waist and bunch around the knees, while simple rope-soled sandals wrap its feet with narrow ochre ties. Fine black hatching follows the wood grain, dense crosshatching darkens the joints, and small cobalt accents visually connect the ceramic head with the trousers. Bright sunlight strikes the upper wood and ceramic while the forward lean leaves narrow warm-black shadows beneath the arms and around the waist.\n\nAt the opening instant, the olivewood courier automaton occupies the central-lower slope exactly as a running figure seen from behind. It faces uphill toward the upper center, with its torso inclined forward, one sandal planted on pale limestone, the rear foot lifted, both knees bent, and both arms pumping close to its sides. The immense village-cliff fills the right and upper area as a continuous overhanging mass of honey-colored stone and chalk-white lime-plaster dwellings. Tiny cobalt shutters, recessed windows, terracotta ledges, painted ceramic drainage jars, scattered olive leaves, and bright bougainvillea punctuate the facade without breaking its solid lower boundary. Its shadowed underside follows the dark rock shelf above the runner. A broad sun-bleached limestone lane remains open on the left, passing around the projecting village-cliff and climbing into the caldera. Sparse grasses, yellow flowers, purple herbs, and low olive scrub emerge from cracks across the foreground. Farther left, stepped olive terraces and small white dwellings cover dry ochre hills beneath a clear turquoise sky crossed by a thin halftone cloud.\n\nThe entire landscape is rendered as a bold comic-book print with forceful warm-black contours, dry-brush shadows, visible Ben-Day dots, slight color-registration offsets, and dense crosshatching across stone and wood. The limited chalk-white, ochre, terracotta, cobalt, olive-green, turquoise, indigo, and warm-black palette remains consistent from foreground plants to distant terraces. Bright late-afternoon light produces crisp graphic shadows. The trailing camera begins low behind the olivewood courier automaton, showing its full running body against the towering village-cliff, then follows its route around the projecting stone while gradually rising and widening toward the open lane."
+ },
+ "Lotus carriage \u00b7 lantern-lit night road": {
+  "id": 8,
+  "chunks": 24,
+  "caption": "Appearance and scene:\n\nThe protagonist is a low, driveable festival carriage shaped like a hand-sculpted glazed-clay lotus bud, with a layered coral-and-jade body, a warm lantern chamber beneath its petals, four exposed dark ceramic spool wheels, and delicate brass-colored clay fittings. The compact body stays low over its wheels, and overlapping coral petals wrap around jade-green inner panels beneath fine raised brass-colored ribs. Curled fittings ornament the rear panel above two small red taillanterns, while the tapered front remains enclosed beneath the converging petals. Subtle fingerprints, modeled seams, uneven glazed edges, and small kiln-darkened recesses are stable identity features. Its warm inner light softly brightens the lower petals and nearby dust. No rider or other occupant is visible. At 0.00s, its unmistakable lanterned rear is nearest the high trailing camera in the lower foreground, while its tapered front points away toward the upper-left reach of the broad road.\n\nAn extremely tiny closed festival wagon occupies the remote upper-right road beyond the river. It has a rounded dark-clay cabin, four minute spool wheels, tiny amber lantern windows, and a compact enclosed festival-wagon silhouette. It remains far smaller than the coral-and-jade lotus carriage and is the only other mobile identity visible.\n\nA wide road of moon-lavender clay dust curves ahead through pinched moss-green clay rosettes and low sculpted shrubs. The road surface is open and traversable, with shallow wheel grooves, scattered clay pebbles, compressed patches, and gently modeled shoulders that remain level with the surrounding ground. A glossy teal resin-clay river follows its exact channel along the road's right side, catching broken cool moon reflections and warm lantern streaks. A low, nearly straight bridge spans the water near the center distance, retaining its modest deck, short supports, broad openings, and small attached lanterns. Slender bamboo-look clay poles carry hanging amber lanterns along both banks and beside the road. Blank parchment-colored shadow-play screens stand among the shrubs without readable markings. Layered violet-brown clay hills, clustered miniature trees, and dark sculpted clouds recede beneath a midnight-indigo sky and a bright textured moon. Warm lantern light, cool moonlight, miniature-set depth, fingerprints, and stop-motion sculpting imperfections remain visible throughout. The high trailing camera begins directly behind and above the coral-and-jade lotus carriage, showing its lanterned rear, the open bend ahead, the river, and the bridge, then follows smoothly while yawing with the carriage's changing heading."
+ },
+ "Crystal-cavern wayfarer \u00b7 dark forest": {
+  "id": 29,
+  "chunks": 24,
+  "caption": "Appearance and scene:\n\nThe protagonist is a rugged adult crystal-cavern wayfarer with a sturdy, athletic human build. His weathered olive-tan skin, strong angular profile, silver-black hair braided into a thick rope down his back, loose gray-streaked temple hairs, and rough dark stubble are stable identity features. A charcoal wool traveling mantle wraps his shoulders beneath a shallow hood, with frayed edges, coarse hand stitching, and a long uneven cloak panel hanging behind him. Under it he wears a fitted dark leather tunic over a smoke-gray undershirt, a broad utility belt with practical iron buckles, layered leather bracers, close leather gloves, reinforced dark trousers, and heavy climbing boots. A compact translucent quartz climbing tool rests in an iron setting at his right hip, secured tightly to the belt. Dust dulls the lower cloak and boots, while cool mineral light glints along the tool, worn leather, buckles, and individual silver strands in his braid. He begins at lower center-left, seen from behind and in three-quarter profile with his torso leaning uphill toward screen-right, his right knee deeply bent and lifted into the next step, his left leg supporting him lower on the incline, and his relaxed right hand hanging beside the quartz tool.\n\nThe only other living identity is the distant prism-winged cave ray high in the upper middle-right, a tiny graceful flyer with a dark tapered body, broad translucent blue-violet mineral fins held in a level horizontal posture, and a fine trailing tail.\n\nAn open calcite path rises from the gritty foreground through center and climbs toward screen-right, its irregular surface formed from worn limestone, damp mineral crust, embedded pebbles, and scattered translucent shards. Dark granular shelves border both sides of the route. Tall smoky-quartz columns emerge sparsely from these shelves, including a broad weathered pillar directly beyond the wayfarer and slender blade-like crystals along the right edge. The ground falls into a recessed shadowed basin at screen-left, where small amber crystal growths catch warm reflected light beneath a heavy black limestone overhang. Farther away, massive blue-veined cavern walls and restrained clusters of sapphire crystal recede through cool haze. A small faceted crystal sanctuary stands embedded in distant rock left of center, partially screened by columns but marked by a pale luminous doorway. Sapphire light pours through fissures in the immense vault and crosses warmer amber reflections on the path, revealing damp stone, mineral staining, airborne grit, natural cloth fibers, skin texture, and imperfect metal. The atmosphere remains that of a physically real crystal-lit cavern captured with restrained cinematic exposure, selective focus, subtle grain, slight motion softness, and mild optical aberration. The medium-wide camera begins near waist height behind-left of the wayfarer, preserving the opening uphill composition, then tracks his advance with a gentle rightward pan and holds its ground as he turns, changing naturally from a trailing view to a frontal view."
+ },
+ "White stag \u00b7 moonlit dirt road": {
+  "id": 31,
+  "chunks": 24,
+  "caption": "Appearance and scene:\n\nThe protagonist is a large adult white stag with a graceful, athletic cervine build. Its long back, deep chest, narrow waist, powerful haunches, slim lower legs, and dark cloven hooves form a physically credible running body. Dense ivory winter fur covers its entire body, with longer uneven guard hairs along the spine, shoulders, rump, and neck. A short pale tail lies above the rounded haunches. Its elongated head has a charcoal nose, dark eyes, alert pointed ears, and slender branching antlers. Faint strands of old black mourning ribbon hang from several antler tines and trail lightly during motion. The dense ivory winter fur, charcoal nose, alert ears, slender antlers, and old black mourning ribbons are stable identity features. Cold moonlight gives the raised fur a restrained pearl sheen while leaving soft blue-gray shadows between the legs and beneath the torso.\n\nAt the opening instant, the large white stag occupies the lower center foreground, seen directly from behind and oriented away from the camera toward the upper-right bend. It is already mid-bound on the compacted silver-gray chalk road, with its torso lifted, one foreleg flexed beneath the chest, its hind legs divided between push-off and recovery, and pale dust scattering immediately below the rear hooves. Its antlers rise clearly above the head, and the narrow ribbons hang from the tines without obscuring their shape.\n\nThe broad chalk road fills the foreground with two worn tracks, scattered pale stones, shallow ruts, and darker strips of compressed dust. A lower branch crosses toward screen-left while the unobstructed main track climbs gently toward screen-right between rough grass verges. Ancient wind-shaped yews gather beside the upper-right bend, while isolated pale birches and low shrubs punctuate the open rolling hills. Abandoned stone keeper lodges and roofless conservatory remnants stand across the middle distance, partially surrounded by low ground mist. A delicate weathered wrought-iron aqueduct extends along the far screen-left hillside. The bright full moon hangs high at screen-left in a deep indigo sky, casting physically plausible cool illumination, long soft-edged shadows, and restrained highlights across fur, chalk, bark, grass, stone, and iron. The air remains clear above the hollows, with subtle high-sensitivity grain and natural motion softness around the fastest hooves and ribbon tips. The documentary wildlife camera begins in a wide rear-facing composition near the stag\u2019s shoulder height, keeping the animal low and central while preserving the moon, road junction, yew-lined bend, ruined lodges, aqueduct, and open hills."
+ },
+ "Route surveyor \u00b7 rainy desert highway": {
+  "id": 36,
+  "chunks": 24,
+  "caption": "Appearance and scene:\n\nThe protagonist is a solitary female route surveyor with a sturdy, practical build. She stands left of center with her back to the camera, facing away along the wet road. Her dark brown hair is gathered into a loose high bun, with several damp strands resting around the nape of her neck; this tied hair, her compact build, and her field-ready silhouette are stable identity features. She wears a charcoal weather shell with a raised folded hood, long sleeves, dark gloves, and matching weatherproof cargo trousers. The trousers and her lower legs continue below the bottom edge, leaving her practical boots, feet, and ground contact out of view at the opening. A pale ceramic field pack covers most of her back, its scuffed hard shell shaped into broad upper shoulders and a rounded lower compartment. Gray webbing passes over both shoulders, narrow seams divide the pack into fitted panels, and a horizontal zippered pocket crosses its lower face. Compact surveying equipment is secured along both sides: a short cylindrical canister hangs low on her right, while a slim upright instrument with a capped circular sensor rises beside her left shoulder. Moisture dulls the shell fabric and gives the ceramic pack a cold, softly reflective sheen.\n\nThe nearest machine is the low enclosed electric bog rover in the right midground, broadside across the far lanes and motionless near the junction. It has a low armored nose, a dark enclosed cabin with angular windows, a long charcoal utility body, and four large deeply treaded wheels under flared guards. Damp moss staining gathers around the lower doors, wheel housings, and sill panels, while rainwater glints across the roof and window edges. The low enclosed electric bog rover faces screen-left toward the curving road, while its rear remains screen-right near the guardrail. Its broad stance, enclosed cabin, four-wheel running structure, and clear front-to-rear organization remain constant. Only faint suspension settling and intermittent beads of water moving down its body disturb its stillness.\n\nThe other visible identities are ordered with the solitary female route surveyor nearest, the low enclosed electric bog rover in the middle distance, the squat six-wheeled maintenance crawler farther ahead near the road center, and the distant black triangular weather drone farthest overhead. The squat six-wheeled maintenance crawler has a compact moss-stained utility body, six low wheels, a dark rear panel, and two small upper work lamps; it is seen nearly end-on from behind and points away along the road. The distant black triangular weather drone is a tiny matte-black wedge with short swept edges, held high against the cloud ceiling. These identities retain their distinct scales and positions throughout the view.\n\nWet charcoal asphalt spreads from the foreground into a broad relay junction, with pale lane lines shining through shallow rainwater and curving left around the basalt plain. Irregular volcanic shoulders carry sparse emerald moss, silver bunchgrass, and black-barked shrubs twisted by wind. A dark rock rise fills the near screen-left verge. Basalt cable stanchions stand along the roads beneath taut lines, while exactly two inclined fog-harvesting vanes occupy the right distance. An elevated circular barometer array rises near the center horizon, and slender vertical-axis wind harvesters repeat along the far ridge. Steel-gray clouds hang over layered mountains, cold cyan haze fills the valleys, and a narrow trace of fading amber remains beneath the cloud bank. Fine mist softens the distance while the wet ground preserves crisp restrained reflections. The trailing camera begins near shoulder height behind the solitary female route surveyor, holding the opening layout with her cropped lower legs, the two rovers, the high weather drone, and the stable near-to-far ordering clearly visible."
+ }
+}

examples/silence_audio.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e54157313bc0cf6f7757785e94cf45a537916441c9fa0a083671eef8e38258f1
+size 53072

materialize_hf_examples.py

@@ -1,369 +0,0 @@
-#!/usr/bin/env python3
-"""Expand the published compact CWM examples into inference-ready inputs."""
-
-from __future__ import annotations
-
-import argparse
-import hashlib
-import json
-import math
-import os
-from pathlib import Path
-import shutil
-import tempfile
-from typing import Any, Iterable
-
-import numpy as np
-from PIL import Image
-
-
-WIDTH = 336
-HEIGHT = 192
-WINDOW_FRAMES = 124
-STRIDE_FRAMES = 90
-NEAR_METRES = 0.3
-FAR_METRES = 256.0
-METRIC_DEPTH = "float32_le_metric_first_surface_metres"
-FIXED_LOG_DEPTH = "uint16_le_fixed_log_gray16_codes"
-SUPPORTED_DEPTH_FORMATS = {METRIC_DEPTH, FIXED_LOG_DEPTH}
-RELEASE_MODELS = {
-    "audio_vae": "../../models/vae/minimax_h3_audio_vae_fp32.safetensors",
-    "dit": "../../models/diffusion_models/minimax_h3_ref2va_bf16.safetensors",
-    "lora": "../../models/lora/minimax-h3-cwm.safetensors",
-    "processor": "../../models/processors/Qwen3-VL-32B-Instruct",
-    "text_encoder": "../../models/text_encoders/qwen3vl_32b_minimax_h3_bf16.safetensors",
-    "video_vae": "../../models/vae/minimax_h3_video_vae_fp16.safetensors",
-}
-RELEASE_SEEDS = {
-    1: 9942001,
-    2: 9800002,
-    3: 9700005,
-    4: 9700006,
-    5: 9700007,
-    6: 9700013,
-    7: 9700020,
-    8: 9700021,
-    9: 9700030,
-    10: 9700036,
-    11: 9700037,
-    12: 9700040,
-    13: 9700044,
-    14: 9700045,
-    15: 9700046,
-    16: 9700047,
-    17: 9700051,
-    18: 9700054,
-    19: 9700055,
-    20: 9700057,
-    21: 9700063,
-    22: 9700065,
-    23: 9700069,
-    24: 9700071,
-    25: 9700078,
-    26: 9700088,
-    27: 9700094,
-    28: 9600014,
-    29: 9600020,
-    30: 9600023,
-    31: 9600028,
-    32: 9600040,
-    33: 9600042,
-    34: 9300038,
-    35: 9400059,
-    36: 9300066,
-    37: 9300069,
-    38: 9800006,
-    39: 9800010,
-    40: 9800013,
-}
-
-
-def require(condition: bool, message: str) -> None:
-    if not condition:
-        raise ValueError(message)
-
-
-def _read_json(path: Path) -> Any:
-    return json.loads(path.read_text(encoding="utf-8"))
-
-
-def _canonical_prompt(value: str) -> str:
-    return value.strip().replace("\r\n", "\n").replace("\r", "\n")
-
-
-def build_release_config(
-    example_root: Path, metadata: dict[str, Any], example_id: int
-) -> dict[str, Any]:
-    windows = []
-    for prompt_item in metadata["prompts"]:
-        prompt = _canonical_prompt(
-            (example_root / prompt_item["file"]).read_text(encoding="utf-8")
-        )
-        # Preserve the text representation used by the released configurations.
-        if example_id != 2:
-            prompt = prompt.replace("\n", "\r\n")
-        windows.append(
-            {
-                "prompt": prompt,
-                "start_frame": prompt_item["conditioning_frame_start"],
-            }
-        )
-    name = f"example_{example_id:02d}"
-    return {
-        "anchor_image": "first_frame.png",
-        "attention_mode": "flash3",
-        "blocks_to_swap": 48,
-        "cache_dir": f"../../outputs/{name}/cache",
-        "condition_root": "conditions",
-        "expected_lora_modules": 200,
-        "models": RELEASE_MODELS,
-        "output": f"../../outputs/{name}/output.mp4",
-        "seed": RELEASE_SEEDS[example_id],
-        "steps": 20,
-        "windows": windows,
-    }
-
-
-def ensure_release_config(
-    config_path: Path, example_root: Path, metadata: dict[str, Any], example_id: int
-) -> None:
-    if not config_path.exists():
-        payload = json.dumps(
-            build_release_config(example_root, metadata, example_id),
-            ensure_ascii=False,
-            indent=2,
-            sort_keys=True,
-        )
-        _atomic_bytes(config_path, (payload + "\n").encode("utf-8"))
-    validate_config(config_path, example_root, metadata, example_id)
-
-
-def fixed_log_codes_to_metric(codes: np.ndarray) -> np.ndarray:
-    """Choose float32 depths that round-trip to every published uint16 code."""
-    require(codes.dtype == np.dtype("uint16"), f"fixed-log depth must be uint16, got {codes.dtype}")
-    metric = np.zeros(codes.shape, dtype="<f4")
-    valid = codes > 0
-    if bool(np.any(valid)):
-        log_near = math.log(NEAR_METRES)
-        log_far = math.log(FAR_METRES)
-        normalized = codes[valid].astype(np.float64) / 65535.0
-        metric[valid] = np.exp(log_far - normalized * (log_far - log_near)).astype(np.float32)
-    return np.ascontiguousarray(metric)
-
-
-def metric_to_fixed_log_codes(metric: np.ndarray) -> np.ndarray:
-    """Reference implementation matching the inference loader."""
-    metric = np.asarray(metric, dtype=np.float32)
-    valid = metric > np.float32(0.001)
-    codes = np.zeros(metric.shape, dtype=np.uint16)
-    if bool(np.any(valid)):
-        clipped = np.clip(metric[valid].astype(np.float64), NEAR_METRES, FAR_METRES)
-        normalized = (math.log(FAR_METRES) - np.log(clipped)) / (
-            math.log(FAR_METRES) - math.log(NEAR_METRES)
-        )
-        codes[valid] = np.floor(np.clip(normalized, 0.0, 1.0) * 65535.0 + 0.5).astype(
-            np.uint16
-        )
-    return codes
-
-
-def _atomic_bytes(path: Path, payload: bytes) -> None:
-    path.parent.mkdir(parents=True, exist_ok=True)
-    if path.exists():
-        require(path.read_bytes() == payload, f"existing file differs: {path}")
-        return
-    descriptor, temporary_name = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent)
-    temporary = Path(temporary_name)
-    try:
-        with os.fdopen(descriptor, "wb") as handle:
-            handle.write(payload)
-            handle.flush()
-            os.fsync(handle.fileno())
-        temporary.replace(path)
-    finally:
-        temporary.unlink(missing_ok=True)
-
-
-def _atomic_semantic_png(path: Path, semantic_id: np.ndarray) -> None:
-    semantic_id = np.ascontiguousarray(semantic_id, dtype=np.uint8)
-    if path.exists():
-        with Image.open(path) as image:
-            existing = np.asarray(image.convert("L"), dtype=np.uint8)
-        require(np.array_equal(existing, semantic_id), f"existing Semantic-ID differs: {path}")
-        return
-    path.parent.mkdir(parents=True, exist_ok=True)
-    descriptor, temporary_name = tempfile.mkstemp(prefix=f".{path.name}.", suffix=".png", dir=path.parent)
-    os.close(descriptor)
-    temporary = Path(temporary_name)
-    try:
-        Image.fromarray(semantic_id, mode="L").save(temporary, format="PNG", compress_level=6)
-        temporary.replace(path)
-    finally:
-        temporary.unlink(missing_ok=True)
-
-
-def _copy_exact(source: Path, destination: Path) -> None:
-    require(source.is_file(), f"missing source file: {source}")
-    payload = source.read_bytes()
-    _atomic_bytes(destination, payload)
-
-
-def validate_metadata(example_root: Path, metadata: dict[str, Any], example_id: int) -> None:
-    require(metadata.get("id") == example_id, f"example {example_id}: metadata id mismatch")
-    require(metadata.get("fps") == 24, f"example {example_id}: fps must be 24")
-    require(metadata.get("resolution") == [WIDTH, HEIGHT], f"example {example_id}: bad resolution")
-    require(metadata.get("depth_format") in SUPPORTED_DEPTH_FORMATS, f"example {example_id}: unsupported depth format")
-    frames = metadata.get("frames")
-    require(type(frames) is int and frames >= WINDOW_FRAMES, f"example {example_id}: invalid frame count")
-    prompts = metadata.get("prompts")
-    require(isinstance(prompts, list) and prompts, f"example {example_id}: prompts missing")
-    require(frames == WINDOW_FRAMES + STRIDE_FRAMES * (len(prompts) - 1), f"example {example_id}: frame/window mismatch")
-    for index, item in enumerate(prompts):
-        require(item.get("index") == index, f"example {example_id}: prompt index mismatch")
-        require(item.get("conditioning_frame_start") == index * STRIDE_FRAMES, f"example {example_id}: prompt start mismatch")
-        require(item.get("conditioning_frame_end_exclusive") == index * STRIDE_FRAMES + WINDOW_FRAMES, f"example {example_id}: prompt end mismatch")
-        prompt_path = example_root / item.get("file", "")
-        require(prompt_path.is_file(), f"example {example_id}: missing prompt {prompt_path}")
-    chunks = metadata.get("condition_chunks")
-    require(isinstance(chunks, list) and chunks, f"example {example_id}: condition chunks missing")
-    cursor = 0
-    for item in chunks:
-        start = item.get("start_frame")
-        end = item.get("end_frame_exclusive")
-        require(start == cursor and type(end) is int and end > start, f"example {example_id}: non-contiguous chunks")
-        require(item.get("frames") == end - start, f"example {example_id}: chunk frame count mismatch")
-        chunk_path = example_root / item.get("file", "")
-        require(chunk_path.is_file(), f"example {example_id}: missing chunk {chunk_path}")
-        cursor = end
-    require(cursor == frames, f"example {example_id}: chunks do not cover all frames")
-
-
-def validate_config(config_path: Path, example_root: Path, metadata: dict[str, Any], example_id: int) -> None:
-    require(config_path.is_file(), f"missing released config: {config_path}")
-    config = _read_json(config_path)
-    require(config.get("anchor_image") == "first_frame.png", f"example {example_id}: config anchor path mismatch")
-    require(config.get("condition_root") == "conditions", f"example {example_id}: config condition path mismatch")
-    require(config.get("seed") == RELEASE_SEEDS[example_id], f"example {example_id}: released seed mismatch")
-    require(config.get("steps") == 20, f"example {example_id}: steps mismatch")
-    require(config.get("attention_mode") == "flash3", f"example {example_id}: attention mode mismatch")
-    require(config.get("blocks_to_swap") == 48, f"example {example_id}: block swap count mismatch")
-    require(config.get("expected_lora_modules") == 200, f"example {example_id}: LoRA module count mismatch")
-    require(config.get("models") == RELEASE_MODELS, f"example {example_id}: model paths mismatch")
-    name = f"example_{example_id:02d}"
-    require(config.get("cache_dir") == f"../../outputs/{name}/cache", f"example {example_id}: cache path mismatch")
-    require(config.get("output") == f"../../outputs/{name}/output.mp4", f"example {example_id}: output path mismatch")
-    windows = config.get("windows")
-    prompt_items = metadata["prompts"]
-    require(isinstance(windows, list) and len(windows) == len(prompt_items), f"example {example_id}: config window count mismatch")
-    for window, prompt_item in zip(windows, prompt_items, strict=True):
-        require(window.get("start_frame") == prompt_item["conditioning_frame_start"], f"example {example_id}: config window start mismatch")
-        published = (example_root / prompt_item["file"]).read_text(encoding="utf-8")
-        require(_canonical_prompt(window.get("prompt", "")) == _canonical_prompt(published), f"example {example_id}: config prompt differs from published prompt")
-
-
-def _iter_chunks(example_root: Path, metadata: dict[str, Any]) -> Iterable[tuple[int, np.ndarray, np.ndarray]]:
-    depth_format = metadata["depth_format"]
-    for chunk in metadata["condition_chunks"]:
-        path = example_root / chunk["file"]
-        with np.load(path, allow_pickle=False) as payload:
-            require(set(payload.files) == {"depth", "semantic_id"}, f"unexpected arrays in {path}")
-            depth = payload["depth"]
-            semantic_id = payload["semantic_id"]
-        expected_shape = (chunk["frames"], HEIGHT, WIDTH)
-        require(depth.shape == expected_shape, f"bad depth shape in {path}: {depth.shape}")
-        require(semantic_id.shape == expected_shape, f"bad Semantic-ID shape in {path}: {semantic_id.shape}")
-        require(semantic_id.dtype == np.dtype("uint8"), f"bad Semantic-ID dtype in {path}: {semantic_id.dtype}")
-        require(int(semantic_id.max(initial=0)) <= 11, f"Semantic-ID outside [0,11] in {path}")
-        if depth_format == METRIC_DEPTH:
-            require(depth.dtype == np.dtype("float32"), f"bad metric depth dtype in {path}: {depth.dtype}")
-            require(bool(np.all(np.isfinite(depth))) and not bool(np.any(depth < 0)), f"invalid metric depth in {path}")
-            metric = np.ascontiguousarray(depth, dtype="<f4")
-        else:
-            require(depth.dtype == np.dtype("uint16"), f"bad fixed-log depth dtype in {path}: {depth.dtype}")
-            metric = fixed_log_codes_to_metric(depth)
-            require(np.array_equal(metric_to_fixed_log_codes(metric), depth), f"fixed-log round-trip failed in {path}")
-        start = chunk["start_frame"]
-        for offset in range(chunk["frames"]):
-            yield start + offset, metric[offset], semantic_id[offset]
-
-
-def materialize_example(
-    source_root: Path,
-    output_root: Path,
-    example_id: int,
-    *,
-    require_config: bool,
-    frame_limit: int | None = None,
-) -> dict[str, Any]:
-    example_root = source_root / str(example_id)
-    metadata_path = example_root / "metadata.json"
-    require(metadata_path.is_file(), f"missing metadata: {metadata_path}")
-    metadata = _read_json(metadata_path)
-    validate_metadata(example_root, metadata, example_id)
-    destination = output_root / "examples" / f"example_{example_id:02d}"
-    config_path = destination / "config.json"
-    if require_config:
-        ensure_release_config(config_path, example_root, metadata, example_id)
-    _copy_exact(example_root / metadata["first_frame"], destination / "first_frame.png")
-    written = 0
-    for ordinal, depth, semantic_id in _iter_chunks(example_root, metadata):
-        if frame_limit is not None and ordinal >= frame_limit:
-            break
-        condition_root = destination / "conditions"
-        _atomic_bytes(condition_root / f"{ordinal:06d}.depth.f32", np.ascontiguousarray(depth, dtype="<f4").tobytes(order="C"))
-        _atomic_semantic_png(condition_root / f"{ordinal:06d}.semantic_id.png", semantic_id)
-        written += 1
-    expected = min(metadata["frames"], frame_limit) if frame_limit is not None else metadata["frames"]
-    require(written == expected, f"example {example_id}: materialized {written}, expected {expected}")
-    return {
-        "id": example_id,
-        "depth_format": metadata["depth_format"],
-        "frames": metadata["frames"],
-        "materialized_frames": written,
-        "windows": len(metadata["prompts"]),
-    }
-
-
-def parse_ids(values: list[int] | None) -> list[int]:
-    ids = list(range(1, 41)) if not values else sorted(set(values))
-    require(ids and all(1 <= value <= 40 for value in ids), "example ids must be in [1,40]")
-    return ids
-
-
-def main(argv: list[str] | None = None) -> int:
-    parser = argparse.ArgumentParser(description=__doc__, allow_abbrev=False)
-    parser.add_argument("--source", required=True, type=Path, help="downloaded code-world-model-inference-examples-40 directory")
-    parser.add_argument("--output", required=True, type=Path, help="release root containing models/, examples/, and outputs/")
-    parser.add_argument("--example", action="append", type=int, dest="examples", help="materialize one public example id; repeat as needed")
-    parser.add_argument("--without-config-check", action="store_true", help=argparse.SUPPRESS)
-    parser.add_argument("--frame-limit", type=int, help=argparse.SUPPRESS)
-    args = parser.parse_args(argv)
-    source = args.source.resolve()
-    output = args.output.resolve()
-    manifest = _read_json(source / "manifest.json")
-    require(manifest.get("schema") == "code_world_model_inference_examples_v3", "unsupported dataset manifest")
-    require(manifest.get("count") == 40 and manifest.get("expected_count") == 40, "dataset manifest does not contain 40 examples")
-    results = []
-    for example_id in parse_ids(args.examples):
-        result = materialize_example(
-            source,
-            output,
-            example_id,
-            require_config=not args.without_config_check,
-            frame_limit=args.frame_limit,
-        )
-        results.append(result)
-        print(f"example_{example_id:02d}: {result['materialized_frames']} frames", flush=True)
-    receipt = {
-        "schema": "cwm_h3_materialized_examples_v1",
-        "source_manifest_sha256": hashlib.sha256((source / "manifest.json").read_bytes()).hexdigest(),
-        "examples": results,
-    }
-    if args.frame_limit is None and len(results) == 40:
-        path = output / "examples" / "materialization.json"
-        _atomic_bytes(path, (json.dumps(receipt, indent=2, sort_keys=True) + "\n").encode("utf-8"))
-    return 0
-
-
-if __name__ == "__main__":
-    raise SystemExit(main())

requirements.txt

@@ -1,8 +1,4 @@
-torch==2.9.1
-torchvision==0.24.1
-transformers==4.57.6
-accelerate==1.6.0
-huggingface-hub==0.34.3
+accelerate
 safetensors
 av
 numpy

streaming_runtime.py

@@ -0,0 +1,187 @@
+"""Runtime adaptations of the rtcwm streaming sampler for ZeroGPU (RTX PRO 6000, sm_120). See app.py."""
+import time
+
+import torch
+import torch.nn.functional as F
+from torch import nn
+
+from musubi_tuner.minimax_h3 import model as h3model
+from musubi_tuner.minimax_h3.packing import _build_modulation_segments
+from rtcwm.model import attention as rt_attention
+from rtcwm.model import packing as rt_packing
+from rtcwm.model import sampler as rt_sampler
+
+STEPS, SHIFT = 4, 5.0
+
+class FP8Linear(nn.Module):
+    """rtcwm.model.fp8.FP8Linear (per-output-channel e4m3 weights, per-row e4m3 activations, torch._scaled_mm),
+    with an emulated fallback should the rowwise-scaled fp8 GEMM be unavailable on this GPU."""
+    use_scaled_mm = True
+
+    def __init__(self, weight_bf16: torch.Tensor):
+        super().__init__()
+        w = weight_bf16.float()
+        scale = (w.abs().amax(dim=1, keepdim=True) / 448.0).clamp_min(1e-12)
+        self.register_buffer("weight_fp8", (w / scale).to(torch.float8_e4m3fn))
+        self.register_buffer("weight_scale", scale.t().contiguous())
+        self.out_features = w.shape[0]
+
+    def forward(self, x):
+        shape = x.shape
+        x2 = x.reshape(-1, shape[-1])
+        scale = (x2.abs().amax(dim=1, keepdim=True).float() / 448.0).clamp_min(1e-12)
+        xq = (x2.float() / scale).to(torch.float8_e4m3fn)
+        if FP8Linear.use_scaled_mm:
+            try:
+                y = torch._scaled_mm(xq, self.weight_fp8.t(), scale_a=scale, scale_b=self.weight_scale,
+                                     out_dtype=torch.bfloat16)
+                return y.reshape(*shape[:-1], self.out_features)
+            except Exception as e:  # noqa: BLE001
+                print(f"_scaled_mm unavailable ({e!r}); using the emulated FP8 GEMM", flush=True)
+                FP8Linear.use_scaled_mm = False
+        y = (xq.to(torch.bfloat16) @ self.weight_fp8.to(torch.bfloat16).t()).float() * scale * self.weight_scale
+        return y.to(torch.bfloat16).reshape(*shape[:-1], self.out_features)
+
+
+_ADALN_T = [None]          # the unique times of the forward in flight (set by run_blocks)
+
+
+def _tkey(t: float) -> int:
+    return int(round(float(t) * 1_000_000))
+
+
+class CachedAdaln(nn.Module):
+    """A DiT block's AdalnProj evaluated once per known time: same output as
+    linear(silu(time_embedder(t).bf16)), looked up by t."""
+
+    def __init__(self, table: torch.Tensor, keys: list, modalities: int, expand: int, hidden: int):
+        super().__init__()
+        self.register_buffer("table", table)                     # [n_t, expand*hidden*modalities] bf16
+        self.index = {k: i for i, k in enumerate(keys)}
+        self.modalities, self.expand, self.hidden = modalities, expand, hidden
+
+    def forward(self, _temb):
+        rows = [self.index[_tkey(t)] for t in _ADALN_T[0]]
+        p = self.table[rows]
+        p = p.reshape(p.shape[0] * self.modalities, self.expand * self.hidden)
+        return p.chunk(self.expand, dim=-1)
+
+
+def sampling_times() -> list:
+    """Every model time a 4-step x0 stream evaluates: condition rows (1.0 text / audio / clean history, 0.999
+    Picture 1 + proxy) and 1 - sigma of each step, computed like the sampler does (float32 rows)."""
+    sig = rt_sampler.schedule(STEPS, "cpu", SHIFT)
+    ts = [1.0, 0.999] + [1.0 - float(sig[i]) for i in range(STEPS)]
+    return sorted({float(torch.tensor(t, dtype=torch.float32)) for t in ts})
+
+
+def prepare_transformer(tr, lora: dict, fp8: bool = True, cache_adaln: bool = True):
+    """Merge the streaming LoRA into the blocks' linears (musubi merge_to, multiplier 1, bf16), put them in FP8
+    (rtcwm.model.fp8), precompute the blocks' AdaLN outputs for the sampling times, and install the rtcwm
+    attention (with the SDPA stand-ins)."""
+    t0 = time.time()
+    tr.requires_grad_(False)
+    used = set()
+    times = sampling_times()
+    with torch.no_grad():
+        temb = tr.time_embedder(torch.tensor(times, dtype=torch.float32)).to(torch.bfloat16)
+        act = F.silu(temb).float()
+        keys = [_tkey(t) for t in times]
+        for i, blk in enumerate(tr.blocks):
+            # LoRA merge (musubi LoRAModule.merge_to, multiplier 1, bf16) + FP8 (rtcwm.model.fp8.quantize_blocks)
+            for parent, prefix, names in ((blk.attn, "attn", ("qkv_proj", "out_proj")), (blk.mlp, "mlp", ("fc1", "fc2"))):
+                for n in names:
+                    lin = getattr(parent, n)
+                    assert lin.bias is None
+                    key = f"lora_unet_blocks_{i}_{prefix}_{n}"
+                    down = lora[f"{key}.lora_down.weight"].float()
+                    up = lora[f"{key}.lora_up.weight"].float()
+                    scale = float(lora[f"{key}.alpha"].float()) / down.shape[0]
+                    w = (lin.weight.float() + (up @ down) * scale).to(torch.bfloat16)
+                    if fp8:
+                        setattr(parent, n, FP8Linear(w))
+                    else:
+                        lin.weight.data = w
+                    used.update({f"{key}.lora_down.weight", f"{key}.lora_up.weight", f"{key}.alpha"})
+                    del w, lin
+            if not cache_adaln:
+                continue
+            ad = blk.adaln_proj
+            table = (act @ ad.linear.weight.float().t() + ad.linear.bias.float()).to(torch.bfloat16)
+            blk.adaln_proj = CachedAdaln(table, keys, ad.modalities, ad.expand, ad.hidden_size)
+            del ad
+            if i % 10 == 0:
+                print(f"block {i} prepared ({time.time() - t0:.0f}s)", flush=True)
+    assert used == set(lora), f"unused LoRA keys: {sorted(set(lora) - used)[:5]}"
+    print(f"merged {len(used) // 3} LoRA modules, FP8 blocks ready in {time.time() - t0:.0f}s", flush=True)
+    install(tr)
+    return tr.eval()
+
+
+def install(tr):
+    rt_attention.install(tr)
+    h3model.Attention.forward = _attention_forward
+
+
+# --------------------------------------------------------------------------------------------- attention
+def _sdpa(q, k, v):
+    """FlashAttention-3 stand-in: q, k, v [B,S,H,D] -> [B,S,H,D], unmasked."""
+    return F.scaled_dot_product_attention(q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2)).transpose(1, 2)
+
+
+rt_attention._fa3 = _sdpa
+
+
+class DenseMask:
+    """rtcwm.model.masks.df_mask as prefix attention calls (rtcwm.model.attention.dense_df) -- for the condition
+    rows alone (n_chunks 0) or together with chunk 0 (n_chunks 1)."""
+
+    def __init__(self, head_rows, duv_rows, audio_rows, n_chunks, static_sees_first):
+        self.head_rows, self.duv_rows, self.audio_rows = head_rows, duv_rows, audio_rows
+        self.n_chunks, self.static_sees_first = n_chunks, static_sees_first
+
+
+def _df_mask(head_rows, duv_rows, audio_rows, n_chunks, device, first_chunk=0, static_sees_first=True):
+    assert first_chunk == 0
+    return DenseMask(head_rows, duv_rows, audio_rows, n_chunks, static_sees_first)
+
+
+rt_sampler.df_mask = _df_mask
+_rtcwm_forward = rt_attention._forward
+
+
+def _attention_forward(self, hidden_states, rotation_table=None):
+    mask, kv = rt_attention._state["mask"], rt_attention._state["kv"]
+    if not isinstance(mask, DenseMask):
+        return _rtcwm_forward(self, hidden_states, rotation_table)
+    b, s, _ = hidden_states.shape
+    q, k, v = self.qkv_proj(hidden_states).split(self.inner_dim, dim=-1)
+    q = self.q_norm(q.reshape(b, s, self.num_heads, self.head_dim))
+    k = self.k_norm(k.reshape(b, s, self.num_heads, self.head_dim))
+    v = v.reshape(b, s, self.num_heads, self.head_dim)
+    if rotation_table is not None:
+        q = rt_attention.rope(q, rotation_table)
+        k = rt_attention.rope(k, rotation_table)
+    q, k, v = q.to(torch.bfloat16), k.to(torch.bfloat16), v.to(torch.bfloat16)
+    if kv is not None and kv.get("store") is not None:
+        kv["store"].append((k.transpose(1, 2).contiguous(), v.transpose(1, 2).contiguous()))
+    out = rt_attention.dense_df(q, k, v, mask)
+    return self.out_proj(out.reshape(b, s, self.inner_dim))
+
+
+def run_blocks(model, hidden, t_rows, tags, positions, *, mask=None, kv=None, train=False, offload=False):
+    """rtcwm.model.packing.run_blocks (inference), publishing the forward's times to CachedAdaln."""
+    uniq, inv = torch.unique(t_rows, sorted=True, return_inverse=True)
+    segments = (_build_modulation_segments(3 * inv + tags),)
+    temb = model.time_embedder(uniq.to(hidden.device)).to(model.dtype)
+    rot = model._rotation_table(positions.unsqueeze(0).to(hidden.device), hidden.dtype)
+    _ADALN_T[0] = uniq.tolist()
+    for block in model.blocks:
+        with rt_attention.active(mask, kv):
+            hidden = block(hidden, temb, segments, rot)
+    return hidden, temb, inv
+
+
+rt_packing.run_blocks = run_blocks
+rt_sampler.run_blocks = run_blocks
+