Add files using upload-large-folder tool

.gitattributes

@@ -33,3 +33,73 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+eval_out/generated_videos/cf_ffff/overall_consistency/041.mp4 filter=lfs diff=lfs merge=lfs -text
+eval_out/generated_videos/cf_ffff/overall_consistency/044.mp4 filter=lfs diff=lfs merge=lfs -text
+eval_out/generated_videos/cf_ffff/overall_consistency/078.mp4 filter=lfs diff=lfs merge=lfs -text
+eval_out/generated_videos/cf_ffff/overall_consistency/064.mp4 filter=lfs diff=lfs merge=lfs -text
+eval_out/generated_videos/cf_ffff/overall_consistency/000.mp4 filter=lfs diff=lfs merge=lfs -text
+eval_out/generated_videos/cf_ffff/overall_consistency/049.mp4 filter=lfs diff=lfs merge=lfs -text
+eval_out/generated_videos/cf_ffff/overall_consistency/005.mp4 filter=lfs diff=lfs merge=lfs -text
+eval_out/generated_videos/cf_ffff/overall_consistency/007.mp4 filter=lfs diff=lfs merge=lfs -text
+eval_out/generated_videos/cf_ffff/overall_consistency/001.mp4 filter=lfs diff=lfs merge=lfs -text
+eval_out/generated_videos/cf_ffff/overall_consistency/002.mp4 filter=lfs diff=lfs merge=lfs -text
+eval_out/generated_videos/cf_ffff/overall_consistency/003.mp4 filter=lfs diff=lfs merge=lfs -text
+eval_out/generated_videos/cf_ffff/overall_consistency/010.mp4 filter=lfs diff=lfs merge=lfs -text
+eval_out/generated_videos/cf_ffff/overall_consistency/012.mp4 filter=lfs diff=lfs merge=lfs -text
+eval_out/generated_videos/cf_ffff/overall_consistency/083.mp4 filter=lfs diff=lfs merge=lfs -text
+eval_out/generated_videos/cf_ffff/overall_consistency/014.mp4 filter=lfs diff=lfs merge=lfs -text
+eval_out/generated_videos/cf_ffff/overall_consistency/013.mp4 filter=lfs diff=lfs merge=lfs -text
+eval_out/generated_videos/cf_ffff/overall_consistency/009.mp4 filter=lfs diff=lfs merge=lfs -text
+eval_out/generated_videos/cf_ffff/overall_consistency/015.mp4 filter=lfs diff=lfs merge=lfs -text
+eval_out/generated_videos/cf_ffff/overall_consistency/047.mp4 filter=lfs diff=lfs merge=lfs -text
+eval_out/generated_videos/cf_ffff/overall_consistency/016.mp4 filter=lfs diff=lfs merge=lfs -text
+eval_out/generated_videos/cf_ffff/overall_consistency/011.mp4 filter=lfs diff=lfs merge=lfs -text
+eval_out/generated_videos/cf_ffff/overall_consistency/021.mp4 filter=lfs diff=lfs merge=lfs -text
+eval_out/generated_videos/cf_ffff/overall_consistency/004.mp4 filter=lfs diff=lfs merge=lfs -text
+eval_out/generated_videos/cf_ffff/overall_consistency/062.mp4 filter=lfs diff=lfs merge=lfs -text
+eval_out/generated_videos/cf_ffff/overall_consistency/008.mp4 filter=lfs diff=lfs merge=lfs -text
+eval_out/generated_videos/cf_ffff/overall_consistency/019.mp4 filter=lfs diff=lfs merge=lfs -text
+eval_out/generated_videos/cf_ffff/overall_consistency/023.mp4 filter=lfs diff=lfs merge=lfs -text
+eval_out/generated_videos/cf_ffff/overall_consistency/061.mp4 filter=lfs diff=lfs merge=lfs -text
+eval_out/generated_videos/cf_ffff/overall_consistency/022.mp4 filter=lfs diff=lfs merge=lfs -text
+eval_out/generated_videos/cf_ffff/overall_consistency/028.mp4 filter=lfs diff=lfs merge=lfs -text
+eval_out/generated_videos/cf_ffff/overall_consistency/027.mp4 filter=lfs diff=lfs merge=lfs -text
+eval_out/generated_videos/cf_ffff/overall_consistency/017.mp4 filter=lfs diff=lfs merge=lfs -text
+eval_out/generated_videos/cf_ffff/overall_consistency/054.mp4 filter=lfs diff=lfs merge=lfs -text
+eval_out/generated_videos/cf_ffff/overall_consistency/034.mp4 filter=lfs diff=lfs merge=lfs -text
+eval_out/generated_videos/cf_ffff/overall_consistency/033.mp4 filter=lfs diff=lfs merge=lfs -text
+eval_out/generated_videos/cf_ffff/overall_consistency/032.mp4 filter=lfs diff=lfs merge=lfs -text
+eval_out/generated_videos/cf_ffff/overall_consistency/092.mp4 filter=lfs diff=lfs merge=lfs -text
+eval_out/generated_videos/cf_ffff/overall_consistency/037.mp4 filter=lfs diff=lfs merge=lfs -text
+eval_out/generated_videos/cf_ffff/overall_consistency/026.mp4 filter=lfs diff=lfs merge=lfs -text
+eval_out/generated_videos/cf_ffff/overall_consistency/035.mp4 filter=lfs diff=lfs merge=lfs -text
+eval_out/generated_videos/cf_ffff/overall_consistency/040.mp4 filter=lfs diff=lfs merge=lfs -text
+eval_out/generated_videos/cf_ffff/overall_consistency/091.mp4 filter=lfs diff=lfs merge=lfs -text
+eval_out/generated_videos/cf_ffff/overall_consistency/020.mp4 filter=lfs diff=lfs merge=lfs -text
+eval_out/generated_videos/cf_ffff/overall_consistency/039.mp4 filter=lfs diff=lfs merge=lfs -text
+eval_out/generated_videos/cf_ffff/overall_consistency/042.mp4 filter=lfs diff=lfs merge=lfs -text
+eval_out/generated_videos/cf_ffff/overall_consistency/006.mp4 filter=lfs diff=lfs merge=lfs -text
+eval_out/generated_videos/cf_ffff/overall_consistency/024.mp4 filter=lfs diff=lfs merge=lfs -text
+eval_out/generated_videos/cf_ffff/overall_consistency/050.mp4 filter=lfs diff=lfs merge=lfs -text
+eval_out/generated_videos/cf_ffff/overall_consistency/048.mp4 filter=lfs diff=lfs merge=lfs -text
+eval_out/generated_videos/cf_ffff/overall_consistency/059.mp4 filter=lfs diff=lfs merge=lfs -text
+eval_out/generated_videos/cf_ffff/overall_consistency/060.mp4 filter=lfs diff=lfs merge=lfs -text
+eval_out/generated_videos/cf_ffff/overall_consistency/057.mp4 filter=lfs diff=lfs merge=lfs -text
+eval_out/generated_videos/cf_ffff/overall_consistency/088.mp4 filter=lfs diff=lfs merge=lfs -text
+eval_out/generated_videos/cf_ffff/overall_consistency/051.mp4 filter=lfs diff=lfs merge=lfs -text
+eval_out/generated_videos/cf_ffff/overall_consistency/053.mp4 filter=lfs diff=lfs merge=lfs -text
+eval_out/generated_videos/cf_ffff/overall_consistency/090.mp4 filter=lfs diff=lfs merge=lfs -text
+eval_out/generated_videos/cf_ffff/overall_consistency/052.mp4 filter=lfs diff=lfs merge=lfs -text
+eval_out/generated_videos/cf_ffff/overall_consistency/058.mp4 filter=lfs diff=lfs merge=lfs -text
+eval_out/generated_videos/cf_ffff/overall_consistency/055.mp4 filter=lfs diff=lfs merge=lfs -text
+eval_out/generated_videos/cf_ffff/overall_consistency/056.mp4 filter=lfs diff=lfs merge=lfs -text
+eval_out/generated_videos/cf_ffff/overall_consistency/089.mp4 filter=lfs diff=lfs merge=lfs -text
+eval_out/generated_videos/cf_ffff/overall_consistency/079.mp4 filter=lfs diff=lfs merge=lfs -text
+eval_out/generated_videos/cf_ffff/overall_consistency/063.mp4 filter=lfs diff=lfs merge=lfs -text
+eval_out/generated_videos/cf_ffff/overall_consistency/030.mp4 filter=lfs diff=lfs merge=lfs -text
+eval_out/generated_videos/cf_ffff/overall_consistency/068.mp4 filter=lfs diff=lfs merge=lfs -text
+eval_out/generated_videos/cf_ffff/overall_consistency/066.mp4 filter=lfs diff=lfs merge=lfs -text
+eval_out/generated_videos/cf_ffff/overall_consistency/065.mp4 filter=lfs diff=lfs merge=lfs -text
+eval_out/generated_videos/cf_ffff/overall_consistency/070.mp4 filter=lfs diff=lfs merge=lfs -text
+eval_out/generated_videos/cf_ffff/overall_consistency/067.mp4 filter=lfs diff=lfs merge=lfs -text
+eval_out/generated_videos/cf_ffff/overall_consistency/072.mp4 filter=lfs diff=lfs merge=lfs -text

assets/lb_actions/a_d_125f/intrinsics.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a526a861f16b3bdd64d12b1946d2438925b0f68d88822599fc1d6b14a324d905
+size 2128

assets/lb_actions/a_d_125f/poses.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f4cebbc48954133ce27c8adc7c71327f3ab6e6b8f1d735441dc6931bb931b5a9
+size 8128

assets/lb_actions/left_right_125f/intrinsics.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a526a861f16b3bdd64d12b1946d2438925b0f68d88822599fc1d6b14a324d905
+size 2128

assets/lb_actions/left_right_125f/poses.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:03161c72fae5167c67aed4e3fa61cddb4dee23f008712d1ebc6a73f4848c631f
+size 8128

assets/lb_actions/up_down_125f/intrinsics.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a526a861f16b3bdd64d12b1946d2438925b0f68d88822599fc1d6b14a324d905
+size 2128

assets/lb_actions/up_down_125f/poses.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f9b64db0d0cea64c0b9b587f065bd6c2d0d76a1512f0ba128dcb4b41b75c04c0
+size 8128

assets/lb_actions/w_s_125f/intrinsics.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a526a861f16b3bdd64d12b1946d2438925b0f68d88822599fc1d6b14a324d905
+size 2128

assets/lb_actions/w_s_125f/poses.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f0624f5ca69206ecd7045c2f2b863b8e80bb8b5792d6b09f67370dab41d0dbb5
+size 8128

assets/lb_actions/w_s_61f/intrinsics.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d5b972022c390258bd820986b146a9cfb86654b84be31ef161cd5c51b0ff746b
+size 1104

assets/lb_actions/w_s_61f/poses.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:423844e4fd3f7900c8e07acf72544399c9ec1efe2f0ffe2243e0c66a64c56be3
+size 4032

cachelib/selective.py

@@ -0,0 +1,148 @@
+"""Selective-token forward for the block-causal Wan DiT.
+
+MotionCache (token granularity) and FlowCache (frame-group granularity) both need
+to run the transformer stack on a *subset* of the current chunk's tokens while the
+remaining tokens keep the activations cached from the previous computed step.
+
+For a KV-cached causal model that means, per self-attention layer:
+  * compute q/k/v only for the selected rows,
+  * scatter the fresh k/v into the slots the current chunk owns in the KV cache,
+    leaving the unselected slots holding the previous step's k/v,
+  * attend the selected queries against the *full* cache (history + current chunk).
+
+This mirrors ``forward_selective`` in
+``MotionCache4SkyReels-V2/skyreels_v2_infer/modules/transformer_motion.py``,
+adapted from SkyReels' persistent ``k_cache/v_cache`` buffers to Self-Forcing's
+rolling ``kv_cache[block]`` dict.
+"""
+
+import math
+
+import torch
+
+from wan.modules.attention import attention
+
+
+def build_causal_freqs(grid_sizes, freqs, start_frame):
+    """Return the per-token RoPE multipliers [seq_len, 1, c] for one chunk.
+
+    Same construction as ``causal_rope_apply`` in ``wan/modules/causal_model.py``,
+    factored out so it can be indexed by a token subset and reused across layers.
+    """
+    c = freqs.size(1)
+    parts = freqs.split([c - 2 * (c // 3), c // 3, c // 3], dim=1)
+    f, h, w = grid_sizes[0].tolist()
+    seq_len = f * h * w
+    freqs_i = torch.cat(
+        [
+            parts[0][start_frame:start_frame + f].view(f, 1, 1, -1).expand(f, h, w, -1),
+            parts[1][:h].view(1, h, 1, -1).expand(f, h, w, -1),
+            parts[2][:w].view(1, 1, w, -1).expand(f, h, w, -1),
+        ],
+        dim=-1,
+    ).reshape(seq_len, 1, -1)
+    return freqs_i
+
+
+def rope_apply_rows(x, freqs_rows):
+    """Apply precomputed RoPE multipliers to x [B, S, n, d] given freqs [S, 1, d/2]."""
+    b, s, n, _ = x.shape
+    x_c = torch.view_as_complex(x.to(torch.float64).reshape(b, s, n, -1, 2))
+    out = torch.view_as_real(x_c * freqs_rows.unsqueeze(0)).flatten(3)
+    return out.type_as(x)
+
+
+def self_attn_selective(attn, x_sel, sel_idx, freqs_rows, kv_cache,
+                        current_start, block_seq_len):
+    """Self-attention over a token subset, refreshing only those KV slots.
+
+    Args:
+        attn: a ``CausalWanSelfAttention`` module.
+        x_sel: [B, n_sel, C] modulated hidden states for the selected tokens.
+        sel_idx: LongTensor [n_sel], token offsets inside the current chunk.
+        freqs_rows: [block_seq_len, 1, d/2] RoPE multipliers for the whole chunk.
+        kv_cache: this layer's ``{"k", "v", "global_end_index", "local_end_index"}``.
+        current_start: token offset of the chunk in global sequence coordinates.
+        block_seq_len: number of tokens the chunk owns (all frames of the chunk).
+    """
+    b, n_sel = x_sel.shape[:2]
+    n, d = attn.num_heads, attn.head_dim
+
+    q = attn.norm_q(attn.q(x_sel)).view(b, n_sel, n, d)
+    k = attn.norm_k(attn.k(x_sel)).view(b, n_sel, n, d)
+    v = attn.v(x_sel).view(b, n_sel, n, d)
+
+    rows = freqs_rows[sel_idx]
+    roped_q = rope_apply_rows(q, rows)
+    roped_k = rope_apply_rows(k, rows).type_as(v)
+
+    # Index arithmetic must use the *full* chunk length: the chunk still occupies
+    # the same KV slots, we are only refreshing some of them.
+    current_end = current_start + block_seq_len
+    local_end_index = (kv_cache["local_end_index"].item() + current_end
+                       - kv_cache["global_end_index"].item())
+    local_start_index = local_end_index - block_seq_len
+
+    slots = local_start_index + sel_idx
+    kv_cache["k"].index_copy_(1, slots, roped_k)
+    kv_cache["v"].index_copy_(1, slots, v)
+
+    lo = max(0, local_end_index - attn.max_attention_size)
+    out = attention(roped_q, kv_cache["k"][:, lo:local_end_index],
+                    kv_cache["v"][:, lo:local_end_index])
+
+    kv_cache["global_end_index"].fill_(current_end)
+    kv_cache["local_end_index"].fill_(local_end_index)
+
+    return attn.o(out.flatten(2))
+
+
+def block_forward_selective(block, x_sel, sel_idx, frame_of_sel, e0, freqs_rows,
+                            context, kv_cache, crossattn_cache,
+                            current_start, block_seq_len):
+    """One ``CausalWanAttentionBlock`` evaluated on a token subset.
+
+    ``frame_of_sel`` maps each selected token to its frame index inside the chunk,
+    so the per-frame AdaLN modulation can be gathered for the selected rows only.
+    """
+    # e0: [B, F, 6, C] -> six tensors of [B, F, 1, C]; gather per selected token.
+    e = (block.modulation.unsqueeze(1) + e0).chunk(6, dim=2)
+    e = [t.squeeze(2)[:, frame_of_sel] for t in e]  # each [B, n_sel, C]
+
+    y = self_attn_selective(
+        block.self_attn,
+        block.norm1(x_sel) * (1 + e[1]) + e[0],
+        sel_idx, freqs_rows, kv_cache, current_start, block_seq_len,
+    )
+    x_sel = x_sel + y * e[2]
+
+    x_sel = x_sel + block.cross_attn(block.norm3(x_sel), context, None,
+                                     crossattn_cache=crossattn_cache)
+    y = block.ffn(block.norm2(x_sel) * (1 + e[4]) + e[3])
+    x_sel = x_sel + y * e[5]
+    return x_sel
+
+
+def run_blocks_selective(model, x, sel_idx, e0, kwargs, kv_cache, crossattn_cache,
+                         current_start, cache_start, grid_sizes):
+    """Run all transformer blocks on ``x[:, sel_idx]`` and return the updated rows.
+
+    Returns a tensor [B, n_sel, C] holding the new hidden states for the selected
+    tokens (the caller merges them into the cached full-length tensor).
+    """
+    f, h, w = grid_sizes[0].tolist()
+    frame_seqlen = h * w
+    block_seq_len = f * frame_seqlen
+    frame_of_sel = torch.div(sel_idx, frame_seqlen, rounding_mode="floor")
+
+    freqs_rows = build_causal_freqs(grid_sizes, kwargs["freqs"],
+                                    current_start // frame_seqlen)
+
+    x_sel = x[:, sel_idx]
+    for i, block in enumerate(model.blocks):
+        x_sel = block_forward_selective(
+            block, x_sel, sel_idx, frame_of_sel, e0, freqs_rows,
+            kwargs["context"], kv_cache[i], crossattn_cache[i],
+            current_start, block_seq_len,
+        )
+    return x_sel

eval/aggregate.py

@@ -0,0 +1,331 @@
+#!/usr/bin/env python
+"""Final summary for the Self-Forcing Extended-251 Full Evaluation.
+
+Implements protocol sections 7 (normalize + aggregate), 8.3 (pixel aggregation),
+9.4 (matched-FFFF speedup) and 10 (output tables), plus the section 11/15
+completeness gate: a strategy that is not a full 251 is never summarised.
+
+    python eval/aggregate.py --out-root eval_out
+"""
+
+import argparse
+import re
+import csv
+import glob
+import hashlib
+import json
+import math
+import os
+import platform
+import sys
+from collections import defaultdict
+
+ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.insert(0, ROOT)
+
+MAPPING = os.path.join(ROOT, "assets/vbench8_extended_subset_mapping.json")
+
+DIMENSIONS = ["subject_consistency", "background_consistency", "motion_smoothness",
+              "dynamic_degree", "aesthetic_quality", "imaging_quality", "scene",
+              "overall_consistency"]
+
+# Protocol section 7.1 -- fixed empirical ranges, never re-estimated from the run.
+NORMALIZE_RANGE = {
+    "subject_consistency":    (0.1462, 1.0),
+    "background_consistency": (0.2615, 1.0),
+    "motion_smoothness":      (0.7060, 0.9975),
+    "dynamic_degree":         (0.0,    1.0),
+    "aesthetic_quality":      (0.0,    1.0),
+    "imaging_quality":        (0.0,    1.0),
+    "scene":                  (0.0,    0.8222),
+    "overall_consistency":    (0.0,    0.3640),
+}
+
+REFERENCE_OF = {"sf": "sf_ffff", "cf": "cf_ffff", "cfa": "cfa_ffff"}
+
+
+def normalize(raw):
+    out = {}
+    for d, v in raw.items():
+        lo, hi = NORMALIZE_RANGE[d]
+        out[d] = (v - lo) / (hi - lo)  # protocol does not clip
+    return out
+
+
+def quality_score(n):
+    return (n["subject_consistency"] + n["background_consistency"]
+            + n["motion_smoothness"] + 0.5 * n["dynamic_degree"]
+            + n["aesthetic_quality"] + n["imaging_quality"]) / 5.5
+
+
+def semantic_score(n):
+    return (n["scene"] + n["overall_consistency"]) / 2.0
+
+
+def selected_score(q, s):
+    return (4.0 * q + s) / 5.0
+
+
+def sha256(path):
+    h = hashlib.sha256()
+    with open(path, "rb") as f:
+        for chunk in iter(lambda: f.read(1 << 20), b""):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def load_per_prompt(out_root, strategy):
+    recs = []
+    for p in sorted(glob.glob(os.path.join(out_root, "per_prompt", strategy, "*.json"))):
+        with open(p) as f:
+            r = json.load(f)
+        if r.get("status") == "complete":
+            recs.append(r)
+    return recs
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--out-root", default="eval_out")
+    ap.add_argument("--expect", type=int, default=251)
+    ap.add_argument("--allow-incomplete", action="store_true",
+                    help="Report partial strategies instead of refusing (diagnostics only)")
+    ap.add_argument("--latency-subsets", default=os.path.join(ROOT, "results/latency_subsets.json"),
+                    help="JSON {strategy_regex: {shard, num_shards, note}}: time these "
+                         "strategies on one prompt shard only (the one that ran on an "
+                         "uncontended GPU); quality still uses every record")
+    ap.add_argument("--only", default=None,
+                    help="Regex: summarise only strategies whose name matches (other "
+                         "strategies in per_prompt/ are ignored, e.g. while they are "
+                         "still being generated)")
+    args = ap.parse_args()
+
+    out_root = (args.out_root if os.path.isabs(args.out_root)
+                else os.path.join(ROOT, args.out_root))
+
+    with open(MAPPING) as f:
+        mapping = json.load(f)
+
+    strategies = sorted(os.path.basename(p) for p in
+                        glob.glob(os.path.join(out_root, "per_prompt", "*")))
+    if args.only:
+        strategies = [s for s in strategies if re.search(args.only, s)]
+    if not strategies:
+        print("no per-prompt records found")
+        return 1
+
+    per_strategy, problems = {}, []
+    for s in strategies:
+        recs = load_per_prompt(out_root, s)
+        if len(recs) != args.expect:
+            problems.append(f"{s}: {len(recs)} complete records, expected {args.expect}")
+        per_strategy[s] = recs
+
+    if problems and not args.allow_incomplete:
+        print("REFUSING TO SUMMARISE -- incomplete strategies (protocol 11.9):")
+        for p in problems:
+            print("  -", p)
+        return 1
+
+    # --- latency, matched to each base model's own FFFF -----------------------
+    mean_latency, mean_context, timed_records = {}, {}, {}
+    for s, recs in per_strategy.items():
+        # Records flagged latency_excluded keep their pixel metrics but carry no
+        # usable timing (see their latency_note); they stay out of the means.
+        lat = [r for r in recs if not r.get("latency_excluded")]
+        if not lat:
+            continue
+        timed_records[s] = lat
+        mean_latency[s] = sum(r["policy_latency_ms"] for r in lat) / len(lat)
+        mean_context[s] = sum(r["excluded_context_kv_latency_ms"] for r in lat) / len(lat)
+
+    # Strategies generated while another job shared the GPU are timed only on the
+    # prompt shard that ran alone (sharding is rows[shard::num_shards] over the
+    # mapping order, see generate_eval.py); their FFFF reference is restricted to
+    # the same prompts so the ratio stays paired.
+    subsets = {}
+    if args.latency_subsets and os.path.exists(args.latency_subsets):
+        with open(args.latency_subsets) as f:
+            subsets = json.load(f)
+    with open(os.path.join(ROOT, "assets/vbench8_extended_subset_mapping.json")) as f:
+        mapping_rows = json.load(f)["rows"]
+    subset_of = {}
+    for s in per_strategy:
+        for pat, spec in subsets.items():
+            if re.search(pat, s):
+                keep = {r["global_index"] for r in mapping_rows[spec["shard"]::spec["num_shards"]]}
+                subset_of[s] = (keep, spec)
+                break
+
+    rows = []
+    for s, recs in per_strategy.items():
+        if not recs:
+            continue
+        ref = recs[0].get("reference_strategy") or REFERENCE_OF[s.split("_")[0]]
+        if ref not in mean_latency:
+            problems.append(f"{s}: reference {ref} missing")
+            continue
+
+        # Ratio of means (protocol 9.4).  Equal sample counts make this the same
+        # as the ratio of sums; a mean of per-prompt percentages is not used.
+        # A strategy whose records carry the FFFF latency measured in the same
+        # process (retime_eval.py) is compared against that, not against the
+        # FFFF run's own records: contention differs between runs, and only a
+        # same-run pairing is "matched FFFF" in the protocol's sense.
+        timed = timed_records[s]
+        matched = [r.get("matched_ffff_policy_latency_ms") for r in timed]
+        strat_latency = mean_latency[s]
+        latency_n = len(timed)
+        if s in subset_of:
+            keep, spec = subset_of[s]
+            sub = [r for r in recs if r["global_index"] in keep]
+            ref_sub = [r for r in per_strategy[ref] if r["global_index"] in keep]
+            assert sub and ref_sub, f"{s}: empty latency subset"
+            strat_latency = sum(r["policy_latency_ms"] for r in sub) / len(sub)
+            ref_latency = sum(r["policy_latency_ms"] for r in ref_sub) / len(ref_sub)
+            latency_n = len(sub)
+            latency_source = (f"generation run, shard {spec['shard']}/{spec['num_shards']} only "
+                              f"({len(sub)} prompts; {spec.get('note', '')})")
+        elif all(m is not None for m in matched):
+            ref_latency = sum(matched) / len(matched)
+            latency_source = "paired retime (same process as FFFF)"
+        elif all(r.get("latency_retimed") for r in recs):
+            ref_latency = mean_latency[ref]
+            latency_source = ("strategy retime: all records from one denoise-only pass on an idle GPU "
+                              "(FFFF from its own records)")
+        elif any(r.get("latency_retimed") for r in recs):
+            n_re = sum(1 for r in recs if r.get("latency_retimed"))
+            ref_latency = mean_latency[ref]
+            latency_source = (f"generation run; {n_re}/{len(recs)} records re-timed "
+                              "(strategy-only denoise rerun on an idle GPU; FFFF from its own records)")
+        else:
+            ref_latency = mean_latency[ref]
+            latency_source = "generation run (FFFF from its own records)"
+            if len(timed) < len(recs):
+                latency_source += (f"; {len(timed)}/{len(recs)} prompts with intact timing "
+                                   "(latency_excluded records dropped)")
+        speedup_pct = 100.0 * (1.0 - strat_latency / ref_latency)
+
+        # PSNR from the mean of per-prompt mean MSE, then converted once (8.3).
+        mses = [r["pixel_metrics_vs_ffff"]["mean_mse"] for r in recs]
+        mean_mse = sum(mses) / len(mses)
+        psnr = -10.0 * math.log10(max(mean_mse, 1e-12))
+        ssim = sum(r["pixel_metrics_vs_ffff"]["ssim"] for r in recs) / len(recs)
+        lpips = sum(r["pixel_metrics_vs_ffff"]["lpips"] for r in recs) / len(recs)
+        frames = {r["pixel_metrics_vs_ffff"]["num_frames"] for r in recs}
+        if frames != {81}:
+            problems.append(f"{s}: pixel metrics used frame counts {sorted(frames)}, expected 81")
+
+        row = {
+            "strategy": s,
+            "base_model": recs[0]["base_model"],
+            "method": recs[0]["method"],
+            "target_speedup": recs[0]["target_speedup"],
+            "schedule": recs[0].get("schedule"),
+            "num_inference_steps": recs[0].get("num_inference_steps"),
+            "num_videos": len(recs),
+            "mp4_reference_records": sum(1 for r in recs if r.get("reference_source") == "ffff_mp4"),
+            "policy_latency_ms": strat_latency,
+            "latency_num_videos": latency_n,
+            "speedup_percent_vs_ffff": 0.0 if s == ref else speedup_pct,
+            "latency_ratio_vs_ffff": strat_latency / ref_latency,
+            "matched_ffff_policy_latency_ms": ref_latency,
+            "latency_source": latency_source,
+            "excluded_context_kv_latency_ms": mean_context[s],
+            "psnr": psnr,
+            "ssim": ssim,
+            "lpips": lpips,
+            "mean_compute_equivalent_forwards": (
+                sum(r["cache_diagnostics"]["compute_equivalent_forwards"] for r in recs)
+                / len(recs)),
+        }
+
+        score_path = os.path.join(out_root, "vbench", "scores", f"{s}.json")
+        if os.path.exists(score_path):
+            with open(score_path) as f:
+                sc = json.load(f)
+            raw = {d: sc["raw"][d] for d in DIMENSIONS if d in sc["raw"]}
+            if len(raw) == len(DIMENSIONS):
+                bad = {d: v for d, v in raw.items() if not (0.0 <= v <= 1.0)}
+                if bad:
+                    problems.append(f"{s}: raw scores outside [0,1]: {bad}")
+                n = normalize(raw)
+                q, sem = quality_score(n), semantic_score(n)
+                sel = selected_score(q, sem)
+                row.update({f"raw_{d}": raw[d] for d in DIMENSIONS})
+                row.update({f"normalized_{d}": n[d] for d in DIMENSIONS})
+                row.update({"quality_score": q, "semantic_score": sem,
+                            "selected_vbench_score": sel,
+                            "selected_vbench_percent": sel * 100.0})
+            else:
+                problems.append(f"{s}: only {len(raw)}/8 VBench dimensions")
+        else:
+            problems.append(f"{s}: no VBench scores yet")
+        rows.append(row)
+
+    rows.sort(key=lambda r: (r["base_model"], r["method"], r["target_speedup"]))
+
+    summary = {
+        "protocol": "Self-Forcing Extended-251 Full Evaluation",
+        "vbench_long": False,
+        "num_strategies": len(rows),
+        "prompts_per_strategy": args.expect,
+        "seed": 0,
+        "video_spec": {"frames": 81, "height": 480, "width": 832, "fps": 16},
+        "precision": "bfloat16",
+        "normalize_range": NORMALIZE_RANGE,
+        "aggregation": {
+            "quality": "(n_sc + n_bc + n_ms + 0.5*n_dd + n_aq + n_iq) / 5.5",
+            "semantic": "(n_scene + n_oc) / 2",
+            "selected": "(4*quality + semantic) / 5",
+            "psnr": "-10*log10(mean(per_prompt_mean_mse)), clamp 1e-12",
+            "speedup": "100 * (1 - mean(strategy) / mean(matched_ffff))",
+        },
+        "latency_includes": "all denoise policy GPU compute incl. cache control modules",
+        "latency_excludes": "context/KV-cache DiT, text encoder, VAE decode, metrics, I/O",
+        "sources": mapping["sources"],
+        "mapping": {"path": MAPPING, "sha256": sha256(MAPPING)},
+        "environment": {
+            "generation_python": "/local/zoubin/cz/envs/self_forcing/bin/python",
+            "vbench_python": "/local/zoubin/cz/envs/vbench_eval/bin/python",
+            "vbench_source": "/local/zoubin/cz/projects/VBench",
+            "vbench_cache_dir": "/local/zoubin/cz/.cache/vbench",
+            "platform": platform.platform(),
+        },
+        "problems": problems,
+        "rows": rows,
+    }
+
+    os.makedirs(os.path.join(out_root, "summaries"), exist_ok=True)
+    jpath = os.path.join(out_root, "summaries", "final_summary.json")
+    with open(jpath, "w") as f:
+        json.dump(summary, f, indent=2)
+
+    cpath = os.path.join(out_root, "summaries", "final_summary.csv")
+    if rows:
+        keys = sorted({k for r in rows for k in r},
+                      key=lambda k: (k not in ("strategy", "base_model", "method"), k))
+        with open(cpath, "w", newline="") as f:
+            w = csv.DictWriter(f, fieldnames=keys)
+            w.writeheader()
+            w.writerows(rows)
+
+    hdr = (f"{'strategy':24s} {'videos':>6s} {'lat_ms':>9s} {'speedup%':>9s} "
+           f"{'psnr':>7s} {'ssim':>7s} {'lpips':>7s} {'sel_vbench%':>12s}")
+    print(hdr)
+    print("-" * len(hdr))
+    for r in rows:
+        sel = r.get("selected_vbench_percent")
+        print(f"{r['strategy']:24s} {r['num_videos']:6d} {r['policy_latency_ms']:9.1f} "
+              f"{r['speedup_percent_vs_ffff']:9.2f} {r['psnr']:7.2f} {r['ssim']:7.4f} "
+              f"{r['lpips']:7.4f} " + (f"{sel:12.3f}" if sel is not None else f"{'-':>12s}"))
+    if problems:
+        print("\nnotes:")
+        for p in problems:
+            print("  -", p)
+    print(f"\nwrote {jpath}\nwrote {cpath}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

eval/build_mapping.py

@@ -0,0 +1,139 @@
+#!/usr/bin/env python
+"""Build and validate assets/vbench8_extended_subset_mapping.json (Extended-251).
+
+Protocol section 4: take the subject_consistency (72), overall_consistency (93)
+and scene (86) suites out of the canonical 946-prompt VBench order, and carry the
+Self-Forcing rewritten extended prompt for each.
+
+Selection is strictly by index into VBench_full_info.json -- the short prompt list
+contains two duplicate texts, so text matching is not safe, and the protocol
+forbids it anyway.
+
+    python eval/build_mapping.py --out assets/vbench8_extended_subset_mapping.json
+"""
+
+import argparse
+import hashlib
+import json
+import os
+import sys
+
+ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+SUITES = ["subject_consistency", "overall_consistency", "scene"]
+EXPECTED = {"subject_consistency": 72, "overall_consistency": 93, "scene": 86}
+
+DEFAULT_FULL_INFO = "/local/zoubin/cz/projects/VBench/vbench/VBench_full_info.json"
+DEFAULT_PROMPT_DIR = "/local/zoubin/cz/projects/Self-Forcing/prompts/vbench"
+
+
+def sha256(path):
+    h = hashlib.sha256()
+    with open(path, "rb") as f:
+        for chunk in iter(lambda: f.read(1 << 20), b""):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--full-info", default=DEFAULT_FULL_INFO)
+    ap.add_argument("--prompt-dir", default=DEFAULT_PROMPT_DIR)
+    ap.add_argument("--out", default=os.path.join(ROOT, "assets/vbench8_extended_subset_mapping.json"))
+    args = ap.parse_args()
+
+    short_path = os.path.join(args.prompt_dir, "all_dimension.txt")
+    ext_path = os.path.join(args.prompt_dir, "all_dimension_extended.txt")
+
+    with open(args.full_info) as f:
+        info = json.load(f)
+    with open(short_path, encoding="utf-8") as f:
+        short = [l.rstrip("\n") for l in f]
+    with open(ext_path, encoding="utf-8") as f:
+        ext = [l.rstrip("\n") for l in f]
+
+    errors = []
+    if len(short) != 946:
+        errors.append(f"short prompts == {len(short)}, expected 946")
+    if len(ext) != 946:
+        errors.append(f"extended prompts == {len(ext)}, expected 946")
+    if len(info) != 946:
+        errors.append(f"VBench_full_info rows == {len(info)}, expected 946")
+    for i, (row, s) in enumerate(zip(info, short)):
+        if row["prompt_en"].strip() != s.strip():
+            errors.append(f"short prompt order differs from VBench_full_info at line {i}")
+            break
+    for i, e in enumerate(ext):
+        if not e.strip():
+            errors.append(f"empty extended prompt at line {i}")
+            break
+
+    rows = []
+    for suite in SUITES:
+        suite_index = 0
+        for gi, row in enumerate(info):
+            if suite not in row["dimension"]:
+                continue
+            entry = {
+                "global_index": gi,
+                "prompt_suite": suite,
+                "suite_index": suite_index,
+                "original_prompt": short[gi],
+                "extended_prompt": ext[gi],
+                "official_dimensions": list(row["dimension"]),
+            }
+            # Standard VBench 0.1.5 needs the official scene keywords to score `scene`.
+            for key in ("auxiliary_info",):
+                if key in row:
+                    entry[key] = row[key]
+            rows.append(entry)
+            suite_index += 1
+
+    counts = {s: sum(1 for r in rows if r["prompt_suite"] == s) for s in SUITES}
+    for s, n in EXPECTED.items():
+        if counts.get(s) != n:
+            errors.append(f"{s} == {counts.get(s)}, expected {n}")
+    if len(rows) != 251:
+        errors.append(f"mapping rows == {len(rows)}, expected 251")
+    if len({r["global_index"] for r in rows}) != len(rows):
+        errors.append("duplicate global_index")
+    for s in SUITES:
+        idx = [r["suite_index"] for r in rows if r["prompt_suite"] == s]
+        if idx != list(range(len(idx))):
+            errors.append(f"{s} suite_index not contiguous")
+
+    if errors:
+        print("MAPPING VALIDATION FAILED:")
+        for e in errors:
+            print("  -", e)
+        return 1
+
+    payload = {
+        "protocol": "Self-Forcing Extended-251 Full Evaluation",
+        "num_rows": len(rows),
+        "counts": counts,
+        "sources": {
+            "vbench_full_info": {"path": args.full_info, "sha256": sha256(args.full_info)},
+            "short_prompts": {"path": short_path, "sha256": sha256(short_path)},
+            "extended_prompts": {"path": ext_path, "sha256": sha256(ext_path)},
+        },
+        "rows": rows,
+    }
+    os.makedirs(os.path.dirname(args.out), exist_ok=True)
+    with open(args.out, "w") as f:
+        json.dump(payload, f, indent=2)
+
+    print("mapping OK")
+    print(f"  rows            {len(rows)}")
+    for s in SUITES:
+        print(f"  {s:22s} {counts[s]}")
+    for k, v in payload["sources"].items():
+        print(f"  sha256 {k:18s} {v['sha256'][:16]}...")
+    print(f"  scene rows with auxiliary_info: "
+          f"{sum(1 for r in rows if r['prompt_suite'] == 'scene' and 'auxiliary_info' in r)}")
+    print(f"wrote {args.out}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

eval/generate_eval.py

@@ -0,0 +1,259 @@
+#!/usr/bin/env python
+"""Generate the Extended-251 videos for every strategy of one base model.
+
+Work is sharded by prompt, not by strategy, because the protocol requires each
+strategy's clip to be paired with the FFFF clip for the *same* prompt and seed
+(section 8.1). A shard therefore generates FFFF first, keeps its decoded frames,
+and immediately scores every cache strategy against them -- no 25 GB of reference
+frames on disk, and the pairing cannot get mismatched.
+
+    CUDA_VISIBLE_DEVICES=0 python eval/generate_eval.py \
+        --base self_forcing --shard 0 --num-shards 4 --out-root eval_out
+
+Re-running skips prompts whose per-strategy record is already complete.
+"""
+
+import argparse
+import json
+import os
+import sys
+import time
+
+ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.insert(0, ROOT)
+
+from eval.strategies import load_strategies  # noqa: E402
+from harness import denoising_steps_for, BASES, enter_base, load_pipeline, use_denoising_steps  # noqa: E402
+
+MAPPING = os.path.join(ROOT, "assets/vbench8_extended_subset_mapping.json")
+
+
+def parse_args():
+    p = argparse.ArgumentParser()
+    p.add_argument("--base", choices=sorted(BASES), required=True)
+    p.add_argument("--shard", type=int, default=0)
+    p.add_argument("--num-shards", type=int, default=1)
+    p.add_argument("--out-root", default="eval_out")
+    p.add_argument("--seed", type=int, default=0)
+    p.add_argument("--num-latent-frames", type=int, default=21)
+    p.add_argument("--fps", type=int, default=16)
+    p.add_argument("--limit", type=int, default=None, help="Smoke-test: first N prompts")
+    p.add_argument("--only-strategy", default=None, help="Smoke-test: one strategy name")
+    p.add_argument("--strategies", default=None,
+                   help="Comma-separated subset to generate (the FFFF reference is always kept)")
+    p.add_argument("--overwrite", action="store_true")
+    p.add_argument("--regenerate-ffff", action="store_true",
+                   help="Regenerate the FFFF reference instead of reading its MP4")
+    return p.parse_args()
+
+
+def main():
+    args = parse_args()
+    out_root = (args.out_root if os.path.isabs(args.out_root)
+                else os.path.join(ROOT, args.out_root))
+
+    with open(MAPPING) as f:
+        mapping = json.load(f)
+    rows = mapping["rows"]
+    if args.limit:
+        rows = rows[:args.limit]
+    shard_rows = rows[args.shard::args.num_shards]
+
+    strategies = load_strategies(args.base)
+    if args.only_strategy:
+        keep = {strategies[0]["name"], args.only_strategy}
+        strategies = [s for s in strategies if s["name"] in keep]
+    if args.strategies:
+        keep = set(args.strategies.split(","))
+        strategies = [s for s in strategies if s["is_reference"] or s["name"] in keep]
+    ref_name = strategies[0]["name"]
+    assert strategies[0]["is_reference"], "first strategy must be FFFF"
+
+    print(f"base={args.base} shard {args.shard}/{args.num_shards}: "
+          f"{len(shard_rows)} prompts x {len(strategies)} strategies", flush=True)
+
+    base = enter_base(args.base)
+
+    import torch
+    from einops import rearrange
+    from torchvision.io import read_video, write_video
+    from utils.misc import set_seed
+
+    from cachelib import (CacheController, build_method, cached_inference, install,
+                          parse_schedule)
+    from eval.pixel_metrics import PixelMetrics
+
+    torch.set_grad_enabled(False)
+    device = torch.device("cuda")
+    pipeline = load_pipeline(base)
+    num_steps = len(pipeline.denoising_step_list)
+    metrics = PixelMetrics(device)
+
+    def record_path(strategy, row):
+        d = os.path.join(out_root, "per_prompt", strategy)
+        os.makedirs(d, exist_ok=True)
+        return os.path.join(d, f"{row['prompt_suite']}_{row['suite_index']:03d}.json")
+
+    def video_path(strategy, row):
+        d = os.path.join(out_root, "generated_videos", strategy, row["prompt_suite"])
+        os.makedirs(d, exist_ok=True)
+        return os.path.join(d, f"{row['suite_index']:03d}.mp4")
+
+    def ref_cache_path(row):
+        d = os.path.join(out_root, "ref_cache", args.base)
+        os.makedirs(d, exist_ok=True)
+        return os.path.join(d, f"{row['prompt_suite']}_{row['suite_index']:03d}.pt")
+
+    def is_done(strategy, row):
+        p = record_path(strategy, row)
+        if args.overwrite or not os.path.exists(p):
+            return False
+        try:
+            with open(p) as f:
+                rec = json.load(f)
+            return rec.get("status") == "complete" and os.path.exists(rec.get("video", ""))
+        except Exception:
+            return False
+
+    def write_json(path, payload):
+        tmp = path + ".tmp"
+        with open(tmp, "w") as f:
+            json.dump(payload, f, indent=2)
+        os.replace(tmp, path)  # atomic: a partial file is never seen as complete
+
+    def run_one(strategy, row):
+        kw = dict(indicator=strategy["indicator"], coefficients=strategy["coefficients"])
+        if strategy["method"] != "none" and strategy.get("param"):
+            kw[strategy["param"]] = strategy["value"]
+        n_steps = strategy.get("num_inference_steps") or num_steps
+        original_steps = use_denoising_steps(pipeline, n_steps)
+        sched = strategy.get("schedule") or ("F" * n_steps)
+        first = strategy.get("first_chunk_schedule")
+        # Chunk 0 may have a longer schedule than the rest (e.g. FFFF over naive 2-step).
+        first_steps = (denoising_steps_for(original_steps, len(first))
+                       if first and len(first) != n_steps else None)
+        ctrl = CacheController(
+            build_method(strategy["method"], **kw), num_steps=n_steps,
+            forced_steps=parse_schedule(sched, n_steps),
+            first_chunk_forced_steps=(parse_schedule(first, len(first)) if first else None))
+        install(pipeline.generator.model, ctrl)
+
+        # Identical noise for every strategy: same seed, same draw order, same shape.
+        set_seed(args.seed)
+        noise = torch.randn([1, args.num_latent_frames, 16, 60, 104],
+                            device=device, dtype=torch.bfloat16)
+        wall0 = time.time()
+        video, latents, timing = cached_inference(
+            pipeline, ctrl, noise, [row["extended_prompt"]], decode=True,
+            first_chunk_steps=first_steps, sampler=strategy.get("sampler", "renoise"))
+        wall = time.time() - wall0
+        pipeline.vae.model.clear_cache()
+        pipeline.denoising_step_list = original_steps
+        return video, timing, ctrl.summary(), wall
+
+    # Warm up every strategy once on a throwaway prompt and discard the result.
+    # Without this the first recorded prompt carries one-off costs -- notably
+    # TaylorSeer's torch.compile of the forecast, which made its first video look
+    # slower than FFFF.
+    if shard_rows:
+        print("warm-up pass (not recorded)", flush=True)
+        for strategy in strategies:
+            run_one(strategy, shard_rows[0])
+        torch.cuda.empty_cache()
+
+    t_start = time.time()
+    for n, row in enumerate(shard_rows):
+        todo = [s for s in strategies if not is_done(s["name"], row)]
+        if not todo:
+            continue
+        # The reference is needed for pixel metrics even when only some strategies
+        # are outstanding.
+        need_ref = any(not s["is_reference"] for s in todo)
+        ref_frames = None
+        # The FFFF reference frames (pre-MP4, fp16) are cached on disk after their
+        # first generation, so later strategy runs do not regenerate FFFF at all.
+        rc = ref_cache_path(row)
+        ref_from_mp4 = False
+        if need_ref and os.path.exists(rc):
+            ref_frames = torch.load(rc, map_location=device)
+            need_ref = False
+        elif need_ref and os.path.exists(video_path(ref_name, row)) and not args.regenerate_ffff:
+            # Reference from the FFFF run's MP4 (H.264, ~40+ dB above the 12-20 dB
+            # strategy PSNRs, so the compression adds <1 % to the MSE); never
+            # regenerate FFFF just to compare against it.
+            vid, _, _ = read_video(video_path(ref_name, row), pts_unit="sec", output_format="TCHW")
+            ref_frames = (vid.to(device, torch.float16) / 255.0)
+            ref_from_mp4 = True
+            need_ref = False
+
+        for strategy in strategies:
+            if strategy["is_reference"]:
+                if strategy not in todo and not need_ref:
+                    continue
+            elif strategy not in todo:
+                continue
+
+            video, timing, summary, wall = run_one(strategy, row)
+            frames = video[0]  # [T, C, H, W] in [0, 1]
+
+            if strategy["is_reference"]:
+                ref_frames = frames.detach().to(torch.float16)
+                if not os.path.exists(rc):
+                    torch.save(ref_frames.cpu(), rc + ".tmp")
+                    os.replace(rc + ".tmp", rc)
+                pix = PixelMetrics.identity(frames.shape[0])
+            else:
+                pix = metrics.compute(frames, ref_frames)
+
+            if strategy in todo:
+                vp = video_path(strategy["name"], row)
+                arr = (255.0 * rearrange(frames, "t c h w -> t h w c")).clamp(0, 255)
+                write_video(vp, arr.to(torch.uint8).cpu(), fps=args.fps)
+
+                write_json(record_path(strategy["name"], row), {
+                    "status": "complete",
+                    "protocol": "Self-Forcing Extended-251 Full Evaluation",
+                    "strategy": strategy["name"],
+                    "base_model": args.base,
+                    "method": strategy["method"],
+                    "param": strategy["param"],
+                    "param_value": strategy["value"],
+                    "target_speedup": strategy["target"],
+                    "schedule": strategy.get("schedule") or ("F" * (strategy.get("num_inference_steps") or 4)),
+                    "num_inference_steps": strategy.get("num_inference_steps") or 4,
+                    "first_chunk_schedule": strategy.get("first_chunk_schedule"),
+                    "sampler": strategy.get("sampler", "renoise"),
+                    "global_index": row["global_index"],
+                    "prompt_suite": row["prompt_suite"],
+                    "suite_index": row["suite_index"],
+                    "prompt": row["extended_prompt"],
+                    "original_prompt": row["original_prompt"],
+                    "seed": args.seed,
+                    "video": vp,
+                    "num_frames": int(frames.shape[0]),
+                    "height": int(frames.shape[2]),
+                    "width": int(frames.shape[3]),
+                    "fps": args.fps,
+                    "policy_latency_ms": timing["denoise_dit_ms"],
+                    "excluded_context_kv_latency_ms": timing["context_kv_dit_ms"],
+                    "wall_generation_s": wall,
+                    "reference_strategy": ref_name,
+                    "pixel_metrics_vs_ffff": pix,
+                    "reference_source": "ffff_mp4" if ref_from_mp4 else "ffff_frames",
+                    "cache_diagnostics": dict(summary),
+                })
+            del frames, video
+
+        ref_frames = None
+        done = n + 1
+        rate = (time.time() - t_start) / done
+        print(f"[{args.base} shard {args.shard}] {done}/{len(shard_rows)} prompts "
+              f"({row['prompt_suite']}/{row['suite_index']:03d})  "
+              f"{rate:.1f}s/prompt  eta {(len(shard_rows) - done) * rate / 60:.0f} min",
+              flush=True)
+
+    print(f"shard {args.shard} done in {(time.time() - t_start) / 60:.1f} min", flush=True)
+
+
+if __name__ == "__main__":
+    main()

eval/generate_predictor_eval.py

@@ -0,0 +1,532 @@
+#!/usr/bin/env python
+"""Extended-251 videos for the Causal-Forcing-a single-block Predictors.
+
+Same protocol as generate_eval.py: work is sharded by prompt, FFFF is generated
+first in the same process and kept as the pixel-metric reference, every strategy
+sees the identical noise draw, and the per-prompt record has the layout
+aggregate.py / run_vbench.py expect.
+
+Strategies (prefix ``cfa`` = Causal-Forcing-a stack):
+
+    cfa_ffff                      reference, 28 full DiT forwards
+    cfa_<model>_fpff              chunks 1..6: F P F F   (22 full +  6 Predictor)
+    cfa_<model>_fppf              chunks 1..6: F P P F   (16 full + 12 Predictor)
+    cfa_<model>_fppp              chunks 1..6: F P P P   (10 full + 18 Predictor)
+
+where <model> is one of atc_chunk, atc_last_frame, disca (Stage-1 Layer-17
+Predictors) or atc_chunk_s2 (Stage-2 DMD EMA).  Chunk 0 is always fully denoised.
+``<model>@<step>`` selects another checkpoint of that run and names the strategy
+cfa_<model>_s<step>..., e.g. ``disca@500`` -> cfa_disca_s500_fppf; a bare model
+uses --checkpoint-step (default 2000) and the unsuffixed name.
+
+    cfa_naive2step                chunks 1..: the 2-step subset of the grid (t = 1000 / 250,
+                                  the same subset as cf_naive2step_c0FFFF), no Predictor
+
+    CUDA_VISIBLE_DEVICES=3 python eval/generate_predictor_eval.py --shard 0 --num-shards 4
+"""
+
+import argparse
+import json
+import os
+import sys
+import time
+
+ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+REPO = os.environ.get("CFA_REPO", "/local/zoubin/cz/projects/Causal-Forcing-a")
+MAPPING = os.path.join(ROOT, "assets/vbench8_extended_subset_mapping.json")
+
+BASE = {
+    "repo": REPO,
+    "config": "configs/causal_forcing_dmd_chunkwise.yaml",
+    "checkpoint": "checkpoints/chunkwise/causal_forcing.pt",
+}
+MODELS = {
+    "atc_chunk": "output/layer17_stage1_atc_chunk_1000p_21f_seed0_4gpu_b16_acc1_2000steps",
+    "atc_last_frame": "output/layer17_stage1_atc_last_frame_1000p_21f_seed0_4gpu_b16_acc1_2000steps",
+    "disca": "output/layer17_stage1_disca_1000p_21f_seed0_4gpu_b16_acc1_2000steps",
+    # Stage-2 random-exit DMD (LoRA critic), EMA weights (a (run dir, file name) pair;
+    # a plain directory gets checkpoint_step_<N>/predictor.safetensors).
+    "atc_chunk_s2": ("output/layer17_stage2_dmd_atc_chunk_lora128_4gpu_b2_2000steps", "predictor_ema.safetensors"),
+}
+NAIVE = {"naive2step": [0, 3]}   # pseudo-models: reduced step grid, no Predictor
+
+
+def checkpoint_weights(model, step):
+    """Weights file of ``model`` at training step ``step`` (Stage-1 runs zero-pad the
+    checkpoint directory, Stage-2 does not)."""
+    spec = MODELS[model]
+    run_dir, fname = spec if isinstance(spec, tuple) else (spec, "predictor.safetensors")
+    for d in (f"checkpoint_step_{step:04d}", f"checkpoint_step_{step}"):
+        w = os.path.join(REPO, run_dir, d, fname)
+        if os.path.exists(w):
+            return w
+    raise FileNotFoundError(f"{model} step {step}: no checkpoint under {os.path.join(REPO, run_dir)}")
+PATTERNS = {"FPFF": {1}, "FPPF": {1, 2}, "FPPP": {1, 2, 3}}
+# Reduced schedules: which step indices of the 4-step grid a chunk (after chunk 0)
+# actually visits.  FPF keeps steps 0 and 3 (t = 1000 / 250) as Full and runs one
+# Predictor step at step 1 (t = 750, anchored on step 0 as in training), skipping
+# step 2 entirely: 3 denoising positions per chunk, 2 Full + 1 Predictor.
+RUN_STEPS = {"FPF": [0, 1, 3]}
+PATTERNS["FPF"] = {1}
+NUM_CHUNKS, FRAMES_PER_CHUNK, NUM_STEPS = 7, 3, 4
+# A Predictor step runs 1 of the 30 Teacher blocks (plus a small fusion MLP);
+# its compute-equivalent cost is booked as 1/30 of a full forward.
+PREDICTOR_COMPUTE_EQUIV = 1.0 / 30.0
+
+
+def parse_args():
+    p = argparse.ArgumentParser()
+    p.add_argument("--shard", type=int, default=0)
+    p.add_argument("--num-shards", type=int, default=1)
+    p.add_argument("--out-root", default="eval_out")
+    p.add_argument("--seed", type=int, default=0)
+    p.add_argument("--fps", type=int, default=16)
+    p.add_argument("--models", default=",".join(MODELS))
+    p.add_argument("--patterns", default="FPPF,FPPP")
+    p.add_argument("--checkpoint-step", type=int, default=2000)
+    p.add_argument("--num-chunks", type=int, default=NUM_CHUNKS,
+                   help="Video length in 3-latent-frame chunks (7 = the 81-frame protocol; longer "
+                        "videos use a rolling 21-frame KV window like Self-Forcing's long-video mode)")
+    p.add_argument("--ref-tag", default="",
+                   help="Inserted into the reference name, e.g. _14c -> cfa_14c_ffff (a separate FFFF "
+                        "reference for a different video length)")
+    p.add_argument("--tag", default="",
+                   help="Inserted into strategy names after the model, e.g. _s1000 -> "
+                        "cfa_disca_s1000_fppf, so another checkpoint does not overwrite the default rows")
+    p.add_argument("--regenerate-ffff", action="store_true",
+                   help="Regenerate the FFFF reference instead of reading cfa_ffff's MP4")
+    p.add_argument("--limit", type=int, default=None, help="Smoke-test: first N prompts")
+    p.add_argument("--overwrite", action="store_true")
+    p.add_argument("--retime-only", action="store_true",
+                   help="Do not regenerate videos/metrics; re-time FFFF and each strategy in the same "
+                        "process per prompt and update policy_latency_ms / matched_ffff_policy_latency_ms.")
+    p.add_argument("--retime-max-ffff-ms", type=float, default=3500.0,
+                   help="With --retime-only, records whose paired FFFF exceeds this are re-timed again "
+                        "(up to --retime-passes passes); clean records are left alone.")
+    p.add_argument("--retime-passes", type=int, default=3)
+    p.add_argument("--retime-strategy-only", action="store_true",
+                   help="Re-time only the strategies (no FFFF run): rerun each selected record's "
+                        "rollout on an idle GPU and replace policy_latency_ms; the generation-run "
+                        "value is kept under generation_run_latency.")
+    p.add_argument("--written-before", type=float, default=None,
+                   help="With --retime-strategy-only: only records whose file mtime is before this "
+                        "epoch second (the window in which another job shared the GPU)")
+    return p.parse_args()
+
+
+def build_strategies(args):
+    out = [{"name": f"cfa{args.ref_tag}_ffff", "model": None, "pattern": "FFFF", "is_reference": True}]
+    for token in args.models.split(","):
+        if token in NAIVE:
+            out.append({"name": f"cfa_{token}{args.tag}", "model": None, "pattern": token.upper(),
+                        "naive": token, "is_reference": False})
+            continue
+        model, _, step = token.partition("@")
+        step = int(step) if step else args.checkpoint_step
+        key = f"{model}@{step}"   # one loaded Predictor per (model, step)
+        label = f"{model}_s{step}" if "@" in token else model
+        for pattern in args.patterns.split(","):
+            out.append({"name": f"cfa_{label}{args.tag}_{pattern.lower()}", "model": key,
+                        "base_model": model, "checkpoint_step": step,
+                        "pattern": pattern.upper(), "is_reference": False})
+    return out
+
+
+class FinalHiddenCapture:
+    """Grab the Teacher's final hidden (input of its output head) on a Full step."""
+
+    def __init__(self, teacher):
+        self.enabled, self.value = False, None
+        self.handle = teacher.head.register_forward_pre_hook(self._hook)
+
+    def _hook(self, _module, inputs):
+        if self.enabled:
+            self.value = inputs[0].detach()
+
+    def start(self):
+        self.value, self.enabled = None, True
+
+    def finish(self):
+        self.enabled = False
+        if self.value is None:
+            raise RuntimeError("Teacher final hidden was not captured")
+        value, self.value = self.value, None
+        return value
+
+
+def main():
+    args = parse_args()
+    out_root = args.out_root if os.path.isabs(args.out_root) else os.path.join(ROOT, args.out_root)
+    with open(MAPPING) as f:
+        rows = json.load(f)["rows"]
+    if args.limit:
+        rows = rows[:args.limit]
+    shard_rows = rows[args.shard::args.num_shards]
+    strategies = build_strategies(args)
+    ref_name = strategies[0]["name"]
+    print(f"cfa shard {args.shard}/{args.num_shards}: {len(shard_rows)} prompts x "
+          f"{len(strategies)} strategies", flush=True)
+
+    os.chdir(REPO)
+    sys.path.insert(0, ROOT)
+    sys.path.insert(0, REPO)  # pipeline/, utils/, predictor_training/ from Causal-Forcing-a
+
+    import torch
+    from einops import rearrange
+    from safetensors.torch import load_file
+    from torchvision.io import read_video, write_video
+
+    from eval.pixel_metrics import PixelMetrics
+    from harness import load_pipeline
+    from predictor_training.metadata import build_single_block_predictor, read_predictor_config
+    from predictor_training.online import online_predictor_step as _online_predictor_step
+    from predictor_training.offline_data import TOKENS_PER_CHUNK
+
+    def online_predictor_step(*, history_cache, chunk, **kw):
+        """Stage-1 contract: the Predictor sees the clean K/V of the *earlier* chunks.
+        With the rolling 21-frame window (long videos) the cache holds the last six
+        clean chunks plus the current noisy one, so the history is everything before
+        the current chunk's tokens rather than ``chunk * TOKENS_PER_CHUNK`` (which
+        the stock helper asserts and which exceeds the window from chunk 7 on).
+        For chunk <= 6 the two are identical."""
+        if chunk * TOKENS_PER_CHUNK <= int(history_cache["local_end_index"].item()):
+            return _online_predictor_step(history_cache=history_cache, chunk=chunk, **kw)
+        end = int(history_cache["local_end_index"].item()) - TOKENS_PER_CHUNK
+        window = {"k": history_cache["k"][:, :end], "v": history_cache["v"][:, :end],
+                  "local_end_index": torch.tensor([end], device=history_cache["k"].device)}
+        # left-pad to the absolute length the stock helper slices; the attention
+        # window (max_attention_size = 21 frames) never reaches the padding
+        pad = chunk * TOKENS_PER_CHUNK - end
+        if pad > 0:
+            z = history_cache["k"].new_zeros(history_cache["k"].shape[0], pad, *history_cache["k"].shape[2:])
+            window = {"k": torch.cat([z, window["k"]], 1), "v": torch.cat([z, window["v"]], 1),
+                      "local_end_index": torch.tensor([chunk * TOKENS_PER_CHUNK], device=z.device)}
+        return _online_predictor_step(history_cache=window, chunk=chunk, **kw)
+    from utils.misc import set_seed
+
+    torch.set_grad_enabled(False)
+    device = torch.device("cuda")
+    pipeline = load_pipeline(BASE)
+    n_chunks = args.num_chunks
+    if n_chunks * FRAMES_PER_CHUNK > 21:
+        # Longer than the 21-latent-frame training context: roll the KV cache over a
+        # 21-frame local window (the model's long-video mechanism; RoPE is relative
+        # so positions inside the window stay in the trained range).
+        model = pipeline.generator.model
+        for m in [model] + [b for b in model.blocks] + [b.self_attn for b in model.blocks]:
+            m.local_attn_size = 21
+            if hasattr(m, "max_attention_size"):
+                m.max_attention_size = 21 * pipeline.frame_seq_length
+        pipeline.local_attn_size = 21
+        pipeline.kv_cache1 = None
+        print(f"long video: {n_chunks} chunks = {n_chunks * FRAMES_PER_CHUNK} latent frames, "
+              f"rolling KV window 21 frames", flush=True)
+    teacher = pipeline.generator.model
+    capture = FinalHiddenCapture(teacher)
+    metrics = PixelMetrics(device)
+    timesteps = pipeline.denoising_step_list.to(device)
+    assert len(timesteps) == NUM_STEPS and pipeline.num_frame_per_block == FRAMES_PER_CHUNK
+
+    predictors, predictor_configs = {}, {}
+    for model in {s["model"] for s in strategies if s["model"]}:
+        base, step = model.split("@")
+        weights = checkpoint_weights(base, int(step))
+        config = read_predictor_config(weights)
+        predictor = build_single_block_predictor(teacher, config, gradient_checkpointing=False)
+        predictor.load_state_dict(load_file(weights, device="cpu"), strict=True)
+        predictors[model] = predictor.to(device).eval().requires_grad_(False)
+        predictor_configs[model] = {**config, "weights": weights}
+        print(f"loaded {model}: {config}", flush=True)
+
+    def reset_caches():
+        if pipeline.kv_cache1 is None:
+            pipeline._initialize_kv_cache(1, torch.bfloat16, device)
+            pipeline._initialize_crossattn_cache(1, torch.bfloat16, device)
+        for cache in pipeline.kv_cache1:
+            cache["global_end_index"] = torch.tensor([0], dtype=torch.long, device=device)
+            cache["local_end_index"] = torch.tensor([0], dtype=torch.long, device=device)
+        for cache in pipeline.crossattn_cache:
+            cache["is_init"] = False
+
+    def rollout(conditional, noise, predictor=None, predictor_steps=frozenset(), run_steps=None):
+        """Mirror of cachelib.runner.cached_inference with Predictor steps spliced in."""
+        reset_caches()
+        output = torch.zeros_like(noise)
+        denoise_events, context_events = [], []
+        full_calls = predictor_calls = 0
+        previous_hidden = None
+        for chunk in range(n_chunks):
+            start_frame = chunk * FRAMES_PER_CHUNK
+            token_start = start_frame * pipeline.frame_seq_length
+            noisy_input = noise[:, start_frame:start_frame + FRAMES_PER_CHUNK]
+            current_hidden = [None] * NUM_STEPS
+            # chunk 0 always runs the full grid; later chunks may visit a subset
+            steps = list(range(NUM_STEPS)) if (run_steps is None or chunk == 0) else list(run_steps)
+            for si, step in enumerate(steps):
+                timestep = torch.ones([1, FRAMES_PER_CHUNK], device=device, dtype=torch.int64) * timesteps[step]
+                start_ev = torch.cuda.Event(enable_timing=True)
+                end_ev = torch.cuda.Event(enable_timing=True)
+                start_ev.record()
+                if predictor is not None and chunk > 0 and step in predictor_steps:
+                    hidden, flow = online_predictor_step(
+                        predictor=predictor, teacher=teacher, noisy_input=noisy_input,
+                        timestep=timestep, anchor_timestep=torch.ones_like(timestep) * timesteps[step - 1],
+                        anchor_hidden=current_hidden[step - 1], previous_hidden=previous_hidden[step],
+                        history_cache=pipeline.kv_cache1[predictor.source_layer],
+                        cross_cache=pipeline.crossattn_cache[predictor.source_layer], chunk=chunk)
+                    denoised_pred = pipeline.generator._convert_flow_pred_to_x0(
+                        flow_pred=flow.flatten(0, 1), xt=noisy_input.flatten(0, 1),
+                        timestep=timestep.flatten(0, 1)).unflatten(0, flow.shape[:2])
+                    current_hidden[step] = hidden
+                    predictor_calls += 1
+                else:
+                    capture.start()
+                    _, denoised_pred = pipeline.generator(
+                        noisy_image_or_video=noisy_input, conditional_dict=conditional,
+                        timestep=timestep, kv_cache=pipeline.kv_cache1,
+                        crossattn_cache=pipeline.crossattn_cache, current_start=token_start)
+                    current_hidden[step] = capture.finish()
+                    full_calls += 1
+                end_ev.record()
+                denoise_events.append((start_ev, end_ev))
+                if si < len(steps) - 1:
+                    # Same RNG draw order as the protocol runner, so every strategy
+                    # re-noises with identical samples.
+                    noisy_input = pipeline.scheduler.add_noise(
+                        denoised_pred.flatten(0, 1), torch.randn_like(denoised_pred.flatten(0, 1)),
+                        timesteps[steps[si + 1]] * torch.ones([FRAMES_PER_CHUNK], device=device, dtype=torch.long),
+                    ).unflatten(0, denoised_pred.shape[:2])
+            output[:, start_frame:start_frame + FRAMES_PER_CHUNK] = denoised_pred
+            # Clean-context KV refresh: full DiT, timed separately, never added in.
+            ctx_start = torch.cuda.Event(enable_timing=True)
+            ctx_end = torch.cuda.Event(enable_timing=True)
+            ctx_start.record()
+            pipeline.generator(
+                noisy_image_or_video=denoised_pred, conditional_dict=conditional,
+                timestep=torch.ones_like(timestep) * pipeline.args.context_noise,
+                kv_cache=pipeline.kv_cache1, crossattn_cache=pipeline.crossattn_cache,
+                current_start=token_start)
+            ctx_end.record()
+            context_events.append((ctx_start, ctx_end))
+            previous_hidden = current_hidden
+        torch.cuda.synchronize()
+        timing = {
+            "denoise_dit_ms": sum(s.elapsed_time(e) for s, e in denoise_events),
+            "context_kv_dit_ms": sum(s.elapsed_time(e) for s, e in context_events),
+            "num_denoise_forwards": len(denoise_events),
+        }
+        video = pipeline.vae.decode_to_pixel(output, use_cache=False)
+        video = (video * 0.5 + 0.5).clamp(0, 1)
+        pipeline.vae.model.clear_cache()
+        return video, timing, {"full_forwards": full_calls, "predictor_forwards": predictor_calls}
+
+    def run_one(strategy, row):
+        set_seed(args.seed)
+        noise = torch.randn([1, n_chunks * FRAMES_PER_CHUNK, 16, 60, 104], device=device, dtype=torch.bfloat16)
+        conditional = pipeline.text_encoder(text_prompts=[row["extended_prompt"]])
+        wall0 = time.time()
+        if strategy["is_reference"]:
+            video, timing, counts = rollout(conditional, noise)
+        elif strategy.get("naive"):
+            video, timing, counts = rollout(conditional, noise, None, frozenset(),
+                                            NAIVE[strategy["naive"]])
+        else:
+            video, timing, counts = rollout(conditional, noise, predictors[strategy["model"]],
+                                            PATTERNS[strategy["pattern"]],
+                                            RUN_STEPS.get(strategy["pattern"]))
+        return video, timing, counts, time.time() - wall0
+
+    def record_path(strategy, row):
+        d = os.path.join(out_root, "per_prompt", strategy)
+        os.makedirs(d, exist_ok=True)
+        return os.path.join(d, f"{row['prompt_suite']}_{row['suite_index']:03d}.json")
+
+    def video_path(strategy, row):
+        d = os.path.join(out_root, "generated_videos", strategy, row["prompt_suite"])
+        os.makedirs(d, exist_ok=True)
+        return os.path.join(d, f"{row['suite_index']:03d}.mp4")
+
+    def is_done(strategy, row):
+        p = record_path(strategy, row)
+        if args.overwrite or not os.path.exists(p):
+            return False
+        try:
+            with open(p) as f:
+                rec = json.load(f)
+            return rec.get("status") == "complete" and os.path.exists(rec.get("video", ""))
+        except Exception:
+            return False
+
+    def write_json(path, payload):
+        tmp = path + ".tmp"
+        with open(tmp, "w") as f:
+            json.dump(payload, f, indent=2)
+        os.replace(tmp, path)
+
+    if shard_rows:
+        print("warm-up pass (not recorded)", flush=True)
+        for strategy in strategies:
+            run_one(strategy, shard_rows[0])
+        torch.cuda.empty_cache()
+
+    if args.retime_strategy_only:
+        targets = [s for s in strategies if not s["is_reference"]]
+        n_done = 0
+        for n, row in enumerate(shard_rows):
+            for strategy in targets:
+                rp = record_path(strategy["name"], row)
+                if not os.path.exists(rp):
+                    continue
+                if args.written_before and os.path.getmtime(rp) >= args.written_before:
+                    continue
+                rec = json.load(open(rp))
+                if rec.get("status") != "complete" or "generation_run_latency" in rec:
+                    continue
+                _, timing, counts, _ = run_one(strategy, row)
+                assert counts["full_forwards"] == rec["cache_diagnostics"]["full_forwards"], rp
+                rec["generation_run_latency"] = {
+                    "policy_latency_ms": rec["policy_latency_ms"],
+                    "excluded_context_kv_latency_ms": rec["excluded_context_kv_latency_ms"]}
+                rec["policy_latency_ms"] = timing["denoise_dit_ms"]
+                rec["excluded_context_kv_latency_ms"] = timing["context_kv_dit_ms"]
+                rec["latency_retimed"] = "strategy-only rerun on an idle GPU (FFFF not re-run)"
+                write_json(rp, rec)
+                n_done += 1
+            if (n + 1) % 20 == 0:
+                print(f"[strategy retime shard {args.shard}] {n + 1}/{len(shard_rows)}", flush=True)
+        print(f"shard {args.shard}: {n_done} records re-timed", flush=True)
+        return
+
+    if args.retime_only:
+        # Protocol "matched FFFF": pair every strategy with an FFFF run of the same
+        # prompt in the same process, seconds apart, so contention cancels in the
+        # ratio.  Only records whose pairing is missing or dirty are touched.
+        targets = [s for s in strategies if not s["is_reference"]]
+        for pass_index in range(args.retime_passes):
+            dirty = 0
+            for n, row in enumerate(shard_rows):
+                todo = []
+                for strategy in targets:
+                    rp = record_path(strategy["name"], row)
+                    if not os.path.exists(rp):
+                        continue
+                    rec = json.load(open(rp))
+                    m = rec.get("matched_ffff_policy_latency_ms")
+                    if m is None or m > args.retime_max_ffff_ms:
+                        todo.append((strategy, rp, rec))
+                if not todo:
+                    continue
+                dirty += len(todo)
+                _, ref_timing, _, _ = run_one(strategies[0], row)
+                for strategy, rp, rec in todo:
+                    _, timing, _, _ = run_one(strategy, row)
+                    rec["policy_latency_ms"] = timing["denoise_dit_ms"]
+                    rec["excluded_context_kv_latency_ms"] = timing["context_kv_dit_ms"]
+                    rec["matched_ffff_policy_latency_ms"] = ref_timing["denoise_dit_ms"]
+                    rec["matched_ffff_context_kv_latency_ms"] = ref_timing["context_kv_dit_ms"]
+                    rec["latency_source"] = "paired retime (same process as FFFF)"
+                    write_json(rp, rec)
+                if (n + 1) % 10 == 0:
+                    print(f"[retime pass {pass_index} shard {args.shard}] {n + 1}/{len(shard_rows)} "
+                          f"ffff={ref_timing['denoise_dit_ms']:.0f}ms", flush=True)
+            print(f"[retime pass {pass_index} shard {args.shard}] re-timed {dirty} records", flush=True)
+            if dirty == 0:
+                break
+        return
+
+    t_start = time.time()
+    for n, row in enumerate(shard_rows):
+        todo = [s for s in strategies if not is_done(s["name"], row)]
+        if not todo:
+            continue
+        need_ref = any(not s["is_reference"] for s in todo)
+        ref_frames = None
+        ref_timing = None
+        ref_from_mp4 = False
+        if need_ref and os.path.exists(video_path(ref_name, row)) and not args.regenerate_ffff:
+            # Reference from the cfa_ffff run's MP4 (H.264, 40+ dB above the strategy
+            # PSNRs); FFFF is not regenerated, so there is no same-process FFFF timing
+            # and the speedup is taken against cfa_ffff's own records (aggregate.py).
+            vid, _, _ = read_video(video_path(ref_name, row), pts_unit="sec", output_format="TCHW")
+            ref_frames = vid.to(device, torch.float16) / 255.0
+            ref_from_mp4 = True
+            need_ref = False
+        for strategy in strategies:
+            if strategy["is_reference"]:
+                if strategy not in todo and not need_ref:
+                    continue
+            elif strategy not in todo:
+                continue
+            video, timing, counts, wall = run_one(strategy, row)
+            frames = video[0]
+            if strategy["is_reference"]:
+                ref_frames = frames.detach().to(torch.float16)
+                ref_timing = timing
+                pix = PixelMetrics.identity(frames.shape[0])
+            else:
+                pix = metrics.compute(frames, ref_frames)
+            if strategy in todo:
+                vp = video_path(strategy["name"], row)
+                arr = (255.0 * rearrange(frames, "t c h w -> t h w c")).clamp(0, 255)
+                write_video(vp, arr.to(torch.uint8).cpu(), fps=args.fps)
+                compute_equiv = counts["full_forwards"] + PREDICTOR_COMPUTE_EQUIV * counts["predictor_forwards"]
+                model = strategy["model"]
+                naive = strategy.get("naive")
+                schedule = "F" * len(NAIVE[naive]) if naive else strategy["pattern"]
+                write_json(record_path(strategy["name"], row), {
+                    "status": "complete",
+                    "protocol": "Self-Forcing Extended-251 Full Evaluation",
+                    "strategy": strategy["name"],
+                    "base_model": "causal_forcing_a",
+                    "method": "none" if model is None else f"predictor_{strategy['base_model']}",
+                    "param": None if model is None else "pattern",
+                    "param_value": None if model is None else strategy["pattern"],
+                    "target_speedup": (n_chunks * NUM_STEPS) / compute_equiv,
+                    "schedule": schedule,
+                    "num_inference_steps": len(NAIVE[naive]) if naive else NUM_STEPS,
+                    "first_chunk_schedule": "FFFF" if (naive or model) else None,
+                    "predictor_config": None if model is None else predictor_configs[model],
+                    "global_index": row["global_index"],
+                    "prompt_suite": row["prompt_suite"],
+                    "suite_index": row["suite_index"],
+                    "prompt": row["extended_prompt"],
+                    "original_prompt": row["original_prompt"],
+                    "seed": args.seed,
+                    "video": vp,
+                    "num_frames": int(frames.shape[0]),
+                    "height": int(frames.shape[2]),
+                    "width": int(frames.shape[3]),
+                    "fps": args.fps,
+                    "policy_latency_ms": timing["denoise_dit_ms"],
+                    "excluded_context_kv_latency_ms": timing["context_kv_dit_ms"],
+                    # Same-process FFFF timing of this prompt (protocol "matched FFFF").
+                    "matched_ffff_policy_latency_ms": ref_timing["denoise_dit_ms"] if ref_timing else None,
+                    "matched_ffff_context_kv_latency_ms": ref_timing["context_kv_dit_ms"] if ref_timing else None,
+                    "checkpoint_step": strategy.get("checkpoint_step"),
+                    "num_chunks": n_chunks,
+                    "wall_generation_s": wall,
+                    "reference_strategy": ref_name,
+                    "pixel_metrics_vs_ffff": pix,
+                    "reference_source": "ffff_mp4" if ref_from_mp4 else "ffff_frames",
+                    "cache_diagnostics": {
+                        "denoise_forwards": timing["num_denoise_forwards"],
+                        "full_forwards": counts["full_forwards"],
+                        "predictor_forwards": counts["predictor_forwards"],
+                        "predictor_compute_equivalent": PREDICTOR_COMPUTE_EQUIV,
+                        "compute_equivalent_forwards": compute_equiv,
+                        "middle_steps": n_chunks * 2,
+                        "middle_compute_equivalent": compute_equiv - n_chunks * 2.0,
+                    },
+                })
+            del frames, video
+        ref_frames = None
+        done = n + 1
+        rate = (time.time() - t_start) / done
+        print(f"[cfa shard {args.shard}] {done}/{len(shard_rows)} prompts "
+              f"({row['prompt_suite']}/{row['suite_index']:03d})  {rate:.1f}s/prompt  "
+              f"eta {(len(shard_rows) - done) * rate / 60:.0f} min", flush=True)
+    print(f"shard {args.shard} done in {(time.time() - t_start) / 60:.1f} min", flush=True)
+
+
+if __name__ == "__main__":
+    main()

eval/pixel_metrics.py

@@ -0,0 +1,85 @@
+"""PSNR / SSIM / LPIPS between a strategy's frames and its matched FFFF frames.
+
+Computed on the decoded RGB tensors in [0, 1], before MP4 encoding, as the
+protocol requires (section 8.1) -- video compression noise would otherwise leak
+into the numbers.
+
+SSIM is implemented here rather than taken from skimage: skimage needs ~30 ms per
+480x832 frame on CPU, and this evaluation compares ~500k frames.
+"""
+
+import torch
+import torch.nn.functional as F
+
+
+def _gaussian_window(size=11, sigma=1.5, device=None, dtype=torch.float32):
+    coords = torch.arange(size, device=device, dtype=dtype) - (size - 1) / 2.0
+    g = torch.exp(-(coords ** 2) / (2 * sigma ** 2))
+    g = g / g.sum()
+    return g[:, None] @ g[None, :]
+
+
+class PixelMetrics:
+    """Frame-wise metrics against a reference clip. Call once per (strategy, prompt)."""
+
+    def __init__(self, device, lpips_net="alex", ssim_window=11, ssim_sigma=1.5):
+        self.device = device
+        self.window = _gaussian_window(ssim_window, ssim_sigma, device=device)[None, None]
+        self.window_size = ssim_window
+        import lpips
+        self.lpips = lpips.LPIPS(net=lpips_net).to(device).eval()
+        for p in self.lpips.parameters():
+            p.requires_grad_(False)
+
+    def ssim(self, x, y):
+        """x, y: [T, C, H, W] in [0, 1]. Returns the mean over frames and channels."""
+        t, c, h, w = x.shape
+        win = self.window.expand(c, 1, self.window_size, self.window_size)
+        pad = self.window_size // 2
+        mu_x = F.conv2d(x, win, padding=pad, groups=c)
+        mu_y = F.conv2d(y, win, padding=pad, groups=c)
+        mu_x2, mu_y2, mu_xy = mu_x * mu_x, mu_y * mu_y, mu_x * mu_y
+        sig_x = F.conv2d(x * x, win, padding=pad, groups=c) - mu_x2
+        sig_y = F.conv2d(y * y, win, padding=pad, groups=c) - mu_y2
+        sig_xy = F.conv2d(x * y, win, padding=pad, groups=c) - mu_xy
+        c1, c2 = 0.01 ** 2, 0.03 ** 2
+        s = ((2 * mu_xy + c1) * (2 * sig_xy + c2)) / ((mu_x2 + mu_y2 + c1) * (sig_x + sig_y + c2))
+        # Drop the padded border, as skimage does with its default crop.
+        s = s[..., pad:-pad, pad:-pad]
+        return s.mean(dim=(1, 2, 3))  # per frame
+
+    @torch.no_grad()
+    def compute(self, frames, ref, chunk=16):
+        """frames, ref: [T, C, H, W] float in [0, 1] on any device.
+
+        Returns per-prompt mean MSE (for the protocol's ratio-of-means PSNR),
+        plus mean SSIM and mean LPIPS over frames.
+        """
+        frames = frames.to(self.device, torch.float32)
+        ref = ref.to(self.device, torch.float32)
+        assert frames.shape == ref.shape, (frames.shape, ref.shape)
+
+        mse_terms, ssim_terms, lpips_terms = [], [], []
+        for i in range(0, frames.shape[0], chunk):
+            a, b = frames[i:i + chunk], ref[i:i + chunk]
+            mse_terms.append(((a - b) ** 2).mean(dim=(1, 2, 3)))
+            ssim_terms.append(self.ssim(a, b))
+            lpips_terms.append(self.lpips(a * 2 - 1, b * 2 - 1).flatten())
+
+        mse = torch.cat(mse_terms)
+        ssim = torch.cat(ssim_terms)
+        lp = torch.cat(lpips_terms)
+        mean_mse = mse.mean().item()
+        return {
+            "mean_mse": mean_mse,
+            "psnr": -10.0 * torch.log10(torch.tensor(max(mean_mse, 1e-12))).item(),
+            "ssim": ssim.mean().item(),
+            "lpips": lp.mean().item(),
+            "num_frames": int(frames.shape[0]),
+        }
+
+    @staticmethod
+    def identity(num_frames):
+        """Fixed self-comparison values the protocol pins for FFFF vs itself."""
+        return {"mean_mse": 1e-12, "psnr": 120.0, "ssim": 1.0, "lpips": 0.0,
+                "num_frames": int(num_frames)}

eval/retime_eval.py

@@ -0,0 +1,188 @@
+#!/usr/bin/env python
+"""Paired re-timing of evaluated strategies against FFFF, prompt by prompt.
+
+``generate_eval.py`` records each strategy's denoise latency, but when it is
+re-run for *additional* strategies the FFFF reference is regenerated only for
+the pixel metrics and its latency is not recorded, so the new records would be
+compared against FFFF timings from a different run under different contention.
+The protocol's speedup is "matched FFFF" (section 9.4): same prompt, same seed,
+same conditions.  This pass restores that: for every prompt it runs FFFF and
+then each requested strategy back to back (denoise only, no decode, no metrics)
+and writes the strategy's latency together with the FFFF latency measured
+seconds earlier into the strategy's per-prompt record.
+
+    CUDA_VISIBLE_DEVICES=0 python eval/retime_eval.py --base self_forcing \
+        --strategies sf_taylorseer_FFxx_x1.3,sf_teacache_Fxxx_x3 \
+        --shard 0 --num-shards 4 --out-root eval_out
+
+Records already carrying ``latency_source`` are skipped, so the pass resumes.
+"""
+
+import argparse
+import json
+import os
+import sys
+import time
+
+ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.insert(0, ROOT)
+
+from eval.strategies import load_strategies  # noqa: E402
+from harness import BASES, enter_base, load_pipeline  # noqa: E402
+
+MAPPING = os.path.join(ROOT, "assets/vbench8_extended_subset_mapping.json")
+SOURCE = "retime_eval: paired with FFFF in the same process, denoise only"
+
+
+def parse_args():
+    p = argparse.ArgumentParser()
+    p.add_argument("--base", choices=sorted(BASES), required=True)
+    p.add_argument("--strategies", required=True,
+                   help="Comma-separated strategy names, or 'all' for every "
+                        "non-reference strategy of the base")
+    p.add_argument("--shard", type=int, default=0)
+    p.add_argument("--num-shards", type=int, default=1)
+    p.add_argument("--out-root", default="eval_out")
+    p.add_argument("--seed", type=int, default=0)
+    p.add_argument("--num-latent-frames", type=int, default=21)
+    p.add_argument("--limit", type=int, default=None)
+    p.add_argument("--force", action="store_true", help="Re-time records already re-timed")
+    p.add_argument("--redo-ratio-bounds", default=None,
+                   help="lo,hi: also re-time records whose measured/compute speedup "
+                        "ratio falls outside this band -- a spike that hit the strategy "
+                        "half of the pair rather than the FFFF half")
+    p.add_argument("--redo-above-ms", type=float, default=None,
+                   help="Also re-time records whose matched FFFF latency exceeds this: "
+                        "a neighbour's job hit that pair, and the pair is only fair "
+                        "when both halves ran under the same conditions")
+    return p.parse_args()
+
+
+def main():
+    args = parse_args()
+    out_root = (args.out_root if os.path.isabs(args.out_root)
+                else os.path.join(ROOT, args.out_root))
+
+    with open(MAPPING) as f:
+        rows = json.load(f)["rows"]
+    if args.limit:
+        rows = rows[:args.limit]
+    shard_rows = rows[args.shard::args.num_shards]
+
+    all_strategies = load_strategies(args.base)
+    reference = all_strategies[0]
+    assert reference["is_reference"]
+    if args.strategies == "all":
+        wanted = [s for s in all_strategies if not s["is_reference"]]
+    else:
+        names = set(args.strategies.split(","))
+        wanted = [s for s in all_strategies if s["name"] in names]
+        missing = names - {s["name"] for s in wanted}
+        if missing:
+            sys.exit(f"unknown strategies for {args.base}: {sorted(missing)}")
+
+    print(f"base={args.base} shard {args.shard}/{args.num_shards}: "
+          f"{len(shard_rows)} prompts x ({reference['name']} + {len(wanted)} strategies)",
+          flush=True)
+
+    base = enter_base(args.base)
+
+    import torch
+    from utils.misc import set_seed
+
+    from cachelib import (CacheController, build_method, cached_inference, install,
+                          parse_schedule)
+
+    torch.set_grad_enabled(False)
+    device = torch.device("cuda")
+    pipeline = load_pipeline(base)
+    num_steps = len(pipeline.denoising_step_list)
+
+    def record_path(strategy, row):
+        return os.path.join(out_root, "per_prompt", strategy,
+                            f"{row['prompt_suite']}_{row['suite_index']:03d}.json")
+
+    def time_one(strategy, row):
+        kw = dict(indicator=strategy["indicator"], coefficients=strategy["coefficients"])
+        if strategy["method"] != "none":
+            kw[strategy["param"]] = strategy["value"]
+        first = strategy.get("first_chunk_schedule")
+        ctrl = CacheController(
+            build_method(strategy["method"], **kw), num_steps=num_steps,
+            forced_steps=parse_schedule(strategy.get("schedule", "FxxF"), num_steps),
+            first_chunk_forced_steps=(parse_schedule(first, num_steps) if first else None))
+        install(pipeline.generator.model, ctrl)
+        set_seed(args.seed)
+        noise = torch.randn([1, args.num_latent_frames, 16, 60, 104],
+                            device=device, dtype=torch.bfloat16)
+        _, _, timing = cached_inference(pipeline, ctrl, noise, [row["extended_prompt"]],
+                                        decode=False)
+        return timing, ctrl.summary()
+
+    def write_json(path, payload):
+        tmp = path + ".tmp"
+        with open(tmp, "w") as f:
+            json.dump(payload, f, indent=2)
+        os.replace(tmp, path)
+
+    def needs(strategy, row):
+        p = record_path(strategy["name"], row)
+        if not os.path.exists(p):
+            return False  # nothing to attach the timing to; generation must run first
+        with open(p) as f:
+            rec = json.load(f)
+        if rec.get("status") != "complete":
+            return False
+        if args.force or "latency_source" not in rec:
+            return True
+        if (args.redo_above_ms is not None
+                and rec.get("matched_ffff_policy_latency_ms", 0) > args.redo_above_ms):
+            return True
+        if args.redo_ratio_bounds:
+            lo, hi = (float(v) for v in args.redo_ratio_bounds.split(","))
+            d = rec["cache_diagnostics"]
+            compute_x = d["denoise_forwards"] / max(d["compute_equivalent_forwards"], 1e-6)
+            measured_x = rec["matched_ffff_policy_latency_ms"] / rec["policy_latency_ms"]
+            if not (lo <= measured_x / compute_x <= hi):
+                return True
+        return False
+
+    if shard_rows:
+        print("warm-up pass (not recorded)", flush=True)
+        for s in [reference] + wanted:
+            time_one(s, shard_rows[0])
+        torch.cuda.empty_cache()
+
+    t0 = time.time()
+    for n, row in enumerate(shard_rows):
+        todo = [s for s in wanted if needs(s, row)]
+        if not todo:
+            continue
+        ref_timing, _ = time_one(reference, row)
+        for s in todo:
+            timing, summary = time_one(s, row)
+            p = record_path(s["name"], row)
+            with open(p) as f:
+                rec = json.load(f)
+            rec["policy_latency_ms_generation_run"] = rec.get("policy_latency_ms")
+            rec["policy_latency_ms"] = timing["denoise_dit_ms"]
+            rec["excluded_context_kv_latency_ms"] = timing["context_kv_dit_ms"]
+            rec["matched_ffff_policy_latency_ms"] = ref_timing["denoise_dit_ms"]
+            rec["latency_source"] = SOURCE
+            # The compute fraction is deterministic; recording it again guards
+            # against a mismatch between the two runs.
+            rec["cache_diagnostics"]["compute_equivalent_forwards_retime"] = summary.get(
+                "compute_equivalent_forwards")
+            write_json(p, rec)
+        done = n + 1
+        rate = (time.time() - t0) / done
+        print(f"[{args.base} retime shard {args.shard}] {done}/{len(shard_rows)} "
+              f"({row['prompt_suite']}/{row['suite_index']:03d})  ffff={ref_timing['denoise_dit_ms']:.0f}ms  "
+              f"{rate:.1f}s/prompt  eta {(len(shard_rows) - done) * rate / 60:.0f} min",
+              flush=True)
+
+    print(f"retime shard {args.shard} done in {(time.time() - t0) / 60:.1f} min", flush=True)
+
+
+if __name__ == "__main__":
+    main()

eval/retime_until_clean.sh

@@ -0,0 +1,96 @@
+#!/usr/bin/env bash
+# Self-healing driver for the paired re-timing on a shared node.
+#
+# Neighbours' jobs both inflate timings and, twice now, took our processes down
+# with them.  So: loop.  Each pass counts the records that still need a fair
+# pairing (never timed, FFFF half contended, or measured/compute ratio outside a
+# sane band), launches one shard per *currently free* GPU among $GPUS, waits, and
+# repeats until nothing is left or MAX_PASSES is hit.  Then aggregate.
+set -u
+C=/local/zoubin/cz/projects/comparison
+PY=/local/zoubin/cz/envs/self_forcing/bin/python
+LOG="$1"
+GPUS=(${GPUS:-4 5 6 7})
+THRESH_MS=${THRESH_MS:-3500}
+RATIO=${RATIO:-0.5,1.1}
+MAX_PASSES=${MAX_PASSES:-12}
+MIN_FREE=${MIN_FREE:-2}     # do not start a pass on fewer free GPUs than this
+WAIT=${WAIT:-120}           # seconds between scans while waiting for GPUs
+NEW_SF=sf_teacache_Fxxx_x3,sf_flowcache_Fxxx_x3,sf_taylorseer_FFxx_x1.3,sf_taylorseer_FFxx_x1.6,sf_taylorseer_FFxx_x2,sf_motioncache_Fxxx_x3
+NEW_CF=cf_teacache_Fxxx_x3,cf_flowcache_Fxxx_x3,cf_taylorseer_FFxx_x1.3,cf_taylorseer_FFxx_x1.6,cf_taylorseer_FFxx_x2,cf_motioncache_Fxxx_x3
+
+export SSL_CERT_FILE=/etc/pki/tls/certs/ca-bundle.crt
+export HF_HOME=/local/zoubin/cz/.cache/huggingface HF_HUB_CACHE=/local/zoubin/cz/.cache/huggingface/hub
+export TRANSFORMERS_CACHE=/local/zoubin/cz/.cache/huggingface/hub TORCH_HOME=/local/zoubin/cz/.cache/torch
+export XDG_CACHE_HOME=/local/zoubin/cz/.cache
+cd "$C"
+
+pending() {  # how many records of $1's strategies still need work
+  $PY - "$1" "$THRESH_MS" "$RATIO" <<'PYEOF'
+import glob, json, sys
+strats, thresh, ratio = sys.argv[1].split(","), float(sys.argv[2]), [float(v) for v in sys.argv[3].split(",")]
+n = 0
+for s in strats:
+    for p in glob.glob(f"eval_out/per_prompt/{s}/*.json"):
+        r = json.load(open(p))
+        if "latency_source" not in r or r["matched_ffff_policy_latency_ms"] > thresh:
+            n += 1; continue
+        d = r["cache_diagnostics"]
+        cx = d["denoise_forwards"] / max(d["compute_equivalent_forwards"], 1e-6)
+        mx = r["matched_ffff_policy_latency_ms"] / r["policy_latency_ms"]
+        if not (ratio[0] <= mx / cx <= ratio[1]):
+            n += 1
+print(n)
+PYEOF
+}
+
+free_gpus() {  # members of GPUS that are actually quiet: little memory AND no compute
+  # Memory alone is not enough -- a VBench scorer holds 2 GB and saturates the SMs,
+  # and that alone inflated a whole pass by 25-80%.  Sample utilisation 3x over 3 s.
+  for g in "${GPUS[@]}"; do
+    used=$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits -i "$g" 2>/dev/null | tr -d ' ')
+    [ -z "$used" ] || [ "$used" -ge 3000 ] && continue
+    peak=0
+    for _ in 1 2 3; do
+      u=$(nvidia-smi --query-gpu=utilization.gpu --format=csv,noheader,nounits -i "$g" 2>/dev/null | tr -d ' ')
+      [ -n "$u" ] && [ "$u" -gt "$peak" ] && peak=$u
+      sleep 1
+    done
+    [ "$peak" -lt 10 ] && echo "$g"
+  done
+}
+
+# Only GPUs nobody else is using are ever touched; waiting never counts as a pass.
+pass=0
+while [ "$pass" -lt "$MAX_PASSES" ]; do
+  n_sf=$(pending "$NEW_SF"); n_cf=$(pending "$NEW_CF")
+  echo "=== scan ($(date +%H:%M:%S)): pending sf=$n_sf cf=$n_cf, passes so far $pass ==="
+  [ "$n_sf" -eq 0 ] && [ "$n_cf" -eq 0 ] && break
+  mapfile -t FREE < <(free_gpus)
+  if [ "${#FREE[@]}" -lt "$MIN_FREE" ]; then
+    echo "only ${#FREE[@]} free GPU(s) among ${GPUS[*]} (need $MIN_FREE); waiting ${WAIT}s"
+    sleep "$WAIT"; continue
+  fi
+  pass=$((pass+1))
+  echo "using GPUs: ${FREE[*]}"
+  for BASE in self_forcing causal_forcing; do
+    if [ "$BASE" = self_forcing ]; then S=$NEW_SF; n=$n_sf; else S=$NEW_CF; n=$n_cf; fi
+    [ "$n" -eq 0 ] && continue
+    pids=()
+    for i in "${!FREE[@]}"; do
+      CUDA_VISIBLE_DEVICES=${FREE[$i]} $PY eval/retime_eval.py --base "$BASE" --strategies "$S" \
+        --shard "$i" --num-shards "${#FREE[@]}" --out-root eval_out \
+        --redo-above-ms "$THRESH_MS" --redo-ratio-bounds "$RATIO" \
+        > "$LOG/retime_${BASE}_pass${pass}_$i.log" 2>&1 &
+      pids+=($!)
+    done
+    for p in "${pids[@]}"; do wait "$p"; done
+    echo "  $BASE pass $pass finished ($(date +%H:%M:%S))"
+  done
+done
+echo "=== final pending: sf=$(pending "$NEW_SF") cf=$(pending "$NEW_CF") ==="
+echo "=== aggregating ($(date +%H:%M:%S)) ==="
+# --only: the predictor evaluation shares eval_out/ and may still be incomplete.
+$PY eval/aggregate.py --out-root "$C/eval_out" --only '^(sf|cf)_' 2>&1 | tee "$LOG/aggregate.log"
+python3 experiment_md.py 2>&1 | tee "$LOG/experiment_md.log"
+echo "ALL RETIME DONE $(date +%H:%M:%S)"

eval/run_all_phases.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+# Waits for the running generation phase, then scores and aggregates.
+set -u
+C=/local/zoubin/cz/projects/comparison
+LOG="$1"
+
+echo "waiting for generation to finish ..."
+while ! grep -q "ALL GENERATION DONE" "$LOG/generation_driver.log" 2>/dev/null; do
+  if ! pgrep -f "[g]enerate_eval.py" > /dev/null 2>&1 \
+     && ! pgrep -f "[r]un_generation.sh" > /dev/null 2>&1; then
+    echo "generation processes gone without the done marker; stopping."
+    exit 1
+  fi
+  sleep 60
+done
+echo "generation complete $(date +%H:%M:%S)"
+
+"$C/eval/run_vbench_all.sh" "$LOG" 2>&1 | tee "$LOG/vbench_driver.log"
+
+echo "=== aggregating $(date +%H:%M:%S) ==="
+/local/zoubin/cz/envs/self_forcing/bin/python "$C/eval/aggregate.py" \
+  --out-root "$C/eval_out" 2>&1 | tee "$LOG/aggregate.log"
+echo "ALL PHASES DONE $(date +%H:%M:%S)"

eval/run_generation.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+# Phase 1: generate the Extended-251 videos for every strategy, one base at a time,
+# sharded by prompt across the GPUs in $GPUS (default 0-3).  Generation runs in the base self_forcing env --
+# the same interpreter and package set the speed study used -- so the eval videos
+# are produced by exactly the model stack that was benchmarked.
+set -u
+C=/local/zoubin/cz/projects/comparison
+PY=/local/zoubin/cz/envs/self_forcing/bin/python
+LOG="$1"; shift
+NSHARDS=${NSHARDS:-4}
+GPUS=(${GPUS:-0 1 2 3})
+
+export SSL_CERT_FILE=/etc/pki/tls/certs/ca-bundle.crt
+export TORCH_HOME=/local/zoubin/cz/.cache/torch
+export HF_HOME=/local/zoubin/cz/.cache/huggingface
+export HF_HUB_CACHE=/local/zoubin/cz/.cache/huggingface/hub
+export TRANSFORMERS_CACHE=/local/zoubin/cz/.cache/huggingface/hub
+export XDG_CACHE_HOME=/local/zoubin/cz/.cache
+
+cd "$C"
+for BASE in ${BASES:-self_forcing causal_forcing}; do
+  echo "=== generation: $BASE ($(date +%H:%M:%S)) ==="
+  pids=()
+  for i in $(seq 0 $((NSHARDS-1))); do
+    CUDA_VISIBLE_DEVICES=${GPUS[$i]} nohup $PY eval/generate_eval.py \
+      --base "$BASE" --shard $i --num-shards $NSHARDS --out-root eval_out \
+      > "$LOG/gen_${BASE}_$i.log" 2>&1 &
+    pids+=($!)
+  done
+  for p in "${pids[@]}"; do wait "$p"; done
+  echo "=== generation done: $BASE ($(date +%H:%M:%S)) ==="
+done
+echo "ALL GENERATION DONE"

eval/run_predictor_eval.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+# Extended-251 evaluation of the Causal-Forcing-a Predictors (cfa_* strategies):
+# phase 1 generation sharded across $GPUS, phase 2 VBench on the not-yet-scored
+# strategies (run_vbench_all.sh skips scored ones), phase 3 aggregate.
+#   GPUS="0 1 2 3" eval/run_predictor_eval.sh logs_cfa
+set -u
+C=/local/zoubin/cz/projects/comparison
+PY=/local/zoubin/cz/envs/self_forcing/bin/python
+LOG="$(cd "$(dirname "$1")" && pwd)/$(basename "$1")"; shift
+GPUS=(${GPUS:-0 1 2 3})
+NSHARDS=${#GPUS[@]}
+mkdir -p "$LOG"
+
+export SSL_CERT_FILE=/etc/pki/tls/certs/ca-bundle.crt
+export TORCH_HOME=/local/zoubin/cz/.cache/torch
+export HF_HOME=/local/zoubin/cz/.cache/huggingface
+export HF_HUB_CACHE=/local/zoubin/cz/.cache/huggingface/hub
+export TRANSFORMERS_CACHE=/local/zoubin/cz/.cache/huggingface/hub
+export XDG_CACHE_HOME=/local/zoubin/cz/.cache
+
+cd "$C"
+echo "=== cfa generation ($(date +%H:%M:%S)) gpus=${GPUS[*]} ==="
+pids=()
+for i in $(seq 0 $((NSHARDS-1))); do
+  CUDA_VISIBLE_DEVICES=${GPUS[$i]} $PY eval/generate_predictor_eval.py \
+    --shard $i --num-shards $NSHARDS --out-root eval_out "$@" \
+    > "$LOG/gen_cfa_$i.log" 2>&1 &
+  pids+=($!)
+done
+status=0
+for p in "${pids[@]}"; do wait "$p" || status=1; done
+echo "=== cfa generation done ($(date +%H:%M:%S)) status=$status ==="
+[[ $status == 0 ]] || { echo "generation failed; not scoring"; exit 1; }
+
+GPUS="${GPUS[*]}" NGPU=$NSHARDS bash "$C/eval/run_vbench_all.sh" "$LOG" 2>&1 | tee "$LOG/vbench_driver.log"
+
+echo "=== aggregating ($(date +%H:%M:%S)) ==="
+$PY "$C/eval/aggregate.py" --out-root "$C/eval_out" 2>&1 | tee "$LOG/aggregate.log"
+echo "ALL CFA PHASES DONE $(date +%H:%M:%S)"

eval/run_predictor_eval_s2.sh

@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+# Extended-251 evaluation of the Stage-2 EMA Predictor: generation sharded over
+# $GPUS, VBench only for the strategies generated here, then aggregate.
+#   GPUS="0 1 2" bash eval/run_predictor_eval_s2.sh logs_cfa_s2
+set -u
+C=/local/zoubin/cz/projects/comparison
+V=/local/zoubin/cz/projects/VBench
+PY=/local/zoubin/cz/envs/self_forcing/bin/python
+VPY=/local/zoubin/cz/envs/vbench_eval/bin/python
+LOG="$(mkdir -p "$1" && cd "$1" && pwd)"
+MODELS="${MODELS:-atc_chunk_s2}"
+PATTERNS="${PATTERNS:-FPFF,FPPF,FPPP}"
+GPUS=(${GPUS:-0 1 2})
+NSHARDS=${#GPUS[@]}
+
+export SSL_CERT_FILE=/etc/pki/tls/certs/ca-bundle.crt
+export TORCH_HOME=/local/zoubin/cz/.cache/torch
+export HF_HOME=/local/zoubin/cz/.cache/huggingface
+export HF_HUB_CACHE=/local/zoubin/cz/.cache/huggingface/hub
+export TRANSFORMERS_CACHE=/local/zoubin/cz/.cache/huggingface/hub
+export XDG_CACHE_HOME=/local/zoubin/cz/.cache
+export VBENCH_CACHE_DIR=/local/zoubin/cz/.cache/vbench
+
+cd "$C"
+echo "=== generation ($(date +%H:%M:%S)) gpus=${GPUS[*]} models=$MODELS patterns=$PATTERNS ==="
+pids=()
+for i in $(seq 0 $((NSHARDS-1))); do
+  CUDA_VISIBLE_DEVICES=${GPUS[$i]} $PY eval/generate_predictor_eval.py \
+    --shard $i --num-shards $NSHARDS --out-root eval_out --models "$MODELS" --patterns "$PATTERNS" \
+    > "$LOG/gen_$i.log" 2>&1 &
+  pids+=($!)
+done
+status=0; for p in "${pids[@]}"; do wait "$p" || status=1; done
+echo "=== generation done ($(date +%H:%M:%S)) status=$status ==="
+[[ $status == 0 ]] || { echo "generation failed; not scoring"; exit 1; }
+
+strats=()
+for m in ${MODELS//,/ }; do for pat in ${PATTERNS//,/ }; do strats+=("cfa_${m}_$(echo "$pat" | tr 'A-Z' 'a-z')"); done; done
+echo "=== vbench ($(date +%H:%M:%S)) strategies=${strats[*]} ==="
+export PYTHONPATH="$V:$C"
+cd "$V"
+pids=()
+for g in $(seq 0 $((NSHARDS-1))); do
+  (
+    for i in $(seq $g $NSHARDS $(( ${#strats[@]} - 1 ))); do
+      s="${strats[$i]}"
+      [[ -f "$C/eval_out/vbench/scores/$s.json" ]] && { echo "skip $s"; continue; }
+      echo "[gpu ${GPUS[$g]}] $s $(date +%H:%M:%S)"
+      CUDA_VISIBLE_DEVICES=${GPUS[$g]} $VPY "$C/eval/run_vbench.py" --strategy "$s" --out-root "$C/eval_out" \
+        > "$LOG/vbench_$s.log" 2>&1 || echo "[gpu ${GPUS[$g]}] FAILED $s"
+    done
+  ) &
+  pids+=($!)
+done
+for p in "${pids[@]}"; do wait "$p"; done
+echo "=== aggregating ($(date +%H:%M:%S)) ==="
+cd "$C"
+$PY eval/aggregate.py --out-root "$C/eval_out" --allow-incomplete > "$LOG/aggregate.log" 2>&1
+echo "ALL S2 PHASES DONE $(date +%H:%M:%S)"

eval/run_predictor_retime_s2.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+# Paired re-timing of the Stage-2 predictor strategies against FFFF (same process).
+set -u
+C=/local/zoubin/cz/projects/comparison
+PY=/local/zoubin/cz/envs/self_forcing/bin/python
+LOG="$(mkdir -p "$1" && cd "$1" && pwd)"
+GPUS=(${GPUS:-0 1 2}); NSHARDS=${#GPUS[@]}
+export SSL_CERT_FILE=/etc/pki/tls/certs/ca-bundle.crt TORCH_HOME=/local/zoubin/cz/.cache/torch HF_HOME=/local/zoubin/cz/.cache/huggingface HF_HUB_CACHE=/local/zoubin/cz/.cache/huggingface/hub XDG_CACHE_HOME=/local/zoubin/cz/.cache
+cd "$C"
+echo "=== retime ($(date +%H:%M:%S)) gpus=${GPUS[*]} ==="
+pids=()
+for i in $(seq 0 $((NSHARDS-1))); do
+  CUDA_VISIBLE_DEVICES=${GPUS[$i]} $PY eval/generate_predictor_eval.py --shard $i --num-shards $NSHARDS --out-root eval_out \
+    --models "${MODELS:-atc_chunk_s2}" --patterns "${PATTERNS:-FPFF,FPPF,FPPP}" --retime-only > "$LOG/retime_$i.log" 2>&1 &
+  pids+=($!)
+done
+status=0; for p in "${pids[@]}"; do wait "$p" || status=1; done
+echo "=== retime done ($(date +%H:%M:%S)) status=$status ==="
+$PY eval/aggregate.py --out-root "$C/eval_out" --allow-incomplete > "$LOG/aggregate_retime.log" 2>&1
+$PY experiment_md.py > "$LOG/experiment_md.log" 2>&1
+echo "RETIME DONE $(date +%H:%M:%S)"

eval/run_predictor_vbench_phase.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+# VBench + aggregate only (generation already complete).
+set -u
+C=/local/zoubin/cz/projects/comparison
+PY=/local/zoubin/cz/envs/self_forcing/bin/python
+LOG="$(cd "$(dirname "$1")" && pwd)/$(basename "$1")"
+GPUS="${GPUS:-0 1 2 3}"
+read -r -a gpu_list <<< "$GPUS"
+cd "$C"
+GPUS="$GPUS" NGPU=${#gpu_list[@]} bash "$C/eval/run_vbench_all.sh" "$LOG" 2>&1 | tee "$LOG/vbench_driver.log"
+echo "=== aggregating ($(date +%H:%M:%S)) ==="
+$PY "$C/eval/aggregate.py" --out-root "$C/eval_out" 2>&1 | tee "$LOG/aggregate.log"
+echo "ALL CFA PHASES DONE $(date +%H:%M:%S)"

eval/run_retime.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+# Phase 3: paired re-timing of strategies added after the first generation run
+# (their FFFF reference was regenerated for the pixel metrics but not timed).
+# Runs after VBench so the GPUs carry nothing else.  Usage:
+#   eval/run_retime.sh <logdir> <comma-separated strategy names, per base via
+#                       STRATS_SF / STRATS_CF env, or "all">
+set -u
+# DISABLED (Sep 6): the user asked for no more FFFF re-timing passes.  Speedups
+# are now the ratio of mean latencies against the FFFF strategy's own records;
+# where those come from a different run the absolute ms are not comparable and
+# the table says so.  Kept for reference only.
+echo "eval/run_retime.sh is disabled; re-timing is no longer part of the pipeline." >&2
+exit 1
+C=/local/zoubin/cz/projects/comparison
+PY=/local/zoubin/cz/envs/self_forcing/bin/python
+LOG="$1"; shift
+NSHARDS=${NSHARDS:-4}
+GPUS=(${GPUS:-0 1 2 3})
+EXTRA=${EXTRA:-}
+
+export SSL_CERT_FILE=/etc/pki/tls/certs/ca-bundle.crt
+export HF_HOME=/local/zoubin/cz/.cache/huggingface
+export HF_HUB_CACHE=/local/zoubin/cz/.cache/huggingface/hub
+export TRANSFORMERS_CACHE=/local/zoubin/cz/.cache/huggingface/hub
+export TORCH_HOME=/local/zoubin/cz/.cache/torch
+export XDG_CACHE_HOME=/local/zoubin/cz/.cache
+
+cd "$C"
+for BASE in self_forcing causal_forcing; do
+  if [ "$BASE" = self_forcing ]; then S="${STRATS_SF:-all}"; else S="${STRATS_CF:-all}"; fi
+  echo "=== retime: $BASE [$S] ($(date +%H:%M:%S)) ==="
+  pids=()
+  for i in $(seq 0 $((NSHARDS-1))); do
+    CUDA_VISIBLE_DEVICES=${GPUS[$i]} nohup $PY eval/retime_eval.py \
+      --base "$BASE" --strategies "$S" --shard $i --num-shards $NSHARDS \
+      --out-root eval_out $EXTRA > "$LOG/retime_${BASE}_$i.log" 2>&1 &
+    pids+=($!)
+  done
+  for p in "${pids[@]}"; do wait "$p"; done
+  echo "=== retime done: $BASE ($(date +%H:%M:%S)) ==="
+done
+echo "ALL RETIME DONE"

eval/run_vbench.py

@@ -0,0 +1,163 @@
+#!/usr/bin/env python
+"""Score one strategy on the eight protocol VBench dimensions.
+
+Standard VBench 0.1.5 metric code is used unmodified. What is built here is only
+the evaluation metadata file, because the protocol needs two things the stock
+`vbench_standard` mode cannot express:
+
+  * `prompt_en` must be the *extended* prompt actually fed to the generator
+    (section 6.1), not the short canonical one;
+  * `scene` must keep the canonical `auxiliary_info` keywords.
+
+The file has exactly the shape `build_full_info_json` produces, and each
+`compute_<dimension>` selects the rows that list it, so per-suite scoping
+(72 / 93 / 86 videos) falls out of the mapping rather than being hard-coded.
+
+    python eval/run_vbench.py --strategy sf_ffff --out-root eval_out
+"""
+
+import argparse
+import json
+import os
+import sys
+import time
+
+ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.insert(0, ROOT)
+
+MAPPING = os.path.join(ROOT, "assets/vbench8_extended_subset_mapping.json")
+
+SUITE_DIMENSIONS = {
+    "subject_consistency": ["subject_consistency", "motion_smoothness", "dynamic_degree"],
+    "overall_consistency": ["overall_consistency", "aesthetic_quality", "imaging_quality"],
+    "scene": ["scene", "background_consistency"],
+}
+DIMENSIONS = [d for dims in SUITE_DIMENSIONS.values() for d in dims]
+
+
+def parse_args():
+    p = argparse.ArgumentParser()
+    p.add_argument("--strategy", required=True)
+    p.add_argument("--out-root", default="eval_out")
+    p.add_argument("--dimensions", default=None,
+                   help="Comma-separated subset (default: all eight)")
+    p.add_argument("--allow-partial", action="store_true",
+                   help="Score even if fewer than 251 videos exist (smoke tests only)")
+    p.add_argument("--overwrite", action="store_true",
+                   help="Recompute dimensions whose raw result file already exists")
+    p.add_argument("--threads", type=int, default=None,
+                   help="torch/OpenMP CPU threads for this process (scoring is CPU-bound; "
+                        "several scorers on one node oversubscribe the cores)")
+    return p.parse_args()
+
+
+def main():
+    args = parse_args()
+    out_root = (args.out_root if os.path.isabs(args.out_root)
+                else os.path.join(ROOT, args.out_root))
+    dims = args.dimensions.split(",") if args.dimensions else list(DIMENSIONS)
+
+    with open(MAPPING) as f:
+        mapping = json.load(f)
+
+    video_root = os.path.join(out_root, "generated_videos", args.strategy)
+    rows, missing = [], []
+    for r in mapping["rows"]:
+        vp = os.path.join(video_root, r["prompt_suite"], f"{r['suite_index']:03d}.mp4")
+        if not os.path.exists(vp):
+            missing.append(vp)
+            continue
+        entry = {
+            # The extended prompt is what the video was generated from, so it is
+            # what overall_consistency and scene must be scored against.
+            "prompt_en": r["extended_prompt"],
+            "dimension": [d for d in SUITE_DIMENSIONS[r["prompt_suite"]]],
+            "video_list": [vp],
+            "global_index": r["global_index"],
+            "prompt_suite": r["prompt_suite"],
+            "suite_index": r["suite_index"],
+        }
+        if "auxiliary_info" in r:
+            entry["auxiliary_info"] = r["auxiliary_info"]
+        rows.append(entry)
+
+    if missing and not args.allow_partial:
+        print(f"ERROR: {len(missing)} of {len(mapping['rows'])} videos missing for "
+              f"{args.strategy}; refusing to score an incomplete strategy.")
+        for m in missing[:5]:
+            print("   ", m)
+        return 1
+    print(f"{args.strategy}: {len(rows)} videos, dimensions={dims}", flush=True)
+
+    scores_dir = os.path.join(out_root, "vbench", "raw_results", args.strategy)
+    os.makedirs(scores_dir, exist_ok=True)
+    info_path = os.path.join(scores_dir, "full_info.json")
+    # Atomic: several per-dimension processes may start on the same strategy.
+    with open(info_path + f".{os.getpid()}", "w") as f:
+        json.dump(rows, f, indent=2)
+    os.replace(info_path + f".{os.getpid()}", info_path)
+
+    import importlib
+
+    import torch
+    if args.threads:
+        torch.set_num_threads(args.threads)
+    from vbench.utils import init_submodules
+
+    device = torch.device("cuda")
+
+    def raw_path(d):
+        return os.path.join(scores_dir, f"{d}.json")
+
+    # A dimension already scored (its raw file exists) is reused, so dimensions can
+    # be spread over several processes and a killed run resumes where it stopped.
+    todo = [d for d in dims if args.overwrite or not os.path.exists(raw_path(d))]
+    submodules = init_submodules(todo, local=True, read_frame=False) if todo else {}
+
+    for d in todo:
+        t0 = time.time()
+        module = importlib.import_module(f"vbench.{d}")
+        fn = getattr(module, f"compute_{d}")
+        score, per_video = fn(info_path, device, submodules[d])
+        tmp = raw_path(d) + ".tmp"
+        with open(tmp, "w") as f:
+            json.dump({"dimension": d, "score": score, "video_results": per_video,
+                       "seconds": time.time() - t0}, f, indent=2)
+        os.replace(tmp, raw_path(d))
+        print(f"  {d:24s} {score:.6f}   ({len(per_video)} videos, "
+              f"{time.time() - t0:.0f}s)", flush=True)
+        torch.cuda.empty_cache()
+
+    # The strategy's score file is written only once all eight dimensions exist.
+    missing_dims = [d for d in DIMENSIONS if not os.path.exists(raw_path(d))]
+    if missing_dims:
+        print(f"{args.strategy}: {len(DIMENSIONS) - len(missing_dims)}/{len(DIMENSIONS)} "
+              f"dimensions scored; missing {missing_dims}", flush=True)
+        return 0
+    results = {}
+    for d in DIMENSIONS:
+        with open(raw_path(d)) as f:
+            raw = json.load(f)
+        results[d] = {"score": raw["score"], "num_videos": len(raw["video_results"]),
+                      "seconds": raw.get("seconds")}
+
+    out_path = os.path.join(out_root, "vbench", "scores", f"{args.strategy}.json")
+    os.makedirs(os.path.dirname(out_path), exist_ok=True)
+    payload = {
+        "strategy": args.strategy,
+        "vbench_long": False,
+        "num_videos": len(rows),
+        "raw": {d: results[d]["score"] for d in DIMENSIONS},
+        "detail": results,
+        "full_info": info_path,
+    }
+    tmp = out_path + ".tmp"
+    with open(tmp, "w") as f:
+        json.dump(payload, f, indent=2)
+    os.replace(tmp, out_path)
+    print(f"wrote {out_path}", flush=True)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

eval/run_vbench_all.sh

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+# Phase 2: score every strategy on the eight protocol dimensions.
+# VBench runs in the vbench_eval venv (torchao 0.7.0 / transformers 4.46.3 /
+# fairscale), because the base env's torchao 0.17.0 is unimportable under torch
+# 2.5.1 and takes transformers' BertModel -- and therefore Tag2Text/scene -- with
+# it.  Generation deliberately stays in the base env.
+set -u
+C=/local/zoubin/cz/projects/comparison
+V=/local/zoubin/cz/projects/VBench
+PY=/local/zoubin/cz/envs/vbench_eval/bin/python
+LOG="$1"; shift
+NGPU=${NGPU:-4}
+GPUS=(${GPUS:-0 1 2 3})
+
+export PYTHONPATH="$V:$C"
+export VBENCH_CACHE_DIR=/local/zoubin/cz/.cache/vbench
+export HF_HOME=/local/zoubin/cz/.cache/huggingface
+export HF_HUB_CACHE=/local/zoubin/cz/.cache/huggingface/hub
+export TRANSFORMERS_CACHE=/local/zoubin/cz/.cache/huggingface/hub
+export XDG_CACHE_HOME=/local/zoubin/cz/.cache
+export SSL_CERT_FILE=/etc/pki/tls/certs/ca-bundle.crt
+export TORCH_HOME=/local/zoubin/cz/.cache/torch
+
+mapfile -t STRATS < <(ls "$C/eval_out/per_prompt")
+echo "scoring ${#STRATS[@]} strategies across $NGPU GPUs"
+
+cd "$V"
+pids=()
+for g in $(seq 0 $((NGPU-1))); do
+  (
+    for i in $(seq $g $NGPU $(( ${#STRATS[@]} - 1 ))); do
+      s="${STRATS[$i]}"
+      if [ -f "$C/eval_out/vbench/scores/$s.json" ]; then
+        echo "skip $s (already scored)"; continue
+      fi
+      echo "[gpu $g] $s  $(date +%H:%M:%S)"
+      CUDA_VISIBLE_DEVICES=${GPUS[$g]} $PY "$C/eval/run_vbench.py" \
+        --strategy "$s" --out-root "$C/eval_out" > "$LOG/vbench_$s.log" 2>&1 \
+        || echo "[gpu $g] FAILED $s"
+    done
+  ) &
+  pids+=($!)
+done
+for p in "${pids[@]}"; do wait "$p"; done
+echo "ALL VBENCH DONE"

eval/setup_vbench_models.py

@@ -0,0 +1,29 @@
+#!/usr/bin/env python
+"""Download the VBench weights the eight protocol dimensions need.
+
+Everything lands in $VBENCH_CACHE_DIR so the run does not depend on whatever a
+user's ~/.cache happens to hold.  `local=True` is used wherever VBench supports
+it, so CLIP and DINO come from the cache directory too rather than from the clip
+package's own download path.
+"""
+import os
+import sys
+
+DIMS = ["subject_consistency", "background_consistency", "motion_smoothness",
+        "dynamic_degree", "aesthetic_quality", "imaging_quality", "scene",
+        "overall_consistency"]
+
+def main():
+    from vbench.utils import init_submodules, CACHE_DIR
+    print("CACHE_DIR =", CACHE_DIR, flush=True)
+    for d in DIMS:
+        print(f"--- {d} ---", flush=True)
+        try:
+            got = init_submodules([d], local=True, read_frame=False)
+            print(f"    ok: {got.get(d)}", flush=True)
+        except Exception as e:
+            print(f"    FAILED: {type(e).__name__}: {e}", flush=True)
+    print("done", flush=True)
+
+if __name__ == "__main__":
+    sys.exit(main())

eval/strategies.py

@@ -0,0 +1,82 @@
+"""The strategy list for the Extended-251 evaluation.
+
+FFFF (every denoising step full) is the reference for each base model; the cache
+strategies are the operating points settled by finalize.py / retime.py, read back
+from results/ so the evaluation cannot silently drift from the speed study.
+"""
+
+import glob
+import json
+import os
+
+ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+METHOD_ORDER = ["teacache", "flowcache", "taylorseer", "motioncache"]
+# FlowCache is frozen: finished operating points stay in the tables, no new ones
+# are added.  ``final_*_v2.json`` files (the measured-speedup targeted points)
+# therefore never contain it.
+DROPPED_FOR_NEW = {"flowcache"}
+BASES = ["self_forcing", "causal_forcing"]
+SHORT = {"self_forcing": "sf", "causal_forcing": "cf"}
+
+
+def load_strategies(base):
+    """[{name, base, method, param, value, target, coefficients, indicator}, ...]
+
+    The first entry is always FFFF.
+    """
+    coeff_path = os.path.join(ROOT, f"results/coeff_{base}.json")
+    coefficients = None
+    if os.path.exists(coeff_path):
+        with open(coeff_path) as f:
+            coefficients = json.load(f)["coefficients"]
+
+    out = [{
+        "name": f"{SHORT[base]}_ffff",
+        "base": base, "method": "none", "param": None, "value": None,
+        "target": 1.0, "coefficients": None, "indicator": "modulated_input",
+        "schedule": "FxxF", "is_reference": True, "sampler": "renoise",
+    }]
+
+    rows = []
+    for p in sorted(glob.glob(os.path.join(ROOT, f"results/final_{base}*.json"))):
+        with open(p) as f:
+            rows += json.load(f)["rows"]
+    rows.sort(key=lambda r: (METHOD_ORDER.index(r["method"])
+                             if r["method"] in METHOD_ORDER else 9,
+                             r.get("schedule", "FxxF") != "FxxF",
+                             r["target_speedup"]))
+    for r in rows:
+        # The original FxxF study keeps its unsuffixed names; other schedules
+        # carry the schedule so they never collide with those records.
+        sched = r.get("schedule", "FxxF")
+        tag = "" if sched.upper() == "FXXF" else f"_{sched}"
+        out.append({
+            "name": r.get("name") or f"{SHORT[base]}_{r['method']}{tag}_x{r['target_speedup']:g}",
+            "schedule": sched,
+            "num_inference_steps": r.get("num_inference_steps"),
+            "first_chunk_schedule": r.get("first_chunk_schedule"),
+            "sampler": r.get("sampler", "renoise"),
+            "base": base,
+            "method": r["method"],
+            "param": r["param"],
+            "value": r["value"],
+            "target": r["target_speedup"],
+            "coefficients": coefficients,
+            "indicator": "modulated_input",
+            "is_reference": False,
+            "compute_equivalent_forwards": r.get("compute_equivalent_forwards"),
+        })
+    return out
+
+
+def all_strategies():
+    return {b: load_strategies(b) for b in BASES}
+
+
+if __name__ == "__main__":
+    for b, s in all_strategies().items():
+        print(f"{b}: {len(s)} strategies")
+        for x in s:
+            v = "-" if x["value"] is None else f"{x['param']}={x['value']:.6g}"
+            print(f"   {x['name']:28s} {v}")

eval/test_protocol.py

@@ -0,0 +1,172 @@
+#!/usr/bin/env python
+"""Unit tests for the evaluation protocol's section 12 checklist.
+
+    python eval/test_protocol.py
+"""
+
+import json
+import os
+import sys
+import tempfile
+
+ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.insert(0, ROOT)
+
+from eval import aggregate as agg  # noqa: E402
+
+MAPPING = os.path.join(ROOT, "assets/vbench8_extended_subset_mapping.json")
+FULL_INFO = "/local/zoubin/cz/projects/VBench/vbench/VBench_full_info.json"
+PROMPT_DIR = "/local/zoubin/cz/projects/Self-Forcing/prompts/vbench"
+
+FAILURES = []
+
+
+def check(name, cond, detail=""):
+    print(f"  {'PASS' if cond else 'FAIL'}  {name}" + (f"   {detail}" if detail else ""))
+    if not cond:
+        FAILURES.append(name)
+
+
+def test_mapping():
+    print("946 -> 251 mapping")
+    with open(MAPPING) as f:
+        m = json.load(f)
+    rows = m["rows"]
+    check("mapping has 251 rows", len(rows) == 251, f"got {len(rows)}")
+    counts = {s: sum(1 for r in rows if r["prompt_suite"] == s)
+              for s in ("subject_consistency", "overall_consistency", "scene")}
+    check("suite counts 72/93/86",
+          counts == {"subject_consistency": 72, "overall_consistency": 93, "scene": 86},
+          str(counts))
+    check("global_index unique",
+          len({r["global_index"] for r in rows}) == len(rows))
+    for s in counts:
+        idx = [r["suite_index"] for r in rows if r["prompt_suite"] == s]
+        check(f"{s} suite_index contiguous", idx == list(range(len(idx))))
+    check("every scene row keeps auxiliary_info",
+          all("auxiliary_info" in r for r in rows if r["prompt_suite"] == "scene"))
+    check("extended prompts non-empty",
+          all(r["extended_prompt"].strip() for r in rows))
+
+
+def test_canonical_order():
+    print("canonical prompt order is enforced")
+    with open(FULL_INFO) as f:
+        info = json.load(f)
+    with open(os.path.join(PROMPT_DIR, "all_dimension.txt"), encoding="utf-8") as f:
+        short = [l.rstrip("\n") for l in f]
+    check("946 short prompts", len(short) == 946, f"got {len(short)}")
+    check("short prompt order == VBench_full_info",
+          all(a["prompt_en"].strip() == b.strip() for a, b in zip(info, short)))
+
+    # A mismatched prompt file must be refused, not silently fuzzy-matched.
+    with open(MAPPING) as f:
+        rows = json.load(f)["rows"]
+    with open(os.path.join(PROMPT_DIR, "all_dimension_extended.txt"), encoding="utf-8") as f:
+        ext = [l.rstrip("\n") for l in f]
+    check("mapping extended prompt matches its global_index",
+          all(r["extended_prompt"] == ext[r["global_index"]] for r in rows))
+    check("mapping original prompt matches its global_index",
+          all(r["original_prompt"] == short[r["global_index"]] for r in rows))
+
+
+def test_normalize_and_aggregate():
+    print("normalize + Quality/Semantic/Selected aggregation")
+    lo_raw = {d: agg.NORMALIZE_RANGE[d][0] for d in agg.DIMENSIONS}
+    hi_raw = {d: agg.NORMALIZE_RANGE[d][1] for d in agg.DIMENSIONS}
+    check("normalize maps min -> 0",
+          all(abs(v) < 1e-12 for v in agg.normalize(lo_raw).values()))
+    check("normalize maps max -> 1",
+          all(abs(v - 1.0) < 1e-12 for v in agg.normalize(hi_raw).values()))
+
+    n = agg.normalize(hi_raw)
+    q, s = agg.quality_score(n), agg.semantic_score(n)
+    check("quality of all-ones == 1", abs(q - 1.0) < 1e-12, f"got {q}")
+    check("semantic of all-ones == 1", abs(s - 1.0) < 1e-12, f"got {s}")
+    check("selected of all-ones == 1", abs(agg.selected_score(q, s) - 1.0) < 1e-12)
+
+    # dynamic_degree carries weight 0.5, everything else in Quality carries 1.0.
+    n0 = {d: 0.0 for d in agg.DIMENSIONS}
+    n_dd = dict(n0, dynamic_degree=1.0)
+    check("dynamic_degree weight is 0.5/5.5",
+          abs(agg.quality_score(n_dd) - 0.5 / 5.5) < 1e-12)
+    n_sc = dict(n0, subject_consistency=1.0)
+    check("other quality dims weight 1/5.5",
+          abs(agg.quality_score(n_sc) - 1.0 / 5.5) < 1e-12)
+    check("selected weights Quality:Semantic 4:1",
+          abs(agg.selected_score(1.0, 0.0) - 0.8) < 1e-12
+          and abs(agg.selected_score(0.0, 1.0) - 0.2) < 1e-12)
+    check("selected is not the plain mean of 8 normalized dims",
+          abs(agg.selected_score(agg.quality_score(n_dd), agg.semantic_score(n_dd))
+              - sum(n_dd.values()) / 8.0) > 1e-6)
+
+
+def _rec(strategy, idx, lat, ctx, mse, frames=81):
+    return {
+        "status": "complete", "strategy": strategy, "base_model": "self_forcing",
+        "method": "none" if strategy.endswith("ffff") else "teacache",
+        "target_speedup": 1.0, "global_index": idx, "prompt_suite": "scene",
+        "suite_index": idx, "policy_latency_ms": lat,
+        "excluded_context_kv_latency_ms": ctx,
+        "pixel_metrics_vs_ffff": {"mean_mse": mse, "psnr": 0.0, "ssim": 1.0,
+                                  "lpips": 0.0, "num_frames": frames},
+        "cache_diagnostics": {"compute_equivalent_forwards": 28.0},
+    }
+
+
+def test_latency_and_completeness():
+    print("latency accounting and the completeness gate")
+    import math
+
+    # ratio of means, not the mean of per-prompt ratios
+    ffff = [_rec("sf_ffff", i, lat, 800.0, 1e-12) for i, lat in enumerate([100.0, 300.0])]
+    cand = [_rec("sf_x", i, lat, 800.0, 1e-4) for i, lat in enumerate([50.0, 250.0])]
+    ratio_of_means = 100.0 * (1 - (sum(r["policy_latency_ms"] for r in cand) / 2)
+                              / (sum(r["policy_latency_ms"] for r in ffff) / 2))
+    mean_of_ratios = 100.0 * (1 - ((50.0 / 100.0) + (250.0 / 300.0)) / 2)
+    check("speedup uses ratio of means", abs(ratio_of_means - 25.0) < 1e-9,
+          f"{ratio_of_means:.3f}%")
+    check("ratio of means differs from mean of ratios",
+          abs(ratio_of_means - mean_of_ratios) > 1.0)
+
+    check("context/KV DiT is a separate field, never inside policy latency",
+          all("excluded_context_kv_latency_ms" in r
+              and r["policy_latency_ms"] != r["policy_latency_ms"]
+              + r["excluded_context_kv_latency_ms"] for r in cand))
+
+    mse = sum(r["pixel_metrics_vs_ffff"]["mean_mse"] for r in cand) / len(cand)
+    check("PSNR aggregates MSE first, then converts once",
+          abs(-10.0 * math.log10(max(mse, 1e-12)) - 40.0) < 1e-9)
+    check("pixel metrics span all 81 frames",
+          {r["pixel_metrics_vs_ffff"]["num_frames"] for r in cand} == {81})
+
+    with tempfile.TemporaryDirectory() as td:
+        for name, recs in (("sf_ffff", ffff), ("sf_x", cand)):
+            d = os.path.join(td, "per_prompt", name)
+            os.makedirs(d)
+            for r in recs:
+                with open(os.path.join(d, f"scene_{r['suite_index']:03d}.json"), "w") as f:
+                    json.dump(r, f)
+        argv = sys.argv
+        sys.argv = ["aggregate", "--out-root", td, "--expect", "251"]
+        try:
+            rc = agg.main()
+        finally:
+            sys.argv = argv
+        check("aggregation refuses a strategy that is not 251", rc == 1, f"rc={rc}")
+
+
+def main():
+    for fn in (test_mapping, test_canonical_order, test_normalize_and_aggregate,
+               test_latency_and_completeness):
+        fn()
+        print()
+    if FAILURES:
+        print(f"{len(FAILURES)} FAILED: {FAILURES}")
+        return 1
+    print("all protocol unit tests passed")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

eval_out/generated_videos/cf_ffff/overall_consistency/000.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d3b9313a8d35ed942c8056953a8a24e8ad5c2eb6d51b84dbbb7263b3a135d9ee
+size 718017

eval_out/generated_videos/cf_ffff/overall_consistency/001.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:97434fa492a782f473db53f5325943455e38f44c2ec0b765a9e94facc1b41f64
+size 678798

eval_out/generated_videos/cf_ffff/overall_consistency/002.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8cc8240a9fc4555de544e0131304dd301803dab2a1867f8efe73d36a6686ed27
+size 665068

eval_out/generated_videos/cf_ffff/overall_consistency/003.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dfeeb51a34c8db083f840497bc64bd6850f361e57b03435d436c0625cf1aa614
+size 652636

eval_out/generated_videos/cf_ffff/overall_consistency/004.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1bcf14ef30751d4cc7ca646615758207443e7848e99c61e91a62566a17301b3a
+size 624330

eval_out/generated_videos/cf_ffff/overall_consistency/005.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5a003819fae2c958584f41480ae48397ed99cb2afcb0bee20a30aa8359adf985
+size 619500

eval_out/generated_videos/cf_ffff/overall_consistency/006.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:510c4b85f51b3605bdb24602644f27c2865ad8cfb7d0f6d105fc1c2fe5027bbf
+size 667321

eval_out/generated_videos/cf_ffff/overall_consistency/007.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cb5b16a9bdf0ba7990fcf6ad75f18a2a0dc8142ca10f2276716c8136ace79e3c
+size 692210

eval_out/generated_videos/cf_ffff/overall_consistency/008.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:93f6f41136b394e1e642a060d2783281692481f08e725a4933e872cf321be58d
+size 538371

eval_out/generated_videos/cf_ffff/overall_consistency/009.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5669c892dd2fdbce0fcfb2ee2f0221645ad620c72562b1bfd3ad1d8a3ad40202
+size 649393

eval_out/generated_videos/cf_ffff/overall_consistency/010.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d05a3ee90af4a46d5cef630591125ad542ec7695b9c25f5279572b9d536d6568
+size 640856

eval_out/generated_videos/cf_ffff/overall_consistency/011.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9aceedf023cd61aac971d5933f802ad3e75770296b10888a2bd5d8fb2294f211
+size 673335

eval_out/generated_videos/cf_ffff/overall_consistency/012.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1742fc927aee1bea4a58cfb703a771181f5558ac95eec019bf82bbb60684ebe6
+size 643474

eval_out/generated_videos/cf_ffff/overall_consistency/013.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8dc175733a2dd86a2a86a5cd181f2990003b1532149aaec82623d841fa710276
+size 697151

eval_out/generated_videos/cf_ffff/overall_consistency/014.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:53b34417aa633a8a16f67d5569cd9acdaae30016bca4b0a12e89297af99a09ab
+size 564467

eval_out/generated_videos/cf_ffff/overall_consistency/015.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5a0bbec98e10b7de4a5edb3ba14d7318683afef3c89587c44f66468c9bbfd705
+size 607298

eval_out/generated_videos/cf_ffff/overall_consistency/016.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d9feefa47ae677af86818c9e3dee84155fe4cc7374a425c9a6d828195fb72361
+size 693258

eval_out/generated_videos/cf_ffff/overall_consistency/017.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c1f29a4bb28547296142bcde8935031005095a0d0fd35f7d46c9c4f0c7ef77f4
+size 647002

eval_out/generated_videos/cf_ffff/overall_consistency/019.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:938b5f61acfd80ca568e800a618032bcb090ed37130378b32e51284d41979519
+size 635121