diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..3f4d529835cb7744b66ee5e11986625a5e9a05e3 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,73 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +eval_out/generated_videos/cf_ffff/overall_consistency/041.mp4 filter=lfs diff=lfs merge=lfs -text +eval_out/generated_videos/cf_ffff/overall_consistency/044.mp4 filter=lfs diff=lfs merge=lfs -text +eval_out/generated_videos/cf_ffff/overall_consistency/078.mp4 filter=lfs diff=lfs merge=lfs -text +eval_out/generated_videos/cf_ffff/overall_consistency/064.mp4 filter=lfs diff=lfs merge=lfs -text +eval_out/generated_videos/cf_ffff/overall_consistency/000.mp4 filter=lfs diff=lfs merge=lfs -text +eval_out/generated_videos/cf_ffff/overall_consistency/049.mp4 filter=lfs diff=lfs merge=lfs -text +eval_out/generated_videos/cf_ffff/overall_consistency/005.mp4 filter=lfs diff=lfs merge=lfs -text +eval_out/generated_videos/cf_ffff/overall_consistency/007.mp4 filter=lfs diff=lfs merge=lfs -text +eval_out/generated_videos/cf_ffff/overall_consistency/001.mp4 filter=lfs diff=lfs merge=lfs -text +eval_out/generated_videos/cf_ffff/overall_consistency/002.mp4 filter=lfs diff=lfs merge=lfs -text +eval_out/generated_videos/cf_ffff/overall_consistency/003.mp4 filter=lfs diff=lfs merge=lfs -text +eval_out/generated_videos/cf_ffff/overall_consistency/010.mp4 filter=lfs diff=lfs merge=lfs -text +eval_out/generated_videos/cf_ffff/overall_consistency/012.mp4 filter=lfs diff=lfs merge=lfs -text +eval_out/generated_videos/cf_ffff/overall_consistency/083.mp4 filter=lfs diff=lfs merge=lfs -text +eval_out/generated_videos/cf_ffff/overall_consistency/014.mp4 filter=lfs diff=lfs merge=lfs -text +eval_out/generated_videos/cf_ffff/overall_consistency/013.mp4 filter=lfs diff=lfs merge=lfs -text +eval_out/generated_videos/cf_ffff/overall_consistency/009.mp4 filter=lfs diff=lfs merge=lfs -text +eval_out/generated_videos/cf_ffff/overall_consistency/015.mp4 filter=lfs diff=lfs merge=lfs -text +eval_out/generated_videos/cf_ffff/overall_consistency/047.mp4 filter=lfs diff=lfs merge=lfs -text +eval_out/generated_videos/cf_ffff/overall_consistency/016.mp4 filter=lfs diff=lfs merge=lfs -text +eval_out/generated_videos/cf_ffff/overall_consistency/011.mp4 filter=lfs diff=lfs merge=lfs -text +eval_out/generated_videos/cf_ffff/overall_consistency/021.mp4 filter=lfs diff=lfs merge=lfs -text +eval_out/generated_videos/cf_ffff/overall_consistency/004.mp4 filter=lfs diff=lfs merge=lfs -text +eval_out/generated_videos/cf_ffff/overall_consistency/062.mp4 filter=lfs diff=lfs merge=lfs -text +eval_out/generated_videos/cf_ffff/overall_consistency/008.mp4 filter=lfs diff=lfs merge=lfs -text +eval_out/generated_videos/cf_ffff/overall_consistency/019.mp4 filter=lfs diff=lfs merge=lfs -text +eval_out/generated_videos/cf_ffff/overall_consistency/023.mp4 filter=lfs diff=lfs merge=lfs -text +eval_out/generated_videos/cf_ffff/overall_consistency/061.mp4 filter=lfs diff=lfs merge=lfs -text +eval_out/generated_videos/cf_ffff/overall_consistency/022.mp4 filter=lfs diff=lfs merge=lfs -text +eval_out/generated_videos/cf_ffff/overall_consistency/028.mp4 filter=lfs diff=lfs merge=lfs -text +eval_out/generated_videos/cf_ffff/overall_consistency/027.mp4 filter=lfs diff=lfs merge=lfs -text +eval_out/generated_videos/cf_ffff/overall_consistency/017.mp4 filter=lfs diff=lfs merge=lfs -text +eval_out/generated_videos/cf_ffff/overall_consistency/054.mp4 filter=lfs diff=lfs merge=lfs -text +eval_out/generated_videos/cf_ffff/overall_consistency/034.mp4 filter=lfs diff=lfs merge=lfs -text +eval_out/generated_videos/cf_ffff/overall_consistency/033.mp4 filter=lfs diff=lfs merge=lfs -text +eval_out/generated_videos/cf_ffff/overall_consistency/032.mp4 filter=lfs diff=lfs merge=lfs -text +eval_out/generated_videos/cf_ffff/overall_consistency/092.mp4 filter=lfs diff=lfs merge=lfs -text +eval_out/generated_videos/cf_ffff/overall_consistency/037.mp4 filter=lfs diff=lfs merge=lfs -text +eval_out/generated_videos/cf_ffff/overall_consistency/026.mp4 filter=lfs diff=lfs merge=lfs -text +eval_out/generated_videos/cf_ffff/overall_consistency/035.mp4 filter=lfs diff=lfs merge=lfs -text +eval_out/generated_videos/cf_ffff/overall_consistency/040.mp4 filter=lfs diff=lfs merge=lfs -text +eval_out/generated_videos/cf_ffff/overall_consistency/091.mp4 filter=lfs diff=lfs merge=lfs -text +eval_out/generated_videos/cf_ffff/overall_consistency/020.mp4 filter=lfs diff=lfs merge=lfs -text +eval_out/generated_videos/cf_ffff/overall_consistency/039.mp4 filter=lfs diff=lfs merge=lfs -text +eval_out/generated_videos/cf_ffff/overall_consistency/042.mp4 filter=lfs diff=lfs merge=lfs -text +eval_out/generated_videos/cf_ffff/overall_consistency/006.mp4 filter=lfs diff=lfs merge=lfs -text +eval_out/generated_videos/cf_ffff/overall_consistency/024.mp4 filter=lfs diff=lfs merge=lfs -text +eval_out/generated_videos/cf_ffff/overall_consistency/050.mp4 filter=lfs diff=lfs merge=lfs -text +eval_out/generated_videos/cf_ffff/overall_consistency/048.mp4 filter=lfs diff=lfs merge=lfs -text +eval_out/generated_videos/cf_ffff/overall_consistency/059.mp4 filter=lfs diff=lfs merge=lfs -text +eval_out/generated_videos/cf_ffff/overall_consistency/060.mp4 filter=lfs diff=lfs merge=lfs -text +eval_out/generated_videos/cf_ffff/overall_consistency/057.mp4 filter=lfs diff=lfs merge=lfs -text +eval_out/generated_videos/cf_ffff/overall_consistency/088.mp4 filter=lfs diff=lfs merge=lfs -text +eval_out/generated_videos/cf_ffff/overall_consistency/051.mp4 filter=lfs diff=lfs merge=lfs -text +eval_out/generated_videos/cf_ffff/overall_consistency/053.mp4 filter=lfs diff=lfs merge=lfs -text +eval_out/generated_videos/cf_ffff/overall_consistency/090.mp4 filter=lfs diff=lfs merge=lfs -text +eval_out/generated_videos/cf_ffff/overall_consistency/052.mp4 filter=lfs diff=lfs merge=lfs -text +eval_out/generated_videos/cf_ffff/overall_consistency/058.mp4 filter=lfs diff=lfs merge=lfs -text +eval_out/generated_videos/cf_ffff/overall_consistency/055.mp4 filter=lfs diff=lfs merge=lfs -text +eval_out/generated_videos/cf_ffff/overall_consistency/056.mp4 filter=lfs diff=lfs merge=lfs -text +eval_out/generated_videos/cf_ffff/overall_consistency/089.mp4 filter=lfs diff=lfs merge=lfs -text +eval_out/generated_videos/cf_ffff/overall_consistency/079.mp4 filter=lfs diff=lfs merge=lfs -text +eval_out/generated_videos/cf_ffff/overall_consistency/063.mp4 filter=lfs diff=lfs merge=lfs -text +eval_out/generated_videos/cf_ffff/overall_consistency/030.mp4 filter=lfs diff=lfs merge=lfs -text +eval_out/generated_videos/cf_ffff/overall_consistency/068.mp4 filter=lfs diff=lfs merge=lfs -text +eval_out/generated_videos/cf_ffff/overall_consistency/066.mp4 filter=lfs diff=lfs merge=lfs -text +eval_out/generated_videos/cf_ffff/overall_consistency/065.mp4 filter=lfs diff=lfs merge=lfs -text +eval_out/generated_videos/cf_ffff/overall_consistency/070.mp4 filter=lfs diff=lfs merge=lfs -text +eval_out/generated_videos/cf_ffff/overall_consistency/067.mp4 filter=lfs diff=lfs merge=lfs -text +eval_out/generated_videos/cf_ffff/overall_consistency/072.mp4 filter=lfs diff=lfs merge=lfs -text diff --git a/assets/lb_actions/a_d_125f/intrinsics.npy b/assets/lb_actions/a_d_125f/intrinsics.npy new file mode 100644 index 0000000000000000000000000000000000000000..21302723238ef422a95e7274566b3abe51ac2da6 --- /dev/null +++ b/assets/lb_actions/a_d_125f/intrinsics.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a526a861f16b3bdd64d12b1946d2438925b0f68d88822599fc1d6b14a324d905 +size 2128 diff --git a/assets/lb_actions/a_d_125f/poses.npy b/assets/lb_actions/a_d_125f/poses.npy new file mode 100644 index 0000000000000000000000000000000000000000..b5b41d4fc577b2e5165131edc59af453741fe30d --- /dev/null +++ b/assets/lb_actions/a_d_125f/poses.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f4cebbc48954133ce27c8adc7c71327f3ab6e6b8f1d735441dc6931bb931b5a9 +size 8128 diff --git a/assets/lb_actions/left_right_125f/intrinsics.npy b/assets/lb_actions/left_right_125f/intrinsics.npy new file mode 100644 index 0000000000000000000000000000000000000000..21302723238ef422a95e7274566b3abe51ac2da6 --- /dev/null +++ b/assets/lb_actions/left_right_125f/intrinsics.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a526a861f16b3bdd64d12b1946d2438925b0f68d88822599fc1d6b14a324d905 +size 2128 diff --git a/assets/lb_actions/left_right_125f/poses.npy b/assets/lb_actions/left_right_125f/poses.npy new file mode 100644 index 0000000000000000000000000000000000000000..f32b366fd5ebb24331e17ab153306b223bc7a91e --- /dev/null +++ b/assets/lb_actions/left_right_125f/poses.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03161c72fae5167c67aed4e3fa61cddb4dee23f008712d1ebc6a73f4848c631f +size 8128 diff --git a/assets/lb_actions/up_down_125f/intrinsics.npy b/assets/lb_actions/up_down_125f/intrinsics.npy new file mode 100644 index 0000000000000000000000000000000000000000..21302723238ef422a95e7274566b3abe51ac2da6 --- /dev/null +++ b/assets/lb_actions/up_down_125f/intrinsics.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a526a861f16b3bdd64d12b1946d2438925b0f68d88822599fc1d6b14a324d905 +size 2128 diff --git a/assets/lb_actions/up_down_125f/poses.npy b/assets/lb_actions/up_down_125f/poses.npy new file mode 100644 index 0000000000000000000000000000000000000000..344094212952dda01ec0e132674ce83049fae2a3 --- /dev/null +++ b/assets/lb_actions/up_down_125f/poses.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f9b64db0d0cea64c0b9b587f065bd6c2d0d76a1512f0ba128dcb4b41b75c04c0 +size 8128 diff --git a/assets/lb_actions/w_s_125f/intrinsics.npy b/assets/lb_actions/w_s_125f/intrinsics.npy new file mode 100644 index 0000000000000000000000000000000000000000..21302723238ef422a95e7274566b3abe51ac2da6 --- /dev/null +++ b/assets/lb_actions/w_s_125f/intrinsics.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a526a861f16b3bdd64d12b1946d2438925b0f68d88822599fc1d6b14a324d905 +size 2128 diff --git a/assets/lb_actions/w_s_125f/poses.npy b/assets/lb_actions/w_s_125f/poses.npy new file mode 100644 index 0000000000000000000000000000000000000000..87be8f8e256489d5df558af554397fa14b90bc06 --- /dev/null +++ b/assets/lb_actions/w_s_125f/poses.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0624f5ca69206ecd7045c2f2b863b8e80bb8b5792d6b09f67370dab41d0dbb5 +size 8128 diff --git a/assets/lb_actions/w_s_61f/intrinsics.npy b/assets/lb_actions/w_s_61f/intrinsics.npy new file mode 100644 index 0000000000000000000000000000000000000000..c67bbd979baac76e8126b2df13628202d0ca1e69 --- /dev/null +++ b/assets/lb_actions/w_s_61f/intrinsics.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d5b972022c390258bd820986b146a9cfb86654b84be31ef161cd5c51b0ff746b +size 1104 diff --git a/assets/lb_actions/w_s_61f/poses.npy b/assets/lb_actions/w_s_61f/poses.npy new file mode 100644 index 0000000000000000000000000000000000000000..8a4bdaa8199b68ddd1f5c5ac27833bf1e131819e --- /dev/null +++ b/assets/lb_actions/w_s_61f/poses.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:423844e4fd3f7900c8e07acf72544399c9ec1efe2f0ffe2243e0c66a64c56be3 +size 4032 diff --git a/cachelib/selective.py b/cachelib/selective.py new file mode 100644 index 0000000000000000000000000000000000000000..75199b6a397cf43cc2f86c6223288be329fede83 --- /dev/null +++ b/cachelib/selective.py @@ -0,0 +1,148 @@ +"""Selective-token forward for the block-causal Wan DiT. + +MotionCache (token granularity) and FlowCache (frame-group granularity) both need +to run the transformer stack on a *subset* of the current chunk's tokens while the +remaining tokens keep the activations cached from the previous computed step. + +For a KV-cached causal model that means, per self-attention layer: + * compute q/k/v only for the selected rows, + * scatter the fresh k/v into the slots the current chunk owns in the KV cache, + leaving the unselected slots holding the previous step's k/v, + * attend the selected queries against the *full* cache (history + current chunk). + +This mirrors ``forward_selective`` in +``MotionCache4SkyReels-V2/skyreels_v2_infer/modules/transformer_motion.py``, +adapted from SkyReels' persistent ``k_cache/v_cache`` buffers to Self-Forcing's +rolling ``kv_cache[block]`` dict. +""" + +import math + +import torch + +from wan.modules.attention import attention + + +def build_causal_freqs(grid_sizes, freqs, start_frame): + """Return the per-token RoPE multipliers [seq_len, 1, c] for one chunk. + + Same construction as ``causal_rope_apply`` in ``wan/modules/causal_model.py``, + factored out so it can be indexed by a token subset and reused across layers. + """ + c = freqs.size(1) + parts = freqs.split([c - 2 * (c // 3), c // 3, c // 3], dim=1) + f, h, w = grid_sizes[0].tolist() + seq_len = f * h * w + freqs_i = torch.cat( + [ + parts[0][start_frame:start_frame + f].view(f, 1, 1, -1).expand(f, h, w, -1), + parts[1][:h].view(1, h, 1, -1).expand(f, h, w, -1), + parts[2][:w].view(1, 1, w, -1).expand(f, h, w, -1), + ], + dim=-1, + ).reshape(seq_len, 1, -1) + return freqs_i + + +def rope_apply_rows(x, freqs_rows): + """Apply precomputed RoPE multipliers to x [B, S, n, d] given freqs [S, 1, d/2].""" + b, s, n, _ = x.shape + x_c = torch.view_as_complex(x.to(torch.float64).reshape(b, s, n, -1, 2)) + out = torch.view_as_real(x_c * freqs_rows.unsqueeze(0)).flatten(3) + return out.type_as(x) + + +def self_attn_selective(attn, x_sel, sel_idx, freqs_rows, kv_cache, + current_start, block_seq_len): + """Self-attention over a token subset, refreshing only those KV slots. + + Args: + attn: a ``CausalWanSelfAttention`` module. + x_sel: [B, n_sel, C] modulated hidden states for the selected tokens. + sel_idx: LongTensor [n_sel], token offsets inside the current chunk. + freqs_rows: [block_seq_len, 1, d/2] RoPE multipliers for the whole chunk. + kv_cache: this layer's ``{"k", "v", "global_end_index", "local_end_index"}``. + current_start: token offset of the chunk in global sequence coordinates. + block_seq_len: number of tokens the chunk owns (all frames of the chunk). + """ + b, n_sel = x_sel.shape[:2] + n, d = attn.num_heads, attn.head_dim + + q = attn.norm_q(attn.q(x_sel)).view(b, n_sel, n, d) + k = attn.norm_k(attn.k(x_sel)).view(b, n_sel, n, d) + v = attn.v(x_sel).view(b, n_sel, n, d) + + rows = freqs_rows[sel_idx] + roped_q = rope_apply_rows(q, rows) + roped_k = rope_apply_rows(k, rows).type_as(v) + + # Index arithmetic must use the *full* chunk length: the chunk still occupies + # the same KV slots, we are only refreshing some of them. + current_end = current_start + block_seq_len + local_end_index = (kv_cache["local_end_index"].item() + current_end + - kv_cache["global_end_index"].item()) + local_start_index = local_end_index - block_seq_len + + slots = local_start_index + sel_idx + kv_cache["k"].index_copy_(1, slots, roped_k) + kv_cache["v"].index_copy_(1, slots, v) + + lo = max(0, local_end_index - attn.max_attention_size) + out = attention(roped_q, kv_cache["k"][:, lo:local_end_index], + kv_cache["v"][:, lo:local_end_index]) + + kv_cache["global_end_index"].fill_(current_end) + kv_cache["local_end_index"].fill_(local_end_index) + + return attn.o(out.flatten(2)) + + +def block_forward_selective(block, x_sel, sel_idx, frame_of_sel, e0, freqs_rows, + context, kv_cache, crossattn_cache, + current_start, block_seq_len): + """One ``CausalWanAttentionBlock`` evaluated on a token subset. + + ``frame_of_sel`` maps each selected token to its frame index inside the chunk, + so the per-frame AdaLN modulation can be gathered for the selected rows only. + """ + # e0: [B, F, 6, C] -> six tensors of [B, F, 1, C]; gather per selected token. + e = (block.modulation.unsqueeze(1) + e0).chunk(6, dim=2) + e = [t.squeeze(2)[:, frame_of_sel] for t in e] # each [B, n_sel, C] + + y = self_attn_selective( + block.self_attn, + block.norm1(x_sel) * (1 + e[1]) + e[0], + sel_idx, freqs_rows, kv_cache, current_start, block_seq_len, + ) + x_sel = x_sel + y * e[2] + + x_sel = x_sel + block.cross_attn(block.norm3(x_sel), context, None, + crossattn_cache=crossattn_cache) + y = block.ffn(block.norm2(x_sel) * (1 + e[4]) + e[3]) + x_sel = x_sel + y * e[5] + return x_sel + + +def run_blocks_selective(model, x, sel_idx, e0, kwargs, kv_cache, crossattn_cache, + current_start, cache_start, grid_sizes): + """Run all transformer blocks on ``x[:, sel_idx]`` and return the updated rows. + + Returns a tensor [B, n_sel, C] holding the new hidden states for the selected + tokens (the caller merges them into the cached full-length tensor). + """ + f, h, w = grid_sizes[0].tolist() + frame_seqlen = h * w + block_seq_len = f * frame_seqlen + frame_of_sel = torch.div(sel_idx, frame_seqlen, rounding_mode="floor") + + freqs_rows = build_causal_freqs(grid_sizes, kwargs["freqs"], + current_start // frame_seqlen) + + x_sel = x[:, sel_idx] + for i, block in enumerate(model.blocks): + x_sel = block_forward_selective( + block, x_sel, sel_idx, frame_of_sel, e0, freqs_rows, + kwargs["context"], kv_cache[i], crossattn_cache[i], + current_start, block_seq_len, + ) + return x_sel diff --git a/eval/aggregate.py b/eval/aggregate.py new file mode 100644 index 0000000000000000000000000000000000000000..56840e432be88abf8c5bcef815e1a17d6dedc8f4 --- /dev/null +++ b/eval/aggregate.py @@ -0,0 +1,331 @@ +#!/usr/bin/env python +"""Final summary for the Self-Forcing Extended-251 Full Evaluation. + +Implements protocol sections 7 (normalize + aggregate), 8.3 (pixel aggregation), +9.4 (matched-FFFF speedup) and 10 (output tables), plus the section 11/15 +completeness gate: a strategy that is not a full 251 is never summarised. + + python eval/aggregate.py --out-root eval_out +""" + +import argparse +import re +import csv +import glob +import hashlib +import json +import math +import os +import platform +import sys +from collections import defaultdict + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, ROOT) + +MAPPING = os.path.join(ROOT, "assets/vbench8_extended_subset_mapping.json") + +DIMENSIONS = ["subject_consistency", "background_consistency", "motion_smoothness", + "dynamic_degree", "aesthetic_quality", "imaging_quality", "scene", + "overall_consistency"] + +# Protocol section 7.1 -- fixed empirical ranges, never re-estimated from the run. +NORMALIZE_RANGE = { + "subject_consistency": (0.1462, 1.0), + "background_consistency": (0.2615, 1.0), + "motion_smoothness": (0.7060, 0.9975), + "dynamic_degree": (0.0, 1.0), + "aesthetic_quality": (0.0, 1.0), + "imaging_quality": (0.0, 1.0), + "scene": (0.0, 0.8222), + "overall_consistency": (0.0, 0.3640), +} + +REFERENCE_OF = {"sf": "sf_ffff", "cf": "cf_ffff", "cfa": "cfa_ffff"} + + +def normalize(raw): + out = {} + for d, v in raw.items(): + lo, hi = NORMALIZE_RANGE[d] + out[d] = (v - lo) / (hi - lo) # protocol does not clip + return out + + +def quality_score(n): + return (n["subject_consistency"] + n["background_consistency"] + + n["motion_smoothness"] + 0.5 * n["dynamic_degree"] + + n["aesthetic_quality"] + n["imaging_quality"]) / 5.5 + + +def semantic_score(n): + return (n["scene"] + n["overall_consistency"]) / 2.0 + + +def selected_score(q, s): + return (4.0 * q + s) / 5.0 + + +def sha256(path): + h = hashlib.sha256() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + h.update(chunk) + return h.hexdigest() + + +def load_per_prompt(out_root, strategy): + recs = [] + for p in sorted(glob.glob(os.path.join(out_root, "per_prompt", strategy, "*.json"))): + with open(p) as f: + r = json.load(f) + if r.get("status") == "complete": + recs.append(r) + return recs + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--out-root", default="eval_out") + ap.add_argument("--expect", type=int, default=251) + ap.add_argument("--allow-incomplete", action="store_true", + help="Report partial strategies instead of refusing (diagnostics only)") + ap.add_argument("--latency-subsets", default=os.path.join(ROOT, "results/latency_subsets.json"), + help="JSON {strategy_regex: {shard, num_shards, note}}: time these " + "strategies on one prompt shard only (the one that ran on an " + "uncontended GPU); quality still uses every record") + ap.add_argument("--only", default=None, + help="Regex: summarise only strategies whose name matches (other " + "strategies in per_prompt/ are ignored, e.g. while they are " + "still being generated)") + args = ap.parse_args() + + out_root = (args.out_root if os.path.isabs(args.out_root) + else os.path.join(ROOT, args.out_root)) + + with open(MAPPING) as f: + mapping = json.load(f) + + strategies = sorted(os.path.basename(p) for p in + glob.glob(os.path.join(out_root, "per_prompt", "*"))) + if args.only: + strategies = [s for s in strategies if re.search(args.only, s)] + if not strategies: + print("no per-prompt records found") + return 1 + + per_strategy, problems = {}, [] + for s in strategies: + recs = load_per_prompt(out_root, s) + if len(recs) != args.expect: + problems.append(f"{s}: {len(recs)} complete records, expected {args.expect}") + per_strategy[s] = recs + + if problems and not args.allow_incomplete: + print("REFUSING TO SUMMARISE -- incomplete strategies (protocol 11.9):") + for p in problems: + print(" -", p) + return 1 + + # --- latency, matched to each base model's own FFFF ----------------------- + mean_latency, mean_context, timed_records = {}, {}, {} + for s, recs in per_strategy.items(): + # Records flagged latency_excluded keep their pixel metrics but carry no + # usable timing (see their latency_note); they stay out of the means. + lat = [r for r in recs if not r.get("latency_excluded")] + if not lat: + continue + timed_records[s] = lat + mean_latency[s] = sum(r["policy_latency_ms"] for r in lat) / len(lat) + mean_context[s] = sum(r["excluded_context_kv_latency_ms"] for r in lat) / len(lat) + + # Strategies generated while another job shared the GPU are timed only on the + # prompt shard that ran alone (sharding is rows[shard::num_shards] over the + # mapping order, see generate_eval.py); their FFFF reference is restricted to + # the same prompts so the ratio stays paired. + subsets = {} + if args.latency_subsets and os.path.exists(args.latency_subsets): + with open(args.latency_subsets) as f: + subsets = json.load(f) + with open(os.path.join(ROOT, "assets/vbench8_extended_subset_mapping.json")) as f: + mapping_rows = json.load(f)["rows"] + subset_of = {} + for s in per_strategy: + for pat, spec in subsets.items(): + if re.search(pat, s): + keep = {r["global_index"] for r in mapping_rows[spec["shard"]::spec["num_shards"]]} + subset_of[s] = (keep, spec) + break + + rows = [] + for s, recs in per_strategy.items(): + if not recs: + continue + ref = recs[0].get("reference_strategy") or REFERENCE_OF[s.split("_")[0]] + if ref not in mean_latency: + problems.append(f"{s}: reference {ref} missing") + continue + + # Ratio of means (protocol 9.4). Equal sample counts make this the same + # as the ratio of sums; a mean of per-prompt percentages is not used. + # A strategy whose records carry the FFFF latency measured in the same + # process (retime_eval.py) is compared against that, not against the + # FFFF run's own records: contention differs between runs, and only a + # same-run pairing is "matched FFFF" in the protocol's sense. + timed = timed_records[s] + matched = [r.get("matched_ffff_policy_latency_ms") for r in timed] + strat_latency = mean_latency[s] + latency_n = len(timed) + if s in subset_of: + keep, spec = subset_of[s] + sub = [r for r in recs if r["global_index"] in keep] + ref_sub = [r for r in per_strategy[ref] if r["global_index"] in keep] + assert sub and ref_sub, f"{s}: empty latency subset" + strat_latency = sum(r["policy_latency_ms"] for r in sub) / len(sub) + ref_latency = sum(r["policy_latency_ms"] for r in ref_sub) / len(ref_sub) + latency_n = len(sub) + latency_source = (f"generation run, shard {spec['shard']}/{spec['num_shards']} only " + f"({len(sub)} prompts; {spec.get('note', '')})") + elif all(m is not None for m in matched): + ref_latency = sum(matched) / len(matched) + latency_source = "paired retime (same process as FFFF)" + elif all(r.get("latency_retimed") for r in recs): + ref_latency = mean_latency[ref] + latency_source = ("strategy retime: all records from one denoise-only pass on an idle GPU " + "(FFFF from its own records)") + elif any(r.get("latency_retimed") for r in recs): + n_re = sum(1 for r in recs if r.get("latency_retimed")) + ref_latency = mean_latency[ref] + latency_source = (f"generation run; {n_re}/{len(recs)} records re-timed " + "(strategy-only denoise rerun on an idle GPU; FFFF from its own records)") + else: + ref_latency = mean_latency[ref] + latency_source = "generation run (FFFF from its own records)" + if len(timed) < len(recs): + latency_source += (f"; {len(timed)}/{len(recs)} prompts with intact timing " + "(latency_excluded records dropped)") + speedup_pct = 100.0 * (1.0 - strat_latency / ref_latency) + + # PSNR from the mean of per-prompt mean MSE, then converted once (8.3). + mses = [r["pixel_metrics_vs_ffff"]["mean_mse"] for r in recs] + mean_mse = sum(mses) / len(mses) + psnr = -10.0 * math.log10(max(mean_mse, 1e-12)) + ssim = sum(r["pixel_metrics_vs_ffff"]["ssim"] for r in recs) / len(recs) + lpips = sum(r["pixel_metrics_vs_ffff"]["lpips"] for r in recs) / len(recs) + frames = {r["pixel_metrics_vs_ffff"]["num_frames"] for r in recs} + if frames != {81}: + problems.append(f"{s}: pixel metrics used frame counts {sorted(frames)}, expected 81") + + row = { + "strategy": s, + "base_model": recs[0]["base_model"], + "method": recs[0]["method"], + "target_speedup": recs[0]["target_speedup"], + "schedule": recs[0].get("schedule"), + "num_inference_steps": recs[0].get("num_inference_steps"), + "num_videos": len(recs), + "mp4_reference_records": sum(1 for r in recs if r.get("reference_source") == "ffff_mp4"), + "policy_latency_ms": strat_latency, + "latency_num_videos": latency_n, + "speedup_percent_vs_ffff": 0.0 if s == ref else speedup_pct, + "latency_ratio_vs_ffff": strat_latency / ref_latency, + "matched_ffff_policy_latency_ms": ref_latency, + "latency_source": latency_source, + "excluded_context_kv_latency_ms": mean_context[s], + "psnr": psnr, + "ssim": ssim, + "lpips": lpips, + "mean_compute_equivalent_forwards": ( + sum(r["cache_diagnostics"]["compute_equivalent_forwards"] for r in recs) + / len(recs)), + } + + score_path = os.path.join(out_root, "vbench", "scores", f"{s}.json") + if os.path.exists(score_path): + with open(score_path) as f: + sc = json.load(f) + raw = {d: sc["raw"][d] for d in DIMENSIONS if d in sc["raw"]} + if len(raw) == len(DIMENSIONS): + bad = {d: v for d, v in raw.items() if not (0.0 <= v <= 1.0)} + if bad: + problems.append(f"{s}: raw scores outside [0,1]: {bad}") + n = normalize(raw) + q, sem = quality_score(n), semantic_score(n) + sel = selected_score(q, sem) + row.update({f"raw_{d}": raw[d] for d in DIMENSIONS}) + row.update({f"normalized_{d}": n[d] for d in DIMENSIONS}) + row.update({"quality_score": q, "semantic_score": sem, + "selected_vbench_score": sel, + "selected_vbench_percent": sel * 100.0}) + else: + problems.append(f"{s}: only {len(raw)}/8 VBench dimensions") + else: + problems.append(f"{s}: no VBench scores yet") + rows.append(row) + + rows.sort(key=lambda r: (r["base_model"], r["method"], r["target_speedup"])) + + summary = { + "protocol": "Self-Forcing Extended-251 Full Evaluation", + "vbench_long": False, + "num_strategies": len(rows), + "prompts_per_strategy": args.expect, + "seed": 0, + "video_spec": {"frames": 81, "height": 480, "width": 832, "fps": 16}, + "precision": "bfloat16", + "normalize_range": NORMALIZE_RANGE, + "aggregation": { + "quality": "(n_sc + n_bc + n_ms + 0.5*n_dd + n_aq + n_iq) / 5.5", + "semantic": "(n_scene + n_oc) / 2", + "selected": "(4*quality + semantic) / 5", + "psnr": "-10*log10(mean(per_prompt_mean_mse)), clamp 1e-12", + "speedup": "100 * (1 - mean(strategy) / mean(matched_ffff))", + }, + "latency_includes": "all denoise policy GPU compute incl. cache control modules", + "latency_excludes": "context/KV-cache DiT, text encoder, VAE decode, metrics, I/O", + "sources": mapping["sources"], + "mapping": {"path": MAPPING, "sha256": sha256(MAPPING)}, + "environment": { + "generation_python": "/local/zoubin/cz/envs/self_forcing/bin/python", + "vbench_python": "/local/zoubin/cz/envs/vbench_eval/bin/python", + "vbench_source": "/local/zoubin/cz/projects/VBench", + "vbench_cache_dir": "/local/zoubin/cz/.cache/vbench", + "platform": platform.platform(), + }, + "problems": problems, + "rows": rows, + } + + os.makedirs(os.path.join(out_root, "summaries"), exist_ok=True) + jpath = os.path.join(out_root, "summaries", "final_summary.json") + with open(jpath, "w") as f: + json.dump(summary, f, indent=2) + + cpath = os.path.join(out_root, "summaries", "final_summary.csv") + if rows: + keys = sorted({k for r in rows for k in r}, + key=lambda k: (k not in ("strategy", "base_model", "method"), k)) + with open(cpath, "w", newline="") as f: + w = csv.DictWriter(f, fieldnames=keys) + w.writeheader() + w.writerows(rows) + + hdr = (f"{'strategy':24s} {'videos':>6s} {'lat_ms':>9s} {'speedup%':>9s} " + f"{'psnr':>7s} {'ssim':>7s} {'lpips':>7s} {'sel_vbench%':>12s}") + print(hdr) + print("-" * len(hdr)) + for r in rows: + sel = r.get("selected_vbench_percent") + print(f"{r['strategy']:24s} {r['num_videos']:6d} {r['policy_latency_ms']:9.1f} " + f"{r['speedup_percent_vs_ffff']:9.2f} {r['psnr']:7.2f} {r['ssim']:7.4f} " + f"{r['lpips']:7.4f} " + (f"{sel:12.3f}" if sel is not None else f"{'-':>12s}")) + if problems: + print("\nnotes:") + for p in problems: + print(" -", p) + print(f"\nwrote {jpath}\nwrote {cpath}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/eval/build_mapping.py b/eval/build_mapping.py new file mode 100644 index 0000000000000000000000000000000000000000..f5bd670f018a48cedaacc6714bfa39493b9751de --- /dev/null +++ b/eval/build_mapping.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python +"""Build and validate assets/vbench8_extended_subset_mapping.json (Extended-251). + +Protocol section 4: take the subject_consistency (72), overall_consistency (93) +and scene (86) suites out of the canonical 946-prompt VBench order, and carry the +Self-Forcing rewritten extended prompt for each. + +Selection is strictly by index into VBench_full_info.json -- the short prompt list +contains two duplicate texts, so text matching is not safe, and the protocol +forbids it anyway. + + python eval/build_mapping.py --out assets/vbench8_extended_subset_mapping.json +""" + +import argparse +import hashlib +import json +import os +import sys + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +SUITES = ["subject_consistency", "overall_consistency", "scene"] +EXPECTED = {"subject_consistency": 72, "overall_consistency": 93, "scene": 86} + +DEFAULT_FULL_INFO = "/local/zoubin/cz/projects/VBench/vbench/VBench_full_info.json" +DEFAULT_PROMPT_DIR = "/local/zoubin/cz/projects/Self-Forcing/prompts/vbench" + + +def sha256(path): + h = hashlib.sha256() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + h.update(chunk) + return h.hexdigest() + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--full-info", default=DEFAULT_FULL_INFO) + ap.add_argument("--prompt-dir", default=DEFAULT_PROMPT_DIR) + ap.add_argument("--out", default=os.path.join(ROOT, "assets/vbench8_extended_subset_mapping.json")) + args = ap.parse_args() + + short_path = os.path.join(args.prompt_dir, "all_dimension.txt") + ext_path = os.path.join(args.prompt_dir, "all_dimension_extended.txt") + + with open(args.full_info) as f: + info = json.load(f) + with open(short_path, encoding="utf-8") as f: + short = [l.rstrip("\n") for l in f] + with open(ext_path, encoding="utf-8") as f: + ext = [l.rstrip("\n") for l in f] + + errors = [] + if len(short) != 946: + errors.append(f"short prompts == {len(short)}, expected 946") + if len(ext) != 946: + errors.append(f"extended prompts == {len(ext)}, expected 946") + if len(info) != 946: + errors.append(f"VBench_full_info rows == {len(info)}, expected 946") + for i, (row, s) in enumerate(zip(info, short)): + if row["prompt_en"].strip() != s.strip(): + errors.append(f"short prompt order differs from VBench_full_info at line {i}") + break + for i, e in enumerate(ext): + if not e.strip(): + errors.append(f"empty extended prompt at line {i}") + break + + rows = [] + for suite in SUITES: + suite_index = 0 + for gi, row in enumerate(info): + if suite not in row["dimension"]: + continue + entry = { + "global_index": gi, + "prompt_suite": suite, + "suite_index": suite_index, + "original_prompt": short[gi], + "extended_prompt": ext[gi], + "official_dimensions": list(row["dimension"]), + } + # Standard VBench 0.1.5 needs the official scene keywords to score `scene`. + for key in ("auxiliary_info",): + if key in row: + entry[key] = row[key] + rows.append(entry) + suite_index += 1 + + counts = {s: sum(1 for r in rows if r["prompt_suite"] == s) for s in SUITES} + for s, n in EXPECTED.items(): + if counts.get(s) != n: + errors.append(f"{s} == {counts.get(s)}, expected {n}") + if len(rows) != 251: + errors.append(f"mapping rows == {len(rows)}, expected 251") + if len({r["global_index"] for r in rows}) != len(rows): + errors.append("duplicate global_index") + for s in SUITES: + idx = [r["suite_index"] for r in rows if r["prompt_suite"] == s] + if idx != list(range(len(idx))): + errors.append(f"{s} suite_index not contiguous") + + if errors: + print("MAPPING VALIDATION FAILED:") + for e in errors: + print(" -", e) + return 1 + + payload = { + "protocol": "Self-Forcing Extended-251 Full Evaluation", + "num_rows": len(rows), + "counts": counts, + "sources": { + "vbench_full_info": {"path": args.full_info, "sha256": sha256(args.full_info)}, + "short_prompts": {"path": short_path, "sha256": sha256(short_path)}, + "extended_prompts": {"path": ext_path, "sha256": sha256(ext_path)}, + }, + "rows": rows, + } + os.makedirs(os.path.dirname(args.out), exist_ok=True) + with open(args.out, "w") as f: + json.dump(payload, f, indent=2) + + print("mapping OK") + print(f" rows {len(rows)}") + for s in SUITES: + print(f" {s:22s} {counts[s]}") + for k, v in payload["sources"].items(): + print(f" sha256 {k:18s} {v['sha256'][:16]}...") + print(f" scene rows with auxiliary_info: " + f"{sum(1 for r in rows if r['prompt_suite'] == 'scene' and 'auxiliary_info' in r)}") + print(f"wrote {args.out}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/eval/generate_eval.py b/eval/generate_eval.py new file mode 100644 index 0000000000000000000000000000000000000000..fbb71872278936d005c65b232c1a22759a66fba5 --- /dev/null +++ b/eval/generate_eval.py @@ -0,0 +1,259 @@ +#!/usr/bin/env python +"""Generate the Extended-251 videos for every strategy of one base model. + +Work is sharded by prompt, not by strategy, because the protocol requires each +strategy's clip to be paired with the FFFF clip for the *same* prompt and seed +(section 8.1). A shard therefore generates FFFF first, keeps its decoded frames, +and immediately scores every cache strategy against them -- no 25 GB of reference +frames on disk, and the pairing cannot get mismatched. + + CUDA_VISIBLE_DEVICES=0 python eval/generate_eval.py \ + --base self_forcing --shard 0 --num-shards 4 --out-root eval_out + +Re-running skips prompts whose per-strategy record is already complete. +""" + +import argparse +import json +import os +import sys +import time + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, ROOT) + +from eval.strategies import load_strategies # noqa: E402 +from harness import denoising_steps_for, BASES, enter_base, load_pipeline, use_denoising_steps # noqa: E402 + +MAPPING = os.path.join(ROOT, "assets/vbench8_extended_subset_mapping.json") + + +def parse_args(): + p = argparse.ArgumentParser() + p.add_argument("--base", choices=sorted(BASES), required=True) + p.add_argument("--shard", type=int, default=0) + p.add_argument("--num-shards", type=int, default=1) + p.add_argument("--out-root", default="eval_out") + p.add_argument("--seed", type=int, default=0) + p.add_argument("--num-latent-frames", type=int, default=21) + p.add_argument("--fps", type=int, default=16) + p.add_argument("--limit", type=int, default=None, help="Smoke-test: first N prompts") + p.add_argument("--only-strategy", default=None, help="Smoke-test: one strategy name") + p.add_argument("--strategies", default=None, + help="Comma-separated subset to generate (the FFFF reference is always kept)") + p.add_argument("--overwrite", action="store_true") + p.add_argument("--regenerate-ffff", action="store_true", + help="Regenerate the FFFF reference instead of reading its MP4") + return p.parse_args() + + +def main(): + args = parse_args() + out_root = (args.out_root if os.path.isabs(args.out_root) + else os.path.join(ROOT, args.out_root)) + + with open(MAPPING) as f: + mapping = json.load(f) + rows = mapping["rows"] + if args.limit: + rows = rows[:args.limit] + shard_rows = rows[args.shard::args.num_shards] + + strategies = load_strategies(args.base) + if args.only_strategy: + keep = {strategies[0]["name"], args.only_strategy} + strategies = [s for s in strategies if s["name"] in keep] + if args.strategies: + keep = set(args.strategies.split(",")) + strategies = [s for s in strategies if s["is_reference"] or s["name"] in keep] + ref_name = strategies[0]["name"] + assert strategies[0]["is_reference"], "first strategy must be FFFF" + + print(f"base={args.base} shard {args.shard}/{args.num_shards}: " + f"{len(shard_rows)} prompts x {len(strategies)} strategies", flush=True) + + base = enter_base(args.base) + + import torch + from einops import rearrange + from torchvision.io import read_video, write_video + from utils.misc import set_seed + + from cachelib import (CacheController, build_method, cached_inference, install, + parse_schedule) + from eval.pixel_metrics import PixelMetrics + + torch.set_grad_enabled(False) + device = torch.device("cuda") + pipeline = load_pipeline(base) + num_steps = len(pipeline.denoising_step_list) + metrics = PixelMetrics(device) + + def record_path(strategy, row): + d = os.path.join(out_root, "per_prompt", strategy) + os.makedirs(d, exist_ok=True) + return os.path.join(d, f"{row['prompt_suite']}_{row['suite_index']:03d}.json") + + def video_path(strategy, row): + d = os.path.join(out_root, "generated_videos", strategy, row["prompt_suite"]) + os.makedirs(d, exist_ok=True) + return os.path.join(d, f"{row['suite_index']:03d}.mp4") + + def ref_cache_path(row): + d = os.path.join(out_root, "ref_cache", args.base) + os.makedirs(d, exist_ok=True) + return os.path.join(d, f"{row['prompt_suite']}_{row['suite_index']:03d}.pt") + + def is_done(strategy, row): + p = record_path(strategy, row) + if args.overwrite or not os.path.exists(p): + return False + try: + with open(p) as f: + rec = json.load(f) + return rec.get("status") == "complete" and os.path.exists(rec.get("video", "")) + except Exception: + return False + + def write_json(path, payload): + tmp = path + ".tmp" + with open(tmp, "w") as f: + json.dump(payload, f, indent=2) + os.replace(tmp, path) # atomic: a partial file is never seen as complete + + def run_one(strategy, row): + kw = dict(indicator=strategy["indicator"], coefficients=strategy["coefficients"]) + if strategy["method"] != "none" and strategy.get("param"): + kw[strategy["param"]] = strategy["value"] + n_steps = strategy.get("num_inference_steps") or num_steps + original_steps = use_denoising_steps(pipeline, n_steps) + sched = strategy.get("schedule") or ("F" * n_steps) + first = strategy.get("first_chunk_schedule") + # Chunk 0 may have a longer schedule than the rest (e.g. FFFF over naive 2-step). + first_steps = (denoising_steps_for(original_steps, len(first)) + if first and len(first) != n_steps else None) + ctrl = CacheController( + build_method(strategy["method"], **kw), num_steps=n_steps, + forced_steps=parse_schedule(sched, n_steps), + first_chunk_forced_steps=(parse_schedule(first, len(first)) if first else None)) + install(pipeline.generator.model, ctrl) + + # Identical noise for every strategy: same seed, same draw order, same shape. + set_seed(args.seed) + noise = torch.randn([1, args.num_latent_frames, 16, 60, 104], + device=device, dtype=torch.bfloat16) + wall0 = time.time() + video, latents, timing = cached_inference( + pipeline, ctrl, noise, [row["extended_prompt"]], decode=True, + first_chunk_steps=first_steps, sampler=strategy.get("sampler", "renoise")) + wall = time.time() - wall0 + pipeline.vae.model.clear_cache() + pipeline.denoising_step_list = original_steps + return video, timing, ctrl.summary(), wall + + # Warm up every strategy once on a throwaway prompt and discard the result. + # Without this the first recorded prompt carries one-off costs -- notably + # TaylorSeer's torch.compile of the forecast, which made its first video look + # slower than FFFF. + if shard_rows: + print("warm-up pass (not recorded)", flush=True) + for strategy in strategies: + run_one(strategy, shard_rows[0]) + torch.cuda.empty_cache() + + t_start = time.time() + for n, row in enumerate(shard_rows): + todo = [s for s in strategies if not is_done(s["name"], row)] + if not todo: + continue + # The reference is needed for pixel metrics even when only some strategies + # are outstanding. + need_ref = any(not s["is_reference"] for s in todo) + ref_frames = None + # The FFFF reference frames (pre-MP4, fp16) are cached on disk after their + # first generation, so later strategy runs do not regenerate FFFF at all. + rc = ref_cache_path(row) + ref_from_mp4 = False + if need_ref and os.path.exists(rc): + ref_frames = torch.load(rc, map_location=device) + need_ref = False + elif need_ref and os.path.exists(video_path(ref_name, row)) and not args.regenerate_ffff: + # Reference from the FFFF run's MP4 (H.264, ~40+ dB above the 12-20 dB + # strategy PSNRs, so the compression adds <1 % to the MSE); never + # regenerate FFFF just to compare against it. + vid, _, _ = read_video(video_path(ref_name, row), pts_unit="sec", output_format="TCHW") + ref_frames = (vid.to(device, torch.float16) / 255.0) + ref_from_mp4 = True + need_ref = False + + for strategy in strategies: + if strategy["is_reference"]: + if strategy not in todo and not need_ref: + continue + elif strategy not in todo: + continue + + video, timing, summary, wall = run_one(strategy, row) + frames = video[0] # [T, C, H, W] in [0, 1] + + if strategy["is_reference"]: + ref_frames = frames.detach().to(torch.float16) + if not os.path.exists(rc): + torch.save(ref_frames.cpu(), rc + ".tmp") + os.replace(rc + ".tmp", rc) + pix = PixelMetrics.identity(frames.shape[0]) + else: + pix = metrics.compute(frames, ref_frames) + + if strategy in todo: + vp = video_path(strategy["name"], row) + arr = (255.0 * rearrange(frames, "t c h w -> t h w c")).clamp(0, 255) + write_video(vp, arr.to(torch.uint8).cpu(), fps=args.fps) + + write_json(record_path(strategy["name"], row), { + "status": "complete", + "protocol": "Self-Forcing Extended-251 Full Evaluation", + "strategy": strategy["name"], + "base_model": args.base, + "method": strategy["method"], + "param": strategy["param"], + "param_value": strategy["value"], + "target_speedup": strategy["target"], + "schedule": strategy.get("schedule") or ("F" * (strategy.get("num_inference_steps") or 4)), + "num_inference_steps": strategy.get("num_inference_steps") or 4, + "first_chunk_schedule": strategy.get("first_chunk_schedule"), + "sampler": strategy.get("sampler", "renoise"), + "global_index": row["global_index"], + "prompt_suite": row["prompt_suite"], + "suite_index": row["suite_index"], + "prompt": row["extended_prompt"], + "original_prompt": row["original_prompt"], + "seed": args.seed, + "video": vp, + "num_frames": int(frames.shape[0]), + "height": int(frames.shape[2]), + "width": int(frames.shape[3]), + "fps": args.fps, + "policy_latency_ms": timing["denoise_dit_ms"], + "excluded_context_kv_latency_ms": timing["context_kv_dit_ms"], + "wall_generation_s": wall, + "reference_strategy": ref_name, + "pixel_metrics_vs_ffff": pix, + "reference_source": "ffff_mp4" if ref_from_mp4 else "ffff_frames", + "cache_diagnostics": dict(summary), + }) + del frames, video + + ref_frames = None + done = n + 1 + rate = (time.time() - t_start) / done + print(f"[{args.base} shard {args.shard}] {done}/{len(shard_rows)} prompts " + f"({row['prompt_suite']}/{row['suite_index']:03d}) " + f"{rate:.1f}s/prompt eta {(len(shard_rows) - done) * rate / 60:.0f} min", + flush=True) + + print(f"shard {args.shard} done in {(time.time() - t_start) / 60:.1f} min", flush=True) + + +if __name__ == "__main__": + main() diff --git a/eval/generate_predictor_eval.py b/eval/generate_predictor_eval.py new file mode 100644 index 0000000000000000000000000000000000000000..33f0bca59240022426b2a7e1433384b56ed5f0d9 --- /dev/null +++ b/eval/generate_predictor_eval.py @@ -0,0 +1,532 @@ +#!/usr/bin/env python +"""Extended-251 videos for the Causal-Forcing-a single-block Predictors. + +Same protocol as generate_eval.py: work is sharded by prompt, FFFF is generated +first in the same process and kept as the pixel-metric reference, every strategy +sees the identical noise draw, and the per-prompt record has the layout +aggregate.py / run_vbench.py expect. + +Strategies (prefix ``cfa`` = Causal-Forcing-a stack): + + cfa_ffff reference, 28 full DiT forwards + cfa__fpff chunks 1..6: F P F F (22 full + 6 Predictor) + cfa__fppf chunks 1..6: F P P F (16 full + 12 Predictor) + cfa__fppp chunks 1..6: F P P P (10 full + 18 Predictor) + +where is one of atc_chunk, atc_last_frame, disca (Stage-1 Layer-17 +Predictors) or atc_chunk_s2 (Stage-2 DMD EMA). Chunk 0 is always fully denoised. +``@`` selects another checkpoint of that run and names the strategy +cfa__s..., e.g. ``disca@500`` -> cfa_disca_s500_fppf; a bare model +uses --checkpoint-step (default 2000) and the unsuffixed name. + + cfa_naive2step chunks 1..: the 2-step subset of the grid (t = 1000 / 250, + the same subset as cf_naive2step_c0FFFF), no Predictor + + CUDA_VISIBLE_DEVICES=3 python eval/generate_predictor_eval.py --shard 0 --num-shards 4 +""" + +import argparse +import json +import os +import sys +import time + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +REPO = os.environ.get("CFA_REPO", "/local/zoubin/cz/projects/Causal-Forcing-a") +MAPPING = os.path.join(ROOT, "assets/vbench8_extended_subset_mapping.json") + +BASE = { + "repo": REPO, + "config": "configs/causal_forcing_dmd_chunkwise.yaml", + "checkpoint": "checkpoints/chunkwise/causal_forcing.pt", +} +MODELS = { + "atc_chunk": "output/layer17_stage1_atc_chunk_1000p_21f_seed0_4gpu_b16_acc1_2000steps", + "atc_last_frame": "output/layer17_stage1_atc_last_frame_1000p_21f_seed0_4gpu_b16_acc1_2000steps", + "disca": "output/layer17_stage1_disca_1000p_21f_seed0_4gpu_b16_acc1_2000steps", + # Stage-2 random-exit DMD (LoRA critic), EMA weights (a (run dir, file name) pair; + # a plain directory gets checkpoint_step_/predictor.safetensors). + "atc_chunk_s2": ("output/layer17_stage2_dmd_atc_chunk_lora128_4gpu_b2_2000steps", "predictor_ema.safetensors"), +} +NAIVE = {"naive2step": [0, 3]} # pseudo-models: reduced step grid, no Predictor + + +def checkpoint_weights(model, step): + """Weights file of ``model`` at training step ``step`` (Stage-1 runs zero-pad the + checkpoint directory, Stage-2 does not).""" + spec = MODELS[model] + run_dir, fname = spec if isinstance(spec, tuple) else (spec, "predictor.safetensors") + for d in (f"checkpoint_step_{step:04d}", f"checkpoint_step_{step}"): + w = os.path.join(REPO, run_dir, d, fname) + if os.path.exists(w): + return w + raise FileNotFoundError(f"{model} step {step}: no checkpoint under {os.path.join(REPO, run_dir)}") +PATTERNS = {"FPFF": {1}, "FPPF": {1, 2}, "FPPP": {1, 2, 3}} +# Reduced schedules: which step indices of the 4-step grid a chunk (after chunk 0) +# actually visits. FPF keeps steps 0 and 3 (t = 1000 / 250) as Full and runs one +# Predictor step at step 1 (t = 750, anchored on step 0 as in training), skipping +# step 2 entirely: 3 denoising positions per chunk, 2 Full + 1 Predictor. +RUN_STEPS = {"FPF": [0, 1, 3]} +PATTERNS["FPF"] = {1} +NUM_CHUNKS, FRAMES_PER_CHUNK, NUM_STEPS = 7, 3, 4 +# A Predictor step runs 1 of the 30 Teacher blocks (plus a small fusion MLP); +# its compute-equivalent cost is booked as 1/30 of a full forward. +PREDICTOR_COMPUTE_EQUIV = 1.0 / 30.0 + + +def parse_args(): + p = argparse.ArgumentParser() + p.add_argument("--shard", type=int, default=0) + p.add_argument("--num-shards", type=int, default=1) + p.add_argument("--out-root", default="eval_out") + p.add_argument("--seed", type=int, default=0) + p.add_argument("--fps", type=int, default=16) + p.add_argument("--models", default=",".join(MODELS)) + p.add_argument("--patterns", default="FPPF,FPPP") + p.add_argument("--checkpoint-step", type=int, default=2000) + p.add_argument("--num-chunks", type=int, default=NUM_CHUNKS, + help="Video length in 3-latent-frame chunks (7 = the 81-frame protocol; longer " + "videos use a rolling 21-frame KV window like Self-Forcing's long-video mode)") + p.add_argument("--ref-tag", default="", + help="Inserted into the reference name, e.g. _14c -> cfa_14c_ffff (a separate FFFF " + "reference for a different video length)") + p.add_argument("--tag", default="", + help="Inserted into strategy names after the model, e.g. _s1000 -> " + "cfa_disca_s1000_fppf, so another checkpoint does not overwrite the default rows") + p.add_argument("--regenerate-ffff", action="store_true", + help="Regenerate the FFFF reference instead of reading cfa_ffff's MP4") + p.add_argument("--limit", type=int, default=None, help="Smoke-test: first N prompts") + p.add_argument("--overwrite", action="store_true") + p.add_argument("--retime-only", action="store_true", + help="Do not regenerate videos/metrics; re-time FFFF and each strategy in the same " + "process per prompt and update policy_latency_ms / matched_ffff_policy_latency_ms.") + p.add_argument("--retime-max-ffff-ms", type=float, default=3500.0, + help="With --retime-only, records whose paired FFFF exceeds this are re-timed again " + "(up to --retime-passes passes); clean records are left alone.") + p.add_argument("--retime-passes", type=int, default=3) + p.add_argument("--retime-strategy-only", action="store_true", + help="Re-time only the strategies (no FFFF run): rerun each selected record's " + "rollout on an idle GPU and replace policy_latency_ms; the generation-run " + "value is kept under generation_run_latency.") + p.add_argument("--written-before", type=float, default=None, + help="With --retime-strategy-only: only records whose file mtime is before this " + "epoch second (the window in which another job shared the GPU)") + return p.parse_args() + + +def build_strategies(args): + out = [{"name": f"cfa{args.ref_tag}_ffff", "model": None, "pattern": "FFFF", "is_reference": True}] + for token in args.models.split(","): + if token in NAIVE: + out.append({"name": f"cfa_{token}{args.tag}", "model": None, "pattern": token.upper(), + "naive": token, "is_reference": False}) + continue + model, _, step = token.partition("@") + step = int(step) if step else args.checkpoint_step + key = f"{model}@{step}" # one loaded Predictor per (model, step) + label = f"{model}_s{step}" if "@" in token else model + for pattern in args.patterns.split(","): + out.append({"name": f"cfa_{label}{args.tag}_{pattern.lower()}", "model": key, + "base_model": model, "checkpoint_step": step, + "pattern": pattern.upper(), "is_reference": False}) + return out + + +class FinalHiddenCapture: + """Grab the Teacher's final hidden (input of its output head) on a Full step.""" + + def __init__(self, teacher): + self.enabled, self.value = False, None + self.handle = teacher.head.register_forward_pre_hook(self._hook) + + def _hook(self, _module, inputs): + if self.enabled: + self.value = inputs[0].detach() + + def start(self): + self.value, self.enabled = None, True + + def finish(self): + self.enabled = False + if self.value is None: + raise RuntimeError("Teacher final hidden was not captured") + value, self.value = self.value, None + return value + + +def main(): + args = parse_args() + out_root = args.out_root if os.path.isabs(args.out_root) else os.path.join(ROOT, args.out_root) + with open(MAPPING) as f: + rows = json.load(f)["rows"] + if args.limit: + rows = rows[:args.limit] + shard_rows = rows[args.shard::args.num_shards] + strategies = build_strategies(args) + ref_name = strategies[0]["name"] + print(f"cfa shard {args.shard}/{args.num_shards}: {len(shard_rows)} prompts x " + f"{len(strategies)} strategies", flush=True) + + os.chdir(REPO) + sys.path.insert(0, ROOT) + sys.path.insert(0, REPO) # pipeline/, utils/, predictor_training/ from Causal-Forcing-a + + import torch + from einops import rearrange + from safetensors.torch import load_file + from torchvision.io import read_video, write_video + + from eval.pixel_metrics import PixelMetrics + from harness import load_pipeline + from predictor_training.metadata import build_single_block_predictor, read_predictor_config + from predictor_training.online import online_predictor_step as _online_predictor_step + from predictor_training.offline_data import TOKENS_PER_CHUNK + + def online_predictor_step(*, history_cache, chunk, **kw): + """Stage-1 contract: the Predictor sees the clean K/V of the *earlier* chunks. + With the rolling 21-frame window (long videos) the cache holds the last six + clean chunks plus the current noisy one, so the history is everything before + the current chunk's tokens rather than ``chunk * TOKENS_PER_CHUNK`` (which + the stock helper asserts and which exceeds the window from chunk 7 on). + For chunk <= 6 the two are identical.""" + if chunk * TOKENS_PER_CHUNK <= int(history_cache["local_end_index"].item()): + return _online_predictor_step(history_cache=history_cache, chunk=chunk, **kw) + end = int(history_cache["local_end_index"].item()) - TOKENS_PER_CHUNK + window = {"k": history_cache["k"][:, :end], "v": history_cache["v"][:, :end], + "local_end_index": torch.tensor([end], device=history_cache["k"].device)} + # left-pad to the absolute length the stock helper slices; the attention + # window (max_attention_size = 21 frames) never reaches the padding + pad = chunk * TOKENS_PER_CHUNK - end + if pad > 0: + z = history_cache["k"].new_zeros(history_cache["k"].shape[0], pad, *history_cache["k"].shape[2:]) + window = {"k": torch.cat([z, window["k"]], 1), "v": torch.cat([z, window["v"]], 1), + "local_end_index": torch.tensor([chunk * TOKENS_PER_CHUNK], device=z.device)} + return _online_predictor_step(history_cache=window, chunk=chunk, **kw) + from utils.misc import set_seed + + torch.set_grad_enabled(False) + device = torch.device("cuda") + pipeline = load_pipeline(BASE) + n_chunks = args.num_chunks + if n_chunks * FRAMES_PER_CHUNK > 21: + # Longer than the 21-latent-frame training context: roll the KV cache over a + # 21-frame local window (the model's long-video mechanism; RoPE is relative + # so positions inside the window stay in the trained range). + model = pipeline.generator.model + for m in [model] + [b for b in model.blocks] + [b.self_attn for b in model.blocks]: + m.local_attn_size = 21 + if hasattr(m, "max_attention_size"): + m.max_attention_size = 21 * pipeline.frame_seq_length + pipeline.local_attn_size = 21 + pipeline.kv_cache1 = None + print(f"long video: {n_chunks} chunks = {n_chunks * FRAMES_PER_CHUNK} latent frames, " + f"rolling KV window 21 frames", flush=True) + teacher = pipeline.generator.model + capture = FinalHiddenCapture(teacher) + metrics = PixelMetrics(device) + timesteps = pipeline.denoising_step_list.to(device) + assert len(timesteps) == NUM_STEPS and pipeline.num_frame_per_block == FRAMES_PER_CHUNK + + predictors, predictor_configs = {}, {} + for model in {s["model"] for s in strategies if s["model"]}: + base, step = model.split("@") + weights = checkpoint_weights(base, int(step)) + config = read_predictor_config(weights) + predictor = build_single_block_predictor(teacher, config, gradient_checkpointing=False) + predictor.load_state_dict(load_file(weights, device="cpu"), strict=True) + predictors[model] = predictor.to(device).eval().requires_grad_(False) + predictor_configs[model] = {**config, "weights": weights} + print(f"loaded {model}: {config}", flush=True) + + def reset_caches(): + if pipeline.kv_cache1 is None: + pipeline._initialize_kv_cache(1, torch.bfloat16, device) + pipeline._initialize_crossattn_cache(1, torch.bfloat16, device) + for cache in pipeline.kv_cache1: + cache["global_end_index"] = torch.tensor([0], dtype=torch.long, device=device) + cache["local_end_index"] = torch.tensor([0], dtype=torch.long, device=device) + for cache in pipeline.crossattn_cache: + cache["is_init"] = False + + def rollout(conditional, noise, predictor=None, predictor_steps=frozenset(), run_steps=None): + """Mirror of cachelib.runner.cached_inference with Predictor steps spliced in.""" + reset_caches() + output = torch.zeros_like(noise) + denoise_events, context_events = [], [] + full_calls = predictor_calls = 0 + previous_hidden = None + for chunk in range(n_chunks): + start_frame = chunk * FRAMES_PER_CHUNK + token_start = start_frame * pipeline.frame_seq_length + noisy_input = noise[:, start_frame:start_frame + FRAMES_PER_CHUNK] + current_hidden = [None] * NUM_STEPS + # chunk 0 always runs the full grid; later chunks may visit a subset + steps = list(range(NUM_STEPS)) if (run_steps is None or chunk == 0) else list(run_steps) + for si, step in enumerate(steps): + timestep = torch.ones([1, FRAMES_PER_CHUNK], device=device, dtype=torch.int64) * timesteps[step] + start_ev = torch.cuda.Event(enable_timing=True) + end_ev = torch.cuda.Event(enable_timing=True) + start_ev.record() + if predictor is not None and chunk > 0 and step in predictor_steps: + hidden, flow = online_predictor_step( + predictor=predictor, teacher=teacher, noisy_input=noisy_input, + timestep=timestep, anchor_timestep=torch.ones_like(timestep) * timesteps[step - 1], + anchor_hidden=current_hidden[step - 1], previous_hidden=previous_hidden[step], + history_cache=pipeline.kv_cache1[predictor.source_layer], + cross_cache=pipeline.crossattn_cache[predictor.source_layer], chunk=chunk) + denoised_pred = pipeline.generator._convert_flow_pred_to_x0( + flow_pred=flow.flatten(0, 1), xt=noisy_input.flatten(0, 1), + timestep=timestep.flatten(0, 1)).unflatten(0, flow.shape[:2]) + current_hidden[step] = hidden + predictor_calls += 1 + else: + capture.start() + _, denoised_pred = pipeline.generator( + noisy_image_or_video=noisy_input, conditional_dict=conditional, + timestep=timestep, kv_cache=pipeline.kv_cache1, + crossattn_cache=pipeline.crossattn_cache, current_start=token_start) + current_hidden[step] = capture.finish() + full_calls += 1 + end_ev.record() + denoise_events.append((start_ev, end_ev)) + if si < len(steps) - 1: + # Same RNG draw order as the protocol runner, so every strategy + # re-noises with identical samples. + noisy_input = pipeline.scheduler.add_noise( + denoised_pred.flatten(0, 1), torch.randn_like(denoised_pred.flatten(0, 1)), + timesteps[steps[si + 1]] * torch.ones([FRAMES_PER_CHUNK], device=device, dtype=torch.long), + ).unflatten(0, denoised_pred.shape[:2]) + output[:, start_frame:start_frame + FRAMES_PER_CHUNK] = denoised_pred + # Clean-context KV refresh: full DiT, timed separately, never added in. + ctx_start = torch.cuda.Event(enable_timing=True) + ctx_end = torch.cuda.Event(enable_timing=True) + ctx_start.record() + pipeline.generator( + noisy_image_or_video=denoised_pred, conditional_dict=conditional, + timestep=torch.ones_like(timestep) * pipeline.args.context_noise, + kv_cache=pipeline.kv_cache1, crossattn_cache=pipeline.crossattn_cache, + current_start=token_start) + ctx_end.record() + context_events.append((ctx_start, ctx_end)) + previous_hidden = current_hidden + torch.cuda.synchronize() + timing = { + "denoise_dit_ms": sum(s.elapsed_time(e) for s, e in denoise_events), + "context_kv_dit_ms": sum(s.elapsed_time(e) for s, e in context_events), + "num_denoise_forwards": len(denoise_events), + } + video = pipeline.vae.decode_to_pixel(output, use_cache=False) + video = (video * 0.5 + 0.5).clamp(0, 1) + pipeline.vae.model.clear_cache() + return video, timing, {"full_forwards": full_calls, "predictor_forwards": predictor_calls} + + def run_one(strategy, row): + set_seed(args.seed) + noise = torch.randn([1, n_chunks * FRAMES_PER_CHUNK, 16, 60, 104], device=device, dtype=torch.bfloat16) + conditional = pipeline.text_encoder(text_prompts=[row["extended_prompt"]]) + wall0 = time.time() + if strategy["is_reference"]: + video, timing, counts = rollout(conditional, noise) + elif strategy.get("naive"): + video, timing, counts = rollout(conditional, noise, None, frozenset(), + NAIVE[strategy["naive"]]) + else: + video, timing, counts = rollout(conditional, noise, predictors[strategy["model"]], + PATTERNS[strategy["pattern"]], + RUN_STEPS.get(strategy["pattern"])) + return video, timing, counts, time.time() - wall0 + + def record_path(strategy, row): + d = os.path.join(out_root, "per_prompt", strategy) + os.makedirs(d, exist_ok=True) + return os.path.join(d, f"{row['prompt_suite']}_{row['suite_index']:03d}.json") + + def video_path(strategy, row): + d = os.path.join(out_root, "generated_videos", strategy, row["prompt_suite"]) + os.makedirs(d, exist_ok=True) + return os.path.join(d, f"{row['suite_index']:03d}.mp4") + + def is_done(strategy, row): + p = record_path(strategy, row) + if args.overwrite or not os.path.exists(p): + return False + try: + with open(p) as f: + rec = json.load(f) + return rec.get("status") == "complete" and os.path.exists(rec.get("video", "")) + except Exception: + return False + + def write_json(path, payload): + tmp = path + ".tmp" + with open(tmp, "w") as f: + json.dump(payload, f, indent=2) + os.replace(tmp, path) + + if shard_rows: + print("warm-up pass (not recorded)", flush=True) + for strategy in strategies: + run_one(strategy, shard_rows[0]) + torch.cuda.empty_cache() + + if args.retime_strategy_only: + targets = [s for s in strategies if not s["is_reference"]] + n_done = 0 + for n, row in enumerate(shard_rows): + for strategy in targets: + rp = record_path(strategy["name"], row) + if not os.path.exists(rp): + continue + if args.written_before and os.path.getmtime(rp) >= args.written_before: + continue + rec = json.load(open(rp)) + if rec.get("status") != "complete" or "generation_run_latency" in rec: + continue + _, timing, counts, _ = run_one(strategy, row) + assert counts["full_forwards"] == rec["cache_diagnostics"]["full_forwards"], rp + rec["generation_run_latency"] = { + "policy_latency_ms": rec["policy_latency_ms"], + "excluded_context_kv_latency_ms": rec["excluded_context_kv_latency_ms"]} + rec["policy_latency_ms"] = timing["denoise_dit_ms"] + rec["excluded_context_kv_latency_ms"] = timing["context_kv_dit_ms"] + rec["latency_retimed"] = "strategy-only rerun on an idle GPU (FFFF not re-run)" + write_json(rp, rec) + n_done += 1 + if (n + 1) % 20 == 0: + print(f"[strategy retime shard {args.shard}] {n + 1}/{len(shard_rows)}", flush=True) + print(f"shard {args.shard}: {n_done} records re-timed", flush=True) + return + + if args.retime_only: + # Protocol "matched FFFF": pair every strategy with an FFFF run of the same + # prompt in the same process, seconds apart, so contention cancels in the + # ratio. Only records whose pairing is missing or dirty are touched. + targets = [s for s in strategies if not s["is_reference"]] + for pass_index in range(args.retime_passes): + dirty = 0 + for n, row in enumerate(shard_rows): + todo = [] + for strategy in targets: + rp = record_path(strategy["name"], row) + if not os.path.exists(rp): + continue + rec = json.load(open(rp)) + m = rec.get("matched_ffff_policy_latency_ms") + if m is None or m > args.retime_max_ffff_ms: + todo.append((strategy, rp, rec)) + if not todo: + continue + dirty += len(todo) + _, ref_timing, _, _ = run_one(strategies[0], row) + for strategy, rp, rec in todo: + _, timing, _, _ = run_one(strategy, row) + rec["policy_latency_ms"] = timing["denoise_dit_ms"] + rec["excluded_context_kv_latency_ms"] = timing["context_kv_dit_ms"] + rec["matched_ffff_policy_latency_ms"] = ref_timing["denoise_dit_ms"] + rec["matched_ffff_context_kv_latency_ms"] = ref_timing["context_kv_dit_ms"] + rec["latency_source"] = "paired retime (same process as FFFF)" + write_json(rp, rec) + if (n + 1) % 10 == 0: + print(f"[retime pass {pass_index} shard {args.shard}] {n + 1}/{len(shard_rows)} " + f"ffff={ref_timing['denoise_dit_ms']:.0f}ms", flush=True) + print(f"[retime pass {pass_index} shard {args.shard}] re-timed {dirty} records", flush=True) + if dirty == 0: + break + return + + t_start = time.time() + for n, row in enumerate(shard_rows): + todo = [s for s in strategies if not is_done(s["name"], row)] + if not todo: + continue + need_ref = any(not s["is_reference"] for s in todo) + ref_frames = None + ref_timing = None + ref_from_mp4 = False + if need_ref and os.path.exists(video_path(ref_name, row)) and not args.regenerate_ffff: + # Reference from the cfa_ffff run's MP4 (H.264, 40+ dB above the strategy + # PSNRs); FFFF is not regenerated, so there is no same-process FFFF timing + # and the speedup is taken against cfa_ffff's own records (aggregate.py). + vid, _, _ = read_video(video_path(ref_name, row), pts_unit="sec", output_format="TCHW") + ref_frames = vid.to(device, torch.float16) / 255.0 + ref_from_mp4 = True + need_ref = False + for strategy in strategies: + if strategy["is_reference"]: + if strategy not in todo and not need_ref: + continue + elif strategy not in todo: + continue + video, timing, counts, wall = run_one(strategy, row) + frames = video[0] + if strategy["is_reference"]: + ref_frames = frames.detach().to(torch.float16) + ref_timing = timing + pix = PixelMetrics.identity(frames.shape[0]) + else: + pix = metrics.compute(frames, ref_frames) + if strategy in todo: + vp = video_path(strategy["name"], row) + arr = (255.0 * rearrange(frames, "t c h w -> t h w c")).clamp(0, 255) + write_video(vp, arr.to(torch.uint8).cpu(), fps=args.fps) + compute_equiv = counts["full_forwards"] + PREDICTOR_COMPUTE_EQUIV * counts["predictor_forwards"] + model = strategy["model"] + naive = strategy.get("naive") + schedule = "F" * len(NAIVE[naive]) if naive else strategy["pattern"] + write_json(record_path(strategy["name"], row), { + "status": "complete", + "protocol": "Self-Forcing Extended-251 Full Evaluation", + "strategy": strategy["name"], + "base_model": "causal_forcing_a", + "method": "none" if model is None else f"predictor_{strategy['base_model']}", + "param": None if model is None else "pattern", + "param_value": None if model is None else strategy["pattern"], + "target_speedup": (n_chunks * NUM_STEPS) / compute_equiv, + "schedule": schedule, + "num_inference_steps": len(NAIVE[naive]) if naive else NUM_STEPS, + "first_chunk_schedule": "FFFF" if (naive or model) else None, + "predictor_config": None if model is None else predictor_configs[model], + "global_index": row["global_index"], + "prompt_suite": row["prompt_suite"], + "suite_index": row["suite_index"], + "prompt": row["extended_prompt"], + "original_prompt": row["original_prompt"], + "seed": args.seed, + "video": vp, + "num_frames": int(frames.shape[0]), + "height": int(frames.shape[2]), + "width": int(frames.shape[3]), + "fps": args.fps, + "policy_latency_ms": timing["denoise_dit_ms"], + "excluded_context_kv_latency_ms": timing["context_kv_dit_ms"], + # Same-process FFFF timing of this prompt (protocol "matched FFFF"). + "matched_ffff_policy_latency_ms": ref_timing["denoise_dit_ms"] if ref_timing else None, + "matched_ffff_context_kv_latency_ms": ref_timing["context_kv_dit_ms"] if ref_timing else None, + "checkpoint_step": strategy.get("checkpoint_step"), + "num_chunks": n_chunks, + "wall_generation_s": wall, + "reference_strategy": ref_name, + "pixel_metrics_vs_ffff": pix, + "reference_source": "ffff_mp4" if ref_from_mp4 else "ffff_frames", + "cache_diagnostics": { + "denoise_forwards": timing["num_denoise_forwards"], + "full_forwards": counts["full_forwards"], + "predictor_forwards": counts["predictor_forwards"], + "predictor_compute_equivalent": PREDICTOR_COMPUTE_EQUIV, + "compute_equivalent_forwards": compute_equiv, + "middle_steps": n_chunks * 2, + "middle_compute_equivalent": compute_equiv - n_chunks * 2.0, + }, + }) + del frames, video + ref_frames = None + done = n + 1 + rate = (time.time() - t_start) / done + print(f"[cfa shard {args.shard}] {done}/{len(shard_rows)} prompts " + f"({row['prompt_suite']}/{row['suite_index']:03d}) {rate:.1f}s/prompt " + f"eta {(len(shard_rows) - done) * rate / 60:.0f} min", flush=True) + print(f"shard {args.shard} done in {(time.time() - t_start) / 60:.1f} min", flush=True) + + +if __name__ == "__main__": + main() diff --git a/eval/pixel_metrics.py b/eval/pixel_metrics.py new file mode 100644 index 0000000000000000000000000000000000000000..d1e5395113ba83cd2e9bd4d6b7fc18d09f6bf3be --- /dev/null +++ b/eval/pixel_metrics.py @@ -0,0 +1,85 @@ +"""PSNR / SSIM / LPIPS between a strategy's frames and its matched FFFF frames. + +Computed on the decoded RGB tensors in [0, 1], before MP4 encoding, as the +protocol requires (section 8.1) -- video compression noise would otherwise leak +into the numbers. + +SSIM is implemented here rather than taken from skimage: skimage needs ~30 ms per +480x832 frame on CPU, and this evaluation compares ~500k frames. +""" + +import torch +import torch.nn.functional as F + + +def _gaussian_window(size=11, sigma=1.5, device=None, dtype=torch.float32): + coords = torch.arange(size, device=device, dtype=dtype) - (size - 1) / 2.0 + g = torch.exp(-(coords ** 2) / (2 * sigma ** 2)) + g = g / g.sum() + return g[:, None] @ g[None, :] + + +class PixelMetrics: + """Frame-wise metrics against a reference clip. Call once per (strategy, prompt).""" + + def __init__(self, device, lpips_net="alex", ssim_window=11, ssim_sigma=1.5): + self.device = device + self.window = _gaussian_window(ssim_window, ssim_sigma, device=device)[None, None] + self.window_size = ssim_window + import lpips + self.lpips = lpips.LPIPS(net=lpips_net).to(device).eval() + for p in self.lpips.parameters(): + p.requires_grad_(False) + + def ssim(self, x, y): + """x, y: [T, C, H, W] in [0, 1]. Returns the mean over frames and channels.""" + t, c, h, w = x.shape + win = self.window.expand(c, 1, self.window_size, self.window_size) + pad = self.window_size // 2 + mu_x = F.conv2d(x, win, padding=pad, groups=c) + mu_y = F.conv2d(y, win, padding=pad, groups=c) + mu_x2, mu_y2, mu_xy = mu_x * mu_x, mu_y * mu_y, mu_x * mu_y + sig_x = F.conv2d(x * x, win, padding=pad, groups=c) - mu_x2 + sig_y = F.conv2d(y * y, win, padding=pad, groups=c) - mu_y2 + sig_xy = F.conv2d(x * y, win, padding=pad, groups=c) - mu_xy + c1, c2 = 0.01 ** 2, 0.03 ** 2 + s = ((2 * mu_xy + c1) * (2 * sig_xy + c2)) / ((mu_x2 + mu_y2 + c1) * (sig_x + sig_y + c2)) + # Drop the padded border, as skimage does with its default crop. + s = s[..., pad:-pad, pad:-pad] + return s.mean(dim=(1, 2, 3)) # per frame + + @torch.no_grad() + def compute(self, frames, ref, chunk=16): + """frames, ref: [T, C, H, W] float in [0, 1] on any device. + + Returns per-prompt mean MSE (for the protocol's ratio-of-means PSNR), + plus mean SSIM and mean LPIPS over frames. + """ + frames = frames.to(self.device, torch.float32) + ref = ref.to(self.device, torch.float32) + assert frames.shape == ref.shape, (frames.shape, ref.shape) + + mse_terms, ssim_terms, lpips_terms = [], [], [] + for i in range(0, frames.shape[0], chunk): + a, b = frames[i:i + chunk], ref[i:i + chunk] + mse_terms.append(((a - b) ** 2).mean(dim=(1, 2, 3))) + ssim_terms.append(self.ssim(a, b)) + lpips_terms.append(self.lpips(a * 2 - 1, b * 2 - 1).flatten()) + + mse = torch.cat(mse_terms) + ssim = torch.cat(ssim_terms) + lp = torch.cat(lpips_terms) + mean_mse = mse.mean().item() + return { + "mean_mse": mean_mse, + "psnr": -10.0 * torch.log10(torch.tensor(max(mean_mse, 1e-12))).item(), + "ssim": ssim.mean().item(), + "lpips": lp.mean().item(), + "num_frames": int(frames.shape[0]), + } + + @staticmethod + def identity(num_frames): + """Fixed self-comparison values the protocol pins for FFFF vs itself.""" + return {"mean_mse": 1e-12, "psnr": 120.0, "ssim": 1.0, "lpips": 0.0, + "num_frames": int(num_frames)} diff --git a/eval/retime_eval.py b/eval/retime_eval.py new file mode 100644 index 0000000000000000000000000000000000000000..643e0208b9a3bef2a09854769f8db4bbd2387403 --- /dev/null +++ b/eval/retime_eval.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python +"""Paired re-timing of evaluated strategies against FFFF, prompt by prompt. + +``generate_eval.py`` records each strategy's denoise latency, but when it is +re-run for *additional* strategies the FFFF reference is regenerated only for +the pixel metrics and its latency is not recorded, so the new records would be +compared against FFFF timings from a different run under different contention. +The protocol's speedup is "matched FFFF" (section 9.4): same prompt, same seed, +same conditions. This pass restores that: for every prompt it runs FFFF and +then each requested strategy back to back (denoise only, no decode, no metrics) +and writes the strategy's latency together with the FFFF latency measured +seconds earlier into the strategy's per-prompt record. + + CUDA_VISIBLE_DEVICES=0 python eval/retime_eval.py --base self_forcing \ + --strategies sf_taylorseer_FFxx_x1.3,sf_teacache_Fxxx_x3 \ + --shard 0 --num-shards 4 --out-root eval_out + +Records already carrying ``latency_source`` are skipped, so the pass resumes. +""" + +import argparse +import json +import os +import sys +import time + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, ROOT) + +from eval.strategies import load_strategies # noqa: E402 +from harness import BASES, enter_base, load_pipeline # noqa: E402 + +MAPPING = os.path.join(ROOT, "assets/vbench8_extended_subset_mapping.json") +SOURCE = "retime_eval: paired with FFFF in the same process, denoise only" + + +def parse_args(): + p = argparse.ArgumentParser() + p.add_argument("--base", choices=sorted(BASES), required=True) + p.add_argument("--strategies", required=True, + help="Comma-separated strategy names, or 'all' for every " + "non-reference strategy of the base") + p.add_argument("--shard", type=int, default=0) + p.add_argument("--num-shards", type=int, default=1) + p.add_argument("--out-root", default="eval_out") + p.add_argument("--seed", type=int, default=0) + p.add_argument("--num-latent-frames", type=int, default=21) + p.add_argument("--limit", type=int, default=None) + p.add_argument("--force", action="store_true", help="Re-time records already re-timed") + p.add_argument("--redo-ratio-bounds", default=None, + help="lo,hi: also re-time records whose measured/compute speedup " + "ratio falls outside this band -- a spike that hit the strategy " + "half of the pair rather than the FFFF half") + p.add_argument("--redo-above-ms", type=float, default=None, + help="Also re-time records whose matched FFFF latency exceeds this: " + "a neighbour's job hit that pair, and the pair is only fair " + "when both halves ran under the same conditions") + return p.parse_args() + + +def main(): + args = parse_args() + out_root = (args.out_root if os.path.isabs(args.out_root) + else os.path.join(ROOT, args.out_root)) + + with open(MAPPING) as f: + rows = json.load(f)["rows"] + if args.limit: + rows = rows[:args.limit] + shard_rows = rows[args.shard::args.num_shards] + + all_strategies = load_strategies(args.base) + reference = all_strategies[0] + assert reference["is_reference"] + if args.strategies == "all": + wanted = [s for s in all_strategies if not s["is_reference"]] + else: + names = set(args.strategies.split(",")) + wanted = [s for s in all_strategies if s["name"] in names] + missing = names - {s["name"] for s in wanted} + if missing: + sys.exit(f"unknown strategies for {args.base}: {sorted(missing)}") + + print(f"base={args.base} shard {args.shard}/{args.num_shards}: " + f"{len(shard_rows)} prompts x ({reference['name']} + {len(wanted)} strategies)", + flush=True) + + base = enter_base(args.base) + + import torch + from utils.misc import set_seed + + from cachelib import (CacheController, build_method, cached_inference, install, + parse_schedule) + + torch.set_grad_enabled(False) + device = torch.device("cuda") + pipeline = load_pipeline(base) + num_steps = len(pipeline.denoising_step_list) + + def record_path(strategy, row): + return os.path.join(out_root, "per_prompt", strategy, + f"{row['prompt_suite']}_{row['suite_index']:03d}.json") + + def time_one(strategy, row): + kw = dict(indicator=strategy["indicator"], coefficients=strategy["coefficients"]) + if strategy["method"] != "none": + kw[strategy["param"]] = strategy["value"] + first = strategy.get("first_chunk_schedule") + ctrl = CacheController( + build_method(strategy["method"], **kw), num_steps=num_steps, + forced_steps=parse_schedule(strategy.get("schedule", "FxxF"), num_steps), + first_chunk_forced_steps=(parse_schedule(first, num_steps) if first else None)) + install(pipeline.generator.model, ctrl) + set_seed(args.seed) + noise = torch.randn([1, args.num_latent_frames, 16, 60, 104], + device=device, dtype=torch.bfloat16) + _, _, timing = cached_inference(pipeline, ctrl, noise, [row["extended_prompt"]], + decode=False) + return timing, ctrl.summary() + + def write_json(path, payload): + tmp = path + ".tmp" + with open(tmp, "w") as f: + json.dump(payload, f, indent=2) + os.replace(tmp, path) + + def needs(strategy, row): + p = record_path(strategy["name"], row) + if not os.path.exists(p): + return False # nothing to attach the timing to; generation must run first + with open(p) as f: + rec = json.load(f) + if rec.get("status") != "complete": + return False + if args.force or "latency_source" not in rec: + return True + if (args.redo_above_ms is not None + and rec.get("matched_ffff_policy_latency_ms", 0) > args.redo_above_ms): + return True + if args.redo_ratio_bounds: + lo, hi = (float(v) for v in args.redo_ratio_bounds.split(",")) + d = rec["cache_diagnostics"] + compute_x = d["denoise_forwards"] / max(d["compute_equivalent_forwards"], 1e-6) + measured_x = rec["matched_ffff_policy_latency_ms"] / rec["policy_latency_ms"] + if not (lo <= measured_x / compute_x <= hi): + return True + return False + + if shard_rows: + print("warm-up pass (not recorded)", flush=True) + for s in [reference] + wanted: + time_one(s, shard_rows[0]) + torch.cuda.empty_cache() + + t0 = time.time() + for n, row in enumerate(shard_rows): + todo = [s for s in wanted if needs(s, row)] + if not todo: + continue + ref_timing, _ = time_one(reference, row) + for s in todo: + timing, summary = time_one(s, row) + p = record_path(s["name"], row) + with open(p) as f: + rec = json.load(f) + rec["policy_latency_ms_generation_run"] = rec.get("policy_latency_ms") + rec["policy_latency_ms"] = timing["denoise_dit_ms"] + rec["excluded_context_kv_latency_ms"] = timing["context_kv_dit_ms"] + rec["matched_ffff_policy_latency_ms"] = ref_timing["denoise_dit_ms"] + rec["latency_source"] = SOURCE + # The compute fraction is deterministic; recording it again guards + # against a mismatch between the two runs. + rec["cache_diagnostics"]["compute_equivalent_forwards_retime"] = summary.get( + "compute_equivalent_forwards") + write_json(p, rec) + done = n + 1 + rate = (time.time() - t0) / done + print(f"[{args.base} retime shard {args.shard}] {done}/{len(shard_rows)} " + f"({row['prompt_suite']}/{row['suite_index']:03d}) ffff={ref_timing['denoise_dit_ms']:.0f}ms " + f"{rate:.1f}s/prompt eta {(len(shard_rows) - done) * rate / 60:.0f} min", + flush=True) + + print(f"retime shard {args.shard} done in {(time.time() - t0) / 60:.1f} min", flush=True) + + +if __name__ == "__main__": + main() diff --git a/eval/retime_until_clean.sh b/eval/retime_until_clean.sh new file mode 100644 index 0000000000000000000000000000000000000000..1d4c65e9dcfc502520b95ddb36af4a479a6bff3c --- /dev/null +++ b/eval/retime_until_clean.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +# Self-healing driver for the paired re-timing on a shared node. +# +# Neighbours' jobs both inflate timings and, twice now, took our processes down +# with them. So: loop. Each pass counts the records that still need a fair +# pairing (never timed, FFFF half contended, or measured/compute ratio outside a +# sane band), launches one shard per *currently free* GPU among $GPUS, waits, and +# repeats until nothing is left or MAX_PASSES is hit. Then aggregate. +set -u +C=/local/zoubin/cz/projects/comparison +PY=/local/zoubin/cz/envs/self_forcing/bin/python +LOG="$1" +GPUS=(${GPUS:-4 5 6 7}) +THRESH_MS=${THRESH_MS:-3500} +RATIO=${RATIO:-0.5,1.1} +MAX_PASSES=${MAX_PASSES:-12} +MIN_FREE=${MIN_FREE:-2} # do not start a pass on fewer free GPUs than this +WAIT=${WAIT:-120} # seconds between scans while waiting for GPUs +NEW_SF=sf_teacache_Fxxx_x3,sf_flowcache_Fxxx_x3,sf_taylorseer_FFxx_x1.3,sf_taylorseer_FFxx_x1.6,sf_taylorseer_FFxx_x2,sf_motioncache_Fxxx_x3 +NEW_CF=cf_teacache_Fxxx_x3,cf_flowcache_Fxxx_x3,cf_taylorseer_FFxx_x1.3,cf_taylorseer_FFxx_x1.6,cf_taylorseer_FFxx_x2,cf_motioncache_Fxxx_x3 + +export SSL_CERT_FILE=/etc/pki/tls/certs/ca-bundle.crt +export HF_HOME=/local/zoubin/cz/.cache/huggingface HF_HUB_CACHE=/local/zoubin/cz/.cache/huggingface/hub +export TRANSFORMERS_CACHE=/local/zoubin/cz/.cache/huggingface/hub TORCH_HOME=/local/zoubin/cz/.cache/torch +export XDG_CACHE_HOME=/local/zoubin/cz/.cache +cd "$C" + +pending() { # how many records of $1's strategies still need work + $PY - "$1" "$THRESH_MS" "$RATIO" <<'PYEOF' +import glob, json, sys +strats, thresh, ratio = sys.argv[1].split(","), float(sys.argv[2]), [float(v) for v in sys.argv[3].split(",")] +n = 0 +for s in strats: + for p in glob.glob(f"eval_out/per_prompt/{s}/*.json"): + r = json.load(open(p)) + if "latency_source" not in r or r["matched_ffff_policy_latency_ms"] > thresh: + n += 1; continue + d = r["cache_diagnostics"] + cx = d["denoise_forwards"] / max(d["compute_equivalent_forwards"], 1e-6) + mx = r["matched_ffff_policy_latency_ms"] / r["policy_latency_ms"] + if not (ratio[0] <= mx / cx <= ratio[1]): + n += 1 +print(n) +PYEOF +} + +free_gpus() { # members of GPUS that are actually quiet: little memory AND no compute + # Memory alone is not enough -- a VBench scorer holds 2 GB and saturates the SMs, + # and that alone inflated a whole pass by 25-80%. Sample utilisation 3x over 3 s. + for g in "${GPUS[@]}"; do + used=$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits -i "$g" 2>/dev/null | tr -d ' ') + [ -z "$used" ] || [ "$used" -ge 3000 ] && continue + peak=0 + for _ in 1 2 3; do + u=$(nvidia-smi --query-gpu=utilization.gpu --format=csv,noheader,nounits -i "$g" 2>/dev/null | tr -d ' ') + [ -n "$u" ] && [ "$u" -gt "$peak" ] && peak=$u + sleep 1 + done + [ "$peak" -lt 10 ] && echo "$g" + done +} + +# Only GPUs nobody else is using are ever touched; waiting never counts as a pass. +pass=0 +while [ "$pass" -lt "$MAX_PASSES" ]; do + n_sf=$(pending "$NEW_SF"); n_cf=$(pending "$NEW_CF") + echo "=== scan ($(date +%H:%M:%S)): pending sf=$n_sf cf=$n_cf, passes so far $pass ===" + [ "$n_sf" -eq 0 ] && [ "$n_cf" -eq 0 ] && break + mapfile -t FREE < <(free_gpus) + if [ "${#FREE[@]}" -lt "$MIN_FREE" ]; then + echo "only ${#FREE[@]} free GPU(s) among ${GPUS[*]} (need $MIN_FREE); waiting ${WAIT}s" + sleep "$WAIT"; continue + fi + pass=$((pass+1)) + echo "using GPUs: ${FREE[*]}" + for BASE in self_forcing causal_forcing; do + if [ "$BASE" = self_forcing ]; then S=$NEW_SF; n=$n_sf; else S=$NEW_CF; n=$n_cf; fi + [ "$n" -eq 0 ] && continue + pids=() + for i in "${!FREE[@]}"; do + CUDA_VISIBLE_DEVICES=${FREE[$i]} $PY eval/retime_eval.py --base "$BASE" --strategies "$S" \ + --shard "$i" --num-shards "${#FREE[@]}" --out-root eval_out \ + --redo-above-ms "$THRESH_MS" --redo-ratio-bounds "$RATIO" \ + > "$LOG/retime_${BASE}_pass${pass}_$i.log" 2>&1 & + pids+=($!) + done + for p in "${pids[@]}"; do wait "$p"; done + echo " $BASE pass $pass finished ($(date +%H:%M:%S))" + done +done +echo "=== final pending: sf=$(pending "$NEW_SF") cf=$(pending "$NEW_CF") ===" +echo "=== aggregating ($(date +%H:%M:%S)) ===" +# --only: the predictor evaluation shares eval_out/ and may still be incomplete. +$PY eval/aggregate.py --out-root "$C/eval_out" --only '^(sf|cf)_' 2>&1 | tee "$LOG/aggregate.log" +python3 experiment_md.py 2>&1 | tee "$LOG/experiment_md.log" +echo "ALL RETIME DONE $(date +%H:%M:%S)" diff --git a/eval/run_all_phases.sh b/eval/run_all_phases.sh new file mode 100644 index 0000000000000000000000000000000000000000..2fa7b6a58a3b1a50331d6eb76a59e6ff586dc248 --- /dev/null +++ b/eval/run_all_phases.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# Waits for the running generation phase, then scores and aggregates. +set -u +C=/local/zoubin/cz/projects/comparison +LOG="$1" + +echo "waiting for generation to finish ..." +while ! grep -q "ALL GENERATION DONE" "$LOG/generation_driver.log" 2>/dev/null; do + if ! pgrep -f "[g]enerate_eval.py" > /dev/null 2>&1 \ + && ! pgrep -f "[r]un_generation.sh" > /dev/null 2>&1; then + echo "generation processes gone without the done marker; stopping." + exit 1 + fi + sleep 60 +done +echo "generation complete $(date +%H:%M:%S)" + +"$C/eval/run_vbench_all.sh" "$LOG" 2>&1 | tee "$LOG/vbench_driver.log" + +echo "=== aggregating $(date +%H:%M:%S) ===" +/local/zoubin/cz/envs/self_forcing/bin/python "$C/eval/aggregate.py" \ + --out-root "$C/eval_out" 2>&1 | tee "$LOG/aggregate.log" +echo "ALL PHASES DONE $(date +%H:%M:%S)" diff --git a/eval/run_generation.sh b/eval/run_generation.sh new file mode 100644 index 0000000000000000000000000000000000000000..3f02e4fd0e3e8c4199e21f165f55c36f3587864d --- /dev/null +++ b/eval/run_generation.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# Phase 1: generate the Extended-251 videos for every strategy, one base at a time, +# sharded by prompt across the GPUs in $GPUS (default 0-3). Generation runs in the base self_forcing env -- +# the same interpreter and package set the speed study used -- so the eval videos +# are produced by exactly the model stack that was benchmarked. +set -u +C=/local/zoubin/cz/projects/comparison +PY=/local/zoubin/cz/envs/self_forcing/bin/python +LOG="$1"; shift +NSHARDS=${NSHARDS:-4} +GPUS=(${GPUS:-0 1 2 3}) + +export SSL_CERT_FILE=/etc/pki/tls/certs/ca-bundle.crt +export TORCH_HOME=/local/zoubin/cz/.cache/torch +export HF_HOME=/local/zoubin/cz/.cache/huggingface +export HF_HUB_CACHE=/local/zoubin/cz/.cache/huggingface/hub +export TRANSFORMERS_CACHE=/local/zoubin/cz/.cache/huggingface/hub +export XDG_CACHE_HOME=/local/zoubin/cz/.cache + +cd "$C" +for BASE in ${BASES:-self_forcing causal_forcing}; do + echo "=== generation: $BASE ($(date +%H:%M:%S)) ===" + pids=() + for i in $(seq 0 $((NSHARDS-1))); do + CUDA_VISIBLE_DEVICES=${GPUS[$i]} nohup $PY eval/generate_eval.py \ + --base "$BASE" --shard $i --num-shards $NSHARDS --out-root eval_out \ + > "$LOG/gen_${BASE}_$i.log" 2>&1 & + pids+=($!) + done + for p in "${pids[@]}"; do wait "$p"; done + echo "=== generation done: $BASE ($(date +%H:%M:%S)) ===" +done +echo "ALL GENERATION DONE" diff --git a/eval/run_predictor_eval.sh b/eval/run_predictor_eval.sh new file mode 100644 index 0000000000000000000000000000000000000000..c6af58b24e6ab07227d50e4ef21cfea3e3ffcdf6 --- /dev/null +++ b/eval/run_predictor_eval.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# Extended-251 evaluation of the Causal-Forcing-a Predictors (cfa_* strategies): +# phase 1 generation sharded across $GPUS, phase 2 VBench on the not-yet-scored +# strategies (run_vbench_all.sh skips scored ones), phase 3 aggregate. +# GPUS="0 1 2 3" eval/run_predictor_eval.sh logs_cfa +set -u +C=/local/zoubin/cz/projects/comparison +PY=/local/zoubin/cz/envs/self_forcing/bin/python +LOG="$(cd "$(dirname "$1")" && pwd)/$(basename "$1")"; shift +GPUS=(${GPUS:-0 1 2 3}) +NSHARDS=${#GPUS[@]} +mkdir -p "$LOG" + +export SSL_CERT_FILE=/etc/pki/tls/certs/ca-bundle.crt +export TORCH_HOME=/local/zoubin/cz/.cache/torch +export HF_HOME=/local/zoubin/cz/.cache/huggingface +export HF_HUB_CACHE=/local/zoubin/cz/.cache/huggingface/hub +export TRANSFORMERS_CACHE=/local/zoubin/cz/.cache/huggingface/hub +export XDG_CACHE_HOME=/local/zoubin/cz/.cache + +cd "$C" +echo "=== cfa generation ($(date +%H:%M:%S)) gpus=${GPUS[*]} ===" +pids=() +for i in $(seq 0 $((NSHARDS-1))); do + CUDA_VISIBLE_DEVICES=${GPUS[$i]} $PY eval/generate_predictor_eval.py \ + --shard $i --num-shards $NSHARDS --out-root eval_out "$@" \ + > "$LOG/gen_cfa_$i.log" 2>&1 & + pids+=($!) +done +status=0 +for p in "${pids[@]}"; do wait "$p" || status=1; done +echo "=== cfa generation done ($(date +%H:%M:%S)) status=$status ===" +[[ $status == 0 ]] || { echo "generation failed; not scoring"; exit 1; } + +GPUS="${GPUS[*]}" NGPU=$NSHARDS bash "$C/eval/run_vbench_all.sh" "$LOG" 2>&1 | tee "$LOG/vbench_driver.log" + +echo "=== aggregating ($(date +%H:%M:%S)) ===" +$PY "$C/eval/aggregate.py" --out-root "$C/eval_out" 2>&1 | tee "$LOG/aggregate.log" +echo "ALL CFA PHASES DONE $(date +%H:%M:%S)" diff --git a/eval/run_predictor_eval_s2.sh b/eval/run_predictor_eval_s2.sh new file mode 100644 index 0000000000000000000000000000000000000000..6ee978f6aadba0e7cf3f77374d944ea6ab1d277c --- /dev/null +++ b/eval/run_predictor_eval_s2.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# Extended-251 evaluation of the Stage-2 EMA Predictor: generation sharded over +# $GPUS, VBench only for the strategies generated here, then aggregate. +# GPUS="0 1 2" bash eval/run_predictor_eval_s2.sh logs_cfa_s2 +set -u +C=/local/zoubin/cz/projects/comparison +V=/local/zoubin/cz/projects/VBench +PY=/local/zoubin/cz/envs/self_forcing/bin/python +VPY=/local/zoubin/cz/envs/vbench_eval/bin/python +LOG="$(mkdir -p "$1" && cd "$1" && pwd)" +MODELS="${MODELS:-atc_chunk_s2}" +PATTERNS="${PATTERNS:-FPFF,FPPF,FPPP}" +GPUS=(${GPUS:-0 1 2}) +NSHARDS=${#GPUS[@]} + +export SSL_CERT_FILE=/etc/pki/tls/certs/ca-bundle.crt +export TORCH_HOME=/local/zoubin/cz/.cache/torch +export HF_HOME=/local/zoubin/cz/.cache/huggingface +export HF_HUB_CACHE=/local/zoubin/cz/.cache/huggingface/hub +export TRANSFORMERS_CACHE=/local/zoubin/cz/.cache/huggingface/hub +export XDG_CACHE_HOME=/local/zoubin/cz/.cache +export VBENCH_CACHE_DIR=/local/zoubin/cz/.cache/vbench + +cd "$C" +echo "=== generation ($(date +%H:%M:%S)) gpus=${GPUS[*]} models=$MODELS patterns=$PATTERNS ===" +pids=() +for i in $(seq 0 $((NSHARDS-1))); do + CUDA_VISIBLE_DEVICES=${GPUS[$i]} $PY eval/generate_predictor_eval.py \ + --shard $i --num-shards $NSHARDS --out-root eval_out --models "$MODELS" --patterns "$PATTERNS" \ + > "$LOG/gen_$i.log" 2>&1 & + pids+=($!) +done +status=0; for p in "${pids[@]}"; do wait "$p" || status=1; done +echo "=== generation done ($(date +%H:%M:%S)) status=$status ===" +[[ $status == 0 ]] || { echo "generation failed; not scoring"; exit 1; } + +strats=() +for m in ${MODELS//,/ }; do for pat in ${PATTERNS//,/ }; do strats+=("cfa_${m}_$(echo "$pat" | tr 'A-Z' 'a-z')"); done; done +echo "=== vbench ($(date +%H:%M:%S)) strategies=${strats[*]} ===" +export PYTHONPATH="$V:$C" +cd "$V" +pids=() +for g in $(seq 0 $((NSHARDS-1))); do + ( + for i in $(seq $g $NSHARDS $(( ${#strats[@]} - 1 ))); do + s="${strats[$i]}" + [[ -f "$C/eval_out/vbench/scores/$s.json" ]] && { echo "skip $s"; continue; } + echo "[gpu ${GPUS[$g]}] $s $(date +%H:%M:%S)" + CUDA_VISIBLE_DEVICES=${GPUS[$g]} $VPY "$C/eval/run_vbench.py" --strategy "$s" --out-root "$C/eval_out" \ + > "$LOG/vbench_$s.log" 2>&1 || echo "[gpu ${GPUS[$g]}] FAILED $s" + done + ) & + pids+=($!) +done +for p in "${pids[@]}"; do wait "$p"; done +echo "=== aggregating ($(date +%H:%M:%S)) ===" +cd "$C" +$PY eval/aggregate.py --out-root "$C/eval_out" --allow-incomplete > "$LOG/aggregate.log" 2>&1 +echo "ALL S2 PHASES DONE $(date +%H:%M:%S)" diff --git a/eval/run_predictor_retime_s2.sh b/eval/run_predictor_retime_s2.sh new file mode 100644 index 0000000000000000000000000000000000000000..9a61a396f9009a5d3c3662837c8c0d70ff62fc85 --- /dev/null +++ b/eval/run_predictor_retime_s2.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# Paired re-timing of the Stage-2 predictor strategies against FFFF (same process). +set -u +C=/local/zoubin/cz/projects/comparison +PY=/local/zoubin/cz/envs/self_forcing/bin/python +LOG="$(mkdir -p "$1" && cd "$1" && pwd)" +GPUS=(${GPUS:-0 1 2}); NSHARDS=${#GPUS[@]} +export SSL_CERT_FILE=/etc/pki/tls/certs/ca-bundle.crt TORCH_HOME=/local/zoubin/cz/.cache/torch HF_HOME=/local/zoubin/cz/.cache/huggingface HF_HUB_CACHE=/local/zoubin/cz/.cache/huggingface/hub XDG_CACHE_HOME=/local/zoubin/cz/.cache +cd "$C" +echo "=== retime ($(date +%H:%M:%S)) gpus=${GPUS[*]} ===" +pids=() +for i in $(seq 0 $((NSHARDS-1))); do + CUDA_VISIBLE_DEVICES=${GPUS[$i]} $PY eval/generate_predictor_eval.py --shard $i --num-shards $NSHARDS --out-root eval_out \ + --models "${MODELS:-atc_chunk_s2}" --patterns "${PATTERNS:-FPFF,FPPF,FPPP}" --retime-only > "$LOG/retime_$i.log" 2>&1 & + pids+=($!) +done +status=0; for p in "${pids[@]}"; do wait "$p" || status=1; done +echo "=== retime done ($(date +%H:%M:%S)) status=$status ===" +$PY eval/aggregate.py --out-root "$C/eval_out" --allow-incomplete > "$LOG/aggregate_retime.log" 2>&1 +$PY experiment_md.py > "$LOG/experiment_md.log" 2>&1 +echo "RETIME DONE $(date +%H:%M:%S)" diff --git a/eval/run_predictor_vbench_phase.sh b/eval/run_predictor_vbench_phase.sh new file mode 100644 index 0000000000000000000000000000000000000000..edfc7338d81fd895d917f8e17d3d27914207518b --- /dev/null +++ b/eval/run_predictor_vbench_phase.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# VBench + aggregate only (generation already complete). +set -u +C=/local/zoubin/cz/projects/comparison +PY=/local/zoubin/cz/envs/self_forcing/bin/python +LOG="$(cd "$(dirname "$1")" && pwd)/$(basename "$1")" +GPUS="${GPUS:-0 1 2 3}" +read -r -a gpu_list <<< "$GPUS" +cd "$C" +GPUS="$GPUS" NGPU=${#gpu_list[@]} bash "$C/eval/run_vbench_all.sh" "$LOG" 2>&1 | tee "$LOG/vbench_driver.log" +echo "=== aggregating ($(date +%H:%M:%S)) ===" +$PY "$C/eval/aggregate.py" --out-root "$C/eval_out" 2>&1 | tee "$LOG/aggregate.log" +echo "ALL CFA PHASES DONE $(date +%H:%M:%S)" diff --git a/eval/run_retime.sh b/eval/run_retime.sh new file mode 100644 index 0000000000000000000000000000000000000000..ed87fca99932949d7b8f86ec32eaf4510e00f6b1 --- /dev/null +++ b/eval/run_retime.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Phase 3: paired re-timing of strategies added after the first generation run +# (their FFFF reference was regenerated for the pixel metrics but not timed). +# Runs after VBench so the GPUs carry nothing else. Usage: +# eval/run_retime.sh +set -u +# DISABLED (Sep 6): the user asked for no more FFFF re-timing passes. Speedups +# are now the ratio of mean latencies against the FFFF strategy's own records; +# where those come from a different run the absolute ms are not comparable and +# the table says so. Kept for reference only. +echo "eval/run_retime.sh is disabled; re-timing is no longer part of the pipeline." >&2 +exit 1 +C=/local/zoubin/cz/projects/comparison +PY=/local/zoubin/cz/envs/self_forcing/bin/python +LOG="$1"; shift +NSHARDS=${NSHARDS:-4} +GPUS=(${GPUS:-0 1 2 3}) +EXTRA=${EXTRA:-} + +export SSL_CERT_FILE=/etc/pki/tls/certs/ca-bundle.crt +export HF_HOME=/local/zoubin/cz/.cache/huggingface +export HF_HUB_CACHE=/local/zoubin/cz/.cache/huggingface/hub +export TRANSFORMERS_CACHE=/local/zoubin/cz/.cache/huggingface/hub +export TORCH_HOME=/local/zoubin/cz/.cache/torch +export XDG_CACHE_HOME=/local/zoubin/cz/.cache + +cd "$C" +for BASE in self_forcing causal_forcing; do + if [ "$BASE" = self_forcing ]; then S="${STRATS_SF:-all}"; else S="${STRATS_CF:-all}"; fi + echo "=== retime: $BASE [$S] ($(date +%H:%M:%S)) ===" + pids=() + for i in $(seq 0 $((NSHARDS-1))); do + CUDA_VISIBLE_DEVICES=${GPUS[$i]} nohup $PY eval/retime_eval.py \ + --base "$BASE" --strategies "$S" --shard $i --num-shards $NSHARDS \ + --out-root eval_out $EXTRA > "$LOG/retime_${BASE}_$i.log" 2>&1 & + pids+=($!) + done + for p in "${pids[@]}"; do wait "$p"; done + echo "=== retime done: $BASE ($(date +%H:%M:%S)) ===" +done +echo "ALL RETIME DONE" diff --git a/eval/run_vbench.py b/eval/run_vbench.py new file mode 100644 index 0000000000000000000000000000000000000000..d8470c6c124087db66aa9d8dafbf984257afd1f7 --- /dev/null +++ b/eval/run_vbench.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python +"""Score one strategy on the eight protocol VBench dimensions. + +Standard VBench 0.1.5 metric code is used unmodified. What is built here is only +the evaluation metadata file, because the protocol needs two things the stock +`vbench_standard` mode cannot express: + + * `prompt_en` must be the *extended* prompt actually fed to the generator + (section 6.1), not the short canonical one; + * `scene` must keep the canonical `auxiliary_info` keywords. + +The file has exactly the shape `build_full_info_json` produces, and each +`compute_` selects the rows that list it, so per-suite scoping +(72 / 93 / 86 videos) falls out of the mapping rather than being hard-coded. + + python eval/run_vbench.py --strategy sf_ffff --out-root eval_out +""" + +import argparse +import json +import os +import sys +import time + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, ROOT) + +MAPPING = os.path.join(ROOT, "assets/vbench8_extended_subset_mapping.json") + +SUITE_DIMENSIONS = { + "subject_consistency": ["subject_consistency", "motion_smoothness", "dynamic_degree"], + "overall_consistency": ["overall_consistency", "aesthetic_quality", "imaging_quality"], + "scene": ["scene", "background_consistency"], +} +DIMENSIONS = [d for dims in SUITE_DIMENSIONS.values() for d in dims] + + +def parse_args(): + p = argparse.ArgumentParser() + p.add_argument("--strategy", required=True) + p.add_argument("--out-root", default="eval_out") + p.add_argument("--dimensions", default=None, + help="Comma-separated subset (default: all eight)") + p.add_argument("--allow-partial", action="store_true", + help="Score even if fewer than 251 videos exist (smoke tests only)") + p.add_argument("--overwrite", action="store_true", + help="Recompute dimensions whose raw result file already exists") + p.add_argument("--threads", type=int, default=None, + help="torch/OpenMP CPU threads for this process (scoring is CPU-bound; " + "several scorers on one node oversubscribe the cores)") + return p.parse_args() + + +def main(): + args = parse_args() + out_root = (args.out_root if os.path.isabs(args.out_root) + else os.path.join(ROOT, args.out_root)) + dims = args.dimensions.split(",") if args.dimensions else list(DIMENSIONS) + + with open(MAPPING) as f: + mapping = json.load(f) + + video_root = os.path.join(out_root, "generated_videos", args.strategy) + rows, missing = [], [] + for r in mapping["rows"]: + vp = os.path.join(video_root, r["prompt_suite"], f"{r['suite_index']:03d}.mp4") + if not os.path.exists(vp): + missing.append(vp) + continue + entry = { + # The extended prompt is what the video was generated from, so it is + # what overall_consistency and scene must be scored against. + "prompt_en": r["extended_prompt"], + "dimension": [d for d in SUITE_DIMENSIONS[r["prompt_suite"]]], + "video_list": [vp], + "global_index": r["global_index"], + "prompt_suite": r["prompt_suite"], + "suite_index": r["suite_index"], + } + if "auxiliary_info" in r: + entry["auxiliary_info"] = r["auxiliary_info"] + rows.append(entry) + + if missing and not args.allow_partial: + print(f"ERROR: {len(missing)} of {len(mapping['rows'])} videos missing for " + f"{args.strategy}; refusing to score an incomplete strategy.") + for m in missing[:5]: + print(" ", m) + return 1 + print(f"{args.strategy}: {len(rows)} videos, dimensions={dims}", flush=True) + + scores_dir = os.path.join(out_root, "vbench", "raw_results", args.strategy) + os.makedirs(scores_dir, exist_ok=True) + info_path = os.path.join(scores_dir, "full_info.json") + # Atomic: several per-dimension processes may start on the same strategy. + with open(info_path + f".{os.getpid()}", "w") as f: + json.dump(rows, f, indent=2) + os.replace(info_path + f".{os.getpid()}", info_path) + + import importlib + + import torch + if args.threads: + torch.set_num_threads(args.threads) + from vbench.utils import init_submodules + + device = torch.device("cuda") + + def raw_path(d): + return os.path.join(scores_dir, f"{d}.json") + + # A dimension already scored (its raw file exists) is reused, so dimensions can + # be spread over several processes and a killed run resumes where it stopped. + todo = [d for d in dims if args.overwrite or not os.path.exists(raw_path(d))] + submodules = init_submodules(todo, local=True, read_frame=False) if todo else {} + + for d in todo: + t0 = time.time() + module = importlib.import_module(f"vbench.{d}") + fn = getattr(module, f"compute_{d}") + score, per_video = fn(info_path, device, submodules[d]) + tmp = raw_path(d) + ".tmp" + with open(tmp, "w") as f: + json.dump({"dimension": d, "score": score, "video_results": per_video, + "seconds": time.time() - t0}, f, indent=2) + os.replace(tmp, raw_path(d)) + print(f" {d:24s} {score:.6f} ({len(per_video)} videos, " + f"{time.time() - t0:.0f}s)", flush=True) + torch.cuda.empty_cache() + + # The strategy's score file is written only once all eight dimensions exist. + missing_dims = [d for d in DIMENSIONS if not os.path.exists(raw_path(d))] + if missing_dims: + print(f"{args.strategy}: {len(DIMENSIONS) - len(missing_dims)}/{len(DIMENSIONS)} " + f"dimensions scored; missing {missing_dims}", flush=True) + return 0 + results = {} + for d in DIMENSIONS: + with open(raw_path(d)) as f: + raw = json.load(f) + results[d] = {"score": raw["score"], "num_videos": len(raw["video_results"]), + "seconds": raw.get("seconds")} + + out_path = os.path.join(out_root, "vbench", "scores", f"{args.strategy}.json") + os.makedirs(os.path.dirname(out_path), exist_ok=True) + payload = { + "strategy": args.strategy, + "vbench_long": False, + "num_videos": len(rows), + "raw": {d: results[d]["score"] for d in DIMENSIONS}, + "detail": results, + "full_info": info_path, + } + tmp = out_path + ".tmp" + with open(tmp, "w") as f: + json.dump(payload, f, indent=2) + os.replace(tmp, out_path) + print(f"wrote {out_path}", flush=True) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/eval/run_vbench_all.sh b/eval/run_vbench_all.sh new file mode 100644 index 0000000000000000000000000000000000000000..46c120435580f30d7e710b454fb77d557ccc9c68 --- /dev/null +++ b/eval/run_vbench_all.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# Phase 2: score every strategy on the eight protocol dimensions. +# VBench runs in the vbench_eval venv (torchao 0.7.0 / transformers 4.46.3 / +# fairscale), because the base env's torchao 0.17.0 is unimportable under torch +# 2.5.1 and takes transformers' BertModel -- and therefore Tag2Text/scene -- with +# it. Generation deliberately stays in the base env. +set -u +C=/local/zoubin/cz/projects/comparison +V=/local/zoubin/cz/projects/VBench +PY=/local/zoubin/cz/envs/vbench_eval/bin/python +LOG="$1"; shift +NGPU=${NGPU:-4} +GPUS=(${GPUS:-0 1 2 3}) + +export PYTHONPATH="$V:$C" +export VBENCH_CACHE_DIR=/local/zoubin/cz/.cache/vbench +export HF_HOME=/local/zoubin/cz/.cache/huggingface +export HF_HUB_CACHE=/local/zoubin/cz/.cache/huggingface/hub +export TRANSFORMERS_CACHE=/local/zoubin/cz/.cache/huggingface/hub +export XDG_CACHE_HOME=/local/zoubin/cz/.cache +export SSL_CERT_FILE=/etc/pki/tls/certs/ca-bundle.crt +export TORCH_HOME=/local/zoubin/cz/.cache/torch + +mapfile -t STRATS < <(ls "$C/eval_out/per_prompt") +echo "scoring ${#STRATS[@]} strategies across $NGPU GPUs" + +cd "$V" +pids=() +for g in $(seq 0 $((NGPU-1))); do + ( + for i in $(seq $g $NGPU $(( ${#STRATS[@]} - 1 ))); do + s="${STRATS[$i]}" + if [ -f "$C/eval_out/vbench/scores/$s.json" ]; then + echo "skip $s (already scored)"; continue + fi + echo "[gpu $g] $s $(date +%H:%M:%S)" + CUDA_VISIBLE_DEVICES=${GPUS[$g]} $PY "$C/eval/run_vbench.py" \ + --strategy "$s" --out-root "$C/eval_out" > "$LOG/vbench_$s.log" 2>&1 \ + || echo "[gpu $g] FAILED $s" + done + ) & + pids+=($!) +done +for p in "${pids[@]}"; do wait "$p"; done +echo "ALL VBENCH DONE" diff --git a/eval/setup_vbench_models.py b/eval/setup_vbench_models.py new file mode 100644 index 0000000000000000000000000000000000000000..72f4d0ecffa7c9ed15f5e823c5604a346be3fc3b --- /dev/null +++ b/eval/setup_vbench_models.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python +"""Download the VBench weights the eight protocol dimensions need. + +Everything lands in $VBENCH_CACHE_DIR so the run does not depend on whatever a +user's ~/.cache happens to hold. `local=True` is used wherever VBench supports +it, so CLIP and DINO come from the cache directory too rather than from the clip +package's own download path. +""" +import os +import sys + +DIMS = ["subject_consistency", "background_consistency", "motion_smoothness", + "dynamic_degree", "aesthetic_quality", "imaging_quality", "scene", + "overall_consistency"] + +def main(): + from vbench.utils import init_submodules, CACHE_DIR + print("CACHE_DIR =", CACHE_DIR, flush=True) + for d in DIMS: + print(f"--- {d} ---", flush=True) + try: + got = init_submodules([d], local=True, read_frame=False) + print(f" ok: {got.get(d)}", flush=True) + except Exception as e: + print(f" FAILED: {type(e).__name__}: {e}", flush=True) + print("done", flush=True) + +if __name__ == "__main__": + sys.exit(main()) diff --git a/eval/strategies.py b/eval/strategies.py new file mode 100644 index 0000000000000000000000000000000000000000..db642fd5705088361367086c83cdf2cb566e073a --- /dev/null +++ b/eval/strategies.py @@ -0,0 +1,82 @@ +"""The strategy list for the Extended-251 evaluation. + +FFFF (every denoising step full) is the reference for each base model; the cache +strategies are the operating points settled by finalize.py / retime.py, read back +from results/ so the evaluation cannot silently drift from the speed study. +""" + +import glob +import json +import os + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +METHOD_ORDER = ["teacache", "flowcache", "taylorseer", "motioncache"] +# FlowCache is frozen: finished operating points stay in the tables, no new ones +# are added. ``final_*_v2.json`` files (the measured-speedup targeted points) +# therefore never contain it. +DROPPED_FOR_NEW = {"flowcache"} +BASES = ["self_forcing", "causal_forcing"] +SHORT = {"self_forcing": "sf", "causal_forcing": "cf"} + + +def load_strategies(base): + """[{name, base, method, param, value, target, coefficients, indicator}, ...] + + The first entry is always FFFF. + """ + coeff_path = os.path.join(ROOT, f"results/coeff_{base}.json") + coefficients = None + if os.path.exists(coeff_path): + with open(coeff_path) as f: + coefficients = json.load(f)["coefficients"] + + out = [{ + "name": f"{SHORT[base]}_ffff", + "base": base, "method": "none", "param": None, "value": None, + "target": 1.0, "coefficients": None, "indicator": "modulated_input", + "schedule": "FxxF", "is_reference": True, "sampler": "renoise", + }] + + rows = [] + for p in sorted(glob.glob(os.path.join(ROOT, f"results/final_{base}*.json"))): + with open(p) as f: + rows += json.load(f)["rows"] + rows.sort(key=lambda r: (METHOD_ORDER.index(r["method"]) + if r["method"] in METHOD_ORDER else 9, + r.get("schedule", "FxxF") != "FxxF", + r["target_speedup"])) + for r in rows: + # The original FxxF study keeps its unsuffixed names; other schedules + # carry the schedule so they never collide with those records. + sched = r.get("schedule", "FxxF") + tag = "" if sched.upper() == "FXXF" else f"_{sched}" + out.append({ + "name": r.get("name") or f"{SHORT[base]}_{r['method']}{tag}_x{r['target_speedup']:g}", + "schedule": sched, + "num_inference_steps": r.get("num_inference_steps"), + "first_chunk_schedule": r.get("first_chunk_schedule"), + "sampler": r.get("sampler", "renoise"), + "base": base, + "method": r["method"], + "param": r["param"], + "value": r["value"], + "target": r["target_speedup"], + "coefficients": coefficients, + "indicator": "modulated_input", + "is_reference": False, + "compute_equivalent_forwards": r.get("compute_equivalent_forwards"), + }) + return out + + +def all_strategies(): + return {b: load_strategies(b) for b in BASES} + + +if __name__ == "__main__": + for b, s in all_strategies().items(): + print(f"{b}: {len(s)} strategies") + for x in s: + v = "-" if x["value"] is None else f"{x['param']}={x['value']:.6g}" + print(f" {x['name']:28s} {v}") diff --git a/eval/test_protocol.py b/eval/test_protocol.py new file mode 100644 index 0000000000000000000000000000000000000000..def155f69f3c0157d45ada04f6ed3f68da4279c0 --- /dev/null +++ b/eval/test_protocol.py @@ -0,0 +1,172 @@ +#!/usr/bin/env python +"""Unit tests for the evaluation protocol's section 12 checklist. + + python eval/test_protocol.py +""" + +import json +import os +import sys +import tempfile + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, ROOT) + +from eval import aggregate as agg # noqa: E402 + +MAPPING = os.path.join(ROOT, "assets/vbench8_extended_subset_mapping.json") +FULL_INFO = "/local/zoubin/cz/projects/VBench/vbench/VBench_full_info.json" +PROMPT_DIR = "/local/zoubin/cz/projects/Self-Forcing/prompts/vbench" + +FAILURES = [] + + +def check(name, cond, detail=""): + print(f" {'PASS' if cond else 'FAIL'} {name}" + (f" {detail}" if detail else "")) + if not cond: + FAILURES.append(name) + + +def test_mapping(): + print("946 -> 251 mapping") + with open(MAPPING) as f: + m = json.load(f) + rows = m["rows"] + check("mapping has 251 rows", len(rows) == 251, f"got {len(rows)}") + counts = {s: sum(1 for r in rows if r["prompt_suite"] == s) + for s in ("subject_consistency", "overall_consistency", "scene")} + check("suite counts 72/93/86", + counts == {"subject_consistency": 72, "overall_consistency": 93, "scene": 86}, + str(counts)) + check("global_index unique", + len({r["global_index"] for r in rows}) == len(rows)) + for s in counts: + idx = [r["suite_index"] for r in rows if r["prompt_suite"] == s] + check(f"{s} suite_index contiguous", idx == list(range(len(idx)))) + check("every scene row keeps auxiliary_info", + all("auxiliary_info" in r for r in rows if r["prompt_suite"] == "scene")) + check("extended prompts non-empty", + all(r["extended_prompt"].strip() for r in rows)) + + +def test_canonical_order(): + print("canonical prompt order is enforced") + with open(FULL_INFO) as f: + info = json.load(f) + with open(os.path.join(PROMPT_DIR, "all_dimension.txt"), encoding="utf-8") as f: + short = [l.rstrip("\n") for l in f] + check("946 short prompts", len(short) == 946, f"got {len(short)}") + check("short prompt order == VBench_full_info", + all(a["prompt_en"].strip() == b.strip() for a, b in zip(info, short))) + + # A mismatched prompt file must be refused, not silently fuzzy-matched. + with open(MAPPING) as f: + rows = json.load(f)["rows"] + with open(os.path.join(PROMPT_DIR, "all_dimension_extended.txt"), encoding="utf-8") as f: + ext = [l.rstrip("\n") for l in f] + check("mapping extended prompt matches its global_index", + all(r["extended_prompt"] == ext[r["global_index"]] for r in rows)) + check("mapping original prompt matches its global_index", + all(r["original_prompt"] == short[r["global_index"]] for r in rows)) + + +def test_normalize_and_aggregate(): + print("normalize + Quality/Semantic/Selected aggregation") + lo_raw = {d: agg.NORMALIZE_RANGE[d][0] for d in agg.DIMENSIONS} + hi_raw = {d: agg.NORMALIZE_RANGE[d][1] for d in agg.DIMENSIONS} + check("normalize maps min -> 0", + all(abs(v) < 1e-12 for v in agg.normalize(lo_raw).values())) + check("normalize maps max -> 1", + all(abs(v - 1.0) < 1e-12 for v in agg.normalize(hi_raw).values())) + + n = agg.normalize(hi_raw) + q, s = agg.quality_score(n), agg.semantic_score(n) + check("quality of all-ones == 1", abs(q - 1.0) < 1e-12, f"got {q}") + check("semantic of all-ones == 1", abs(s - 1.0) < 1e-12, f"got {s}") + check("selected of all-ones == 1", abs(agg.selected_score(q, s) - 1.0) < 1e-12) + + # dynamic_degree carries weight 0.5, everything else in Quality carries 1.0. + n0 = {d: 0.0 for d in agg.DIMENSIONS} + n_dd = dict(n0, dynamic_degree=1.0) + check("dynamic_degree weight is 0.5/5.5", + abs(agg.quality_score(n_dd) - 0.5 / 5.5) < 1e-12) + n_sc = dict(n0, subject_consistency=1.0) + check("other quality dims weight 1/5.5", + abs(agg.quality_score(n_sc) - 1.0 / 5.5) < 1e-12) + check("selected weights Quality:Semantic 4:1", + abs(agg.selected_score(1.0, 0.0) - 0.8) < 1e-12 + and abs(agg.selected_score(0.0, 1.0) - 0.2) < 1e-12) + check("selected is not the plain mean of 8 normalized dims", + abs(agg.selected_score(agg.quality_score(n_dd), agg.semantic_score(n_dd)) + - sum(n_dd.values()) / 8.0) > 1e-6) + + +def _rec(strategy, idx, lat, ctx, mse, frames=81): + return { + "status": "complete", "strategy": strategy, "base_model": "self_forcing", + "method": "none" if strategy.endswith("ffff") else "teacache", + "target_speedup": 1.0, "global_index": idx, "prompt_suite": "scene", + "suite_index": idx, "policy_latency_ms": lat, + "excluded_context_kv_latency_ms": ctx, + "pixel_metrics_vs_ffff": {"mean_mse": mse, "psnr": 0.0, "ssim": 1.0, + "lpips": 0.0, "num_frames": frames}, + "cache_diagnostics": {"compute_equivalent_forwards": 28.0}, + } + + +def test_latency_and_completeness(): + print("latency accounting and the completeness gate") + import math + + # ratio of means, not the mean of per-prompt ratios + ffff = [_rec("sf_ffff", i, lat, 800.0, 1e-12) for i, lat in enumerate([100.0, 300.0])] + cand = [_rec("sf_x", i, lat, 800.0, 1e-4) for i, lat in enumerate([50.0, 250.0])] + ratio_of_means = 100.0 * (1 - (sum(r["policy_latency_ms"] for r in cand) / 2) + / (sum(r["policy_latency_ms"] for r in ffff) / 2)) + mean_of_ratios = 100.0 * (1 - ((50.0 / 100.0) + (250.0 / 300.0)) / 2) + check("speedup uses ratio of means", abs(ratio_of_means - 25.0) < 1e-9, + f"{ratio_of_means:.3f}%") + check("ratio of means differs from mean of ratios", + abs(ratio_of_means - mean_of_ratios) > 1.0) + + check("context/KV DiT is a separate field, never inside policy latency", + all("excluded_context_kv_latency_ms" in r + and r["policy_latency_ms"] != r["policy_latency_ms"] + + r["excluded_context_kv_latency_ms"] for r in cand)) + + mse = sum(r["pixel_metrics_vs_ffff"]["mean_mse"] for r in cand) / len(cand) + check("PSNR aggregates MSE first, then converts once", + abs(-10.0 * math.log10(max(mse, 1e-12)) - 40.0) < 1e-9) + check("pixel metrics span all 81 frames", + {r["pixel_metrics_vs_ffff"]["num_frames"] for r in cand} == {81}) + + with tempfile.TemporaryDirectory() as td: + for name, recs in (("sf_ffff", ffff), ("sf_x", cand)): + d = os.path.join(td, "per_prompt", name) + os.makedirs(d) + for r in recs: + with open(os.path.join(d, f"scene_{r['suite_index']:03d}.json"), "w") as f: + json.dump(r, f) + argv = sys.argv + sys.argv = ["aggregate", "--out-root", td, "--expect", "251"] + try: + rc = agg.main() + finally: + sys.argv = argv + check("aggregation refuses a strategy that is not 251", rc == 1, f"rc={rc}") + + +def main(): + for fn in (test_mapping, test_canonical_order, test_normalize_and_aggregate, + test_latency_and_completeness): + fn() + print() + if FAILURES: + print(f"{len(FAILURES)} FAILED: {FAILURES}") + return 1 + print("all protocol unit tests passed") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/eval_out/generated_videos/cf_ffff/overall_consistency/000.mp4 b/eval_out/generated_videos/cf_ffff/overall_consistency/000.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..99cad18329376c8b1c33025bdfbbd5e75a3ccc4d --- /dev/null +++ b/eval_out/generated_videos/cf_ffff/overall_consistency/000.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3b9313a8d35ed942c8056953a8a24e8ad5c2eb6d51b84dbbb7263b3a135d9ee +size 718017 diff --git a/eval_out/generated_videos/cf_ffff/overall_consistency/001.mp4 b/eval_out/generated_videos/cf_ffff/overall_consistency/001.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fb0eaab536ab9cc48cf788aff70ac5e7d057e740 --- /dev/null +++ b/eval_out/generated_videos/cf_ffff/overall_consistency/001.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97434fa492a782f473db53f5325943455e38f44c2ec0b765a9e94facc1b41f64 +size 678798 diff --git a/eval_out/generated_videos/cf_ffff/overall_consistency/002.mp4 b/eval_out/generated_videos/cf_ffff/overall_consistency/002.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..085353af201bea149fb5c5620d8bcc37178c4028 --- /dev/null +++ b/eval_out/generated_videos/cf_ffff/overall_consistency/002.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8cc8240a9fc4555de544e0131304dd301803dab2a1867f8efe73d36a6686ed27 +size 665068 diff --git a/eval_out/generated_videos/cf_ffff/overall_consistency/003.mp4 b/eval_out/generated_videos/cf_ffff/overall_consistency/003.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2ff2944b0642b2cda16b208103e812c8e0451cfa --- /dev/null +++ b/eval_out/generated_videos/cf_ffff/overall_consistency/003.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dfeeb51a34c8db083f840497bc64bd6850f361e57b03435d436c0625cf1aa614 +size 652636 diff --git a/eval_out/generated_videos/cf_ffff/overall_consistency/004.mp4 b/eval_out/generated_videos/cf_ffff/overall_consistency/004.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8b144cb1c7f058993cbb1c131c08f3eee2603b9d --- /dev/null +++ b/eval_out/generated_videos/cf_ffff/overall_consistency/004.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1bcf14ef30751d4cc7ca646615758207443e7848e99c61e91a62566a17301b3a +size 624330 diff --git a/eval_out/generated_videos/cf_ffff/overall_consistency/005.mp4 b/eval_out/generated_videos/cf_ffff/overall_consistency/005.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1e50d4c1e1071bd5541cd10d94a7e2e0f3449024 --- /dev/null +++ b/eval_out/generated_videos/cf_ffff/overall_consistency/005.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a003819fae2c958584f41480ae48397ed99cb2afcb0bee20a30aa8359adf985 +size 619500 diff --git a/eval_out/generated_videos/cf_ffff/overall_consistency/006.mp4 b/eval_out/generated_videos/cf_ffff/overall_consistency/006.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8b25f1785a53ba7173182be75d2b59a3dab77188 --- /dev/null +++ b/eval_out/generated_videos/cf_ffff/overall_consistency/006.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:510c4b85f51b3605bdb24602644f27c2865ad8cfb7d0f6d105fc1c2fe5027bbf +size 667321 diff --git a/eval_out/generated_videos/cf_ffff/overall_consistency/007.mp4 b/eval_out/generated_videos/cf_ffff/overall_consistency/007.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..01e6dfae7a99d6e3b5130f0fa5ea3e98bace1b72 --- /dev/null +++ b/eval_out/generated_videos/cf_ffff/overall_consistency/007.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb5b16a9bdf0ba7990fcf6ad75f18a2a0dc8142ca10f2276716c8136ace79e3c +size 692210 diff --git a/eval_out/generated_videos/cf_ffff/overall_consistency/008.mp4 b/eval_out/generated_videos/cf_ffff/overall_consistency/008.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a20aa5bc2c7c4c6fa41cdebad3e276d73b3a989b --- /dev/null +++ b/eval_out/generated_videos/cf_ffff/overall_consistency/008.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:93f6f41136b394e1e642a060d2783281692481f08e725a4933e872cf321be58d +size 538371 diff --git a/eval_out/generated_videos/cf_ffff/overall_consistency/009.mp4 b/eval_out/generated_videos/cf_ffff/overall_consistency/009.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..775985f043bc936a97bb5a18263e6d74dc3d5dd4 --- /dev/null +++ b/eval_out/generated_videos/cf_ffff/overall_consistency/009.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5669c892dd2fdbce0fcfb2ee2f0221645ad620c72562b1bfd3ad1d8a3ad40202 +size 649393 diff --git a/eval_out/generated_videos/cf_ffff/overall_consistency/010.mp4 b/eval_out/generated_videos/cf_ffff/overall_consistency/010.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e6211b2b0f82af190ffb8905b759cbff5844e167 --- /dev/null +++ b/eval_out/generated_videos/cf_ffff/overall_consistency/010.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d05a3ee90af4a46d5cef630591125ad542ec7695b9c25f5279572b9d536d6568 +size 640856 diff --git a/eval_out/generated_videos/cf_ffff/overall_consistency/011.mp4 b/eval_out/generated_videos/cf_ffff/overall_consistency/011.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1b320c42c72de58fc737f49090c14139b95abb56 --- /dev/null +++ b/eval_out/generated_videos/cf_ffff/overall_consistency/011.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9aceedf023cd61aac971d5933f802ad3e75770296b10888a2bd5d8fb2294f211 +size 673335 diff --git a/eval_out/generated_videos/cf_ffff/overall_consistency/012.mp4 b/eval_out/generated_videos/cf_ffff/overall_consistency/012.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d24a760145e5bfab09105418bc1e8730a3633b87 --- /dev/null +++ b/eval_out/generated_videos/cf_ffff/overall_consistency/012.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1742fc927aee1bea4a58cfb703a771181f5558ac95eec019bf82bbb60684ebe6 +size 643474 diff --git a/eval_out/generated_videos/cf_ffff/overall_consistency/013.mp4 b/eval_out/generated_videos/cf_ffff/overall_consistency/013.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..bcb38106d6a27d447960ff359adf97815c2995a1 --- /dev/null +++ b/eval_out/generated_videos/cf_ffff/overall_consistency/013.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8dc175733a2dd86a2a86a5cd181f2990003b1532149aaec82623d841fa710276 +size 697151 diff --git a/eval_out/generated_videos/cf_ffff/overall_consistency/014.mp4 b/eval_out/generated_videos/cf_ffff/overall_consistency/014.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f5128f94bb1981e8dae9889d3b0b8f9e56828d97 --- /dev/null +++ b/eval_out/generated_videos/cf_ffff/overall_consistency/014.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53b34417aa633a8a16f67d5569cd9acdaae30016bca4b0a12e89297af99a09ab +size 564467 diff --git a/eval_out/generated_videos/cf_ffff/overall_consistency/015.mp4 b/eval_out/generated_videos/cf_ffff/overall_consistency/015.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fa63d71b2e4ecb89f4e66771690f828bc0b7627c --- /dev/null +++ b/eval_out/generated_videos/cf_ffff/overall_consistency/015.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a0bbec98e10b7de4a5edb3ba14d7318683afef3c89587c44f66468c9bbfd705 +size 607298 diff --git a/eval_out/generated_videos/cf_ffff/overall_consistency/016.mp4 b/eval_out/generated_videos/cf_ffff/overall_consistency/016.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2097f541ac4da57b65de7b5555e5a9416e161caa --- /dev/null +++ b/eval_out/generated_videos/cf_ffff/overall_consistency/016.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d9feefa47ae677af86818c9e3dee84155fe4cc7374a425c9a6d828195fb72361 +size 693258 diff --git a/eval_out/generated_videos/cf_ffff/overall_consistency/017.mp4 b/eval_out/generated_videos/cf_ffff/overall_consistency/017.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..98bfa2d652252ad586d56e47b8bee2bf829b2a2d --- /dev/null +++ b/eval_out/generated_videos/cf_ffff/overall_consistency/017.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c1f29a4bb28547296142bcde8935031005095a0d0fd35f7d46c9c4f0c7ef77f4 +size 647002 diff --git a/eval_out/generated_videos/cf_ffff/overall_consistency/019.mp4 b/eval_out/generated_videos/cf_ffff/overall_consistency/019.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..217c749e35929131c57cb84fa29772a74963a81b --- /dev/null +++ b/eval_out/generated_videos/cf_ffff/overall_consistency/019.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:938b5f61acfd80ca568e800a618032bcb090ed37130378b32e51284d41979519 +size 635121 diff --git a/eval_out/generated_videos/cf_ffff/overall_consistency/020.mp4 b/eval_out/generated_videos/cf_ffff/overall_consistency/020.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..15ab0b0afc76d219507316d4e4fc3ba63ebe5a6e --- /dev/null +++ b/eval_out/generated_videos/cf_ffff/overall_consistency/020.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d49382441277e9bd73c2d6f8511699af1b21052549ee65513e074ff0c439556b +size 681829 diff --git a/eval_out/generated_videos/cf_ffff/overall_consistency/021.mp4 b/eval_out/generated_videos/cf_ffff/overall_consistency/021.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..48d779ac5285d5295a7999143bc7f185f3617b73 --- /dev/null +++ b/eval_out/generated_videos/cf_ffff/overall_consistency/021.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8cb05bd4d88ccf2ab6abad9004101f4aeaaf0b27a9754d85074e1bb549d9dc7d +size 621572 diff --git a/eval_out/generated_videos/cf_ffff/overall_consistency/022.mp4 b/eval_out/generated_videos/cf_ffff/overall_consistency/022.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..26d30c72ed0cea239cc5244aeda1b03b4dcd45fd --- /dev/null +++ b/eval_out/generated_videos/cf_ffff/overall_consistency/022.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d413c3b306c822fe46ccdfe42be1755d517375cf2529a3bf6394f5b3d0e63b84 +size 612265 diff --git a/eval_out/generated_videos/cf_ffff/overall_consistency/023.mp4 b/eval_out/generated_videos/cf_ffff/overall_consistency/023.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..738110e2d50f41a3399f6615caa17b338eb07683 --- /dev/null +++ b/eval_out/generated_videos/cf_ffff/overall_consistency/023.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:98f4c2dafa64fb134bd89b54a8a62d171855f92aca254a3fc50d3b610483c147 +size 617616 diff --git a/eval_out/generated_videos/cf_ffff/overall_consistency/024.mp4 b/eval_out/generated_videos/cf_ffff/overall_consistency/024.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3cf6ad003695da52c044852ec4c864c489bba69e --- /dev/null +++ b/eval_out/generated_videos/cf_ffff/overall_consistency/024.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d8ebcdc55a389db046c2a774c36b25507d6f461765bda4c406376fac60737ef +size 653522 diff --git a/eval_out/generated_videos/cf_ffff/overall_consistency/026.mp4 b/eval_out/generated_videos/cf_ffff/overall_consistency/026.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7ba0d0523a45462bf3e55c1f8e1ef5fd78112ec9 --- /dev/null +++ b/eval_out/generated_videos/cf_ffff/overall_consistency/026.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44259df19aca91548dbd8178668d4eb669fecbee8eff53a1876160e537e64c53 +size 632726 diff --git a/eval_out/generated_videos/cf_ffff/overall_consistency/027.mp4 b/eval_out/generated_videos/cf_ffff/overall_consistency/027.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6c356c0ba0b99d788985cbfd731e8a9f4d501027 --- /dev/null +++ b/eval_out/generated_videos/cf_ffff/overall_consistency/027.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ceaaa6d1fe61556e7c97a5dbad476c44d4638ee8b9acc48f06f73ca6d5ba7c1 +size 632909 diff --git a/eval_out/generated_videos/cf_ffff/overall_consistency/028.mp4 b/eval_out/generated_videos/cf_ffff/overall_consistency/028.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..64e4d13f26872555214b4a858e095e11e606ff6d --- /dev/null +++ b/eval_out/generated_videos/cf_ffff/overall_consistency/028.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d43e65da58bcfced9f200a1003139692330618864e9eccc9905067085a8d8b12 +size 676970 diff --git a/eval_out/generated_videos/cf_ffff/overall_consistency/030.mp4 b/eval_out/generated_videos/cf_ffff/overall_consistency/030.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..43cb695fb18fe40eb15f1d4677e0bff4d076c49d --- /dev/null +++ b/eval_out/generated_videos/cf_ffff/overall_consistency/030.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:410ece769510fda8db4f9293bf55abc683e67cad5fbc6c9194d7f287a304456b +size 630621 diff --git a/eval_out/generated_videos/cf_ffff/overall_consistency/032.mp4 b/eval_out/generated_videos/cf_ffff/overall_consistency/032.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d21ee709d6c26d6da362d92b4d8b5632ad961464 --- /dev/null +++ b/eval_out/generated_videos/cf_ffff/overall_consistency/032.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f403986c93325ccc8363391e4f026845d7eb2cddb34ff444acd0081c1314f54 +size 630350 diff --git a/eval_out/generated_videos/cf_ffff/overall_consistency/033.mp4 b/eval_out/generated_videos/cf_ffff/overall_consistency/033.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..27b2e496af762d908d245859b9385ba424e09ebf --- /dev/null +++ b/eval_out/generated_videos/cf_ffff/overall_consistency/033.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b57b30187c168053b37626318f587a4a7cb6d17fa06169c7ce09f3ecae08f9ae +size 565971 diff --git a/eval_out/generated_videos/cf_ffff/overall_consistency/034.mp4 b/eval_out/generated_videos/cf_ffff/overall_consistency/034.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d1ed33abb383f991f97f723c836d9e536b0a3cf4 --- /dev/null +++ b/eval_out/generated_videos/cf_ffff/overall_consistency/034.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:774311f97a6ca75d3c48660140dd908699d0508e089452974a6f247472b418db +size 655819 diff --git a/eval_out/generated_videos/cf_ffff/overall_consistency/035.mp4 b/eval_out/generated_videos/cf_ffff/overall_consistency/035.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..efd89ae7c1f7040a9eb0097fe336a9ff090b3f95 --- /dev/null +++ b/eval_out/generated_videos/cf_ffff/overall_consistency/035.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f7f8bd6c46022cf681a3650a26aa3756b79231654de82cdae141e00eda1e3fb +size 647654 diff --git a/eval_out/generated_videos/cf_ffff/overall_consistency/037.mp4 b/eval_out/generated_videos/cf_ffff/overall_consistency/037.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6f3bda13354aac7bc4f0f547331ad57c53f6a21a --- /dev/null +++ b/eval_out/generated_videos/cf_ffff/overall_consistency/037.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:54545effc03ba3bd2c8f8a4b0ed8c530b36b3a48499002e2c95afaa265300621 +size 686134 diff --git a/eval_out/generated_videos/cf_ffff/overall_consistency/039.mp4 b/eval_out/generated_videos/cf_ffff/overall_consistency/039.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c62e3c33693b12227e550a6ac937cf6f130d22a3 --- /dev/null +++ b/eval_out/generated_videos/cf_ffff/overall_consistency/039.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c4a2db35ef635694663d1be7a7b42044da1cb2b21f1a2b91d169663360cbae83 +size 643137 diff --git a/eval_out/generated_videos/cf_ffff/overall_consistency/040.mp4 b/eval_out/generated_videos/cf_ffff/overall_consistency/040.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fe94c7dc937199b8a280d3ccdbcd63937a3a633f --- /dev/null +++ b/eval_out/generated_videos/cf_ffff/overall_consistency/040.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:202875cf7224bed260d2f17b65a397d5184ae80ba5402bb6785268537145517b +size 630097 diff --git a/eval_out/generated_videos/cf_ffff/overall_consistency/041.mp4 b/eval_out/generated_videos/cf_ffff/overall_consistency/041.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1d7049eac8f4d6c6597b8d292f518c46264f9349 --- /dev/null +++ b/eval_out/generated_videos/cf_ffff/overall_consistency/041.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:571f8ea6d21d60944bc1ffd675b2725816ad401333acf0fa1d42525db619593b +size 640993 diff --git a/eval_out/generated_videos/cf_ffff/overall_consistency/042.mp4 b/eval_out/generated_videos/cf_ffff/overall_consistency/042.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a7f2877fa6108eeda192dd6302640d68535c5d59 --- /dev/null +++ b/eval_out/generated_videos/cf_ffff/overall_consistency/042.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:183cd0a248d3d2336f7fc0e48928eccfe11162950911237ab6dd734518dfc08b +size 616765 diff --git a/eval_out/generated_videos/cf_ffff/overall_consistency/044.mp4 b/eval_out/generated_videos/cf_ffff/overall_consistency/044.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..97808d1b7df1948f6a84808d427b745375ee3de9 --- /dev/null +++ b/eval_out/generated_videos/cf_ffff/overall_consistency/044.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d91b0f0b55dfa66f5df2ec209948656a5759903ab5b8cfc0fe6ef305e65d6b8 +size 651116 diff --git a/eval_out/generated_videos/cf_ffff/overall_consistency/047.mp4 b/eval_out/generated_videos/cf_ffff/overall_consistency/047.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..41788f3c66065dbef76b9676695c026885e11df9 --- /dev/null +++ b/eval_out/generated_videos/cf_ffff/overall_consistency/047.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:73827fc6f9876e2c4dacbf3ea8a81f7cf52dd9415599a3625892f88bc7a92473 +size 654528 diff --git a/eval_out/generated_videos/cf_ffff/overall_consistency/048.mp4 b/eval_out/generated_videos/cf_ffff/overall_consistency/048.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3042d14cdf69976d454c44295a149d875ce85d36 --- /dev/null +++ b/eval_out/generated_videos/cf_ffff/overall_consistency/048.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7574d7f04ca850a7be91077d7c737dc3f03f7d99fa667b7532851d9b87fd431a +size 588795 diff --git a/eval_out/generated_videos/cf_ffff/overall_consistency/049.mp4 b/eval_out/generated_videos/cf_ffff/overall_consistency/049.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..312f38cf4252fb70faff8d5846bb2f738e771888 --- /dev/null +++ b/eval_out/generated_videos/cf_ffff/overall_consistency/049.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1eba5dfcead5a6cc79465db517de49a0eb3f64b6ede8c6c7e271cbea9a3ec129 +size 590630 diff --git a/eval_out/generated_videos/cf_ffff/overall_consistency/050.mp4 b/eval_out/generated_videos/cf_ffff/overall_consistency/050.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4f413d15b2df3f055ba5b6a233edd93f766d53a9 --- /dev/null +++ b/eval_out/generated_videos/cf_ffff/overall_consistency/050.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1955babeac9f2288dc368b2b13ca6c29410f88326967954e4ee6c7aa6a7aabbe +size 595360 diff --git a/eval_out/generated_videos/cf_ffff/overall_consistency/051.mp4 b/eval_out/generated_videos/cf_ffff/overall_consistency/051.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c99cb4201cd956748f26724a40c9253c9f5d0dfc --- /dev/null +++ b/eval_out/generated_videos/cf_ffff/overall_consistency/051.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed2bb80af5f3351229a6462be3a50ebbc159238dfbf5f551b51cc908b4e73362 +size 628367 diff --git a/eval_out/generated_videos/cf_ffff/overall_consistency/052.mp4 b/eval_out/generated_videos/cf_ffff/overall_consistency/052.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1920c50948cbb4bb592a737ce93a4333ae08953d --- /dev/null +++ b/eval_out/generated_videos/cf_ffff/overall_consistency/052.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d40231b52e589fc1392518e6c5e7bb7b3e3f3e11d849558ff2def55f1bca20c3 +size 633282 diff --git a/eval_out/generated_videos/cf_ffff/overall_consistency/053.mp4 b/eval_out/generated_videos/cf_ffff/overall_consistency/053.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ac8765e51b8db4ac8ad0ea9ead3bbb871ae3106b --- /dev/null +++ b/eval_out/generated_videos/cf_ffff/overall_consistency/053.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c276727642c2ff5c0519db65b349d534f2955a195e683f9c310a65c8ef047258 +size 608734 diff --git a/eval_out/generated_videos/cf_ffff/overall_consistency/054.mp4 b/eval_out/generated_videos/cf_ffff/overall_consistency/054.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f77e839291a1a96afdde1206d3bb79458fea876c --- /dev/null +++ b/eval_out/generated_videos/cf_ffff/overall_consistency/054.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:375858824c678ba596de394cc0575dd3c28d6ba64d5fb9576c599dafd8c2c3a6 +size 711594 diff --git a/eval_out/generated_videos/cf_ffff/overall_consistency/055.mp4 b/eval_out/generated_videos/cf_ffff/overall_consistency/055.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7ca4b7fce3b36a16ce75c8ed7df0359f696a26d3 --- /dev/null +++ b/eval_out/generated_videos/cf_ffff/overall_consistency/055.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ca3a63200a0918a87190b4879805b8d3b28edfc32df469804ccca2098147ffe +size 548890 diff --git a/eval_out/generated_videos/cf_ffff/overall_consistency/056.mp4 b/eval_out/generated_videos/cf_ffff/overall_consistency/056.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..dbb32f61a1ee0914548add739c3251df4cb37ea7 --- /dev/null +++ b/eval_out/generated_videos/cf_ffff/overall_consistency/056.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c189c59ab28b81e3fc3bf859caf169c57c7d5efd5a815f55c4fc63325533845f +size 643582 diff --git a/eval_out/generated_videos/cf_ffff/overall_consistency/057.mp4 b/eval_out/generated_videos/cf_ffff/overall_consistency/057.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a75bb894d19cd34be86db9970c2f4ad82db4ff2c --- /dev/null +++ b/eval_out/generated_videos/cf_ffff/overall_consistency/057.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c1332e4c183a0f7b89f5cad53e512ba3843481a7e2c370a11e31d255ed257f6 +size 663036 diff --git a/eval_out/generated_videos/cf_ffff/overall_consistency/058.mp4 b/eval_out/generated_videos/cf_ffff/overall_consistency/058.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d2dec17992372215d2d3b3aee3020e75020900f3 --- /dev/null +++ b/eval_out/generated_videos/cf_ffff/overall_consistency/058.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dddb62686636c90f09f31a761294c35f0bcf171672c71b3329f1ead866cd0109 +size 659057 diff --git a/eval_out/generated_videos/cf_ffff/overall_consistency/059.mp4 b/eval_out/generated_videos/cf_ffff/overall_consistency/059.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..69adb36e79e601e270ef7529599ed442e694c7f4 --- /dev/null +++ b/eval_out/generated_videos/cf_ffff/overall_consistency/059.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5943a0f3eb876f9abe7354aada991499aaa7f62987421014e5ccb0d6670183d4 +size 638476 diff --git a/eval_out/generated_videos/cf_ffff/overall_consistency/060.mp4 b/eval_out/generated_videos/cf_ffff/overall_consistency/060.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0fb8d49586a62dd70873bd1e11abf1538b4a3dcd --- /dev/null +++ b/eval_out/generated_videos/cf_ffff/overall_consistency/060.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e72cb534e872f779114c2220d1baeee28e62cdf9d7162927459071387e8a5151 +size 599696 diff --git a/eval_out/generated_videos/cf_ffff/overall_consistency/061.mp4 b/eval_out/generated_videos/cf_ffff/overall_consistency/061.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..aa4b3f09582f0cb0732729b3534c8dab900c0035 --- /dev/null +++ b/eval_out/generated_videos/cf_ffff/overall_consistency/061.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b2655bb9bdf420235fb0e16523a2ccaa2b132b1ff1aa0d0a5c9e157a32ee3886 +size 568256 diff --git a/eval_out/generated_videos/cf_ffff/overall_consistency/062.mp4 b/eval_out/generated_videos/cf_ffff/overall_consistency/062.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3a36b38c8afc1146368b65343c79cd665d260cbf --- /dev/null +++ b/eval_out/generated_videos/cf_ffff/overall_consistency/062.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:05f4a98f95596e8dcc4fb63ee8d4f61382c4dd25636a0a84d77bef13de44c73f +size 696092 diff --git a/eval_out/generated_videos/cf_ffff/overall_consistency/063.mp4 b/eval_out/generated_videos/cf_ffff/overall_consistency/063.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e0fdfa50d502772ff8dc64dd008108e8352c214f --- /dev/null +++ b/eval_out/generated_videos/cf_ffff/overall_consistency/063.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9705056a1725fd09482064362d9e32380565e824a26cb6ff9019c994c0e5c706 +size 621342 diff --git a/eval_out/generated_videos/cf_ffff/overall_consistency/064.mp4 b/eval_out/generated_videos/cf_ffff/overall_consistency/064.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b50900ec271beebf189ba635cf7aa9205c9d19c8 --- /dev/null +++ b/eval_out/generated_videos/cf_ffff/overall_consistency/064.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:94374322dfdc09b823bc60a3fd0248230847a83775c37f50f3e3e500db5efab6 +size 639927 diff --git a/eval_out/generated_videos/cf_ffff/overall_consistency/065.mp4 b/eval_out/generated_videos/cf_ffff/overall_consistency/065.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..51b859688fd6bc2b13f11e8c53a9ae104a815cbf --- /dev/null +++ b/eval_out/generated_videos/cf_ffff/overall_consistency/065.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:54ffb99cf0d2a332cd7342da947ab60cdcc21fd6d5fecfbd3125c2e202e29463 +size 637606 diff --git a/eval_out/generated_videos/cf_ffff/overall_consistency/066.mp4 b/eval_out/generated_videos/cf_ffff/overall_consistency/066.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6eef7a35e9468a412635ed3e8860fda0fa6fe954 --- /dev/null +++ b/eval_out/generated_videos/cf_ffff/overall_consistency/066.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:916b8039c9dfe2f0186f636c2e6305abf93b4088dde3e97c62ecf9628406c279 +size 597902 diff --git a/eval_out/generated_videos/cf_ffff/overall_consistency/067.mp4 b/eval_out/generated_videos/cf_ffff/overall_consistency/067.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0153f018bb017ba0e7922d1b4a92bb7575cd0435 --- /dev/null +++ b/eval_out/generated_videos/cf_ffff/overall_consistency/067.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7c20e553814e25d754206f21407172c47d551cc7d1d2ca72551a531723c88fc +size 644604 diff --git a/eval_out/generated_videos/cf_ffff/overall_consistency/068.mp4 b/eval_out/generated_videos/cf_ffff/overall_consistency/068.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..715bb27bc2dd96531f10e3de11cd42a197ae7f34 --- /dev/null +++ b/eval_out/generated_videos/cf_ffff/overall_consistency/068.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f20d7242761c717d159559842e20f3a7547609f66fbc6242ec935f868d34e68d +size 628497 diff --git a/eval_out/generated_videos/cf_ffff/overall_consistency/070.mp4 b/eval_out/generated_videos/cf_ffff/overall_consistency/070.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..463a691d29ecde58e44749299de3c30cfa592094 --- /dev/null +++ b/eval_out/generated_videos/cf_ffff/overall_consistency/070.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b6e5d75cec9f4ad7eae871acd74c0bc4a357aa79b814e50b15ee3a5c034123a7 +size 639077 diff --git a/eval_out/generated_videos/cf_ffff/overall_consistency/072.mp4 b/eval_out/generated_videos/cf_ffff/overall_consistency/072.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7cae9bb8bf3497e85fbf7d91366f142e9897240d --- /dev/null +++ b/eval_out/generated_videos/cf_ffff/overall_consistency/072.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab1c13a5f40bd5903afabd6773d8afe4ff2f16b62bc6d8234fbf6356d43c57bd +size 589559 diff --git a/eval_out/generated_videos/cf_ffff/overall_consistency/078.mp4 b/eval_out/generated_videos/cf_ffff/overall_consistency/078.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..73419ccf44b42ced25df3d5ceb57e9efbadb6470 --- /dev/null +++ b/eval_out/generated_videos/cf_ffff/overall_consistency/078.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:080d94895d18a4911e034612391ef4eca3b952c2f4010686df6b605428f4601e +size 550642 diff --git a/eval_out/generated_videos/cf_ffff/overall_consistency/079.mp4 b/eval_out/generated_videos/cf_ffff/overall_consistency/079.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..09ccd574c204a20d18f282aca6a2ca782f30f988 --- /dev/null +++ b/eval_out/generated_videos/cf_ffff/overall_consistency/079.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:38a3694fc319ae43dec4b6a18dbf67f0ce9be8029e8187b6e2bfcfc9739d672a +size 739700 diff --git a/eval_out/generated_videos/cf_ffff/overall_consistency/083.mp4 b/eval_out/generated_videos/cf_ffff/overall_consistency/083.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1a01359d68b40f117eef9dc6da8e0444d4382547 --- /dev/null +++ b/eval_out/generated_videos/cf_ffff/overall_consistency/083.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0247acecc70f733b24e6fea4ebc56f00e32515ab3f07aeebd009e50ca1eb204e +size 687863 diff --git a/eval_out/generated_videos/cf_ffff/overall_consistency/088.mp4 b/eval_out/generated_videos/cf_ffff/overall_consistency/088.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..881e50802ffa2e9e55fea95a2ec6cf80a7b0fb50 --- /dev/null +++ b/eval_out/generated_videos/cf_ffff/overall_consistency/088.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c958c5c43de20db284329f2a17a3731d4c4a358b696b3f6c24caa831cae60170 +size 666820 diff --git a/eval_out/generated_videos/cf_ffff/overall_consistency/089.mp4 b/eval_out/generated_videos/cf_ffff/overall_consistency/089.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f731829069e5c90b67c992bc2fc8931d1836840a --- /dev/null +++ b/eval_out/generated_videos/cf_ffff/overall_consistency/089.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d2cfcecbfbf5f4b6a73e8bca6a31948560f4f8971db72ede534f56b64fd891a8 +size 632661 diff --git a/eval_out/generated_videos/cf_ffff/overall_consistency/090.mp4 b/eval_out/generated_videos/cf_ffff/overall_consistency/090.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..24c5b8a1acf06fcd51594c049ef7663fb1ac665c --- /dev/null +++ b/eval_out/generated_videos/cf_ffff/overall_consistency/090.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da1fdf4a34bc5b9a3b223f67a54dbae82ab18ce1fcf90fa95f65fd3eadc5f4c5 +size 645255 diff --git a/eval_out/generated_videos/cf_ffff/overall_consistency/091.mp4 b/eval_out/generated_videos/cf_ffff/overall_consistency/091.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..29b97b354b24d76b38df759a7d3a01f4fac76a97 --- /dev/null +++ b/eval_out/generated_videos/cf_ffff/overall_consistency/091.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:358d9426723e0bad01085e690577f7f8987f9cd3b0f6c99f074ebf383e57adea +size 525139 diff --git a/eval_out/generated_videos/cf_ffff/overall_consistency/092.mp4 b/eval_out/generated_videos/cf_ffff/overall_consistency/092.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..642d3636537c997f6876956984f96ab0c1c338ad --- /dev/null +++ b/eval_out/generated_videos/cf_ffff/overall_consistency/092.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a98735769ffc4180703af9ba9e67844863fb00efb7db46cdf1c88356a095e530 +size 681730