Add files using upload-large-folder tool
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- eval_out/per_prompt/cf_flowcache_Fxxx_x3/overall_consistency_000.json +43 -0
- eval_out/per_prompt/cf_flowcache_Fxxx_x3/overall_consistency_001.json +43 -0
- eval_out/per_prompt/cf_flowcache_Fxxx_x3/overall_consistency_002.json +43 -0
- eval_out/per_prompt/cf_flowcache_Fxxx_x3/overall_consistency_003.json +43 -0
- eval_out/per_prompt/cf_flowcache_Fxxx_x3/overall_consistency_004.json +43 -0
- eval_out/per_prompt/cf_flowcache_Fxxx_x3/overall_consistency_005.json +43 -0
- eval_out/per_prompt/cf_flowcache_Fxxx_x3/overall_consistency_006.json +43 -0
- eval_out/per_prompt/cf_flowcache_Fxxx_x3/overall_consistency_007.json +43 -0
- eval_out/per_prompt/cf_flowcache_Fxxx_x3/overall_consistency_008.json +43 -0
- eval_out/per_prompt/cf_flowcache_Fxxx_x3/overall_consistency_009.json +43 -0
- eval_out/per_prompt/cf_flowcache_Fxxx_x3/overall_consistency_010.json +43 -0
- eval_out/per_prompt/cf_flowcache_Fxxx_x3/overall_consistency_011.json +43 -0
- eval_out/per_prompt/cf_flowcache_Fxxx_x3/overall_consistency_012.json +43 -0
- eval_out/per_prompt/cf_flowcache_Fxxx_x3/overall_consistency_013.json +43 -0
- eval_out/per_prompt/cf_flowcache_Fxxx_x3/overall_consistency_015.json +43 -0
- eval_out/per_prompt/cf_flowcache_Fxxx_x3/subject_consistency_000.json +43 -0
- eval_out/per_prompt/cf_flowcache_Fxxx_x3/subject_consistency_001.json +43 -0
- eval_out/per_prompt/cf_flowcache_Fxxx_x3/subject_consistency_002.json +43 -0
- eval_out/per_prompt/cf_flowcache_Fxxx_x3/subject_consistency_003.json +43 -0
- eval_out/per_prompt/cf_flowcache_Fxxx_x3/subject_consistency_004.json +43 -0
- eval_out/per_prompt/cf_flowcache_Fxxx_x3/subject_consistency_005.json +43 -0
- eval_out/per_prompt/cf_flowcache_Fxxx_x3/subject_consistency_006.json +43 -0
- eval_out/per_prompt/cf_flowcache_Fxxx_x3/subject_consistency_007.json +43 -0
- eval_out/per_prompt/cf_flowcache_Fxxx_x3/subject_consistency_008.json +43 -0
- eval_out/per_prompt/cf_flowcache_Fxxx_x3/subject_consistency_009.json +43 -0
- eval_out/per_prompt/cf_flowcache_Fxxx_x3/subject_consistency_010.json +43 -0
- eval_out/per_prompt/cf_flowcache_Fxxx_x3/subject_consistency_011.json +43 -0
- eval_out/per_prompt/cf_flowcache_Fxxx_x3/subject_consistency_012.json +43 -0
- eval_out/per_prompt/cf_flowcache_Fxxx_x3/subject_consistency_013.json +43 -0
- eval_out/per_prompt/cf_flowcache_Fxxx_x3/subject_consistency_014.json +43 -0
- eval_out/per_prompt/cf_flowcache_Fxxx_x3/subject_consistency_015.json +43 -0
- eval_out/per_prompt/cf_flowcache_Fxxx_x3/subject_consistency_016.json +43 -0
- eval_out/per_prompt/cf_flowcache_Fxxx_x3/subject_consistency_017.json +43 -0
- eval_out/per_prompt/cf_flowcache_Fxxx_x3/subject_consistency_018.json +43 -0
- eval_out/per_prompt/cf_flowcache_Fxxx_x3/subject_consistency_019.json +43 -0
- eval_out/per_prompt/cf_flowcache_Fxxx_x3/subject_consistency_020.json +43 -0
- eval_out/per_prompt/cf_flowcache_Fxxx_x3/subject_consistency_021.json +43 -0
- eval_out/per_prompt/cf_flowcache_Fxxx_x3/subject_consistency_022.json +43 -0
- eval_out/per_prompt/cf_flowcache_Fxxx_x3/subject_consistency_023.json +43 -0
- eval_out/per_prompt/cf_flowcache_Fxxx_x3/subject_consistency_024.json +43 -0
- eval_out/per_prompt/cf_flowcache_Fxxx_x3/subject_consistency_025.json +43 -0
- eval_out/per_prompt/cf_flowcache_Fxxx_x3/subject_consistency_026.json +43 -0
- eval_out/per_prompt/cf_flowcache_Fxxx_x3/subject_consistency_027.json +43 -0
- eval_out/per_prompt/cf_flowcache_Fxxx_x3/subject_consistency_028.json +43 -0
- eval_out/per_prompt/cf_flowcache_Fxxx_x3/subject_consistency_029.json +43 -0
- eval_out/per_prompt/cf_flowcache_Fxxx_x3/subject_consistency_030.json +43 -0
- eval_out/per_prompt/cf_flowcache_Fxxx_x3/subject_consistency_031.json +43 -0
- eval_out/per_prompt/cf_flowcache_Fxxx_x3/subject_consistency_032.json +43 -0
- eval_out/per_prompt/cf_flowcache_Fxxx_x3/subject_consistency_033.json +43 -0
- eval_out/per_prompt/cf_flowcache_Fxxx_x3/subject_consistency_034.json +43 -0
eval_out/per_prompt/cf_flowcache_Fxxx_x3/overall_consistency_000.json
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "complete",
|
| 3 |
+
"protocol": "Self-Forcing Extended-251 Full Evaluation",
|
| 4 |
+
"strategy": "cf_flowcache_Fxxx_x3",
|
| 5 |
+
"base_model": "causal_forcing",
|
| 6 |
+
"method": "flowcache",
|
| 7 |
+
"param": "thresh",
|
| 8 |
+
"param_value": 1.69921875,
|
| 9 |
+
"target_speedup": 3.0,
|
| 10 |
+
"schedule": "Fxxx",
|
| 11 |
+
"global_index": 683,
|
| 12 |
+
"prompt_suite": "overall_consistency",
|
| 13 |
+
"suite_index": 0,
|
| 14 |
+
"prompt": "A close-up view of a cluster of vibrant green grapes on a rotating glass table under soft, diffused lighting. The grapes are large and plump, reflecting the gentle light as they rotate slowly, showcasing their smooth, shiny surfaces. The background is blurred, focusing attention solely on the grapes and their subtle reflections. The camera remains static, capturing the serene and detailed motion of the rotating grapes. Close-up shot.",
|
| 15 |
+
"original_prompt": "Close up of grapes on a rotating table.",
|
| 16 |
+
"seed": 0,
|
| 17 |
+
"video": "/local/zoubin/cz/projects/comparison/eval_out/generated_videos/cf_flowcache_Fxxx_x3/overall_consistency/000.mp4",
|
| 18 |
+
"num_frames": 81,
|
| 19 |
+
"height": 480,
|
| 20 |
+
"width": 832,
|
| 21 |
+
"fps": 16,
|
| 22 |
+
"policy_latency_ms": 904.8431755304337,
|
| 23 |
+
"excluded_context_kv_latency_ms": 790.8133850097656,
|
| 24 |
+
"wall_generation_s": 3.45025897026062,
|
| 25 |
+
"reference_strategy": "cf_ffff",
|
| 26 |
+
"pixel_metrics_vs_ffff": {
|
| 27 |
+
"mean_mse": 0.10440772026777267,
|
| 28 |
+
"psnr": 9.812673926353455,
|
| 29 |
+
"ssim": 0.3504944443702698,
|
| 30 |
+
"lpips": 0.7776526808738708,
|
| 31 |
+
"num_frames": 81
|
| 32 |
+
},
|
| 33 |
+
"cache_diagnostics": {
|
| 34 |
+
"denoise_forwards": 28,
|
| 35 |
+
"compute_equivalent_forwards": 7.333333333333333,
|
| 36 |
+
"middle_steps": 21,
|
| 37 |
+
"middle_compute_equivalent": 0.3333333333333333,
|
| 38 |
+
"compute_equivalent_forwards_retime": 7.333333333333333
|
| 39 |
+
},
|
| 40 |
+
"policy_latency_ms_generation_run": 1178.0223089456558,
|
| 41 |
+
"matched_ffff_policy_latency_ms": 3178.67537689209,
|
| 42 |
+
"latency_source": "retime_eval: paired with FFFF in the same process, denoise only"
|
| 43 |
+
}
|
eval_out/per_prompt/cf_flowcache_Fxxx_x3/overall_consistency_001.json
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "complete",
|
| 3 |
+
"protocol": "Self-Forcing Extended-251 Full Evaluation",
|
| 4 |
+
"strategy": "cf_flowcache_Fxxx_x3",
|
| 5 |
+
"base_model": "causal_forcing",
|
| 6 |
+
"method": "flowcache",
|
| 7 |
+
"param": "thresh",
|
| 8 |
+
"param_value": 1.69921875,
|
| 9 |
+
"target_speedup": 3.0,
|
| 10 |
+
"schedule": "Fxxx",
|
| 11 |
+
"global_index": 684,
|
| 12 |
+
"prompt_suite": "overall_consistency",
|
| 13 |
+
"suite_index": 1,
|
| 14 |
+
"prompt": "A single green sea turtle gracefully swimming through the vast, clear waters of the ocean. The turtle has a smooth, rounded shell with brown and olive patterns, and flippers that move rhythmically as it propels itself forward. Schools of colorful fish swim alongside, adding to the vibrant marine life. The sunlight filters through the water, creating gentle beams and highlighting the soft, turquoise hues of the ocean. The background shows a few coral reefs and patches of seaweed swaying gently with the current. Shot from an underwater perspective, medium close-up focusing on the turtle.",
|
| 15 |
+
"original_prompt": "Turtle swimming in ocean.",
|
| 16 |
+
"seed": 0,
|
| 17 |
+
"video": "/local/zoubin/cz/projects/comparison/eval_out/generated_videos/cf_flowcache_Fxxx_x3/overall_consistency/001.mp4",
|
| 18 |
+
"num_frames": 81,
|
| 19 |
+
"height": 480,
|
| 20 |
+
"width": 832,
|
| 21 |
+
"fps": 16,
|
| 22 |
+
"policy_latency_ms": 902.6284517049789,
|
| 23 |
+
"excluded_context_kv_latency_ms": 778.1921615600586,
|
| 24 |
+
"wall_generation_s": 3.469468355178833,
|
| 25 |
+
"reference_strategy": "cf_ffff",
|
| 26 |
+
"pixel_metrics_vs_ffff": {
|
| 27 |
+
"mean_mse": 0.06300566345453262,
|
| 28 |
+
"psnr": 12.00620412826538,
|
| 29 |
+
"ssim": 0.3181286156177521,
|
| 30 |
+
"lpips": 0.669120728969574,
|
| 31 |
+
"num_frames": 81
|
| 32 |
+
},
|
| 33 |
+
"cache_diagnostics": {
|
| 34 |
+
"denoise_forwards": 28,
|
| 35 |
+
"compute_equivalent_forwards": 8.0,
|
| 36 |
+
"middle_steps": 21,
|
| 37 |
+
"middle_compute_equivalent": 1.0,
|
| 38 |
+
"compute_equivalent_forwards_retime": 8.0
|
| 39 |
+
},
|
| 40 |
+
"policy_latency_ms_generation_run": 1126.2802653312683,
|
| 41 |
+
"matched_ffff_policy_latency_ms": 3113.966682434082,
|
| 42 |
+
"latency_source": "retime_eval: paired with FFFF in the same process, denoise only"
|
| 43 |
+
}
|
eval_out/per_prompt/cf_flowcache_Fxxx_x3/overall_consistency_002.json
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "complete",
|
| 3 |
+
"protocol": "Self-Forcing Extended-251 Full Evaluation",
|
| 4 |
+
"strategy": "cf_flowcache_Fxxx_x3",
|
| 5 |
+
"base_model": "causal_forcing",
|
| 6 |
+
"method": "flowcache",
|
| 7 |
+
"param": "thresh",
|
| 8 |
+
"param_value": 1.69921875,
|
| 9 |
+
"target_speedup": 3.0,
|
| 10 |
+
"schedule": "Fxxx",
|
| 11 |
+
"global_index": 685,
|
| 12 |
+
"prompt_suite": "overall_consistency",
|
| 13 |
+
"suite_index": 2,
|
| 14 |
+
"prompt": "A stormtrooper from the Star Wars universe, clad in pristine white armor with a black helmet, is meticulously vacuuming a sandy beach. He bends down slightly, moving the vacuum cleaner back and forth across the sand with purposeful motions. His gloved hand firmly grips the handle of the vacuum as he navigates around rocks and debris. The sun sets behind him, casting long shadows and giving the scene a dramatic, golden glow. The background shows crashing waves and seagulls flying overhead. Medium close-up shot, focusing on the stormtrooper's actions and the sweeping motion of the vacuum.",
|
| 15 |
+
"original_prompt": "A storm trooper vacuuming the beach.",
|
| 16 |
+
"seed": 0,
|
| 17 |
+
"video": "/local/zoubin/cz/projects/comparison/eval_out/generated_videos/cf_flowcache_Fxxx_x3/overall_consistency/002.mp4",
|
| 18 |
+
"num_frames": 81,
|
| 19 |
+
"height": 480,
|
| 20 |
+
"width": 832,
|
| 21 |
+
"fps": 16,
|
| 22 |
+
"policy_latency_ms": 1572.8279753923416,
|
| 23 |
+
"excluded_context_kv_latency_ms": 796.2469635009766,
|
| 24 |
+
"wall_generation_s": 4.127354860305786,
|
| 25 |
+
"reference_strategy": "cf_ffff",
|
| 26 |
+
"pixel_metrics_vs_ffff": {
|
| 27 |
+
"mean_mse": 0.06299798935651779,
|
| 28 |
+
"psnr": 12.006733417510986,
|
| 29 |
+
"ssim": 0.47075748443603516,
|
| 30 |
+
"lpips": 0.5509769320487976,
|
| 31 |
+
"num_frames": 81
|
| 32 |
+
},
|
| 33 |
+
"cache_diagnostics": {
|
| 34 |
+
"denoise_forwards": 28,
|
| 35 |
+
"compute_equivalent_forwards": 13.333333333333332,
|
| 36 |
+
"middle_steps": 21,
|
| 37 |
+
"middle_compute_equivalent": 6.333333333333334,
|
| 38 |
+
"compute_equivalent_forwards_retime": 13.333333333333332
|
| 39 |
+
},
|
| 40 |
+
"policy_latency_ms_generation_run": 1902.1509569883347,
|
| 41 |
+
"matched_ffff_policy_latency_ms": 3168.737564086914,
|
| 42 |
+
"latency_source": "retime_eval: paired with FFFF in the same process, denoise only"
|
| 43 |
+
}
|
eval_out/per_prompt/cf_flowcache_Fxxx_x3/overall_consistency_003.json
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "complete",
|
| 3 |
+
"protocol": "Self-Forcing Extended-251 Full Evaluation",
|
| 4 |
+
"strategy": "cf_flowcache_Fxxx_x3",
|
| 5 |
+
"base_model": "causal_forcing",
|
| 6 |
+
"method": "flowcache",
|
| 7 |
+
"param": "thresh",
|
| 8 |
+
"param_value": 1.69921875,
|
| 9 |
+
"target_speedup": 3.0,
|
| 10 |
+
"schedule": "Fxxx",
|
| 11 |
+
"global_index": 686,
|
| 12 |
+
"prompt_suite": "overall_consistency",
|
| 13 |
+
"suite_index": 3,
|
| 14 |
+
"prompt": "A large, black-and-white panda stands confidently on a surfboard in the golden hour of sunset over the ocean. The panda has a playful expression, with its front paws resting lightly on the board. The waves gently roll around the surfboard, and the sky is painted with warm hues of orange and pink. The water is calm, reflecting the vibrant colors of the setting sun. The scene captures the serene beauty of the ocean at twilight. Medium shot focusing on the panda and the surfboard, with the horizon and sunset visible in the background.",
|
| 15 |
+
"original_prompt": "A panda standing on a surfboard in the ocean in sunset.",
|
| 16 |
+
"seed": 0,
|
| 17 |
+
"video": "/local/zoubin/cz/projects/comparison/eval_out/generated_videos/cf_flowcache_Fxxx_x3/overall_consistency/003.mp4",
|
| 18 |
+
"num_frames": 81,
|
| 19 |
+
"height": 480,
|
| 20 |
+
"width": 832,
|
| 21 |
+
"fps": 16,
|
| 22 |
+
"policy_latency_ms": 1019.9760345220566,
|
| 23 |
+
"excluded_context_kv_latency_ms": 777.0445022583008,
|
| 24 |
+
"wall_generation_s": 3.577324867248535,
|
| 25 |
+
"reference_strategy": "cf_ffff",
|
| 26 |
+
"pixel_metrics_vs_ffff": {
|
| 27 |
+
"mean_mse": 0.0912286713719368,
|
| 28 |
+
"psnr": 10.398685932159424,
|
| 29 |
+
"ssim": 0.36971515417099,
|
| 30 |
+
"lpips": 0.6783936023712158,
|
| 31 |
+
"num_frames": 81
|
| 32 |
+
},
|
| 33 |
+
"cache_diagnostics": {
|
| 34 |
+
"denoise_forwards": 28,
|
| 35 |
+
"compute_equivalent_forwards": 8.333333333333332,
|
| 36 |
+
"middle_steps": 21,
|
| 37 |
+
"middle_compute_equivalent": 1.3333333333333333,
|
| 38 |
+
"compute_equivalent_forwards_retime": 8.333333333333332
|
| 39 |
+
},
|
| 40 |
+
"policy_latency_ms_generation_run": 1261.505482673645,
|
| 41 |
+
"matched_ffff_policy_latency_ms": 3116.6825637817383,
|
| 42 |
+
"latency_source": "retime_eval: paired with FFFF in the same process, denoise only"
|
| 43 |
+
}
|
eval_out/per_prompt/cf_flowcache_Fxxx_x3/overall_consistency_004.json
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "complete",
|
| 3 |
+
"protocol": "Self-Forcing Extended-251 Full Evaluation",
|
| 4 |
+
"strategy": "cf_flowcache_Fxxx_x3",
|
| 5 |
+
"base_model": "causal_forcing",
|
| 6 |
+
"method": "flowcache",
|
| 7 |
+
"param": "thresh",
|
| 8 |
+
"param_value": 1.69921875,
|
| 9 |
+
"target_speedup": 3.0,
|
| 10 |
+
"schedule": "Fxxx",
|
| 11 |
+
"global_index": 687,
|
| 12 |
+
"prompt_suite": "overall_consistency",
|
| 13 |
+
"suite_index": 4,
|
| 14 |
+
"prompt": "A serene sunny afternoon scene featuring an astronaut in a sleek, white spacesuit with a reflective visor, feeding ducks at a tranquil pond. The astronaut bends gently at the waist, holding a handful of bread crumbs towards the ducks, which are swimming close to the surface. The water reflects the clear blue sky and the astronaut's figure, creating a harmonious mirror image. The surrounding environment includes lush green grass and blooming flowers, enhancing the peaceful atmosphere. The camera focuses closely on the interaction between the astronaut and the ducks, capturing the gentle movements and reflections in the water. Medium close-up shot, static camera.",
|
| 15 |
+
"original_prompt": "An astronaut feeding ducks on a sunny afternoon, reflection from the water.",
|
| 16 |
+
"seed": 0,
|
| 17 |
+
"video": "/local/zoubin/cz/projects/comparison/eval_out/generated_videos/cf_flowcache_Fxxx_x3/overall_consistency/004.mp4",
|
| 18 |
+
"num_frames": 81,
|
| 19 |
+
"height": 480,
|
| 20 |
+
"width": 832,
|
| 21 |
+
"fps": 16,
|
| 22 |
+
"policy_latency_ms": 915.4318103790283,
|
| 23 |
+
"excluded_context_kv_latency_ms": 790.5893096923828,
|
| 24 |
+
"wall_generation_s": 3.471740245819092,
|
| 25 |
+
"reference_strategy": "cf_ffff",
|
| 26 |
+
"pixel_metrics_vs_ffff": {
|
| 27 |
+
"mean_mse": 0.07179976999759674,
|
| 28 |
+
"psnr": 11.438769102096558,
|
| 29 |
+
"ssim": 0.32959407567977905,
|
| 30 |
+
"lpips": 0.6650192141532898,
|
| 31 |
+
"num_frames": 81
|
| 32 |
+
},
|
| 33 |
+
"cache_diagnostics": {
|
| 34 |
+
"denoise_forwards": 28,
|
| 35 |
+
"compute_equivalent_forwards": 8.0,
|
| 36 |
+
"middle_steps": 21,
|
| 37 |
+
"middle_compute_equivalent": 1.0,
|
| 38 |
+
"compute_equivalent_forwards_retime": 8.0
|
| 39 |
+
},
|
| 40 |
+
"policy_latency_ms_generation_run": 1266.81742978096,
|
| 41 |
+
"matched_ffff_policy_latency_ms": 3166.6651611328125,
|
| 42 |
+
"latency_source": "retime_eval: paired with FFFF in the same process, denoise only"
|
| 43 |
+
}
|
eval_out/per_prompt/cf_flowcache_Fxxx_x3/overall_consistency_005.json
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "complete",
|
| 3 |
+
"protocol": "Self-Forcing Extended-251 Full Evaluation",
|
| 4 |
+
"strategy": "cf_flowcache_Fxxx_x3",
|
| 5 |
+
"base_model": "causal_forcing",
|
| 6 |
+
"method": "flowcache",
|
| 7 |
+
"param": "thresh",
|
| 8 |
+
"param_value": 1.69921875,
|
| 9 |
+
"target_speedup": 3.0,
|
| 10 |
+
"schedule": "Fxxx",
|
| 11 |
+
"global_index": 688,
|
| 12 |
+
"prompt_suite": "overall_consistency",
|
| 13 |
+
"suite_index": 5,
|
| 14 |
+
"prompt": "A cozy indoor setting with two pandas sitting on plush chairs, engaged in a thoughtful discussion about an academic paper. Both pandas have black patches around their eyes and are wearing small glasses perched on their snouts. They are hunched over a stack of papers filled with complex equations and graphs, occasionally pointing at sections and nodding in agreement. One panda is gesturing animatedly with a pen, while the other listens intently. The room is softly lit with warm, ambient lighting, creating a comfortable atmosphere for scholarly debate. Medium close-up view focusing on the pandas' expressions and gestures.",
|
| 15 |
+
"original_prompt": "Two pandas discussing an academic paper.",
|
| 16 |
+
"seed": 0,
|
| 17 |
+
"video": "/local/zoubin/cz/projects/comparison/eval_out/generated_videos/cf_flowcache_Fxxx_x3/overall_consistency/005.mp4",
|
| 18 |
+
"num_frames": 81,
|
| 19 |
+
"height": 480,
|
| 20 |
+
"width": 832,
|
| 21 |
+
"fps": 16,
|
| 22 |
+
"policy_latency_ms": 967.2534379959106,
|
| 23 |
+
"excluded_context_kv_latency_ms": 776.7176132202148,
|
| 24 |
+
"wall_generation_s": 3.5396063327789307,
|
| 25 |
+
"reference_strategy": "cf_ffff",
|
| 26 |
+
"pixel_metrics_vs_ffff": {
|
| 27 |
+
"mean_mse": 0.08122103661298752,
|
| 28 |
+
"psnr": 10.903314352035522,
|
| 29 |
+
"ssim": 0.3845294713973999,
|
| 30 |
+
"lpips": 0.6529510617256165,
|
| 31 |
+
"num_frames": 81
|
| 32 |
+
},
|
| 33 |
+
"cache_diagnostics": {
|
| 34 |
+
"denoise_forwards": 28,
|
| 35 |
+
"compute_equivalent_forwards": 8.666666666666666,
|
| 36 |
+
"middle_steps": 21,
|
| 37 |
+
"middle_compute_equivalent": 1.6666666666666665,
|
| 38 |
+
"compute_equivalent_forwards_retime": 8.666666666666666
|
| 39 |
+
},
|
| 40 |
+
"policy_latency_ms_generation_run": 1242.5762654542923,
|
| 41 |
+
"matched_ffff_policy_latency_ms": 3117.9992904663086,
|
| 42 |
+
"latency_source": "retime_eval: paired with FFFF in the same process, denoise only"
|
| 43 |
+
}
|
eval_out/per_prompt/cf_flowcache_Fxxx_x3/overall_consistency_006.json
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "complete",
|
| 3 |
+
"protocol": "Self-Forcing Extended-251 Full Evaluation",
|
| 4 |
+
"strategy": "cf_flowcache_Fxxx_x3",
|
| 5 |
+
"base_model": "causal_forcing",
|
| 6 |
+
"method": "flowcache",
|
| 7 |
+
"param": "thresh",
|
| 8 |
+
"param_value": 1.69921875,
|
| 9 |
+
"target_speedup": 3.0,
|
| 10 |
+
"schedule": "Fxxx",
|
| 11 |
+
"global_index": 689,
|
| 12 |
+
"prompt_suite": "overall_consistency",
|
| 13 |
+
"suite_index": 6,
|
| 14 |
+
"prompt": "A sunset time-lapse sequence at a serene beach, capturing the vibrant colors of the sky as the sun sets below the horizon. The sky transitions from bright orange and pink hues to deep purples and blues, with wispy clouds moving gracefully across the frame. The beach is calm, with soft waves gently lapping at the shore. The camera remains stationary, focusing on the dynamic changes in the sky and the interplay of light and shadow over the landscape. Wide shot, capturing the expansive beauty of the beach and sky during sunset.",
|
| 15 |
+
"original_prompt": "Sunset time lapse at the beach with moving clouds and colors in the sky.",
|
| 16 |
+
"seed": 0,
|
| 17 |
+
"video": "/local/zoubin/cz/projects/comparison/eval_out/generated_videos/cf_flowcache_Fxxx_x3/overall_consistency/006.mp4",
|
| 18 |
+
"num_frames": 81,
|
| 19 |
+
"height": 480,
|
| 20 |
+
"width": 832,
|
| 21 |
+
"fps": 16,
|
| 22 |
+
"policy_latency_ms": 916.054802775383,
|
| 23 |
+
"excluded_context_kv_latency_ms": 790.4520263671875,
|
| 24 |
+
"wall_generation_s": 3.4835829734802246,
|
| 25 |
+
"reference_strategy": "cf_ffff",
|
| 26 |
+
"pixel_metrics_vs_ffff": {
|
| 27 |
+
"mean_mse": 0.06271815299987793,
|
| 28 |
+
"psnr": 12.026067972183228,
|
| 29 |
+
"ssim": 0.3196362555027008,
|
| 30 |
+
"lpips": 0.6688971519470215,
|
| 31 |
+
"num_frames": 81
|
| 32 |
+
},
|
| 33 |
+
"cache_diagnostics": {
|
| 34 |
+
"denoise_forwards": 28,
|
| 35 |
+
"compute_equivalent_forwards": 8.0,
|
| 36 |
+
"middle_steps": 21,
|
| 37 |
+
"middle_compute_equivalent": 1.0,
|
| 38 |
+
"compute_equivalent_forwards_retime": 8.0
|
| 39 |
+
},
|
| 40 |
+
"policy_latency_ms_generation_run": 1109.4795693159103,
|
| 41 |
+
"matched_ffff_policy_latency_ms": 3165.3640670776367,
|
| 42 |
+
"latency_source": "retime_eval: paired with FFFF in the same process, denoise only"
|
| 43 |
+
}
|
eval_out/per_prompt/cf_flowcache_Fxxx_x3/overall_consistency_007.json
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "complete",
|
| 3 |
+
"protocol": "Self-Forcing Extended-251 Full Evaluation",
|
| 4 |
+
"strategy": "cf_flowcache_Fxxx_x3",
|
| 5 |
+
"base_model": "causal_forcing",
|
| 6 |
+
"method": "flowcache",
|
| 7 |
+
"param": "thresh",
|
| 8 |
+
"param_value": 1.69921875,
|
| 9 |
+
"target_speedup": 3.0,
|
| 10 |
+
"schedule": "Fxxx",
|
| 11 |
+
"global_index": 690,
|
| 12 |
+
"prompt_suite": "overall_consistency",
|
| 13 |
+
"suite_index": 7,
|
| 14 |
+
"prompt": "A plump, fluffy rabbit donning a voluminous purple robe walks gracefully through a vibrant fantasy landscape. The rabbit has large, expressive eyes and a gentle, curious expression. Its fur is soft and thick, and the robe drapes elegantly over its body. The landscape features rolling hills covered in lush green grass, colorful wildflowers, and towering magical trees with shimmering leaves. In the distance, there are sparkling waterfalls and mystical castles. The scene is bathed in warm, golden sunlight. Medium shot, focusing on the rabbit's walk through the picturesque environment.",
|
| 15 |
+
"original_prompt": "A fat rabbit wearing a purple robe walking through a fantasy landscape.",
|
| 16 |
+
"seed": 0,
|
| 17 |
+
"video": "/local/zoubin/cz/projects/comparison/eval_out/generated_videos/cf_flowcache_Fxxx_x3/overall_consistency/007.mp4",
|
| 18 |
+
"num_frames": 81,
|
| 19 |
+
"height": 480,
|
| 20 |
+
"width": 832,
|
| 21 |
+
"fps": 16,
|
| 22 |
+
"policy_latency_ms": 1050.5099767446518,
|
| 23 |
+
"excluded_context_kv_latency_ms": 777.2849960327148,
|
| 24 |
+
"wall_generation_s": 3.6074771881103516,
|
| 25 |
+
"reference_strategy": "cf_ffff",
|
| 26 |
+
"pixel_metrics_vs_ffff": {
|
| 27 |
+
"mean_mse": 0.08840932697057724,
|
| 28 |
+
"psnr": 10.535019636154175,
|
| 29 |
+
"ssim": 0.26215916872024536,
|
| 30 |
+
"lpips": 0.6043736934661865,
|
| 31 |
+
"num_frames": 81
|
| 32 |
+
},
|
| 33 |
+
"cache_diagnostics": {
|
| 34 |
+
"denoise_forwards": 28,
|
| 35 |
+
"compute_equivalent_forwards": 9.333333333333332,
|
| 36 |
+
"middle_steps": 21,
|
| 37 |
+
"middle_compute_equivalent": 2.3333333333333335,
|
| 38 |
+
"compute_equivalent_forwards_retime": 9.333333333333332
|
| 39 |
+
},
|
| 40 |
+
"policy_latency_ms_generation_run": 1332.2037494182587,
|
| 41 |
+
"matched_ffff_policy_latency_ms": 3110.379180908203,
|
| 42 |
+
"latency_source": "retime_eval: paired with FFFF in the same process, denoise only"
|
| 43 |
+
}
|
eval_out/per_prompt/cf_flowcache_Fxxx_x3/overall_consistency_008.json
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "complete",
|
| 3 |
+
"protocol": "Self-Forcing Extended-251 Full Evaluation",
|
| 4 |
+
"strategy": "cf_flowcache_Fxxx_x3",
|
| 5 |
+
"base_model": "causal_forcing",
|
| 6 |
+
"method": "flowcache",
|
| 7 |
+
"param": "thresh",
|
| 8 |
+
"param_value": 1.69921875,
|
| 9 |
+
"target_speedup": 3.0,
|
| 10 |
+
"schedule": "Fxxx",
|
| 11 |
+
"global_index": 691,
|
| 12 |
+
"prompt_suite": "overall_consistency",
|
| 13 |
+
"suite_index": 8,
|
| 14 |
+
"prompt": "A koala bear playing a grand piano in a lush, dense forest. The koala has soft, grey fur and large, round ears. It sits upright on the piano bench, paws delicately placed on the keys, creating gentle melodies. The forest background is filled with tall eucalyptus trees, dappled sunlight filtering through the leaves, and a carpet of green moss beneath the piano. The scene is calm and serene, with the koala focused intently on its performance. Medium close-up shot, capturing the koala and part of the forest surroundings.",
|
| 15 |
+
"original_prompt": "A koala bear playing piano in the forest.",
|
| 16 |
+
"seed": 0,
|
| 17 |
+
"video": "/local/zoubin/cz/projects/comparison/eval_out/generated_videos/cf_flowcache_Fxxx_x3/overall_consistency/008.mp4",
|
| 18 |
+
"num_frames": 81,
|
| 19 |
+
"height": 480,
|
| 20 |
+
"width": 832,
|
| 21 |
+
"fps": 16,
|
| 22 |
+
"policy_latency_ms": 1520.8138592243195,
|
| 23 |
+
"excluded_context_kv_latency_ms": 789.5379867553711,
|
| 24 |
+
"wall_generation_s": 4.065098762512207,
|
| 25 |
+
"reference_strategy": "cf_ffff",
|
| 26 |
+
"pixel_metrics_vs_ffff": {
|
| 27 |
+
"mean_mse": 0.06088685616850853,
|
| 28 |
+
"psnr": 12.15476393699646,
|
| 29 |
+
"ssim": 0.2992255687713623,
|
| 30 |
+
"lpips": 0.48640838265419006,
|
| 31 |
+
"num_frames": 81
|
| 32 |
+
},
|
| 33 |
+
"cache_diagnostics": {
|
| 34 |
+
"denoise_forwards": 28,
|
| 35 |
+
"compute_equivalent_forwards": 13.0,
|
| 36 |
+
"middle_steps": 21,
|
| 37 |
+
"middle_compute_equivalent": 6.0,
|
| 38 |
+
"compute_equivalent_forwards_retime": 13.0
|
| 39 |
+
},
|
| 40 |
+
"policy_latency_ms_generation_run": 2114.6621522903442,
|
| 41 |
+
"matched_ffff_policy_latency_ms": 3163.6348266601562,
|
| 42 |
+
"latency_source": "retime_eval: paired with FFFF in the same process, denoise only"
|
| 43 |
+
}
|
eval_out/per_prompt/cf_flowcache_Fxxx_x3/overall_consistency_009.json
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "complete",
|
| 3 |
+
"protocol": "Self-Forcing Extended-251 Full Evaluation",
|
| 4 |
+
"strategy": "cf_flowcache_Fxxx_x3",
|
| 5 |
+
"base_model": "causal_forcing",
|
| 6 |
+
"method": "flowcache",
|
| 7 |
+
"param": "thresh",
|
| 8 |
+
"param_value": 1.69921875,
|
| 9 |
+
"target_speedup": 3.0,
|
| 10 |
+
"schedule": "Fxxx",
|
| 11 |
+
"global_index": 692,
|
| 12 |
+
"prompt_suite": "overall_consistency",
|
| 13 |
+
"suite_index": 9,
|
| 14 |
+
"prompt": "A realistic animation of an astronaut floating in the vastness of space. The astronaut is wearing a full spacesuit with reflective gold visor and carrying a jetpack. They are gliding gracefully, arms outstretched, against a backdrop of stars and distant planets. The scene includes streaks of light from passing meteoroids and a faint blue Earth visible in the distance. The astronaut occasionally adjusts their jetpack to maneuver smoothly through space. Wide shot, static camera, emphasizing the serene beauty of the cosmos.",
|
| 15 |
+
"original_prompt": "An astronaut flying in space.",
|
| 16 |
+
"seed": 0,
|
| 17 |
+
"video": "/local/zoubin/cz/projects/comparison/eval_out/generated_videos/cf_flowcache_Fxxx_x3/overall_consistency/009.mp4",
|
| 18 |
+
"num_frames": 81,
|
| 19 |
+
"height": 480,
|
| 20 |
+
"width": 832,
|
| 21 |
+
"fps": 16,
|
| 22 |
+
"policy_latency_ms": 1101.5701503753662,
|
| 23 |
+
"excluded_context_kv_latency_ms": 777.5437316894531,
|
| 24 |
+
"wall_generation_s": 3.6867434978485107,
|
| 25 |
+
"reference_strategy": "cf_ffff",
|
| 26 |
+
"pixel_metrics_vs_ffff": {
|
| 27 |
+
"mean_mse": 0.07036373764276505,
|
| 28 |
+
"psnr": 11.52651071548462,
|
| 29 |
+
"ssim": 0.27602940797805786,
|
| 30 |
+
"lpips": 0.6574572920799255,
|
| 31 |
+
"num_frames": 81
|
| 32 |
+
},
|
| 33 |
+
"cache_diagnostics": {
|
| 34 |
+
"denoise_forwards": 28,
|
| 35 |
+
"compute_equivalent_forwards": 9.666666666666666,
|
| 36 |
+
"middle_steps": 21,
|
| 37 |
+
"middle_compute_equivalent": 2.6666666666666665,
|
| 38 |
+
"compute_equivalent_forwards_retime": 9.666666666666666
|
| 39 |
+
},
|
| 40 |
+
"policy_latency_ms_generation_run": 1390.7565455436707,
|
| 41 |
+
"matched_ffff_policy_latency_ms": 3114.9062271118164,
|
| 42 |
+
"latency_source": "retime_eval: paired with FFFF in the same process, denoise only"
|
| 43 |
+
}
|
eval_out/per_prompt/cf_flowcache_Fxxx_x3/overall_consistency_010.json
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "complete",
|
| 3 |
+
"protocol": "Self-Forcing Extended-251 Full Evaluation",
|
| 4 |
+
"strategy": "cf_flowcache_Fxxx_x3",
|
| 5 |
+
"base_model": "causal_forcing",
|
| 6 |
+
"method": "flowcache",
|
| 7 |
+
"param": "thresh",
|
| 8 |
+
"param_value": 1.69921875,
|
| 9 |
+
"target_speedup": 3.0,
|
| 10 |
+
"schedule": "Fxxx",
|
| 11 |
+
"global_index": 693,
|
| 12 |
+
"prompt_suite": "overall_consistency",
|
| 13 |
+
"suite_index": 10,
|
| 14 |
+
"prompt": "A vibrant display of fireworks lighting up the night sky. The fireworks burst into colorful explosions of red, blue, green, and gold, creating intricate patterns against a dark, starry backdrop. Each explosion is followed by trails of light that slowly fade away. The ground is seen below, with people watching in awe, their faces illuminated momentarily by the bright flashes. The scene captures the excitement and joy of a festive celebration. Wide shot, capturing the expansive sky and the crowd below.",
|
| 15 |
+
"original_prompt": "Fireworks.",
|
| 16 |
+
"seed": 0,
|
| 17 |
+
"video": "/local/zoubin/cz/projects/comparison/eval_out/generated_videos/cf_flowcache_Fxxx_x3/overall_consistency/010.mp4",
|
| 18 |
+
"num_frames": 81,
|
| 19 |
+
"height": 480,
|
| 20 |
+
"width": 832,
|
| 21 |
+
"fps": 16,
|
| 22 |
+
"policy_latency_ms": 1190.8790935277939,
|
| 23 |
+
"excluded_context_kv_latency_ms": 790.3857269287109,
|
| 24 |
+
"wall_generation_s": 3.760356903076172,
|
| 25 |
+
"reference_strategy": "cf_ffff",
|
| 26 |
+
"pixel_metrics_vs_ffff": {
|
| 27 |
+
"mean_mse": 0.09721642732620239,
|
| 28 |
+
"psnr": 10.122603178024292,
|
| 29 |
+
"ssim": 0.24856030941009521,
|
| 30 |
+
"lpips": 0.6236875653266907,
|
| 31 |
+
"num_frames": 81
|
| 32 |
+
},
|
| 33 |
+
"cache_diagnostics": {
|
| 34 |
+
"denoise_forwards": 28,
|
| 35 |
+
"compute_equivalent_forwards": 10.666666666666668,
|
| 36 |
+
"middle_steps": 21,
|
| 37 |
+
"middle_compute_equivalent": 3.6666666666666665,
|
| 38 |
+
"compute_equivalent_forwards_retime": 10.666666666666668
|
| 39 |
+
},
|
| 40 |
+
"policy_latency_ms_generation_run": 1481.9765086174011,
|
| 41 |
+
"matched_ffff_policy_latency_ms": 3164.6699600219727,
|
| 42 |
+
"latency_source": "retime_eval: paired with FFFF in the same process, denoise only"
|
| 43 |
+
}
|
eval_out/per_prompt/cf_flowcache_Fxxx_x3/overall_consistency_011.json
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "complete",
|
| 3 |
+
"protocol": "Self-Forcing Extended-251 Full Evaluation",
|
| 4 |
+
"strategy": "cf_flowcache_Fxxx_x3",
|
| 5 |
+
"base_model": "causal_forcing",
|
| 6 |
+
"method": "flowcache",
|
| 7 |
+
"param": "thresh",
|
| 8 |
+
"param_value": 1.69921875,
|
| 9 |
+
"target_speedup": 3.0,
|
| 10 |
+
"schedule": "Fxxx",
|
| 11 |
+
"global_index": 694,
|
| 12 |
+
"prompt_suite": "overall_consistency",
|
| 13 |
+
"suite_index": 11,
|
| 14 |
+
"prompt": "Animated watercolor painting, soft and fluffy white clouds drifting across a bright blue sky. The clouds move gracefully, changing shape as they float, creating a serene and tranquil atmosphere. The sky is clear with gentle sunlight filtering through, casting a warm glow over the scene. The clouds are rendered with delicate brush strokes, giving them a dreamy, almost ethereal quality. Close-up view focusing on the movement and transformation of the clouds.",
|
| 15 |
+
"original_prompt": "An animated painting of fluffy white clouds moving in sky.",
|
| 16 |
+
"seed": 0,
|
| 17 |
+
"video": "/local/zoubin/cz/projects/comparison/eval_out/generated_videos/cf_flowcache_Fxxx_x3/overall_consistency/011.mp4",
|
| 18 |
+
"num_frames": 81,
|
| 19 |
+
"height": 480,
|
| 20 |
+
"width": 832,
|
| 21 |
+
"fps": 16,
|
| 22 |
+
"policy_latency_ms": 969.6456083059311,
|
| 23 |
+
"excluded_context_kv_latency_ms": 776.8830718994141,
|
| 24 |
+
"wall_generation_s": 3.518270254135132,
|
| 25 |
+
"reference_strategy": "cf_ffff",
|
| 26 |
+
"pixel_metrics_vs_ffff": {
|
| 27 |
+
"mean_mse": 0.060558147728443146,
|
| 28 |
+
"psnr": 12.178274393081665,
|
| 29 |
+
"ssim": 0.4846240282058716,
|
| 30 |
+
"lpips": 0.6642823219299316,
|
| 31 |
+
"num_frames": 81
|
| 32 |
+
},
|
| 33 |
+
"cache_diagnostics": {
|
| 34 |
+
"denoise_forwards": 28,
|
| 35 |
+
"compute_equivalent_forwards": 8.333333333333332,
|
| 36 |
+
"middle_steps": 21,
|
| 37 |
+
"middle_compute_equivalent": 1.3333333333333333,
|
| 38 |
+
"compute_equivalent_forwards_retime": 8.333333333333332
|
| 39 |
+
},
|
| 40 |
+
"policy_latency_ms_generation_run": 1161.9415745735168,
|
| 41 |
+
"matched_ffff_policy_latency_ms": 3120.0264053344727,
|
| 42 |
+
"latency_source": "retime_eval: paired with FFFF in the same process, denoise only"
|
| 43 |
+
}
|
eval_out/per_prompt/cf_flowcache_Fxxx_x3/overall_consistency_012.json
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "complete",
|
| 3 |
+
"protocol": "Self-Forcing Extended-251 Full Evaluation",
|
| 4 |
+
"strategy": "cf_flowcache_Fxxx_x3",
|
| 5 |
+
"base_model": "causal_forcing",
|
| 6 |
+
"method": "flowcache",
|
| 7 |
+
"param": "thresh",
|
| 8 |
+
"param_value": 1.69921875,
|
| 9 |
+
"target_speedup": 3.0,
|
| 10 |
+
"schedule": "Fxxx",
|
| 11 |
+
"global_index": 695,
|
| 12 |
+
"prompt_suite": "overall_consistency",
|
| 13 |
+
"suite_index": 12,
|
| 14 |
+
"prompt": "Aerial view flying through vibrant fantasy landscapes. The camera captures sweeping vistas of floating islands, lush forests, and majestic waterfalls. Enormous mythical creatures like dragons and unicorns roam freely in the distance. The sky transitions from dawn hues to midday sunlight, casting dynamic shadows on the terrain. The landscape is detailed with intricate flora and fauna, creating a rich and immersive environment. The camera moves smoothly, capturing the vastness and beauty of these magical realms. Wide shot aerial perspective throughout the sequence.",
|
| 15 |
+
"original_prompt": "Flying through fantasy landscapes.",
|
| 16 |
+
"seed": 0,
|
| 17 |
+
"video": "/local/zoubin/cz/projects/comparison/eval_out/generated_videos/cf_flowcache_Fxxx_x3/overall_consistency/012.mp4",
|
| 18 |
+
"num_frames": 81,
|
| 19 |
+
"height": 480,
|
| 20 |
+
"width": 832,
|
| 21 |
+
"fps": 16,
|
| 22 |
+
"policy_latency_ms": 1536.5967763662338,
|
| 23 |
+
"excluded_context_kv_latency_ms": 789.407470703125,
|
| 24 |
+
"wall_generation_s": 4.076923847198486,
|
| 25 |
+
"reference_strategy": "cf_ffff",
|
| 26 |
+
"pixel_metrics_vs_ffff": {
|
| 27 |
+
"mean_mse": 0.07512294501066208,
|
| 28 |
+
"psnr": 11.242274045944214,
|
| 29 |
+
"ssim": 0.3613429665565491,
|
| 30 |
+
"lpips": 0.5667085647583008,
|
| 31 |
+
"num_frames": 81
|
| 32 |
+
},
|
| 33 |
+
"cache_diagnostics": {
|
| 34 |
+
"denoise_forwards": 28,
|
| 35 |
+
"compute_equivalent_forwards": 13.0,
|
| 36 |
+
"middle_steps": 21,
|
| 37 |
+
"middle_compute_equivalent": 6.0,
|
| 38 |
+
"compute_equivalent_forwards_retime": 13.0
|
| 39 |
+
},
|
| 40 |
+
"policy_latency_ms_generation_run": 1570.0793944597244,
|
| 41 |
+
"matched_ffff_policy_latency_ms": 3164.1957397460938,
|
| 42 |
+
"latency_source": "retime_eval: paired with FFFF in the same process, denoise only"
|
| 43 |
+
}
|
eval_out/per_prompt/cf_flowcache_Fxxx_x3/overall_consistency_013.json
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "complete",
|
| 3 |
+
"protocol": "Self-Forcing Extended-251 Full Evaluation",
|
| 4 |
+
"strategy": "cf_flowcache_Fxxx_x3",
|
| 5 |
+
"base_model": "causal_forcing",
|
| 6 |
+
"method": "flowcache",
|
| 7 |
+
"param": "thresh",
|
| 8 |
+
"param_value": 1.69921875,
|
| 9 |
+
"target_speedup": 3.0,
|
| 10 |
+
"schedule": "Fxxx",
|
| 11 |
+
"global_index": 696,
|
| 12 |
+
"prompt_suite": "overall_consistency",
|
| 13 |
+
"suite_index": 13,
|
| 14 |
+
"prompt": "A large, hairy Bigfoot creature walking through a heavy snowstorm. The Bigfoot stands at least eight feet tall, covered in shaggy brown fur, with muscular limbs and a stooped posture. Snowflakes swirl around him as he moves slowly and deliberately through the dense forest, his feet sinking slightly into the deep snow. The landscape is bleak and desolate, with bare trees and thick snowdrifts. The atmosphere is eerie and quiet, with only the sound of crunching snow and howling wind. The scene is captured in a mid-shot, focusing on the Bigfoot\u2019s powerful form as he trudges through the storm.",
|
| 15 |
+
"original_prompt": "A bigfoot walking in the snowstorm.",
|
| 16 |
+
"seed": 0,
|
| 17 |
+
"video": "/local/zoubin/cz/projects/comparison/eval_out/generated_videos/cf_flowcache_Fxxx_x3/overall_consistency/013.mp4",
|
| 18 |
+
"num_frames": 81,
|
| 19 |
+
"height": 480,
|
| 20 |
+
"width": 832,
|
| 21 |
+
"fps": 16,
|
| 22 |
+
"policy_latency_ms": 1593.493219614029,
|
| 23 |
+
"excluded_context_kv_latency_ms": 776.9013977050781,
|
| 24 |
+
"wall_generation_s": 4.188819408416748,
|
| 25 |
+
"reference_strategy": "cf_ffff",
|
| 26 |
+
"pixel_metrics_vs_ffff": {
|
| 27 |
+
"mean_mse": 0.06156369298696518,
|
| 28 |
+
"psnr": 12.106753587722778,
|
| 29 |
+
"ssim": 0.40758976340293884,
|
| 30 |
+
"lpips": 0.5298315286636353,
|
| 31 |
+
"num_frames": 81
|
| 32 |
+
},
|
| 33 |
+
"cache_diagnostics": {
|
| 34 |
+
"denoise_forwards": 28,
|
| 35 |
+
"compute_equivalent_forwards": 14.0,
|
| 36 |
+
"middle_steps": 21,
|
| 37 |
+
"middle_compute_equivalent": 7.0,
|
| 38 |
+
"compute_equivalent_forwards_retime": 14.0
|
| 39 |
+
},
|
| 40 |
+
"policy_latency_ms_generation_run": 2008.4685833454132,
|
| 41 |
+
"matched_ffff_policy_latency_ms": 3113.6305923461914,
|
| 42 |
+
"latency_source": "retime_eval: paired with FFFF in the same process, denoise only"
|
| 43 |
+
}
|
eval_out/per_prompt/cf_flowcache_Fxxx_x3/overall_consistency_015.json
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "complete",
|
| 3 |
+
"protocol": "Self-Forcing Extended-251 Full Evaluation",
|
| 4 |
+
"strategy": "cf_flowcache_Fxxx_x3",
|
| 5 |
+
"base_model": "causal_forcing",
|
| 6 |
+
"method": "flowcache",
|
| 7 |
+
"param": "thresh",
|
| 8 |
+
"param_value": 1.69921875,
|
| 9 |
+
"target_speedup": 3.0,
|
| 10 |
+
"schedule": "Fxxx",
|
| 11 |
+
"global_index": 698,
|
| 12 |
+
"prompt_suite": "overall_consistency",
|
| 13 |
+
"suite_index": 15,
|
| 14 |
+
"prompt": "A sleek black cat wearing stylish oversized sunglasses and a bright red lifeguard hat, positioned beside a pool. The cat is sitting upright with its paws resting on the edge of a poolside chair, alert and focused. The cat's fur is smooth and glossy, with a collar that has a small whistle attached. The pool is clear and sparkling, surrounded by lush green grass and palm trees. In the background, there are lounge chairs and umbrellas under a bright blue sky. The scene is set in a warm, sunny day environment. Medium close-up shot, static camera.",
|
| 15 |
+
"original_prompt": "A cat wearing sunglasses and working as a lifeguard at a pool.",
|
| 16 |
+
"seed": 0,
|
| 17 |
+
"video": "/local/zoubin/cz/projects/comparison/eval_out/generated_videos/cf_flowcache_Fxxx_x3/overall_consistency/015.mp4",
|
| 18 |
+
"num_frames": 81,
|
| 19 |
+
"height": 480,
|
| 20 |
+
"width": 832,
|
| 21 |
+
"fps": 16,
|
| 22 |
+
"policy_latency_ms": 901.2893747091293,
|
| 23 |
+
"excluded_context_kv_latency_ms": 778.176383972168,
|
| 24 |
+
"wall_generation_s": 3.4640982151031494,
|
| 25 |
+
"reference_strategy": "cf_ffff",
|
| 26 |
+
"pixel_metrics_vs_ffff": {
|
| 27 |
+
"mean_mse": 0.07208160310983658,
|
| 28 |
+
"psnr": 11.42175555229187,
|
| 29 |
+
"ssim": 0.30946776270866394,
|
| 30 |
+
"lpips": 0.5860425233840942,
|
| 31 |
+
"num_frames": 81
|
| 32 |
+
},
|
| 33 |
+
"cache_diagnostics": {
|
| 34 |
+
"denoise_forwards": 28,
|
| 35 |
+
"compute_equivalent_forwards": 8.0,
|
| 36 |
+
"middle_steps": 21,
|
| 37 |
+
"middle_compute_equivalent": 1.0,
|
| 38 |
+
"compute_equivalent_forwards_retime": 8.0
|
| 39 |
+
},
|
| 40 |
+
"policy_latency_ms_generation_run": 1087.2182960510254,
|
| 41 |
+
"matched_ffff_policy_latency_ms": 3119.299026489258,
|
| 42 |
+
"latency_source": "retime_eval: paired with FFFF in the same process, denoise only"
|
| 43 |
+
}
|
eval_out/per_prompt/cf_flowcache_Fxxx_x3/subject_consistency_000.json
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "complete",
|
| 3 |
+
"protocol": "Self-Forcing Extended-251 Full Evaluation",
|
| 4 |
+
"strategy": "cf_flowcache_Fxxx_x3",
|
| 5 |
+
"base_model": "causal_forcing",
|
| 6 |
+
"method": "flowcache",
|
| 7 |
+
"param": "thresh",
|
| 8 |
+
"param_value": 1.69921875,
|
| 9 |
+
"target_speedup": 3.0,
|
| 10 |
+
"schedule": "Fxxx",
|
| 11 |
+
"global_index": 257,
|
| 12 |
+
"prompt_suite": "subject_consistency",
|
| 13 |
+
"suite_index": 0,
|
| 14 |
+
"prompt": "A person swimming in the ocean, with waves gently crashing around them. The swimmer is fully submerged, with only their head breaking the water's surface, taking deep breaths between strokes. They are wearing a black swimsuit and have wavy, shoulder-length brown hair that floats around their face. The sun is setting, casting a warm golden glow over the water, and the sky is filled with hues of orange and pink. The ocean is vast and serene, with a few seagulls flying overhead. The scene is captured in a medium close-up, focusing on the swimmer's determined facial expressions and fluid arm movements.",
|
| 15 |
+
"original_prompt": "a person swimming in ocean",
|
| 16 |
+
"seed": 0,
|
| 17 |
+
"video": "/local/zoubin/cz/projects/comparison/eval_out/generated_videos/cf_flowcache_Fxxx_x3/subject_consistency/000.mp4",
|
| 18 |
+
"num_frames": 81,
|
| 19 |
+
"height": 480,
|
| 20 |
+
"width": 832,
|
| 21 |
+
"fps": 16,
|
| 22 |
+
"policy_latency_ms": 1092.9017399549484,
|
| 23 |
+
"excluded_context_kv_latency_ms": 787.1849517822266,
|
| 24 |
+
"wall_generation_s": 3.642585039138794,
|
| 25 |
+
"reference_strategy": "cf_ffff",
|
| 26 |
+
"pixel_metrics_vs_ffff": {
|
| 27 |
+
"mean_mse": 0.056151632219552994,
|
| 28 |
+
"psnr": 12.506376504898071,
|
| 29 |
+
"ssim": 0.41219818592071533,
|
| 30 |
+
"lpips": 0.698592483997345,
|
| 31 |
+
"num_frames": 81
|
| 32 |
+
},
|
| 33 |
+
"cache_diagnostics": {
|
| 34 |
+
"denoise_forwards": 28,
|
| 35 |
+
"compute_equivalent_forwards": 8.666666666666666,
|
| 36 |
+
"middle_steps": 21,
|
| 37 |
+
"middle_compute_equivalent": 1.6666666666666665,
|
| 38 |
+
"compute_equivalent_forwards_retime": 8.666666666666666
|
| 39 |
+
},
|
| 40 |
+
"policy_latency_ms_generation_run": 1091.3761380910873,
|
| 41 |
+
"matched_ffff_policy_latency_ms": 3166.0057983398438,
|
| 42 |
+
"latency_source": "retime_eval: paired with FFFF in the same process, denoise only"
|
| 43 |
+
}
|
eval_out/per_prompt/cf_flowcache_Fxxx_x3/subject_consistency_001.json
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "complete",
|
| 3 |
+
"protocol": "Self-Forcing Extended-251 Full Evaluation",
|
| 4 |
+
"strategy": "cf_flowcache_Fxxx_x3",
|
| 5 |
+
"base_model": "causal_forcing",
|
| 6 |
+
"method": "flowcache",
|
| 7 |
+
"param": "thresh",
|
| 8 |
+
"param_value": 1.69921875,
|
| 9 |
+
"target_speedup": 3.0,
|
| 10 |
+
"schedule": "Fxxx",
|
| 11 |
+
"global_index": 258,
|
| 12 |
+
"prompt_suite": "subject_consistency",
|
| 13 |
+
"suite_index": 1,
|
| 14 |
+
"prompt": "A professional setting where a person is giving a presentation to a room full of colleagues. The presenter stands confidently at the front of the room, holding a laser pointer and gesturing towards a large projection screen displaying slides. They wear a well-fitted suit with a neat tie, exuding professionalism. The audience sits attentively in rows of chairs, some taking notes, others engaged in thoughtful expressions. The room has a modern feel with clean lines, ample lighting, and a professional atmosphere. The camera captures the scene from a medium shot, focusing on the presenter and the audience reaction.",
|
| 15 |
+
"original_prompt": "a person giving a presentation to a room full of colleagues",
|
| 16 |
+
"seed": 0,
|
| 17 |
+
"video": "/local/zoubin/cz/projects/comparison/eval_out/generated_videos/cf_flowcache_Fxxx_x3/subject_consistency/001.mp4",
|
| 18 |
+
"num_frames": 81,
|
| 19 |
+
"height": 480,
|
| 20 |
+
"width": 832,
|
| 21 |
+
"fps": 16,
|
| 22 |
+
"policy_latency_ms": 886.2176933288574,
|
| 23 |
+
"excluded_context_kv_latency_ms": 773.4634323120117,
|
| 24 |
+
"wall_generation_s": 3.461256265640259,
|
| 25 |
+
"reference_strategy": "cf_ffff",
|
| 26 |
+
"pixel_metrics_vs_ffff": {
|
| 27 |
+
"mean_mse": 0.08465196192264557,
|
| 28 |
+
"psnr": 10.72363018989563,
|
| 29 |
+
"ssim": 0.3422987163066864,
|
| 30 |
+
"lpips": 0.6920055150985718,
|
| 31 |
+
"num_frames": 81
|
| 32 |
+
},
|
| 33 |
+
"cache_diagnostics": {
|
| 34 |
+
"denoise_forwards": 28,
|
| 35 |
+
"compute_equivalent_forwards": 7.666666666666666,
|
| 36 |
+
"middle_steps": 21,
|
| 37 |
+
"middle_compute_equivalent": 0.6666666666666666,
|
| 38 |
+
"compute_equivalent_forwards_retime": 7.666666666666666
|
| 39 |
+
},
|
| 40 |
+
"policy_latency_ms_generation_run": 1623.6010296344757,
|
| 41 |
+
"matched_ffff_policy_latency_ms": 3099.1262283325195,
|
| 42 |
+
"latency_source": "retime_eval: paired with FFFF in the same process, denoise only"
|
| 43 |
+
}
|
eval_out/per_prompt/cf_flowcache_Fxxx_x3/subject_consistency_002.json
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "complete",
|
| 3 |
+
"protocol": "Self-Forcing Extended-251 Full Evaluation",
|
| 4 |
+
"strategy": "cf_flowcache_Fxxx_x3",
|
| 5 |
+
"base_model": "causal_forcing",
|
| 6 |
+
"method": "flowcache",
|
| 7 |
+
"param": "thresh",
|
| 8 |
+
"param_value": 1.69921875,
|
| 9 |
+
"target_speedup": 3.0,
|
| 10 |
+
"schedule": "Fxxx",
|
| 11 |
+
"global_index": 259,
|
| 12 |
+
"prompt_suite": "subject_consistency",
|
| 13 |
+
"suite_index": 2,
|
| 14 |
+
"prompt": "A person in a cozy kitchen setting is washing dishes at a sink filled with sudsy water. The individual, who appears to be middle-aged with casual attire, is scrubbing a plate with a sponge. Water droplets splash and cling to the dishes as they are washed. The kitchen is warmly lit, with sunlight streaming in from a window nearby. The background shows other kitchen appliances and cabinets. The person is standing with a focused expression, moving the sponge in circular motions across the dish. Medium close-up shot, static scene.",
|
| 15 |
+
"original_prompt": "a person washing the dishes",
|
| 16 |
+
"seed": 0,
|
| 17 |
+
"video": "/local/zoubin/cz/projects/comparison/eval_out/generated_videos/cf_flowcache_Fxxx_x3/subject_consistency/002.mp4",
|
| 18 |
+
"num_frames": 81,
|
| 19 |
+
"height": 480,
|
| 20 |
+
"width": 832,
|
| 21 |
+
"fps": 16,
|
| 22 |
+
"policy_latency_ms": 1620.2439424991608,
|
| 23 |
+
"excluded_context_kv_latency_ms": 788.658317565918,
|
| 24 |
+
"wall_generation_s": 4.1852707862854,
|
| 25 |
+
"reference_strategy": "cf_ffff",
|
| 26 |
+
"pixel_metrics_vs_ffff": {
|
| 27 |
+
"mean_mse": 0.05455402657389641,
|
| 28 |
+
"psnr": 12.63173222541809,
|
| 29 |
+
"ssim": 0.46643000841140747,
|
| 30 |
+
"lpips": 0.5778776407241821,
|
| 31 |
+
"num_frames": 81
|
| 32 |
+
},
|
| 33 |
+
"cache_diagnostics": {
|
| 34 |
+
"denoise_forwards": 28,
|
| 35 |
+
"compute_equivalent_forwards": 14.0,
|
| 36 |
+
"middle_steps": 21,
|
| 37 |
+
"middle_compute_equivalent": 7.0,
|
| 38 |
+
"compute_equivalent_forwards_retime": 14.0
|
| 39 |
+
},
|
| 40 |
+
"policy_latency_ms_generation_run": 2974.706108212471,
|
| 41 |
+
"matched_ffff_policy_latency_ms": 3162.8747024536133,
|
| 42 |
+
"latency_source": "retime_eval: paired with FFFF in the same process, denoise only"
|
| 43 |
+
}
|
eval_out/per_prompt/cf_flowcache_Fxxx_x3/subject_consistency_003.json
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "complete",
|
| 3 |
+
"protocol": "Self-Forcing Extended-251 Full Evaluation",
|
| 4 |
+
"strategy": "cf_flowcache_Fxxx_x3",
|
| 5 |
+
"base_model": "causal_forcing",
|
| 6 |
+
"method": "flowcache",
|
| 7 |
+
"param": "thresh",
|
| 8 |
+
"param_value": 1.69921875,
|
| 9 |
+
"target_speedup": 3.0,
|
| 10 |
+
"schedule": "Fxxx",
|
| 11 |
+
"global_index": 260,
|
| 12 |
+
"prompt_suite": "subject_consistency",
|
| 13 |
+
"suite_index": 3,
|
| 14 |
+
"prompt": "A person enjoying a juicy burger, with a satisfied smile on their face. They are seated at a casual dining table, surrounded by napkins and a drink. The burger is topped with lettuce, tomato, and cheese, and the person is taking a bite, showcasing the delicious layers inside. The scene has a warm, inviting atmosphere with soft lighting and a cozy background. Medium close-up shot focusing on the person's hand holding the burger and their facial expressions as they savor each bite.",
|
| 15 |
+
"original_prompt": "a person eating a burger",
|
| 16 |
+
"seed": 0,
|
| 17 |
+
"video": "/local/zoubin/cz/projects/comparison/eval_out/generated_videos/cf_flowcache_Fxxx_x3/subject_consistency/003.mp4",
|
| 18 |
+
"num_frames": 81,
|
| 19 |
+
"height": 480,
|
| 20 |
+
"width": 832,
|
| 21 |
+
"fps": 16,
|
| 22 |
+
"policy_latency_ms": 1430.9517686367035,
|
| 23 |
+
"excluded_context_kv_latency_ms": 771.5014114379883,
|
| 24 |
+
"wall_generation_s": 3.9953954219818115,
|
| 25 |
+
"reference_strategy": "cf_ffff",
|
| 26 |
+
"pixel_metrics_vs_ffff": {
|
| 27 |
+
"mean_mse": 0.06951364129781723,
|
| 28 |
+
"psnr": 11.579300165176392,
|
| 29 |
+
"ssim": 0.43418461084365845,
|
| 30 |
+
"lpips": 0.5564706325531006,
|
| 31 |
+
"num_frames": 81
|
| 32 |
+
},
|
| 33 |
+
"cache_diagnostics": {
|
| 34 |
+
"denoise_forwards": 28,
|
| 35 |
+
"compute_equivalent_forwards": 12.333333333333334,
|
| 36 |
+
"middle_steps": 21,
|
| 37 |
+
"middle_compute_equivalent": 5.333333333333333,
|
| 38 |
+
"compute_equivalent_forwards_retime": 12.333333333333334
|
| 39 |
+
},
|
| 40 |
+
"policy_latency_ms_generation_run": 2609.540231704712,
|
| 41 |
+
"matched_ffff_policy_latency_ms": 3096.9683685302734,
|
| 42 |
+
"latency_source": "retime_eval: paired with FFFF in the same process, denoise only"
|
| 43 |
+
}
|
eval_out/per_prompt/cf_flowcache_Fxxx_x3/subject_consistency_004.json
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "complete",
|
| 3 |
+
"protocol": "Self-Forcing Extended-251 Full Evaluation",
|
| 4 |
+
"strategy": "cf_flowcache_Fxxx_x3",
|
| 5 |
+
"base_model": "causal_forcing",
|
| 6 |
+
"method": "flowcache",
|
| 7 |
+
"param": "thresh",
|
| 8 |
+
"param_value": 1.69921875,
|
| 9 |
+
"target_speedup": 3.0,
|
| 10 |
+
"schedule": "Fxxx",
|
| 11 |
+
"global_index": 261,
|
| 12 |
+
"prompt_suite": "subject_consistency",
|
| 13 |
+
"suite_index": 4,
|
| 14 |
+
"prompt": "A lone figure, wearing a heavy winter coat with a fur-lined hood, walks through a dense snowstorm. The person is bundled up with gloves and a scarf wrapped tightly around their face, only their eyes visible. Snowflakes swirl around them as they trudge forward, their footsteps muffled by the thick layer of snow. The background shows a dimly lit forest with tall pine trees barely visible through the heavy snowfall. The scene is captured in a medium shot, focusing on the person's determined walk against the harsh conditions.",
|
| 15 |
+
"original_prompt": "a person walking in the snowstorm",
|
| 16 |
+
"seed": 0,
|
| 17 |
+
"video": "/local/zoubin/cz/projects/comparison/eval_out/generated_videos/cf_flowcache_Fxxx_x3/subject_consistency/004.mp4",
|
| 18 |
+
"num_frames": 81,
|
| 19 |
+
"height": 480,
|
| 20 |
+
"width": 832,
|
| 21 |
+
"fps": 16,
|
| 22 |
+
"policy_latency_ms": 1496.9227246046066,
|
| 23 |
+
"excluded_context_kv_latency_ms": 789.7685089111328,
|
| 24 |
+
"wall_generation_s": 4.041562557220459,
|
| 25 |
+
"reference_strategy": "cf_ffff",
|
| 26 |
+
"pixel_metrics_vs_ffff": {
|
| 27 |
+
"mean_mse": 0.05358025059103966,
|
| 28 |
+
"psnr": 12.709952592849731,
|
| 29 |
+
"ssim": 0.4741244912147522,
|
| 30 |
+
"lpips": 0.46590912342071533,
|
| 31 |
+
"num_frames": 81
|
| 32 |
+
},
|
| 33 |
+
"cache_diagnostics": {
|
| 34 |
+
"denoise_forwards": 28,
|
| 35 |
+
"compute_equivalent_forwards": 12.666666666666668,
|
| 36 |
+
"middle_steps": 21,
|
| 37 |
+
"middle_compute_equivalent": 5.666666666666666,
|
| 38 |
+
"compute_equivalent_forwards_retime": 12.666666666666668
|
| 39 |
+
},
|
| 40 |
+
"policy_latency_ms_generation_run": 2817.153255581856,
|
| 41 |
+
"matched_ffff_policy_latency_ms": 3165.155448913574,
|
| 42 |
+
"latency_source": "retime_eval: paired with FFFF in the same process, denoise only"
|
| 43 |
+
}
|
eval_out/per_prompt/cf_flowcache_Fxxx_x3/subject_consistency_005.json
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "complete",
|
| 3 |
+
"protocol": "Self-Forcing Extended-251 Full Evaluation",
|
| 4 |
+
"strategy": "cf_flowcache_Fxxx_x3",
|
| 5 |
+
"base_model": "causal_forcing",
|
| 6 |
+
"method": "flowcache",
|
| 7 |
+
"param": "thresh",
|
| 8 |
+
"param_value": 1.69921875,
|
| 9 |
+
"target_speedup": 3.0,
|
| 10 |
+
"schedule": "Fxxx",
|
| 11 |
+
"global_index": 262,
|
| 12 |
+
"prompt_suite": "subject_consistency",
|
| 13 |
+
"suite_index": 5,
|
| 14 |
+
"prompt": "A cozy, warm caf\u00e9 setting with soft ambient lighting and wooden furnishings. A young adult, casually dressed in a sweater and jeans, sits at a small round table. They hold a steaming cup of coffee in their hand, taking a sip while looking pensively out the window. The caf\u00e9 is moderately busy with other patrons engaged in conversations. The background showcases various coffee drinks and pastries displayed on a counter. The person\u2019s expression is relaxed and content. Medium shot focusing on the person\u2019s face and the coffee cup, capturing the intimate atmosphere of the caf\u00e9.",
|
| 15 |
+
"original_prompt": "a person drinking coffee in a cafe",
|
| 16 |
+
"seed": 0,
|
| 17 |
+
"video": "/local/zoubin/cz/projects/comparison/eval_out/generated_videos/cf_flowcache_Fxxx_x3/subject_consistency/005.mp4",
|
| 18 |
+
"num_frames": 81,
|
| 19 |
+
"height": 480,
|
| 20 |
+
"width": 832,
|
| 21 |
+
"fps": 16,
|
| 22 |
+
"policy_latency_ms": 1039.2287901639938,
|
| 23 |
+
"excluded_context_kv_latency_ms": 775.7002258300781,
|
| 24 |
+
"wall_generation_s": 3.617298126220703,
|
| 25 |
+
"reference_strategy": "cf_ffff",
|
| 26 |
+
"pixel_metrics_vs_ffff": {
|
| 27 |
+
"mean_mse": 0.06829880177974701,
|
| 28 |
+
"psnr": 11.655869483947754,
|
| 29 |
+
"ssim": 0.32951605319976807,
|
| 30 |
+
"lpips": 0.6603900194168091,
|
| 31 |
+
"num_frames": 81
|
| 32 |
+
},
|
| 33 |
+
"cache_diagnostics": {
|
| 34 |
+
"denoise_forwards": 28,
|
| 35 |
+
"compute_equivalent_forwards": 9.333333333333332,
|
| 36 |
+
"middle_steps": 21,
|
| 37 |
+
"middle_compute_equivalent": 2.333333333333333,
|
| 38 |
+
"compute_equivalent_forwards_retime": 9.333333333333332
|
| 39 |
+
},
|
| 40 |
+
"policy_latency_ms_generation_run": 1945.9524539709091,
|
| 41 |
+
"matched_ffff_policy_latency_ms": 3104.9696884155273,
|
| 42 |
+
"latency_source": "retime_eval: paired with FFFF in the same process, denoise only"
|
| 43 |
+
}
|
eval_out/per_prompt/cf_flowcache_Fxxx_x3/subject_consistency_006.json
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "complete",
|
| 3 |
+
"protocol": "Self-Forcing Extended-251 Full Evaluation",
|
| 4 |
+
"strategy": "cf_flowcache_Fxxx_x3",
|
| 5 |
+
"base_model": "causal_forcing",
|
| 6 |
+
"method": "flowcache",
|
| 7 |
+
"param": "thresh",
|
| 8 |
+
"param_value": 1.69921875,
|
| 9 |
+
"target_speedup": 3.0,
|
| 10 |
+
"schedule": "Fxxx",
|
| 11 |
+
"global_index": 263,
|
| 12 |
+
"prompt_suite": "subject_consistency",
|
| 13 |
+
"suite_index": 6,
|
| 14 |
+
"prompt": "A close-up of a young adult playing a classic acoustic guitar, with their fingers deftly plucking the strings. They sit in a cozy, dimly lit room, with soft ambient lighting casting gentle shadows across their face. The person has tousled brown hair, a relaxed expression, and wears a casual outfit consisting of a vintage t-shirt and jeans. Their posture is casual yet focused, leaning slightly forward as they play. The guitar's wooden body reflects the warm glow from a nearby lamp. The scene captures the intimate atmosphere of a musical performance, with the focus on the person's expressive face and skilled hands moving over the fretboard.",
|
| 15 |
+
"original_prompt": "a person playing guitar",
|
| 16 |
+
"seed": 0,
|
| 17 |
+
"video": "/local/zoubin/cz/projects/comparison/eval_out/generated_videos/cf_flowcache_Fxxx_x3/subject_consistency/006.mp4",
|
| 18 |
+
"num_frames": 81,
|
| 19 |
+
"height": 480,
|
| 20 |
+
"width": 832,
|
| 21 |
+
"fps": 16,
|
| 22 |
+
"policy_latency_ms": 913.6868675947189,
|
| 23 |
+
"excluded_context_kv_latency_ms": 786.6017684936523,
|
| 24 |
+
"wall_generation_s": 3.487720489501953,
|
| 25 |
+
"reference_strategy": "cf_ffff",
|
| 26 |
+
"pixel_metrics_vs_ffff": {
|
| 27 |
+
"mean_mse": 0.031851112842559814,
|
| 28 |
+
"psnr": 14.968754053115845,
|
| 29 |
+
"ssim": 0.4154454469680786,
|
| 30 |
+
"lpips": 0.7435934543609619,
|
| 31 |
+
"num_frames": 81
|
| 32 |
+
},
|
| 33 |
+
"cache_diagnostics": {
|
| 34 |
+
"denoise_forwards": 28,
|
| 35 |
+
"compute_equivalent_forwards": 8.0,
|
| 36 |
+
"middle_steps": 21,
|
| 37 |
+
"middle_compute_equivalent": 1.0,
|
| 38 |
+
"compute_equivalent_forwards_retime": 8.0
|
| 39 |
+
},
|
| 40 |
+
"policy_latency_ms_generation_run": 1682.826812028885,
|
| 41 |
+
"matched_ffff_policy_latency_ms": 3160.4890060424805,
|
| 42 |
+
"latency_source": "retime_eval: paired with FFFF in the same process, denoise only"
|
| 43 |
+
}
|
eval_out/per_prompt/cf_flowcache_Fxxx_x3/subject_consistency_007.json
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "complete",
|
| 3 |
+
"protocol": "Self-Forcing Extended-251 Full Evaluation",
|
| 4 |
+
"strategy": "cf_flowcache_Fxxx_x3",
|
| 5 |
+
"base_model": "causal_forcing",
|
| 6 |
+
"method": "flowcache",
|
| 7 |
+
"param": "thresh",
|
| 8 |
+
"param_value": 1.69921875,
|
| 9 |
+
"target_speedup": 3.0,
|
| 10 |
+
"schedule": "Fxxx",
|
| 11 |
+
"global_index": 264,
|
| 12 |
+
"prompt_suite": "subject_consistency",
|
| 13 |
+
"suite_index": 7,
|
| 14 |
+
"prompt": "A vintage black bicycle leaning gently against a large, gnarled oak tree in a serene forest clearing. The bicycle has a classic frame with a round handlebar and a single rear mudguard. Its tires are slightly deflated, giving it a slightly neglected yet charming appearance. The tree has lush green leaves and several low branches that gracefully arch over the bike. The forest floor is covered with soft, brown autumn leaves and patches of moss. The scene is bathed in warm, natural sunlight filtering through the canopy, casting dappled shadows. Medium shot capturing the bicycle and the lower part of the tree trunk.",
|
| 15 |
+
"original_prompt": "a bicycle leaning against a tree",
|
| 16 |
+
"seed": 0,
|
| 17 |
+
"video": "/local/zoubin/cz/projects/comparison/eval_out/generated_videos/cf_flowcache_Fxxx_x3/subject_consistency/007.mp4",
|
| 18 |
+
"num_frames": 81,
|
| 19 |
+
"height": 480,
|
| 20 |
+
"width": 832,
|
| 21 |
+
"fps": 16,
|
| 22 |
+
"policy_latency_ms": 1028.9355922937393,
|
| 23 |
+
"excluded_context_kv_latency_ms": 776.8084869384766,
|
| 24 |
+
"wall_generation_s": 3.58719801902771,
|
| 25 |
+
"reference_strategy": "cf_ffff",
|
| 26 |
+
"pixel_metrics_vs_ffff": {
|
| 27 |
+
"mean_mse": 0.10938073694705963,
|
| 28 |
+
"psnr": 9.610591530799866,
|
| 29 |
+
"ssim": 0.17225052416324615,
|
| 30 |
+
"lpips": 0.5780423283576965,
|
| 31 |
+
"num_frames": 81
|
| 32 |
+
},
|
| 33 |
+
"cache_diagnostics": {
|
| 34 |
+
"denoise_forwards": 28,
|
| 35 |
+
"compute_equivalent_forwards": 9.0,
|
| 36 |
+
"middle_steps": 21,
|
| 37 |
+
"middle_compute_equivalent": 1.9999999999999998,
|
| 38 |
+
"compute_equivalent_forwards_retime": 9.0
|
| 39 |
+
},
|
| 40 |
+
"policy_latency_ms_generation_run": 1654.2737029790878,
|
| 41 |
+
"matched_ffff_policy_latency_ms": 3108.3598403930664,
|
| 42 |
+
"latency_source": "retime_eval: paired with FFFF in the same process, denoise only"
|
| 43 |
+
}
|
eval_out/per_prompt/cf_flowcache_Fxxx_x3/subject_consistency_008.json
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "complete",
|
| 3 |
+
"protocol": "Self-Forcing Extended-251 Full Evaluation",
|
| 4 |
+
"strategy": "cf_flowcache_Fxxx_x3",
|
| 5 |
+
"base_model": "causal_forcing",
|
| 6 |
+
"method": "flowcache",
|
| 7 |
+
"param": "thresh",
|
| 8 |
+
"param_value": 1.69921875,
|
| 9 |
+
"target_speedup": 3.0,
|
| 10 |
+
"schedule": "Fxxx",
|
| 11 |
+
"global_index": 265,
|
| 12 |
+
"prompt_suite": "subject_consistency",
|
| 13 |
+
"suite_index": 8,
|
| 14 |
+
"prompt": "A bicycle glides smoothly through a vast, snowy field under a clear blue sky. The bicycle has a classic design with black tires, a silver frame, and a small white basket attached to the front. The rider, a young adult wearing a red winter coat, black gloves, and a woolen hat, pedals effortlessly with a slight forward lean. Snowflakes gently fall around them, creating a serene and tranquil atmosphere. The background showcases rolling hills covered in pristine snow, with distant pine trees adding a touch of greenery. The scene is captured in a medium shot, focusing on the bicycle and the rider as they move gracefully across the snowy landscape.",
|
| 15 |
+
"original_prompt": "a bicycle gliding through a snowy field",
|
| 16 |
+
"seed": 0,
|
| 17 |
+
"video": "/local/zoubin/cz/projects/comparison/eval_out/generated_videos/cf_flowcache_Fxxx_x3/subject_consistency/008.mp4",
|
| 18 |
+
"num_frames": 81,
|
| 19 |
+
"height": 480,
|
| 20 |
+
"width": 832,
|
| 21 |
+
"fps": 16,
|
| 22 |
+
"policy_latency_ms": 978.1033310890198,
|
| 23 |
+
"excluded_context_kv_latency_ms": 789.4485092163086,
|
| 24 |
+
"wall_generation_s": 3.5297257900238037,
|
| 25 |
+
"reference_strategy": "cf_ffff",
|
| 26 |
+
"pixel_metrics_vs_ffff": {
|
| 27 |
+
"mean_mse": 0.06268158555030823,
|
| 28 |
+
"psnr": 12.028599977493286,
|
| 29 |
+
"ssim": 0.43092477321624756,
|
| 30 |
+
"lpips": 0.6509794592857361,
|
| 31 |
+
"num_frames": 81
|
| 32 |
+
},
|
| 33 |
+
"cache_diagnostics": {
|
| 34 |
+
"denoise_forwards": 28,
|
| 35 |
+
"compute_equivalent_forwards": 8.333333333333334,
|
| 36 |
+
"middle_steps": 21,
|
| 37 |
+
"middle_compute_equivalent": 1.3333333333333333,
|
| 38 |
+
"compute_equivalent_forwards_retime": 8.333333333333334
|
| 39 |
+
},
|
| 40 |
+
"policy_latency_ms_generation_run": 1813.1033440828323,
|
| 41 |
+
"matched_ffff_policy_latency_ms": 3161.0075149536133,
|
| 42 |
+
"latency_source": "retime_eval: paired with FFFF in the same process, denoise only"
|
| 43 |
+
}
|
eval_out/per_prompt/cf_flowcache_Fxxx_x3/subject_consistency_009.json
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "complete",
|
| 3 |
+
"protocol": "Self-Forcing Extended-251 Full Evaluation",
|
| 4 |
+
"strategy": "cf_flowcache_Fxxx_x3",
|
| 5 |
+
"base_model": "causal_forcing",
|
| 6 |
+
"method": "flowcache",
|
| 7 |
+
"param": "thresh",
|
| 8 |
+
"param_value": 1.69921875,
|
| 9 |
+
"target_speedup": 3.0,
|
| 10 |
+
"schedule": "Fxxx",
|
| 11 |
+
"global_index": 266,
|
| 12 |
+
"prompt_suite": "subject_consistency",
|
| 13 |
+
"suite_index": 9,
|
| 14 |
+
"prompt": "A cyclist riding a classic black bicycle slows down gradually as they approach a stop sign. The cyclist has a helmet on, and their posture is upright with one hand on the brake handle, applying gentle pressure. The bicycle's wheels spin slightly slower, creating small dust clouds as they decelerate on a gravel path. The surroundings feature a quiet suburban street lined with green trees and parked cars. The scene transitions from a medium shot to a close-up focusing on the cyclist's face, which shows concentration and determination. Static shot, emphasizing the motion of the bicycle slowing down.",
|
| 15 |
+
"original_prompt": "a bicycle slowing down to stop",
|
| 16 |
+
"seed": 0,
|
| 17 |
+
"video": "/local/zoubin/cz/projects/comparison/eval_out/generated_videos/cf_flowcache_Fxxx_x3/subject_consistency/009.mp4",
|
| 18 |
+
"num_frames": 81,
|
| 19 |
+
"height": 480,
|
| 20 |
+
"width": 832,
|
| 21 |
+
"fps": 16,
|
| 22 |
+
"policy_latency_ms": 1394.7868626117706,
|
| 23 |
+
"excluded_context_kv_latency_ms": 777.4814834594727,
|
| 24 |
+
"wall_generation_s": 3.969381809234619,
|
| 25 |
+
"reference_strategy": "cf_ffff",
|
| 26 |
+
"pixel_metrics_vs_ffff": {
|
| 27 |
+
"mean_mse": 0.10911491513252258,
|
| 28 |
+
"psnr": 9.621158838272095,
|
| 29 |
+
"ssim": 0.27627241611480713,
|
| 30 |
+
"lpips": 0.600887656211853,
|
| 31 |
+
"num_frames": 81
|
| 32 |
+
},
|
| 33 |
+
"cache_diagnostics": {
|
| 34 |
+
"denoise_forwards": 28,
|
| 35 |
+
"compute_equivalent_forwards": 12.0,
|
| 36 |
+
"middle_steps": 21,
|
| 37 |
+
"middle_compute_equivalent": 5.0,
|
| 38 |
+
"compute_equivalent_forwards_retime": 12.0
|
| 39 |
+
},
|
| 40 |
+
"policy_latency_ms_generation_run": 2539.961320757866,
|
| 41 |
+
"matched_ffff_policy_latency_ms": 3120.2527465820312,
|
| 42 |
+
"latency_source": "retime_eval: paired with FFFF in the same process, denoise only"
|
| 43 |
+
}
|
eval_out/per_prompt/cf_flowcache_Fxxx_x3/subject_consistency_010.json
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "complete",
|
| 3 |
+
"protocol": "Self-Forcing Extended-251 Full Evaluation",
|
| 4 |
+
"strategy": "cf_flowcache_Fxxx_x3",
|
| 5 |
+
"base_model": "causal_forcing",
|
| 6 |
+
"method": "flowcache",
|
| 7 |
+
"param": "thresh",
|
| 8 |
+
"param_value": 1.69921875,
|
| 9 |
+
"target_speedup": 3.0,
|
| 10 |
+
"schedule": "Fxxx",
|
| 11 |
+
"global_index": 267,
|
| 12 |
+
"prompt_suite": "subject_consistency",
|
| 13 |
+
"suite_index": 10,
|
| 14 |
+
"prompt": "A bicycle accelerating to gain speed, captured in a smooth time-lapse sequence. The bicycle is a classic red road bike with sleek aerodynamic handlebars and reflective wheels. The rider, a focused individual in cycling gear, leans forward, pedaling vigorously. The background shows a winding asphalt road leading into a forest, with trees and bushes blurring past as the bike speeds up. The camera starts from a mid-shot and gradually zooms in to focus on the rider's determined face and the spinning wheels, emphasizing the acceleration.",
|
| 15 |
+
"original_prompt": "a bicycle accelerating to gain speed",
|
| 16 |
+
"seed": 0,
|
| 17 |
+
"video": "/local/zoubin/cz/projects/comparison/eval_out/generated_videos/cf_flowcache_Fxxx_x3/subject_consistency/010.mp4",
|
| 18 |
+
"num_frames": 81,
|
| 19 |
+
"height": 480,
|
| 20 |
+
"width": 832,
|
| 21 |
+
"fps": 16,
|
| 22 |
+
"policy_latency_ms": 1510.787816286087,
|
| 23 |
+
"excluded_context_kv_latency_ms": 789.0631256103516,
|
| 24 |
+
"wall_generation_s": 4.084575414657593,
|
| 25 |
+
"reference_strategy": "cf_ffff",
|
| 26 |
+
"pixel_metrics_vs_ffff": {
|
| 27 |
+
"mean_mse": 0.08218518644571304,
|
| 28 |
+
"psnr": 10.852065086364746,
|
| 29 |
+
"ssim": 0.351231187582016,
|
| 30 |
+
"lpips": 0.588748037815094,
|
| 31 |
+
"num_frames": 81
|
| 32 |
+
},
|
| 33 |
+
"cache_diagnostics": {
|
| 34 |
+
"denoise_forwards": 28,
|
| 35 |
+
"compute_equivalent_forwards": 12.666666666666666,
|
| 36 |
+
"middle_steps": 21,
|
| 37 |
+
"middle_compute_equivalent": 5.666666666666667,
|
| 38 |
+
"compute_equivalent_forwards_retime": 12.666666666666666
|
| 39 |
+
},
|
| 40 |
+
"policy_latency_ms_generation_run": 2800.0748012065887,
|
| 41 |
+
"matched_ffff_policy_latency_ms": 3166.8505477905273,
|
| 42 |
+
"latency_source": "retime_eval: paired with FFFF in the same process, denoise only"
|
| 43 |
+
}
|
eval_out/per_prompt/cf_flowcache_Fxxx_x3/subject_consistency_011.json
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "complete",
|
| 3 |
+
"protocol": "Self-Forcing Extended-251 Full Evaluation",
|
| 4 |
+
"strategy": "cf_flowcache_Fxxx_x3",
|
| 5 |
+
"base_model": "causal_forcing",
|
| 6 |
+
"method": "flowcache",
|
| 7 |
+
"param": "thresh",
|
| 8 |
+
"param_value": 1.69921875,
|
| 9 |
+
"target_speedup": 3.0,
|
| 10 |
+
"schedule": "Fxxx",
|
| 11 |
+
"global_index": 268,
|
| 12 |
+
"prompt_suite": "subject_consistency",
|
| 13 |
+
"suite_index": 11,
|
| 14 |
+
"prompt": "A realistic depiction of a car stuck in heavy traffic during rush hour. The scene shows multiple cars lined up on a busy urban highway, each vehicle emitting exhaust fumes. The driver in the front car, a middle-aged man with a stressed expression, is leaning forward in his seat, peering ahead through the windshield. The sun is setting, casting a warm golden glow over the scene. The background features tall buildings and billboards. The camera focuses on the car from a medium shot, capturing the interior and exterior details, emphasizing the stationary nature of the vehicles.",
|
| 15 |
+
"original_prompt": "a car stuck in traffic during rush hour",
|
| 16 |
+
"seed": 0,
|
| 17 |
+
"video": "/local/zoubin/cz/projects/comparison/eval_out/generated_videos/cf_flowcache_Fxxx_x3/subject_consistency/011.mp4",
|
| 18 |
+
"num_frames": 81,
|
| 19 |
+
"height": 480,
|
| 20 |
+
"width": 832,
|
| 21 |
+
"fps": 16,
|
| 22 |
+
"policy_latency_ms": 1376.3157824277878,
|
| 23 |
+
"excluded_context_kv_latency_ms": 775.2216949462891,
|
| 24 |
+
"wall_generation_s": 3.9240365028381348,
|
| 25 |
+
"reference_strategy": "cf_ffff",
|
| 26 |
+
"pixel_metrics_vs_ffff": {
|
| 27 |
+
"mean_mse": 0.06713749468326569,
|
| 28 |
+
"psnr": 11.730349063873291,
|
| 29 |
+
"ssim": 0.39366188645362854,
|
| 30 |
+
"lpips": 0.6669381260871887,
|
| 31 |
+
"num_frames": 81
|
| 32 |
+
},
|
| 33 |
+
"cache_diagnostics": {
|
| 34 |
+
"denoise_forwards": 28,
|
| 35 |
+
"compute_equivalent_forwards": 11.333333333333332,
|
| 36 |
+
"middle_steps": 21,
|
| 37 |
+
"middle_compute_equivalent": 4.333333333333333,
|
| 38 |
+
"compute_equivalent_forwards_retime": 11.333333333333332
|
| 39 |
+
},
|
| 40 |
+
"policy_latency_ms_generation_run": 2453.942619562149,
|
| 41 |
+
"matched_ffff_policy_latency_ms": 3122.2659912109375,
|
| 42 |
+
"latency_source": "retime_eval: paired with FFFF in the same process, denoise only"
|
| 43 |
+
}
|
eval_out/per_prompt/cf_flowcache_Fxxx_x3/subject_consistency_012.json
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "complete",
|
| 3 |
+
"protocol": "Self-Forcing Extended-251 Full Evaluation",
|
| 4 |
+
"strategy": "cf_flowcache_Fxxx_x3",
|
| 5 |
+
"base_model": "causal_forcing",
|
| 6 |
+
"method": "flowcache",
|
| 7 |
+
"param": "thresh",
|
| 8 |
+
"param_value": 1.69921875,
|
| 9 |
+
"target_speedup": 3.0,
|
| 10 |
+
"schedule": "Fxxx",
|
| 11 |
+
"global_index": 269,
|
| 12 |
+
"prompt_suite": "subject_consistency",
|
| 13 |
+
"suite_index": 12,
|
| 14 |
+
"prompt": "A sleek, modern sedan driving down a city street, approaching a sharp corner. As the car begins to turn, it smoothly navigates the bend, with the driver maintaining control. The exterior of the vehicle is polished and gleaming, reflecting the surrounding urban environment. In the background, there are tall buildings and street signs, adding to the bustling atmosphere. The scene is captured from a dynamic camera angle that follows the car as it turns, providing a sense of movement and speed. Wide shot, focusing on the car as it makes the turn.",
|
| 15 |
+
"original_prompt": "a car turning a corner",
|
| 16 |
+
"seed": 0,
|
| 17 |
+
"video": "/local/zoubin/cz/projects/comparison/eval_out/generated_videos/cf_flowcache_Fxxx_x3/subject_consistency/012.mp4",
|
| 18 |
+
"num_frames": 81,
|
| 19 |
+
"height": 480,
|
| 20 |
+
"width": 832,
|
| 21 |
+
"fps": 16,
|
| 22 |
+
"policy_latency_ms": 1612.5985959768295,
|
| 23 |
+
"excluded_context_kv_latency_ms": 785.5492477416992,
|
| 24 |
+
"wall_generation_s": 4.159493446350098,
|
| 25 |
+
"reference_strategy": "cf_ffff",
|
| 26 |
+
"pixel_metrics_vs_ffff": {
|
| 27 |
+
"mean_mse": 0.08138371258974075,
|
| 28 |
+
"psnr": 10.894625186920166,
|
| 29 |
+
"ssim": 0.31123775243759155,
|
| 30 |
+
"lpips": 0.5834479928016663,
|
| 31 |
+
"num_frames": 81
|
| 32 |
+
},
|
| 33 |
+
"cache_diagnostics": {
|
| 34 |
+
"denoise_forwards": 28,
|
| 35 |
+
"compute_equivalent_forwards": 14.0,
|
| 36 |
+
"middle_steps": 21,
|
| 37 |
+
"middle_compute_equivalent": 7.0,
|
| 38 |
+
"compute_equivalent_forwards_retime": 14.0
|
| 39 |
+
},
|
| 40 |
+
"policy_latency_ms_generation_run": 2992.6742219924927,
|
| 41 |
+
"matched_ffff_policy_latency_ms": 3148.1932525634766,
|
| 42 |
+
"latency_source": "retime_eval: paired with FFFF in the same process, denoise only"
|
| 43 |
+
}
|
eval_out/per_prompt/cf_flowcache_Fxxx_x3/subject_consistency_013.json
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "complete",
|
| 3 |
+
"protocol": "Self-Forcing Extended-251 Full Evaluation",
|
| 4 |
+
"strategy": "cf_flowcache_Fxxx_x3",
|
| 5 |
+
"base_model": "causal_forcing",
|
| 6 |
+
"method": "flowcache",
|
| 7 |
+
"param": "thresh",
|
| 8 |
+
"param_value": 1.69921875,
|
| 9 |
+
"target_speedup": 3.0,
|
| 10 |
+
"schedule": "Fxxx",
|
| 11 |
+
"global_index": 270,
|
| 12 |
+
"prompt_suite": "subject_consistency",
|
| 13 |
+
"suite_index": 13,
|
| 14 |
+
"prompt": "A sleek, modern sedan gradually slows down as it approaches a stop sign. The car's brake lights illuminate, casting a soft red glow through the rear window. Inside, the driver maintains a calm and focused expression, gently pressing the brake pedal. The wheels slow their rotation, and the vehicle comes to a smooth stop. The background shows a quiet suburban street with a few parked cars and neatly trimmed hedges. The scene is captured from a mid-shot perspective, focusing on the front half of the car.",
|
| 15 |
+
"original_prompt": "a car slowing down to stop",
|
| 16 |
+
"seed": 0,
|
| 17 |
+
"video": "/local/zoubin/cz/projects/comparison/eval_out/generated_videos/cf_flowcache_Fxxx_x3/subject_consistency/013.mp4",
|
| 18 |
+
"num_frames": 81,
|
| 19 |
+
"height": 480,
|
| 20 |
+
"width": 832,
|
| 21 |
+
"fps": 16,
|
| 22 |
+
"policy_latency_ms": 902.8570635318756,
|
| 23 |
+
"excluded_context_kv_latency_ms": 778.0305862426758,
|
| 24 |
+
"wall_generation_s": 3.490159511566162,
|
| 25 |
+
"reference_strategy": "cf_ffff",
|
| 26 |
+
"pixel_metrics_vs_ffff": {
|
| 27 |
+
"mean_mse": 0.07255952060222626,
|
| 28 |
+
"psnr": 11.39305591583252,
|
| 29 |
+
"ssim": 0.3377978503704071,
|
| 30 |
+
"lpips": 0.6946015357971191,
|
| 31 |
+
"num_frames": 81
|
| 32 |
+
},
|
| 33 |
+
"cache_diagnostics": {
|
| 34 |
+
"denoise_forwards": 28,
|
| 35 |
+
"compute_equivalent_forwards": 8.0,
|
| 36 |
+
"middle_steps": 21,
|
| 37 |
+
"middle_compute_equivalent": 1.0,
|
| 38 |
+
"compute_equivalent_forwards_retime": 8.0
|
| 39 |
+
},
|
| 40 |
+
"policy_latency_ms_generation_run": 1627.2647557258606,
|
| 41 |
+
"matched_ffff_policy_latency_ms": 3115.6374282836914,
|
| 42 |
+
"latency_source": "retime_eval: paired with FFFF in the same process, denoise only"
|
| 43 |
+
}
|
eval_out/per_prompt/cf_flowcache_Fxxx_x3/subject_consistency_014.json
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "complete",
|
| 3 |
+
"protocol": "Self-Forcing Extended-251 Full Evaluation",
|
| 4 |
+
"strategy": "cf_flowcache_Fxxx_x3",
|
| 5 |
+
"base_model": "causal_forcing",
|
| 6 |
+
"method": "flowcache",
|
| 7 |
+
"param": "thresh",
|
| 8 |
+
"param_value": 1.69921875,
|
| 9 |
+
"target_speedup": 3.0,
|
| 10 |
+
"schedule": "Fxxx",
|
| 11 |
+
"global_index": 271,
|
| 12 |
+
"prompt_suite": "subject_consistency",
|
| 13 |
+
"suite_index": 14,
|
| 14 |
+
"prompt": "A sleek, modern sports car accelerating rapidly down a straight, empty road, gaining speed with each passing second. The car's engine roars as it speeds up, tires leaving occasional smoke trails on the asphalt. The exterior of the car is a shiny metallic silver with minimalistic design elements. The background showcases a scenic countryside with rolling hills and a clear blue sky. The camera follows the car from a slight distance, capturing the intense acceleration and speed from a dynamic tracking shot.",
|
| 15 |
+
"original_prompt": "a car accelerating to gain speed",
|
| 16 |
+
"seed": 0,
|
| 17 |
+
"video": "/local/zoubin/cz/projects/comparison/eval_out/generated_videos/cf_flowcache_Fxxx_x3/subject_consistency/014.mp4",
|
| 18 |
+
"num_frames": 81,
|
| 19 |
+
"height": 480,
|
| 20 |
+
"width": 832,
|
| 21 |
+
"fps": 16,
|
| 22 |
+
"policy_latency_ms": 1046.7888938188553,
|
| 23 |
+
"excluded_context_kv_latency_ms": 786.2396469116211,
|
| 24 |
+
"wall_generation_s": 3.631049156188965,
|
| 25 |
+
"reference_strategy": "cf_ffff",
|
| 26 |
+
"pixel_metrics_vs_ffff": {
|
| 27 |
+
"mean_mse": 0.05760398507118225,
|
| 28 |
+
"psnr": 12.395474910736084,
|
| 29 |
+
"ssim": 0.4695456624031067,
|
| 30 |
+
"lpips": 0.7190747857093811,
|
| 31 |
+
"num_frames": 81
|
| 32 |
+
},
|
| 33 |
+
"cache_diagnostics": {
|
| 34 |
+
"denoise_forwards": 28,
|
| 35 |
+
"compute_equivalent_forwards": 8.666666666666666,
|
| 36 |
+
"middle_steps": 21,
|
| 37 |
+
"middle_compute_equivalent": 1.6666666666666665,
|
| 38 |
+
"compute_equivalent_forwards_retime": 8.666666666666666
|
| 39 |
+
},
|
| 40 |
+
"policy_latency_ms_generation_run": 1863.2801665067673,
|
| 41 |
+
"matched_ffff_policy_latency_ms": 3147.62109375,
|
| 42 |
+
"latency_source": "retime_eval: paired with FFFF in the same process, denoise only"
|
| 43 |
+
}
|
eval_out/per_prompt/cf_flowcache_Fxxx_x3/subject_consistency_015.json
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "complete",
|
| 3 |
+
"protocol": "Self-Forcing Extended-251 Full Evaluation",
|
| 4 |
+
"strategy": "cf_flowcache_Fxxx_x3",
|
| 5 |
+
"base_model": "causal_forcing",
|
| 6 |
+
"method": "flowcache",
|
| 7 |
+
"param": "thresh",
|
| 8 |
+
"param_value": 1.69921875,
|
| 9 |
+
"target_speedup": 3.0,
|
| 10 |
+
"schedule": "Fxxx",
|
| 11 |
+
"global_index": 272,
|
| 12 |
+
"prompt_suite": "subject_consistency",
|
| 13 |
+
"suite_index": 15,
|
| 14 |
+
"prompt": "A sleek black motorcycle cruising along a scenic coastal highway during sunset. The motorcycle has a polished chrome finish and tinted windows on the helmet. The rider, wearing a black leather jacket and jeans, has their hands firmly on the handlebars and is leaning slightly forward, maintaining a steady posture. Waves crash against the rocky shoreline on one side of the highway, while lush green hills roll into the distance on the other. The sky is painted with vibrant hues of orange and pink, casting a warm glow over the entire scene. The camera follows the motorcycle from behind, capturing the wind-swept road and the dramatic coastal landscape in a wide shot.",
|
| 15 |
+
"original_prompt": "a motorcycle cruising along a coastal highway",
|
| 16 |
+
"seed": 0,
|
| 17 |
+
"video": "/local/zoubin/cz/projects/comparison/eval_out/generated_videos/cf_flowcache_Fxxx_x3/subject_consistency/015.mp4",
|
| 18 |
+
"num_frames": 81,
|
| 19 |
+
"height": 480,
|
| 20 |
+
"width": 832,
|
| 21 |
+
"fps": 16,
|
| 22 |
+
"policy_latency_ms": 1032.1158599853516,
|
| 23 |
+
"excluded_context_kv_latency_ms": 776.5309219360352,
|
| 24 |
+
"wall_generation_s": 3.5878381729125977,
|
| 25 |
+
"reference_strategy": "cf_ffff",
|
| 26 |
+
"pixel_metrics_vs_ffff": {
|
| 27 |
+
"mean_mse": 0.08295726031064987,
|
| 28 |
+
"psnr": 10.811456441879272,
|
| 29 |
+
"ssim": 0.38816338777542114,
|
| 30 |
+
"lpips": 0.6601709723472595,
|
| 31 |
+
"num_frames": 81
|
| 32 |
+
},
|
| 33 |
+
"cache_diagnostics": {
|
| 34 |
+
"denoise_forwards": 28,
|
| 35 |
+
"compute_equivalent_forwards": 8.666666666666668,
|
| 36 |
+
"middle_steps": 21,
|
| 37 |
+
"middle_compute_equivalent": 1.6666666666666665,
|
| 38 |
+
"compute_equivalent_forwards_retime": 8.666666666666668
|
| 39 |
+
},
|
| 40 |
+
"policy_latency_ms_generation_run": 1808.8166434764862,
|
| 41 |
+
"matched_ffff_policy_latency_ms": 3114.3559646606445,
|
| 42 |
+
"latency_source": "retime_eval: paired with FFFF in the same process, denoise only"
|
| 43 |
+
}
|
eval_out/per_prompt/cf_flowcache_Fxxx_x3/subject_consistency_016.json
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "complete",
|
| 3 |
+
"protocol": "Self-Forcing Extended-251 Full Evaluation",
|
| 4 |
+
"strategy": "cf_flowcache_Fxxx_x3",
|
| 5 |
+
"base_model": "causal_forcing",
|
| 6 |
+
"method": "flowcache",
|
| 7 |
+
"param": "thresh",
|
| 8 |
+
"param_value": 1.69921875,
|
| 9 |
+
"target_speedup": 3.0,
|
| 10 |
+
"schedule": "Fxxx",
|
| 11 |
+
"global_index": 273,
|
| 12 |
+
"prompt_suite": "subject_consistency",
|
| 13 |
+
"suite_index": 16,
|
| 14 |
+
"prompt": "A motorcycle turning a corner on a narrow urban street at night. The bike is black with sleek, aerodynamic features, and the rider is wearing a black leather jacket, jeans, and a helmet. The rider leans into the turn, with one hand gripping the handlebars and the other clutching the side of the motorcycle. The streetlights cast a soft glow, illuminating the wet asphalt and creating reflections. In the background, there are dimly lit buildings and parked cars. The scene is captured from a mid-shot perspective, focusing on the motorcycle and the rider as they navigate the corner.",
|
| 15 |
+
"original_prompt": "a motorcycle turning a corner",
|
| 16 |
+
"seed": 0,
|
| 17 |
+
"video": "/local/zoubin/cz/projects/comparison/eval_out/generated_videos/cf_flowcache_Fxxx_x3/subject_consistency/016.mp4",
|
| 18 |
+
"num_frames": 81,
|
| 19 |
+
"height": 480,
|
| 20 |
+
"width": 832,
|
| 21 |
+
"fps": 16,
|
| 22 |
+
"policy_latency_ms": 999.577853679657,
|
| 23 |
+
"excluded_context_kv_latency_ms": 785.729377746582,
|
| 24 |
+
"wall_generation_s": 3.565227746963501,
|
| 25 |
+
"reference_strategy": "cf_ffff",
|
| 26 |
+
"pixel_metrics_vs_ffff": {
|
| 27 |
+
"mean_mse": 0.07378467917442322,
|
| 28 |
+
"psnr": 11.320338249206543,
|
| 29 |
+
"ssim": 0.29043808579444885,
|
| 30 |
+
"lpips": 0.6638583540916443,
|
| 31 |
+
"num_frames": 81
|
| 32 |
+
},
|
| 33 |
+
"cache_diagnostics": {
|
| 34 |
+
"denoise_forwards": 28,
|
| 35 |
+
"compute_equivalent_forwards": 9.0,
|
| 36 |
+
"middle_steps": 21,
|
| 37 |
+
"middle_compute_equivalent": 2.0,
|
| 38 |
+
"compute_equivalent_forwards_retime": 9.0
|
| 39 |
+
},
|
| 40 |
+
"policy_latency_ms_generation_run": 1065.7660394906998,
|
| 41 |
+
"matched_ffff_policy_latency_ms": 3144.2738723754883,
|
| 42 |
+
"latency_source": "retime_eval: paired with FFFF in the same process, denoise only"
|
| 43 |
+
}
|
eval_out/per_prompt/cf_flowcache_Fxxx_x3/subject_consistency_017.json
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "complete",
|
| 3 |
+
"protocol": "Self-Forcing Extended-251 Full Evaluation",
|
| 4 |
+
"strategy": "cf_flowcache_Fxxx_x3",
|
| 5 |
+
"base_model": "causal_forcing",
|
| 6 |
+
"method": "flowcache",
|
| 7 |
+
"param": "thresh",
|
| 8 |
+
"param_value": 1.69921875,
|
| 9 |
+
"target_speedup": 3.0,
|
| 10 |
+
"schedule": "Fxxx",
|
| 11 |
+
"global_index": 274,
|
| 12 |
+
"prompt_suite": "subject_consistency",
|
| 13 |
+
"suite_index": 17,
|
| 14 |
+
"prompt": "A motorcycle gradually slows down as it approaches a stop. The rider, wearing a black helmet and leather jacket, leans slightly forward with one hand on the brake lever and the other on the handlebar. The bike's engine revs decrease steadily, and the rear wheel begins to gently skid on the asphalt. In the background, there are blurred images of urban street signs and parked cars. The scene is captured in a slow-motion shot from a low-angle perspective, focusing on the motorcycle's front wheel and the rider's determined face.",
|
| 15 |
+
"original_prompt": "a motorcycle slowing down to stop",
|
| 16 |
+
"seed": 0,
|
| 17 |
+
"video": "/local/zoubin/cz/projects/comparison/eval_out/generated_videos/cf_flowcache_Fxxx_x3/subject_consistency/017.mp4",
|
| 18 |
+
"num_frames": 81,
|
| 19 |
+
"height": 480,
|
| 20 |
+
"width": 832,
|
| 21 |
+
"fps": 16,
|
| 22 |
+
"policy_latency_ms": 971.7171145677567,
|
| 23 |
+
"excluded_context_kv_latency_ms": 777.3180694580078,
|
| 24 |
+
"wall_generation_s": 3.5567526817321777,
|
| 25 |
+
"reference_strategy": "cf_ffff",
|
| 26 |
+
"pixel_metrics_vs_ffff": {
|
| 27 |
+
"mean_mse": 0.12573683261871338,
|
| 28 |
+
"psnr": 9.005374908447266,
|
| 29 |
+
"ssim": 0.2846040725708008,
|
| 30 |
+
"lpips": 0.6674872040748596,
|
| 31 |
+
"num_frames": 81
|
| 32 |
+
},
|
| 33 |
+
"cache_diagnostics": {
|
| 34 |
+
"denoise_forwards": 28,
|
| 35 |
+
"compute_equivalent_forwards": 8.333333333333332,
|
| 36 |
+
"middle_steps": 21,
|
| 37 |
+
"middle_compute_equivalent": 1.3333333333333333,
|
| 38 |
+
"compute_equivalent_forwards_retime": 8.333333333333332
|
| 39 |
+
},
|
| 40 |
+
"policy_latency_ms_generation_run": 1695.3606140613556,
|
| 41 |
+
"matched_ffff_policy_latency_ms": 3119.5853118896484,
|
| 42 |
+
"latency_source": "retime_eval: paired with FFFF in the same process, denoise only"
|
| 43 |
+
}
|
eval_out/per_prompt/cf_flowcache_Fxxx_x3/subject_consistency_018.json
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "complete",
|
| 3 |
+
"protocol": "Self-Forcing Extended-251 Full Evaluation",
|
| 4 |
+
"strategy": "cf_flowcache_Fxxx_x3",
|
| 5 |
+
"base_model": "causal_forcing",
|
| 6 |
+
"method": "flowcache",
|
| 7 |
+
"param": "thresh",
|
| 8 |
+
"param_value": 1.69921875,
|
| 9 |
+
"target_speedup": 3.0,
|
| 10 |
+
"schedule": "Fxxx",
|
| 11 |
+
"global_index": 275,
|
| 12 |
+
"prompt_suite": "subject_consistency",
|
| 13 |
+
"suite_index": 18,
|
| 14 |
+
"prompt": "A sleek, black motorcycle gliding smoothly through a vast snowy field under a clear winter sky. The motorcycle's headlights cast a soft glow, illuminating the pristine snow as it moves. The rider is dressed in full winter gear, including a black helmet and jacket, with gloves and goggles. The landscape is serene, with occasional patches of untouched snow and distant pine trees. The camera follows the motorcycle in a smooth tracking shot, capturing the fluid motion and the gentle flurry of snow kicked up by the tires. Medium to wide shot, emphasizing the expansive snowy backdrop and the seamless glide of the motorcycle.",
|
| 15 |
+
"original_prompt": "a motorcycle gliding through a snowy field",
|
| 16 |
+
"seed": 0,
|
| 17 |
+
"video": "/local/zoubin/cz/projects/comparison/eval_out/generated_videos/cf_flowcache_Fxxx_x3/subject_consistency/018.mp4",
|
| 18 |
+
"num_frames": 81,
|
| 19 |
+
"height": 480,
|
| 20 |
+
"width": 832,
|
| 21 |
+
"fps": 16,
|
| 22 |
+
"policy_latency_ms": 904.6780908107758,
|
| 23 |
+
"excluded_context_kv_latency_ms": 790.0515899658203,
|
| 24 |
+
"wall_generation_s": 3.4800007343292236,
|
| 25 |
+
"reference_strategy": "cf_ffff",
|
| 26 |
+
"pixel_metrics_vs_ffff": {
|
| 27 |
+
"mean_mse": 0.07364711165428162,
|
| 28 |
+
"psnr": 11.3284432888031,
|
| 29 |
+
"ssim": 0.45013776421546936,
|
| 30 |
+
"lpips": 0.7224014401435852,
|
| 31 |
+
"num_frames": 81
|
| 32 |
+
},
|
| 33 |
+
"cache_diagnostics": {
|
| 34 |
+
"denoise_forwards": 28,
|
| 35 |
+
"compute_equivalent_forwards": 7.666666666666666,
|
| 36 |
+
"middle_steps": 21,
|
| 37 |
+
"middle_compute_equivalent": 0.6666666666666666,
|
| 38 |
+
"compute_equivalent_forwards_retime": 7.666666666666666
|
| 39 |
+
},
|
| 40 |
+
"policy_latency_ms_generation_run": 1588.0743520259857,
|
| 41 |
+
"matched_ffff_policy_latency_ms": 3163.856575012207,
|
| 42 |
+
"latency_source": "retime_eval: paired with FFFF in the same process, denoise only"
|
| 43 |
+
}
|
eval_out/per_prompt/cf_flowcache_Fxxx_x3/subject_consistency_019.json
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "complete",
|
| 3 |
+
"protocol": "Self-Forcing Extended-251 Full Evaluation",
|
| 4 |
+
"strategy": "cf_flowcache_Fxxx_x3",
|
| 5 |
+
"base_model": "causal_forcing",
|
| 6 |
+
"method": "flowcache",
|
| 7 |
+
"param": "thresh",
|
| 8 |
+
"param_value": 1.69921875,
|
| 9 |
+
"target_speedup": 3.0,
|
| 10 |
+
"schedule": "Fxxx",
|
| 11 |
+
"global_index": 276,
|
| 12 |
+
"prompt_suite": "subject_consistency",
|
| 13 |
+
"suite_index": 19,
|
| 14 |
+
"prompt": "A high-speed action scene featuring a sleek black motorcycle accelerating down a winding asphalt road. The motorcycle's engine roars as it gains speed, smoke trailing from the tires. The rider, wearing a black leather jacket and helmet, leans forward with intense focus. The background shows blurred trees and distant hills, emphasizing the speed. The camera follows the motorcycle closely, capturing every detail of the bike and rider as they accelerate. Close-up shot, dynamic camera movement tracking the motorcycle.",
|
| 15 |
+
"original_prompt": "a motorcycle accelerating to gain speed",
|
| 16 |
+
"seed": 0,
|
| 17 |
+
"video": "/local/zoubin/cz/projects/comparison/eval_out/generated_videos/cf_flowcache_Fxxx_x3/subject_consistency/019.mp4",
|
| 18 |
+
"num_frames": 81,
|
| 19 |
+
"height": 480,
|
| 20 |
+
"width": 832,
|
| 21 |
+
"fps": 16,
|
| 22 |
+
"policy_latency_ms": 902.9960578680038,
|
| 23 |
+
"excluded_context_kv_latency_ms": 778.4731521606445,
|
| 24 |
+
"wall_generation_s": 3.465857744216919,
|
| 25 |
+
"reference_strategy": "cf_ffff",
|
| 26 |
+
"pixel_metrics_vs_ffff": {
|
| 27 |
+
"mean_mse": 0.11841674894094467,
|
| 28 |
+
"psnr": 9.265868663787842,
|
| 29 |
+
"ssim": 0.3725886940956116,
|
| 30 |
+
"lpips": 0.6841627359390259,
|
| 31 |
+
"num_frames": 81
|
| 32 |
+
},
|
| 33 |
+
"cache_diagnostics": {
|
| 34 |
+
"denoise_forwards": 28,
|
| 35 |
+
"compute_equivalent_forwards": 8.0,
|
| 36 |
+
"middle_steps": 21,
|
| 37 |
+
"middle_compute_equivalent": 1.0,
|
| 38 |
+
"compute_equivalent_forwards_retime": 8.0
|
| 39 |
+
},
|
| 40 |
+
"policy_latency_ms_generation_run": 1591.665526151657,
|
| 41 |
+
"matched_ffff_policy_latency_ms": 3113.233352661133,
|
| 42 |
+
"latency_source": "retime_eval: paired with FFFF in the same process, denoise only"
|
| 43 |
+
}
|
eval_out/per_prompt/cf_flowcache_Fxxx_x3/subject_consistency_020.json
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "complete",
|
| 3 |
+
"protocol": "Self-Forcing Extended-251 Full Evaluation",
|
| 4 |
+
"strategy": "cf_flowcache_Fxxx_x3",
|
| 5 |
+
"base_model": "causal_forcing",
|
| 6 |
+
"method": "flowcache",
|
| 7 |
+
"param": "thresh",
|
| 8 |
+
"param_value": 1.69921875,
|
| 9 |
+
"target_speedup": 3.0,
|
| 10 |
+
"schedule": "Fxxx",
|
| 11 |
+
"global_index": 277,
|
| 12 |
+
"prompt_suite": "subject_consistency",
|
| 13 |
+
"suite_index": 20,
|
| 14 |
+
"prompt": "A sleek commercial airplane flying through a clear blue sky, with fluffy white clouds scattered in the background. The airplane is shown from a slightly angled perspective, emphasizing its aerodynamic shape and the smooth curvature of its wings. The sun is casting a gentle glow over the plane, highlighting its metallic surface. The scene transitions from a wide shot of the plane against the vast sky to a closer medium shot focusing on the plane's streamlined design as it soars gracefully through the air.",
|
| 15 |
+
"original_prompt": "an airplane soaring through a clear blue sky",
|
| 16 |
+
"seed": 0,
|
| 17 |
+
"video": "/local/zoubin/cz/projects/comparison/eval_out/generated_videos/cf_flowcache_Fxxx_x3/subject_consistency/020.mp4",
|
| 18 |
+
"num_frames": 81,
|
| 19 |
+
"height": 480,
|
| 20 |
+
"width": 832,
|
| 21 |
+
"fps": 16,
|
| 22 |
+
"policy_latency_ms": 964.3721561431885,
|
| 23 |
+
"excluded_context_kv_latency_ms": 789.8158187866211,
|
| 24 |
+
"wall_generation_s": 3.521601915359497,
|
| 25 |
+
"reference_strategy": "cf_ffff",
|
| 26 |
+
"pixel_metrics_vs_ffff": {
|
| 27 |
+
"mean_mse": 0.1034887358546257,
|
| 28 |
+
"psnr": 9.851069450378418,
|
| 29 |
+
"ssim": 0.39533093571662903,
|
| 30 |
+
"lpips": 0.7625374794006348,
|
| 31 |
+
"num_frames": 81
|
| 32 |
+
},
|
| 33 |
+
"cache_diagnostics": {
|
| 34 |
+
"denoise_forwards": 28,
|
| 35 |
+
"compute_equivalent_forwards": 7.666666666666666,
|
| 36 |
+
"middle_steps": 21,
|
| 37 |
+
"middle_compute_equivalent": 0.6666666666666666,
|
| 38 |
+
"compute_equivalent_forwards_retime": 7.666666666666666
|
| 39 |
+
},
|
| 40 |
+
"policy_latency_ms_generation_run": 985.6602519750595,
|
| 41 |
+
"matched_ffff_policy_latency_ms": 3160.1592178344727,
|
| 42 |
+
"latency_source": "retime_eval: paired with FFFF in the same process, denoise only"
|
| 43 |
+
}
|
eval_out/per_prompt/cf_flowcache_Fxxx_x3/subject_consistency_021.json
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "complete",
|
| 3 |
+
"protocol": "Self-Forcing Extended-251 Full Evaluation",
|
| 4 |
+
"strategy": "cf_flowcache_Fxxx_x3",
|
| 5 |
+
"base_model": "causal_forcing",
|
| 6 |
+
"method": "flowcache",
|
| 7 |
+
"param": "thresh",
|
| 8 |
+
"param_value": 1.69921875,
|
| 9 |
+
"target_speedup": 3.0,
|
| 10 |
+
"schedule": "Fxxx",
|
| 11 |
+
"global_index": 278,
|
| 12 |
+
"prompt_suite": "subject_consistency",
|
| 13 |
+
"suite_index": 21,
|
| 14 |
+
"prompt": "A large commercial airplane taking off from a busy airport runway. The airplane is a Boeing 747 with distinctive white and blue livery. It is surrounded by tarmac and lined up with other planes, creating a sense of scale. The engines are roaring, and there is a trail of smoke and dust as it accelerates down the runway. The ground crew stands at a safe distance, watching the plane take flight. The camera captures the moment of lift-off from a low angle, showcasing the power and size of the aircraft as it ascends into the sky. Wide shot, starting from a static view of the runway and panning upwards as the plane lifts off.",
|
| 15 |
+
"original_prompt": "an airplane taking off",
|
| 16 |
+
"seed": 0,
|
| 17 |
+
"video": "/local/zoubin/cz/projects/comparison/eval_out/generated_videos/cf_flowcache_Fxxx_x3/subject_consistency/021.mp4",
|
| 18 |
+
"num_frames": 81,
|
| 19 |
+
"height": 480,
|
| 20 |
+
"width": 832,
|
| 21 |
+
"fps": 16,
|
| 22 |
+
"policy_latency_ms": 1255.2507627010345,
|
| 23 |
+
"excluded_context_kv_latency_ms": 776.3953323364258,
|
| 24 |
+
"wall_generation_s": 3.85227370262146,
|
| 25 |
+
"reference_strategy": "cf_ffff",
|
| 26 |
+
"pixel_metrics_vs_ffff": {
|
| 27 |
+
"mean_mse": 0.08052070438861847,
|
| 28 |
+
"psnr": 10.940924882888794,
|
| 29 |
+
"ssim": 0.47354042530059814,
|
| 30 |
+
"lpips": 0.630456268787384,
|
| 31 |
+
"num_frames": 81
|
| 32 |
+
},
|
| 33 |
+
"cache_diagnostics": {
|
| 34 |
+
"denoise_forwards": 28,
|
| 35 |
+
"compute_equivalent_forwards": 10.666666666666668,
|
| 36 |
+
"middle_steps": 21,
|
| 37 |
+
"middle_compute_equivalent": 3.666666666666667,
|
| 38 |
+
"compute_equivalent_forwards_retime": 10.666666666666668
|
| 39 |
+
},
|
| 40 |
+
"policy_latency_ms_generation_run": 1999.7579536437988,
|
| 41 |
+
"matched_ffff_policy_latency_ms": 3117.8198165893555,
|
| 42 |
+
"latency_source": "retime_eval: paired with FFFF in the same process, denoise only"
|
| 43 |
+
}
|
eval_out/per_prompt/cf_flowcache_Fxxx_x3/subject_consistency_022.json
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "complete",
|
| 3 |
+
"protocol": "Self-Forcing Extended-251 Full Evaluation",
|
| 4 |
+
"strategy": "cf_flowcache_Fxxx_x3",
|
| 5 |
+
"base_model": "causal_forcing",
|
| 6 |
+
"method": "flowcache",
|
| 7 |
+
"param": "thresh",
|
| 8 |
+
"param_value": 1.69921875,
|
| 9 |
+
"target_speedup": 3.0,
|
| 10 |
+
"schedule": "Fxxx",
|
| 11 |
+
"global_index": 279,
|
| 12 |
+
"prompt_suite": "subject_consistency",
|
| 13 |
+
"suite_index": 22,
|
| 14 |
+
"prompt": "A commercial airliner landing smoothly on a well-maintained runway at sunset. The aircraft touches down gently, its wheels making contact with the tarmac, leaving a trail of smoke. The wings are fully extended, and the plane slows down gradually as it taxis towards the terminal area. The sky is painted with warm hues of orange and pink, casting a beautiful glow over the scene. The camera follows the plane from a medium distance, maintaining a static shot to capture the entirety of the smooth landing process. Wide shot.",
|
| 15 |
+
"original_prompt": "an airplane landing smoothly on a runway",
|
| 16 |
+
"seed": 0,
|
| 17 |
+
"video": "/local/zoubin/cz/projects/comparison/eval_out/generated_videos/cf_flowcache_Fxxx_x3/subject_consistency/022.mp4",
|
| 18 |
+
"num_frames": 81,
|
| 19 |
+
"height": 480,
|
| 20 |
+
"width": 832,
|
| 21 |
+
"fps": 16,
|
| 22 |
+
"policy_latency_ms": 986.3661184310913,
|
| 23 |
+
"excluded_context_kv_latency_ms": 789.7821578979492,
|
| 24 |
+
"wall_generation_s": 3.5581936836242676,
|
| 25 |
+
"reference_strategy": "cf_ffff",
|
| 26 |
+
"pixel_metrics_vs_ffff": {
|
| 27 |
+
"mean_mse": 0.04023322835564613,
|
| 28 |
+
"psnr": 13.954150676727295,
|
| 29 |
+
"ssim": 0.43638911843299866,
|
| 30 |
+
"lpips": 0.6641862392425537,
|
| 31 |
+
"num_frames": 81
|
| 32 |
+
},
|
| 33 |
+
"cache_diagnostics": {
|
| 34 |
+
"denoise_forwards": 28,
|
| 35 |
+
"compute_equivalent_forwards": 8.333333333333332,
|
| 36 |
+
"middle_steps": 21,
|
| 37 |
+
"middle_compute_equivalent": 1.3333333333333333,
|
| 38 |
+
"compute_equivalent_forwards_retime": 8.333333333333332
|
| 39 |
+
},
|
| 40 |
+
"policy_latency_ms_generation_run": 1804.9707794189453,
|
| 41 |
+
"matched_ffff_policy_latency_ms": 3164.611602783203,
|
| 42 |
+
"latency_source": "retime_eval: paired with FFFF in the same process, denoise only"
|
| 43 |
+
}
|
eval_out/per_prompt/cf_flowcache_Fxxx_x3/subject_consistency_023.json
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "complete",
|
| 3 |
+
"protocol": "Self-Forcing Extended-251 Full Evaluation",
|
| 4 |
+
"strategy": "cf_flowcache_Fxxx_x3",
|
| 5 |
+
"base_model": "causal_forcing",
|
| 6 |
+
"method": "flowcache",
|
| 7 |
+
"param": "thresh",
|
| 8 |
+
"param_value": 1.69921875,
|
| 9 |
+
"target_speedup": 3.0,
|
| 10 |
+
"schedule": "Fxxx",
|
| 11 |
+
"global_index": 280,
|
| 12 |
+
"prompt_suite": "subject_consistency",
|
| 13 |
+
"suite_index": 23,
|
| 14 |
+
"prompt": "A Boeing 747 commercial airliner accelerating down a runway to gain speed for takeoff. The plane's engines emit powerful bursts of exhaust smoke as it speeds up. The tarmac is smooth and empty, reflecting sunlight. The camera captures the front third of the aircraft from a low angle, emphasizing the plane's massive size and the intense acceleration. The tires leave dark marks on the runway as the plane gathers momentum. Wide shot, focusing on the plane's forward motion and the blur of the tarmac behind it.",
|
| 15 |
+
"original_prompt": "an airplane accelerating to gain speed",
|
| 16 |
+
"seed": 0,
|
| 17 |
+
"video": "/local/zoubin/cz/projects/comparison/eval_out/generated_videos/cf_flowcache_Fxxx_x3/subject_consistency/023.mp4",
|
| 18 |
+
"num_frames": 81,
|
| 19 |
+
"height": 480,
|
| 20 |
+
"width": 832,
|
| 21 |
+
"fps": 16,
|
| 22 |
+
"policy_latency_ms": 1097.5360538959503,
|
| 23 |
+
"excluded_context_kv_latency_ms": 778.2885971069336,
|
| 24 |
+
"wall_generation_s": 3.647850513458252,
|
| 25 |
+
"reference_strategy": "cf_ffff",
|
| 26 |
+
"pixel_metrics_vs_ffff": {
|
| 27 |
+
"mean_mse": 0.07628903537988663,
|
| 28 |
+
"psnr": 11.175378561019897,
|
| 29 |
+
"ssim": 0.4775775074958801,
|
| 30 |
+
"lpips": 0.6651372909545898,
|
| 31 |
+
"num_frames": 81
|
| 32 |
+
},
|
| 33 |
+
"cache_diagnostics": {
|
| 34 |
+
"denoise_forwards": 28,
|
| 35 |
+
"compute_equivalent_forwards": 9.333333333333332,
|
| 36 |
+
"middle_steps": 21,
|
| 37 |
+
"middle_compute_equivalent": 2.333333333333333,
|
| 38 |
+
"compute_equivalent_forwards_retime": 9.333333333333332
|
| 39 |
+
},
|
| 40 |
+
"policy_latency_ms_generation_run": 1908.4767711162567,
|
| 41 |
+
"matched_ffff_policy_latency_ms": 3119.192543029785,
|
| 42 |
+
"latency_source": "retime_eval: paired with FFFF in the same process, denoise only"
|
| 43 |
+
}
|
eval_out/per_prompt/cf_flowcache_Fxxx_x3/subject_consistency_024.json
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "complete",
|
| 3 |
+
"protocol": "Self-Forcing Extended-251 Full Evaluation",
|
| 4 |
+
"strategy": "cf_flowcache_Fxxx_x3",
|
| 5 |
+
"base_model": "causal_forcing",
|
| 6 |
+
"method": "flowcache",
|
| 7 |
+
"param": "thresh",
|
| 8 |
+
"param_value": 1.69921875,
|
| 9 |
+
"target_speedup": 3.0,
|
| 10 |
+
"schedule": "Fxxx",
|
| 11 |
+
"global_index": 281,
|
| 12 |
+
"prompt_suite": "subject_consistency",
|
| 13 |
+
"suite_index": 24,
|
| 14 |
+
"prompt": "A city bus making a turn at a street corner. The bus is painted in bright yellow with large rectangular windows and a door at the front. It has the typical urban bus design with multiple wheels and a high chassis. As the bus navigates the corner, it smoothly turns right, revealing a glimpse of the bustling cityscape beyond, including other vehicles and pedestrians. The camera captures the scene from a medium distance, focusing on the bus as it completes the turn. The environment is vibrant with a mix of modern buildings and green spaces. Static shot.",
|
| 15 |
+
"original_prompt": "a bus turning a corner",
|
| 16 |
+
"seed": 0,
|
| 17 |
+
"video": "/local/zoubin/cz/projects/comparison/eval_out/generated_videos/cf_flowcache_Fxxx_x3/subject_consistency/024.mp4",
|
| 18 |
+
"num_frames": 81,
|
| 19 |
+
"height": 480,
|
| 20 |
+
"width": 832,
|
| 21 |
+
"fps": 16,
|
| 22 |
+
"policy_latency_ms": 976.6183354854584,
|
| 23 |
+
"excluded_context_kv_latency_ms": 789.5039596557617,
|
| 24 |
+
"wall_generation_s": 3.532313823699951,
|
| 25 |
+
"reference_strategy": "cf_ffff",
|
| 26 |
+
"pixel_metrics_vs_ffff": {
|
| 27 |
+
"mean_mse": 0.11845023930072784,
|
| 28 |
+
"psnr": 9.264640808105469,
|
| 29 |
+
"ssim": 0.2271525114774704,
|
| 30 |
+
"lpips": 0.7136176228523254,
|
| 31 |
+
"num_frames": 81
|
| 32 |
+
},
|
| 33 |
+
"cache_diagnostics": {
|
| 34 |
+
"denoise_forwards": 28,
|
| 35 |
+
"compute_equivalent_forwards": 8.333333333333334,
|
| 36 |
+
"middle_steps": 21,
|
| 37 |
+
"middle_compute_equivalent": 1.3333333333333333,
|
| 38 |
+
"compute_equivalent_forwards_retime": 8.333333333333334
|
| 39 |
+
},
|
| 40 |
+
"policy_latency_ms_generation_run": 1153.138698220253,
|
| 41 |
+
"matched_ffff_policy_latency_ms": 3159.353157043457,
|
| 42 |
+
"latency_source": "retime_eval: paired with FFFF in the same process, denoise only"
|
| 43 |
+
}
|
eval_out/per_prompt/cf_flowcache_Fxxx_x3/subject_consistency_025.json
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "complete",
|
| 3 |
+
"protocol": "Self-Forcing Extended-251 Full Evaluation",
|
| 4 |
+
"strategy": "cf_flowcache_Fxxx_x3",
|
| 5 |
+
"base_model": "causal_forcing",
|
| 6 |
+
"method": "flowcache",
|
| 7 |
+
"param": "thresh",
|
| 8 |
+
"param_value": 1.69921875,
|
| 9 |
+
"target_speedup": 3.0,
|
| 10 |
+
"schedule": "Fxxx",
|
| 11 |
+
"global_index": 282,
|
| 12 |
+
"prompt_suite": "subject_consistency",
|
| 13 |
+
"suite_index": 25,
|
| 14 |
+
"prompt": "A busy city street during rush hour, with multiple lanes of traffic filled with cars and buses. Focus on a large yellow school bus stuck in the middle lane, its passengers visible through the windows, looking impatient and occasionally glancing at their phones or talking amongst themselves. The bus is surrounded by honking vehicles, adding to the chaotic atmosphere. The background shows tall buildings and bustling sidewalks. The camera captures the scene from a mid-shot angle, emphasizing the congestion and frustration on the road.",
|
| 15 |
+
"original_prompt": "a bus stuck in traffic during rush hour",
|
| 16 |
+
"seed": 0,
|
| 17 |
+
"video": "/local/zoubin/cz/projects/comparison/eval_out/generated_videos/cf_flowcache_Fxxx_x3/subject_consistency/025.mp4",
|
| 18 |
+
"num_frames": 81,
|
| 19 |
+
"height": 480,
|
| 20 |
+
"width": 832,
|
| 21 |
+
"fps": 16,
|
| 22 |
+
"policy_latency_ms": 1287.433528661728,
|
| 23 |
+
"excluded_context_kv_latency_ms": 776.5405807495117,
|
| 24 |
+
"wall_generation_s": 3.8639838695526123,
|
| 25 |
+
"reference_strategy": "cf_ffff",
|
| 26 |
+
"pixel_metrics_vs_ffff": {
|
| 27 |
+
"mean_mse": 0.11934202909469604,
|
| 28 |
+
"psnr": 9.232065677642822,
|
| 29 |
+
"ssim": 0.2314656376838684,
|
| 30 |
+
"lpips": 0.6379618048667908,
|
| 31 |
+
"num_frames": 81
|
| 32 |
+
},
|
| 33 |
+
"cache_diagnostics": {
|
| 34 |
+
"denoise_forwards": 28,
|
| 35 |
+
"compute_equivalent_forwards": 11.666666666666666,
|
| 36 |
+
"middle_steps": 21,
|
| 37 |
+
"middle_compute_equivalent": 4.666666666666667,
|
| 38 |
+
"compute_equivalent_forwards_retime": 11.666666666666666
|
| 39 |
+
},
|
| 40 |
+
"policy_latency_ms_generation_run": 1753.649449467659,
|
| 41 |
+
"matched_ffff_policy_latency_ms": 3115.4520950317383,
|
| 42 |
+
"latency_source": "retime_eval: paired with FFFF in the same process, denoise only"
|
| 43 |
+
}
|
eval_out/per_prompt/cf_flowcache_Fxxx_x3/subject_consistency_026.json
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "complete",
|
| 3 |
+
"protocol": "Self-Forcing Extended-251 Full Evaluation",
|
| 4 |
+
"strategy": "cf_flowcache_Fxxx_x3",
|
| 5 |
+
"base_model": "causal_forcing",
|
| 6 |
+
"method": "flowcache",
|
| 7 |
+
"param": "thresh",
|
| 8 |
+
"param_value": 1.69921875,
|
| 9 |
+
"target_speedup": 3.0,
|
| 10 |
+
"schedule": "Fxxx",
|
| 11 |
+
"global_index": 283,
|
| 12 |
+
"prompt_suite": "subject_consistency",
|
| 13 |
+
"suite_index": 26,
|
| 14 |
+
"prompt": "A large city bus accelerating down a busy street, gaining speed as it moves forward. The bus is painted in bright yellow with black stripes and displays advertisements on its sides. Inside, passengers are visible through the windows, their faces blurred for privacy. The camera focuses on the front of the bus, capturing the wheels spinning rapidly and the tires gripping the road. The exterior view shows the bus passing other vehicles, with the surroundings blurring slightly due to the increasing speed. Wide shot, static camera focusing on the acceleration process.",
|
| 15 |
+
"original_prompt": "a bus accelerating to gain speed",
|
| 16 |
+
"seed": 0,
|
| 17 |
+
"video": "/local/zoubin/cz/projects/comparison/eval_out/generated_videos/cf_flowcache_Fxxx_x3/subject_consistency/026.mp4",
|
| 18 |
+
"num_frames": 81,
|
| 19 |
+
"height": 480,
|
| 20 |
+
"width": 832,
|
| 21 |
+
"fps": 16,
|
| 22 |
+
"policy_latency_ms": 916.4821189641953,
|
| 23 |
+
"excluded_context_kv_latency_ms": 790.4081954956055,
|
| 24 |
+
"wall_generation_s": 3.483203411102295,
|
| 25 |
+
"reference_strategy": "cf_ffff",
|
| 26 |
+
"pixel_metrics_vs_ffff": {
|
| 27 |
+
"mean_mse": 0.18582218885421753,
|
| 28 |
+
"psnr": 7.309024333953857,
|
| 29 |
+
"ssim": 0.2226180136203766,
|
| 30 |
+
"lpips": 0.7571300268173218,
|
| 31 |
+
"num_frames": 81
|
| 32 |
+
},
|
| 33 |
+
"cache_diagnostics": {
|
| 34 |
+
"denoise_forwards": 28,
|
| 35 |
+
"compute_equivalent_forwards": 8.0,
|
| 36 |
+
"middle_steps": 21,
|
| 37 |
+
"middle_compute_equivalent": 1.0,
|
| 38 |
+
"compute_equivalent_forwards_retime": 8.0
|
| 39 |
+
},
|
| 40 |
+
"policy_latency_ms_generation_run": 1272.1243662834167,
|
| 41 |
+
"matched_ffff_policy_latency_ms": 3164.984130859375,
|
| 42 |
+
"latency_source": "retime_eval: paired with FFFF in the same process, denoise only"
|
| 43 |
+
}
|
eval_out/per_prompt/cf_flowcache_Fxxx_x3/subject_consistency_027.json
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "complete",
|
| 3 |
+
"protocol": "Self-Forcing Extended-251 Full Evaluation",
|
| 4 |
+
"strategy": "cf_flowcache_Fxxx_x3",
|
| 5 |
+
"base_model": "causal_forcing",
|
| 6 |
+
"method": "flowcache",
|
| 7 |
+
"param": "thresh",
|
| 8 |
+
"param_value": 1.69921875,
|
| 9 |
+
"target_speedup": 3.0,
|
| 10 |
+
"schedule": "Fxxx",
|
| 11 |
+
"global_index": 284,
|
| 12 |
+
"prompt_suite": "subject_consistency",
|
| 13 |
+
"suite_index": 27,
|
| 14 |
+
"prompt": "A high-speed train rushes down the railway tracks, its sleek body gleaming under the bright sunlight. The train moves with great momentum, leaving a trail of steam behind as it passes through the landscape. The camera follows the train from a distance, capturing the powerful motion and speed. Surrounding the tracks are lush green fields and tall trees, creating a picturesque backdrop. The video should emphasize the train's velocity and the dynamic motion of the scenery passing by. Wide shot, showing the entirety of the train and the expansive environment surrounding it.",
|
| 15 |
+
"original_prompt": "a train speeding down the tracks",
|
| 16 |
+
"seed": 0,
|
| 17 |
+
"video": "/local/zoubin/cz/projects/comparison/eval_out/generated_videos/cf_flowcache_Fxxx_x3/subject_consistency/027.mp4",
|
| 18 |
+
"num_frames": 81,
|
| 19 |
+
"height": 480,
|
| 20 |
+
"width": 832,
|
| 21 |
+
"fps": 16,
|
| 22 |
+
"policy_latency_ms": 967.8185909986496,
|
| 23 |
+
"excluded_context_kv_latency_ms": 776.7652435302734,
|
| 24 |
+
"wall_generation_s": 3.5247104167938232,
|
| 25 |
+
"reference_strategy": "cf_ffff",
|
| 26 |
+
"pixel_metrics_vs_ffff": {
|
| 27 |
+
"mean_mse": 0.13054589927196503,
|
| 28 |
+
"psnr": 8.84236752986908,
|
| 29 |
+
"ssim": 0.28842607140541077,
|
| 30 |
+
"lpips": 0.7434969544410706,
|
| 31 |
+
"num_frames": 81
|
| 32 |
+
},
|
| 33 |
+
"cache_diagnostics": {
|
| 34 |
+
"denoise_forwards": 28,
|
| 35 |
+
"compute_equivalent_forwards": 8.333333333333332,
|
| 36 |
+
"middle_steps": 21,
|
| 37 |
+
"middle_compute_equivalent": 1.3333333333333333,
|
| 38 |
+
"compute_equivalent_forwards_retime": 8.333333333333332
|
| 39 |
+
},
|
| 40 |
+
"policy_latency_ms_generation_run": 1403.2515460252762,
|
| 41 |
+
"matched_ffff_policy_latency_ms": 3120.294723510742,
|
| 42 |
+
"latency_source": "retime_eval: paired with FFFF in the same process, denoise only"
|
| 43 |
+
}
|
eval_out/per_prompt/cf_flowcache_Fxxx_x3/subject_consistency_028.json
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "complete",
|
| 3 |
+
"protocol": "Self-Forcing Extended-251 Full Evaluation",
|
| 4 |
+
"strategy": "cf_flowcache_Fxxx_x3",
|
| 5 |
+
"base_model": "causal_forcing",
|
| 6 |
+
"method": "flowcache",
|
| 7 |
+
"param": "thresh",
|
| 8 |
+
"param_value": 1.69921875,
|
| 9 |
+
"target_speedup": 3.0,
|
| 10 |
+
"schedule": "Fxxx",
|
| 11 |
+
"global_index": 285,
|
| 12 |
+
"prompt_suite": "subject_consistency",
|
| 13 |
+
"suite_index": 28,
|
| 14 |
+
"prompt": "A realistic, cinematic scene featuring a long-distance passenger train crossing over a tall steel bridge. The train consists of multiple carriages connected seamlessly, moving steadily across the bridge. The bridge spans a vast, green valley below, with lush forests and rolling hills visible in the distance. The train\u2019s wheels clatter rhythmically as it crosses the bridge, and the camera captures the full length of the train from a mid-shot perspective, showcasing the height of the bridge and the expansive landscape beneath. Static shot.",
|
| 15 |
+
"original_prompt": "a train crossing over a tall bridge",
|
| 16 |
+
"seed": 0,
|
| 17 |
+
"video": "/local/zoubin/cz/projects/comparison/eval_out/generated_videos/cf_flowcache_Fxxx_x3/subject_consistency/028.mp4",
|
| 18 |
+
"num_frames": 81,
|
| 19 |
+
"height": 480,
|
| 20 |
+
"width": 832,
|
| 21 |
+
"fps": 16,
|
| 22 |
+
"policy_latency_ms": 916.80726146698,
|
| 23 |
+
"excluded_context_kv_latency_ms": 790.0829238891602,
|
| 24 |
+
"wall_generation_s": 3.467594623565674,
|
| 25 |
+
"reference_strategy": "cf_ffff",
|
| 26 |
+
"pixel_metrics_vs_ffff": {
|
| 27 |
+
"mean_mse": 0.06491214036941528,
|
| 28 |
+
"psnr": 11.876740455627441,
|
| 29 |
+
"ssim": 0.27977636456489563,
|
| 30 |
+
"lpips": 0.6718499660491943,
|
| 31 |
+
"num_frames": 81
|
| 32 |
+
},
|
| 33 |
+
"cache_diagnostics": {
|
| 34 |
+
"denoise_forwards": 28,
|
| 35 |
+
"compute_equivalent_forwards": 8.0,
|
| 36 |
+
"middle_steps": 21,
|
| 37 |
+
"middle_compute_equivalent": 1.0,
|
| 38 |
+
"compute_equivalent_forwards_retime": 8.0
|
| 39 |
+
},
|
| 40 |
+
"policy_latency_ms_generation_run": 1299.8586184978485,
|
| 41 |
+
"matched_ffff_policy_latency_ms": 3157.2815475463867,
|
| 42 |
+
"latency_source": "retime_eval: paired with FFFF in the same process, denoise only"
|
| 43 |
+
}
|
eval_out/per_prompt/cf_flowcache_Fxxx_x3/subject_consistency_029.json
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "complete",
|
| 3 |
+
"protocol": "Self-Forcing Extended-251 Full Evaluation",
|
| 4 |
+
"strategy": "cf_flowcache_Fxxx_x3",
|
| 5 |
+
"base_model": "causal_forcing",
|
| 6 |
+
"method": "flowcache",
|
| 7 |
+
"param": "thresh",
|
| 8 |
+
"param_value": 1.69921875,
|
| 9 |
+
"target_speedup": 3.0,
|
| 10 |
+
"schedule": "Fxxx",
|
| 11 |
+
"global_index": 286,
|
| 12 |
+
"prompt_suite": "subject_consistency",
|
| 13 |
+
"suite_index": 29,
|
| 14 |
+
"prompt": "A high-speed train accelerating on a straight track, gaining speed rapidly. The train consists of several sleek, modern carriages with aerodynamic designs. As the train accelerates, the wheels spin faster, and the scenery outside blurs into streaks of color. Smoke billows from the engine as it works hard to propel the train forward. The camera starts from a mid-shot of the train's front section, gradually zooming in to focus on the powerful engine, emphasizing the intense acceleration. The background shows a passing landscape, with trees and buildings quickly receding into the distance.",
|
| 15 |
+
"original_prompt": "a train accelerating to gain speed",
|
| 16 |
+
"seed": 0,
|
| 17 |
+
"video": "/local/zoubin/cz/projects/comparison/eval_out/generated_videos/cf_flowcache_Fxxx_x3/subject_consistency/029.mp4",
|
| 18 |
+
"num_frames": 81,
|
| 19 |
+
"height": 480,
|
| 20 |
+
"width": 832,
|
| 21 |
+
"fps": 16,
|
| 22 |
+
"policy_latency_ms": 1109.0837692022324,
|
| 23 |
+
"excluded_context_kv_latency_ms": 776.423942565918,
|
| 24 |
+
"wall_generation_s": 3.681239366531372,
|
| 25 |
+
"reference_strategy": "cf_ffff",
|
| 26 |
+
"pixel_metrics_vs_ffff": {
|
| 27 |
+
"mean_mse": 0.12685588002204895,
|
| 28 |
+
"psnr": 8.966894149780273,
|
| 29 |
+
"ssim": 0.32865914702415466,
|
| 30 |
+
"lpips": 0.7240904569625854,
|
| 31 |
+
"num_frames": 81
|
| 32 |
+
},
|
| 33 |
+
"cache_diagnostics": {
|
| 34 |
+
"denoise_forwards": 28,
|
| 35 |
+
"compute_equivalent_forwards": 9.333333333333334,
|
| 36 |
+
"middle_steps": 21,
|
| 37 |
+
"middle_compute_equivalent": 2.333333333333333,
|
| 38 |
+
"compute_equivalent_forwards_retime": 9.333333333333334
|
| 39 |
+
},
|
| 40 |
+
"policy_latency_ms_generation_run": 1600.9579451084137,
|
| 41 |
+
"matched_ffff_policy_latency_ms": 3118.407257080078,
|
| 42 |
+
"latency_source": "retime_eval: paired with FFFF in the same process, denoise only"
|
| 43 |
+
}
|
eval_out/per_prompt/cf_flowcache_Fxxx_x3/subject_consistency_030.json
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "complete",
|
| 3 |
+
"protocol": "Self-Forcing Extended-251 Full Evaluation",
|
| 4 |
+
"strategy": "cf_flowcache_Fxxx_x3",
|
| 5 |
+
"base_model": "causal_forcing",
|
| 6 |
+
"method": "flowcache",
|
| 7 |
+
"param": "thresh",
|
| 8 |
+
"param_value": 1.69921875,
|
| 9 |
+
"target_speedup": 3.0,
|
| 10 |
+
"schedule": "Fxxx",
|
| 11 |
+
"global_index": 287,
|
| 12 |
+
"prompt_suite": "subject_consistency",
|
| 13 |
+
"suite_index": 30,
|
| 14 |
+
"prompt": "A large semi-truck turning a corner on a busy city street during rush hour. The truck has a typical design with multiple mirrors, a distinctive grill, and large tires. It slowly navigates the turn, signaling with its blinker. The truck driver sits attentively at the wheel, looking focused. Surrounding vehicles and pedestrians move around the truck, maintaining their own paths. The environment includes tall buildings and road signs. The scene captures the truck from a mid-shot perspective, emphasizing the truck's size and the dynamics of the turn.",
|
| 15 |
+
"original_prompt": "a truck turning a corner",
|
| 16 |
+
"seed": 0,
|
| 17 |
+
"video": "/local/zoubin/cz/projects/comparison/eval_out/generated_videos/cf_flowcache_Fxxx_x3/subject_consistency/030.mp4",
|
| 18 |
+
"num_frames": 81,
|
| 19 |
+
"height": 480,
|
| 20 |
+
"width": 832,
|
| 21 |
+
"fps": 16,
|
| 22 |
+
"policy_latency_ms": 1459.85484457016,
|
| 23 |
+
"excluded_context_kv_latency_ms": 1015.201301574707,
|
| 24 |
+
"wall_generation_s": 3.6358721256256104,
|
| 25 |
+
"reference_strategy": "cf_ffff",
|
| 26 |
+
"pixel_metrics_vs_ffff": {
|
| 27 |
+
"mean_mse": 0.11195720732212067,
|
| 28 |
+
"psnr": 9.509479403495789,
|
| 29 |
+
"ssim": 0.21206964552402496,
|
| 30 |
+
"lpips": 0.6988887190818787,
|
| 31 |
+
"num_frames": 81
|
| 32 |
+
},
|
| 33 |
+
"cache_diagnostics": {
|
| 34 |
+
"denoise_forwards": 28,
|
| 35 |
+
"compute_equivalent_forwards": 8.666666666666668,
|
| 36 |
+
"middle_steps": 21,
|
| 37 |
+
"middle_compute_equivalent": 1.6666666666666665,
|
| 38 |
+
"compute_equivalent_forwards_retime": 8.666666666666668
|
| 39 |
+
},
|
| 40 |
+
"policy_latency_ms_generation_run": 1073.0273495912552,
|
| 41 |
+
"matched_ffff_policy_latency_ms": 3210.2698135375977,
|
| 42 |
+
"latency_source": "retime_eval: paired with FFFF in the same process, denoise only"
|
| 43 |
+
}
|
eval_out/per_prompt/cf_flowcache_Fxxx_x3/subject_consistency_031.json
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "complete",
|
| 3 |
+
"protocol": "Self-Forcing Extended-251 Full Evaluation",
|
| 4 |
+
"strategy": "cf_flowcache_Fxxx_x3",
|
| 5 |
+
"base_model": "causal_forcing",
|
| 6 |
+
"method": "flowcache",
|
| 7 |
+
"param": "thresh",
|
| 8 |
+
"param_value": 1.69921875,
|
| 9 |
+
"target_speedup": 3.0,
|
| 10 |
+
"schedule": "Fxxx",
|
| 11 |
+
"global_index": 288,
|
| 12 |
+
"prompt_suite": "subject_consistency",
|
| 13 |
+
"suite_index": 31,
|
| 14 |
+
"prompt": "A serene coastal bay at sunset, with gentle waves lapping against the shore. In the center of the frame, a large cargo truck is partially submerged and anchored in the calm waters, its front end resting on the sandy bottom. The truck is surrounded by clear water, with sunlight reflecting off the surface, creating a peaceful yet surreal atmosphere. The sky is painted with soft hues of orange and pink, casting a warm glow over the tranquil scene. The camera is positioned at a medium shot to capture the entirety of the truck and the surrounding bay.",
|
| 15 |
+
"original_prompt": "a truck anchored in a tranquil bay",
|
| 16 |
+
"seed": 0,
|
| 17 |
+
"video": "/local/zoubin/cz/projects/comparison/eval_out/generated_videos/cf_flowcache_Fxxx_x3/subject_consistency/031.mp4",
|
| 18 |
+
"num_frames": 81,
|
| 19 |
+
"height": 480,
|
| 20 |
+
"width": 832,
|
| 21 |
+
"fps": 16,
|
| 22 |
+
"policy_latency_ms": 1029.3065810203552,
|
| 23 |
+
"excluded_context_kv_latency_ms": 776.5553894042969,
|
| 24 |
+
"wall_generation_s": 3.5712172985076904,
|
| 25 |
+
"reference_strategy": "cf_ffff",
|
| 26 |
+
"pixel_metrics_vs_ffff": {
|
| 27 |
+
"mean_mse": 0.04059092700481415,
|
| 28 |
+
"psnr": 13.91571044921875,
|
| 29 |
+
"ssim": 0.42946991324424744,
|
| 30 |
+
"lpips": 0.6857722401618958,
|
| 31 |
+
"num_frames": 81
|
| 32 |
+
},
|
| 33 |
+
"cache_diagnostics": {
|
| 34 |
+
"denoise_forwards": 28,
|
| 35 |
+
"compute_equivalent_forwards": 8.666666666666666,
|
| 36 |
+
"middle_steps": 21,
|
| 37 |
+
"middle_compute_equivalent": 1.6666666666666665,
|
| 38 |
+
"compute_equivalent_forwards_retime": 8.666666666666666
|
| 39 |
+
},
|
| 40 |
+
"policy_latency_ms_generation_run": 1219.522245645523,
|
| 41 |
+
"matched_ffff_policy_latency_ms": 3116.1371154785156,
|
| 42 |
+
"latency_source": "retime_eval: paired with FFFF in the same process, denoise only"
|
| 43 |
+
}
|
eval_out/per_prompt/cf_flowcache_Fxxx_x3/subject_consistency_032.json
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "complete",
|
| 3 |
+
"protocol": "Self-Forcing Extended-251 Full Evaluation",
|
| 4 |
+
"strategy": "cf_flowcache_Fxxx_x3",
|
| 5 |
+
"base_model": "causal_forcing",
|
| 6 |
+
"method": "flowcache",
|
| 7 |
+
"param": "thresh",
|
| 8 |
+
"param_value": 1.69921875,
|
| 9 |
+
"target_speedup": 3.0,
|
| 10 |
+
"schedule": "Fxxx",
|
| 11 |
+
"global_index": 289,
|
| 12 |
+
"prompt_suite": "subject_consistency",
|
| 13 |
+
"suite_index": 32,
|
| 14 |
+
"prompt": "A realistic daytime scene of a large semi-truck stuck in heavy rush hour traffic on a busy highway. The truck is surrounded by other vehicles, including cars and smaller trucks, all moving slowly. The road is congested with multiple lanes of traffic, and there are tall buildings and skyscrapers visible in the background. The sun is bright, casting shadows on the road and vehicles. The truck driver is visible through the window, looking slightly frustrated but patient. The camera captures a mid-shot view of the truck and surrounding vehicles, emphasizing the slow-moving traffic and the congested environment.",
|
| 15 |
+
"original_prompt": "a truck stuck in traffic during rush hour",
|
| 16 |
+
"seed": 0,
|
| 17 |
+
"video": "/local/zoubin/cz/projects/comparison/eval_out/generated_videos/cf_flowcache_Fxxx_x3/subject_consistency/032.mp4",
|
| 18 |
+
"num_frames": 81,
|
| 19 |
+
"height": 480,
|
| 20 |
+
"width": 832,
|
| 21 |
+
"fps": 16,
|
| 22 |
+
"policy_latency_ms": 914.8163568973541,
|
| 23 |
+
"excluded_context_kv_latency_ms": 789.0164566040039,
|
| 24 |
+
"wall_generation_s": 3.4632608890533447,
|
| 25 |
+
"reference_strategy": "cf_ffff",
|
| 26 |
+
"pixel_metrics_vs_ffff": {
|
| 27 |
+
"mean_mse": 0.10832216590642929,
|
| 28 |
+
"psnr": 9.65282678604126,
|
| 29 |
+
"ssim": 0.2763538062572479,
|
| 30 |
+
"lpips": 0.7285743355751038,
|
| 31 |
+
"num_frames": 81
|
| 32 |
+
},
|
| 33 |
+
"cache_diagnostics": {
|
| 34 |
+
"denoise_forwards": 28,
|
| 35 |
+
"compute_equivalent_forwards": 8.0,
|
| 36 |
+
"middle_steps": 21,
|
| 37 |
+
"middle_compute_equivalent": 1.0,
|
| 38 |
+
"compute_equivalent_forwards_retime": 8.0
|
| 39 |
+
},
|
| 40 |
+
"policy_latency_ms_generation_run": 1266.772062778473,
|
| 41 |
+
"matched_ffff_policy_latency_ms": 3166.093017578125,
|
| 42 |
+
"latency_source": "retime_eval: paired with FFFF in the same process, denoise only"
|
| 43 |
+
}
|
eval_out/per_prompt/cf_flowcache_Fxxx_x3/subject_consistency_033.json
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "complete",
|
| 3 |
+
"protocol": "Self-Forcing Extended-251 Full Evaluation",
|
| 4 |
+
"strategy": "cf_flowcache_Fxxx_x3",
|
| 5 |
+
"base_model": "causal_forcing",
|
| 6 |
+
"method": "flowcache",
|
| 7 |
+
"param": "thresh",
|
| 8 |
+
"param_value": 1.69921875,
|
| 9 |
+
"target_speedup": 3.0,
|
| 10 |
+
"schedule": "Fxxx",
|
| 11 |
+
"global_index": 290,
|
| 12 |
+
"prompt_suite": "subject_consistency",
|
| 13 |
+
"suite_index": 33,
|
| 14 |
+
"prompt": "A large semi-truck gradually slows down as it approaches a stop sign. The truck has a classic red color with a silver undercarriage and large tires. The driver is visible through the window, maintaining control of the vehicle with a focused expression. The truck's lights and mirrors reflect the surroundings, indicating an urban street environment with other vehicles and pedestrians nearby. The scene captures the smooth deceleration of the truck, emphasizing the gradual reduction in speed until it comes to a complete stop. Wide shot, static camera, focusing on the truck and its interaction with the surrounding traffic.",
|
| 15 |
+
"original_prompt": "a truck slowing down to stop",
|
| 16 |
+
"seed": 0,
|
| 17 |
+
"video": "/local/zoubin/cz/projects/comparison/eval_out/generated_videos/cf_flowcache_Fxxx_x3/subject_consistency/033.mp4",
|
| 18 |
+
"num_frames": 81,
|
| 19 |
+
"height": 480,
|
| 20 |
+
"width": 832,
|
| 21 |
+
"fps": 16,
|
| 22 |
+
"policy_latency_ms": 1219.833822131157,
|
| 23 |
+
"excluded_context_kv_latency_ms": 776.3369064331055,
|
| 24 |
+
"wall_generation_s": 3.803065776824951,
|
| 25 |
+
"reference_strategy": "cf_ffff",
|
| 26 |
+
"pixel_metrics_vs_ffff": {
|
| 27 |
+
"mean_mse": 0.12235356867313385,
|
| 28 |
+
"psnr": 9.123833775520325,
|
| 29 |
+
"ssim": 0.24352839589118958,
|
| 30 |
+
"lpips": 0.6630001664161682,
|
| 31 |
+
"num_frames": 81
|
| 32 |
+
},
|
| 33 |
+
"cache_diagnostics": {
|
| 34 |
+
"denoise_forwards": 28,
|
| 35 |
+
"compute_equivalent_forwards": 10.666666666666668,
|
| 36 |
+
"middle_steps": 21,
|
| 37 |
+
"middle_compute_equivalent": 3.6666666666666665,
|
| 38 |
+
"compute_equivalent_forwards_retime": 10.666666666666668
|
| 39 |
+
},
|
| 40 |
+
"policy_latency_ms_generation_run": 1585.7231330871582,
|
| 41 |
+
"matched_ffff_policy_latency_ms": 3119.431816101074,
|
| 42 |
+
"latency_source": "retime_eval: paired with FFFF in the same process, denoise only"
|
| 43 |
+
}
|
eval_out/per_prompt/cf_flowcache_Fxxx_x3/subject_consistency_034.json
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "complete",
|
| 3 |
+
"protocol": "Self-Forcing Extended-251 Full Evaluation",
|
| 4 |
+
"strategy": "cf_flowcache_Fxxx_x3",
|
| 5 |
+
"base_model": "causal_forcing",
|
| 6 |
+
"method": "flowcache",
|
| 7 |
+
"param": "thresh",
|
| 8 |
+
"param_value": 1.69921875,
|
| 9 |
+
"target_speedup": 3.0,
|
| 10 |
+
"schedule": "Fxxx",
|
| 11 |
+
"global_index": 291,
|
| 12 |
+
"prompt_suite": "subject_consistency",
|
| 13 |
+
"suite_index": 34,
|
| 14 |
+
"prompt": "A large, modern semi-truck accelerating down a straight, rural highway to gain speed. The truck's engine roars as it moves forward, smoke billowing from the tires. The truck's body is painted in a sleek, metallic gray with the company logo prominently displayed on both sides. The driver is seen in the cab, focused on the road ahead, pressing the accelerator pedal. The camera follows the truck from behind, capturing the dust kicked up by the rear wheels and the surrounding landscape whizzing past. Wide shot, dynamic camera movement following the truck as it accelerates.",
|
| 15 |
+
"original_prompt": "a truck accelerating to gain speed",
|
| 16 |
+
"seed": 0,
|
| 17 |
+
"video": "/local/zoubin/cz/projects/comparison/eval_out/generated_videos/cf_flowcache_Fxxx_x3/subject_consistency/034.mp4",
|
| 18 |
+
"num_frames": 81,
|
| 19 |
+
"height": 480,
|
| 20 |
+
"width": 832,
|
| 21 |
+
"fps": 16,
|
| 22 |
+
"policy_latency_ms": 914.477846622467,
|
| 23 |
+
"excluded_context_kv_latency_ms": 789.6825637817383,
|
| 24 |
+
"wall_generation_s": 3.489763021469116,
|
| 25 |
+
"reference_strategy": "cf_ffff",
|
| 26 |
+
"pixel_metrics_vs_ffff": {
|
| 27 |
+
"mean_mse": 0.09885168820619583,
|
| 28 |
+
"psnr": 10.050158500671387,
|
| 29 |
+
"ssim": 0.3810111880302429,
|
| 30 |
+
"lpips": 0.7290009260177612,
|
| 31 |
+
"num_frames": 81
|
| 32 |
+
},
|
| 33 |
+
"cache_diagnostics": {
|
| 34 |
+
"denoise_forwards": 28,
|
| 35 |
+
"compute_equivalent_forwards": 8.0,
|
| 36 |
+
"middle_steps": 21,
|
| 37 |
+
"middle_compute_equivalent": 1.0,
|
| 38 |
+
"compute_equivalent_forwards_retime": 8.0
|
| 39 |
+
},
|
| 40 |
+
"policy_latency_ms_generation_run": 1421.4493210315704,
|
| 41 |
+
"matched_ffff_policy_latency_ms": 3157.6129608154297,
|
| 42 |
+
"latency_source": "retime_eval: paired with FFFF in the same process, denoise only"
|
| 43 |
+
}
|