Cccccz commited on
Commit
f6d1c09
·
verified ·
1 Parent(s): 5be8366

Add files using upload-large-folder tool

Browse files
Files changed (20) hide show
  1. eval_out/per_prompt/sf_taylorseer_x1.3_c0FFFF/overall_consistency_039.json +51 -0
  2. eval_out/per_prompt/sf_taylorseer_x1.3_c0FFFF/scene_085.json +51 -0
  3. eval_out/per_prompt/sf_taylorseer_x1.3_c0FFFF/subject_consistency_024.json +51 -0
  4. eval_out/per_prompt/sf_taylorseer_x1.3_c0FFFF/subject_consistency_063.json +51 -0
  5. eval_out/per_prompt/sf_taylorseer_x1.3_c0FFFF/subject_consistency_067.json +51 -0
  6. eval_out/per_prompt/sf_taylorseer_x1.3_c0FFFF/subject_consistency_068.json +51 -0
  7. eval_out/per_prompt/sf_taylorseer_x1.3_c0FFFF/subject_consistency_069.json +51 -0
  8. eval_out/per_prompt/sf_taylorseer_x1.75_c0FFFF/overall_consistency_003.json +51 -0
  9. eval_out/per_prompt/sf_taylorseer_x1.75_c0FFFF/overall_consistency_011.json +51 -0
  10. eval_out/per_prompt/sf_taylorseer_x1.75_c0FFFF/subject_consistency_000.json +51 -0
  11. eval_out/per_prompt/sf_taylorseer_x1.75_c0FFFF/subject_consistency_001.json +51 -0
  12. eval_out/per_prompt/sf_taylorseer_x1.75_c0FFFF/subject_consistency_002.json +51 -0
  13. eval_out/per_prompt/sf_taylorseer_x1.75_c0FFFF/subject_consistency_003.json +51 -0
  14. eval_out/per_prompt/sf_taylorseer_x1.75_c0FFFF/subject_consistency_007.json +51 -0
  15. eval_out/per_prompt/sf_taylorseer_x1.75_c0FFFF/subject_consistency_011.json +51 -0
  16. eval_out/per_prompt/sf_taylorseer_x1.75_c0FFFF/subject_consistency_030.json +51 -0
  17. eval_out/per_prompt/sf_taylorseer_x1.75_c0FFFF/subject_consistency_032.json +51 -0
  18. eval_out/per_prompt/sf_taylorseer_x1.75_c0FFFF/subject_consistency_033.json +51 -0
  19. eval_out/per_prompt/sf_taylorseer_x1.75_c0FFFF/subject_consistency_038.json +51 -0
  20. eval_out/per_prompt/sf_taylorseer_x1.75_c0FFFF/subject_consistency_040.json +51 -0
eval_out/per_prompt/sf_taylorseer_x1.3_c0FFFF/overall_consistency_039.json ADDED
@@ -0,0 +1,51 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "status": "complete",
3
+ "protocol": "Self-Forcing Extended-251 Full Evaluation",
4
+ "strategy": "sf_taylorseer_x1.3_c0FFFF",
5
+ "base_model": "self_forcing",
6
+ "method": "taylorseer",
7
+ "param": "interval",
8
+ "param_value": 2.0833333333333335,
9
+ "target_speedup": 1.3,
10
+ "schedule": "FxxF",
11
+ "num_inference_steps": 4,
12
+ "first_chunk_schedule": "FFFF",
13
+ "global_index": 722,
14
+ "prompt_suite": "overall_consistency",
15
+ "suite_index": 39,
16
+ "prompt": "A vintage sewing machine in operation, featuring an antique wooden cabinet and metal parts with a slight patina from age. The needle moves up and down rhythmically as the machine hums softly. A roll of fabric is fed through the machine, creating neat stitches. The background shows an old-fashioned sewing room with wooden floors, a few scattered sewing tools, and a faded floral wallpaper. The focus is on the detailed mechanical workings of the sewing machine. Medium close-up shot, static camera.",
17
+ "original_prompt": "Sewing machine, old sewing machine working.",
18
+ "seed": 0,
19
+ "video": "/local/zoubin/cz/projects/comparison/eval_out/generated_videos/sf_taylorseer_x1.3_c0FFFF/overall_consistency/039.mp4",
20
+ "num_frames": 81,
21
+ "height": 480,
22
+ "width": 832,
23
+ "fps": 16,
24
+ "policy_latency_ms": 2513.988447189331,
25
+ "excluded_context_kv_latency_ms": 785.1402282714844,
26
+ "wall_generation_s": 4.885093450546265,
27
+ "reference_strategy": "sf_ffff",
28
+ "pixel_metrics_vs_ffff": {
29
+ "mean_mse": 0.00941628310829401,
30
+ "psnr": 20.2612042427063,
31
+ "ssim": 0.8555830121040344,
32
+ "lpips": 0.05621269717812538,
33
+ "num_frames": 81
34
+ },
35
+ "cache_diagnostics": {
36
+ "denoise_forwards": 28,
37
+ "compute_equivalent_forwards": 22.0,
38
+ "middle_steps": 12,
39
+ "middle_compute_equivalent": 6.0,
40
+ "active_step_ratio": 0.0,
41
+ "empty_step_ratio": 0.5,
42
+ "full_step_ratio": 0.5,
43
+ "mean_selected_fraction_active": 0.0,
44
+ "flops_speedup_estimate": 1.2727272727272727
45
+ },
46
+ "generation_run_latency": {
47
+ "policy_latency_ms": 2495.3382148742676,
48
+ "excluded_context_kv_latency_ms": 782.0622024536133
49
+ },
50
+ "latency_retimed": "strategy-only denoise rerun on an idle GPU (FFFF not re-run)"
51
+ }
eval_out/per_prompt/sf_taylorseer_x1.3_c0FFFF/scene_085.json ADDED
@@ -0,0 +1,51 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "status": "complete",
3
+ "protocol": "Self-Forcing Extended-251 Full Evaluation",
4
+ "strategy": "sf_taylorseer_x1.3_c0FFFF",
5
+ "base_model": "self_forcing",
6
+ "method": "taylorseer",
7
+ "param": "interval",
8
+ "param_value": 2.0833333333333335,
9
+ "target_speedup": 1.3,
10
+ "schedule": "FxxF",
11
+ "num_inference_steps": 4,
12
+ "first_chunk_schedule": "FFFF",
13
+ "global_index": 861,
14
+ "prompt_suite": "scene",
15
+ "suite_index": 85,
16
+ "prompt": "A serene landscape featuring a picturesque windmill standing tall against a clear blue sky. The windmill's wooden blades slowly rotate in the gentle breeze. The structure is surrounded by rolling green hills and scattered wildflowers. In the background, a quaint village can be seen in the distance. The scene is captured in a peaceful, realistic style with soft lighting. Wide shot, static scene focusing on the windmill and its surroundings.",
17
+ "original_prompt": "windmill",
18
+ "seed": 0,
19
+ "video": "/local/zoubin/cz/projects/comparison/eval_out/generated_videos/sf_taylorseer_x1.3_c0FFFF/scene/085.mp4",
20
+ "num_frames": 81,
21
+ "height": 480,
22
+ "width": 832,
23
+ "fps": 16,
24
+ "policy_latency_ms": 2527.0332522392273,
25
+ "excluded_context_kv_latency_ms": 791.0349502563477,
26
+ "wall_generation_s": 4.920947074890137,
27
+ "reference_strategy": "sf_ffff",
28
+ "pixel_metrics_vs_ffff": {
29
+ "mean_mse": 0.0057836249470710754,
30
+ "psnr": 22.37799882888794,
31
+ "ssim": 0.8083147406578064,
32
+ "lpips": 0.07342062890529633,
33
+ "num_frames": 81
34
+ },
35
+ "cache_diagnostics": {
36
+ "denoise_forwards": 28,
37
+ "compute_equivalent_forwards": 22.0,
38
+ "middle_steps": 12,
39
+ "middle_compute_equivalent": 6.0,
40
+ "active_step_ratio": 0.0,
41
+ "empty_step_ratio": 0.5,
42
+ "full_step_ratio": 0.5,
43
+ "mean_selected_fraction_active": 0.0,
44
+ "flops_speedup_estimate": 1.2727272727272727
45
+ },
46
+ "generation_run_latency": {
47
+ "policy_latency_ms": 2520.671703338623,
48
+ "excluded_context_kv_latency_ms": 788.6137619018555
49
+ },
50
+ "latency_retimed": "strategy-only denoise rerun on an idle GPU (FFFF not re-run)"
51
+ }
eval_out/per_prompt/sf_taylorseer_x1.3_c0FFFF/subject_consistency_024.json ADDED
@@ -0,0 +1,51 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "status": "complete",
3
+ "protocol": "Self-Forcing Extended-251 Full Evaluation",
4
+ "strategy": "sf_taylorseer_x1.3_c0FFFF",
5
+ "base_model": "self_forcing",
6
+ "method": "taylorseer",
7
+ "param": "interval",
8
+ "param_value": 2.0833333333333335,
9
+ "target_speedup": 1.3,
10
+ "schedule": "FxxF",
11
+ "num_inference_steps": 4,
12
+ "first_chunk_schedule": "FFFF",
13
+ "global_index": 281,
14
+ "prompt_suite": "subject_consistency",
15
+ "suite_index": 24,
16
+ "prompt": "A city bus making a turn at a street corner. The bus is painted in bright yellow with large rectangular windows and a door at the front. It has the typical urban bus design with multiple wheels and a high chassis. As the bus navigates the corner, it smoothly turns right, revealing a glimpse of the bustling cityscape beyond, including other vehicles and pedestrians. The camera captures the scene from a medium distance, focusing on the bus as it completes the turn. The environment is vibrant with a mix of modern buildings and green spaces. Static shot.",
17
+ "original_prompt": "a bus turning a corner",
18
+ "seed": 0,
19
+ "video": "/local/zoubin/cz/projects/comparison/eval_out/generated_videos/sf_taylorseer_x1.3_c0FFFF/subject_consistency/024.mp4",
20
+ "num_frames": 81,
21
+ "height": 480,
22
+ "width": 832,
23
+ "fps": 16,
24
+ "policy_latency_ms": 2517.12047290802,
25
+ "excluded_context_kv_latency_ms": 787.6044692993164,
26
+ "wall_generation_s": 11.521724462509155,
27
+ "reference_strategy": "sf_ffff",
28
+ "pixel_metrics_vs_ffff": {
29
+ "mean_mse": 0.03489350900053978,
30
+ "psnr": 14.572553634643555,
31
+ "ssim": 0.5643095970153809,
32
+ "lpips": 0.2656008303165436,
33
+ "num_frames": 81
34
+ },
35
+ "cache_diagnostics": {
36
+ "denoise_forwards": 28,
37
+ "compute_equivalent_forwards": 22.0,
38
+ "middle_steps": 12,
39
+ "middle_compute_equivalent": 6.0,
40
+ "active_step_ratio": 0.0,
41
+ "empty_step_ratio": 0.5,
42
+ "full_step_ratio": 0.5,
43
+ "mean_selected_fraction_active": 0.0,
44
+ "flops_speedup_estimate": 1.2727272727272727
45
+ },
46
+ "generation_run_latency": {
47
+ "policy_latency_ms": 6098.015002250671,
48
+ "excluded_context_kv_latency_ms": 1947.539535522461
49
+ },
50
+ "latency_retimed": "strategy-only denoise rerun on an idle GPU (FFFF not re-run)"
51
+ }
eval_out/per_prompt/sf_taylorseer_x1.3_c0FFFF/subject_consistency_063.json ADDED
@@ -0,0 +1,51 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "status": "complete",
3
+ "protocol": "Self-Forcing Extended-251 Full Evaluation",
4
+ "strategy": "sf_taylorseer_x1.3_c0FFFF",
5
+ "base_model": "self_forcing",
6
+ "method": "taylorseer",
7
+ "param": "interval",
8
+ "param_value": 2.0833333333333335,
9
+ "target_speedup": 1.3,
10
+ "schedule": "FxxF",
11
+ "num_inference_steps": 4,
12
+ "first_chunk_schedule": "FFFF",
13
+ "global_index": 320,
14
+ "prompt_suite": "subject_consistency",
15
+ "suite_index": 63,
16
+ "prompt": "A realistic, close-up view of a large brown bear standing on all fours in a dense forest. The bear's fur is shaggy and brown, with patches of lighter color around its muzzle. It sniffs the air intently, its nostrils flaring as it searches for scents of food. Its ears are perked up, and its body language conveys a sense of alertness and curiosity. The background shows tall trees and lush greenery, adding to the immersive forest atmosphere. The camera remains static, capturing the bear's focused and natural behavior.",
17
+ "original_prompt": "a bear sniffing the air for scents of food",
18
+ "seed": 0,
19
+ "video": "/local/zoubin/cz/projects/comparison/eval_out/generated_videos/sf_taylorseer_x1.3_c0FFFF/subject_consistency/063.mp4",
20
+ "num_frames": 81,
21
+ "height": 480,
22
+ "width": 832,
23
+ "fps": 16,
24
+ "policy_latency_ms": 2516.3756895065308,
25
+ "excluded_context_kv_latency_ms": 787.7337417602539,
26
+ "wall_generation_s": 4.952801942825317,
27
+ "reference_strategy": "sf_ffff",
28
+ "pixel_metrics_vs_ffff": {
29
+ "mean_mse": 0.017908887937664986,
30
+ "psnr": 17.469314336776733,
31
+ "ssim": 0.6026286482810974,
32
+ "lpips": 0.07898098975419998,
33
+ "num_frames": 81
34
+ },
35
+ "cache_diagnostics": {
36
+ "denoise_forwards": 28,
37
+ "compute_equivalent_forwards": 22.0,
38
+ "middle_steps": 12,
39
+ "middle_compute_equivalent": 6.0,
40
+ "active_step_ratio": 0.0,
41
+ "empty_step_ratio": 0.5,
42
+ "full_step_ratio": 0.5,
43
+ "mean_selected_fraction_active": 0.0,
44
+ "flops_speedup_estimate": 1.2727272727272727
45
+ },
46
+ "generation_run_latency": {
47
+ "policy_latency_ms": 2544.755612850189,
48
+ "excluded_context_kv_latency_ms": 794.0722808837891
49
+ },
50
+ "latency_retimed": "strategy-only denoise rerun on an idle GPU (FFFF not re-run)"
51
+ }
eval_out/per_prompt/sf_taylorseer_x1.3_c0FFFF/subject_consistency_067.json ADDED
@@ -0,0 +1,51 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "status": "complete",
3
+ "protocol": "Self-Forcing Extended-251 Full Evaluation",
4
+ "strategy": "sf_taylorseer_x1.3_c0FFFF",
5
+ "base_model": "self_forcing",
6
+ "method": "taylorseer",
7
+ "param": "interval",
8
+ "param_value": 2.0833333333333335,
9
+ "target_speedup": 1.3,
10
+ "schedule": "FxxF",
11
+ "num_inference_steps": 4,
12
+ "first_chunk_schedule": "FFFF",
13
+ "global_index": 324,
14
+ "prompt_suite": "subject_consistency",
15
+ "suite_index": 67,
16
+ "prompt": "A zebra sprinting towards a herd of zebras in a vast African savanna. The zebra is mid-stride, with its legs extended as it accelerates towards the group. It has a sleek black and white striped coat, and its mane flows freely behind it. The herd is scattered across the grasslands, grazing peacefully, with a few zebras lifting their heads to watch the newcomer. The sun is setting, casting a warm golden glow over the landscape. The background includes tall grasses and distant acacia trees. The scene is captured from a low-angle perspective to emphasize the zebra's speed and the expansive environment.",
17
+ "original_prompt": "a zebra running to join a herd of its kind",
18
+ "seed": 0,
19
+ "video": "/local/zoubin/cz/projects/comparison/eval_out/generated_videos/sf_taylorseer_x1.3_c0FFFF/subject_consistency/067.mp4",
20
+ "num_frames": 81,
21
+ "height": 480,
22
+ "width": 832,
23
+ "fps": 16,
24
+ "policy_latency_ms": 2518.7997436523438,
25
+ "excluded_context_kv_latency_ms": 788.1541213989258,
26
+ "wall_generation_s": 4.926305055618286,
27
+ "reference_strategy": "sf_ffff",
28
+ "pixel_metrics_vs_ffff": {
29
+ "mean_mse": 0.010949764400720596,
30
+ "psnr": 19.605952501296997,
31
+ "ssim": 0.6932971477508545,
32
+ "lpips": 0.18801888823509216,
33
+ "num_frames": 81
34
+ },
35
+ "cache_diagnostics": {
36
+ "denoise_forwards": 28,
37
+ "compute_equivalent_forwards": 22.0,
38
+ "middle_steps": 12,
39
+ "middle_compute_equivalent": 6.0,
40
+ "active_step_ratio": 0.0,
41
+ "empty_step_ratio": 0.5,
42
+ "full_step_ratio": 0.5,
43
+ "mean_selected_fraction_active": 0.0,
44
+ "flops_speedup_estimate": 1.2727272727272727
45
+ },
46
+ "generation_run_latency": {
47
+ "policy_latency_ms": 2521.385200023651,
48
+ "excluded_context_kv_latency_ms": 793.2896347045898
49
+ },
50
+ "latency_retimed": "strategy-only denoise rerun on an idle GPU (FFFF not re-run)"
51
+ }
eval_out/per_prompt/sf_taylorseer_x1.3_c0FFFF/subject_consistency_068.json ADDED
@@ -0,0 +1,51 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "status": "complete",
3
+ "protocol": "Self-Forcing Extended-251 Full Evaluation",
4
+ "strategy": "sf_taylorseer_x1.3_c0FFFF",
5
+ "base_model": "self_forcing",
6
+ "method": "taylorseer",
7
+ "param": "interval",
8
+ "param_value": 2.0833333333333335,
9
+ "target_speedup": 1.3,
10
+ "schedule": "FxxF",
11
+ "num_inference_steps": 4,
12
+ "first_chunk_schedule": "FFFF",
13
+ "global_index": 325,
14
+ "prompt_suite": "subject_consistency",
15
+ "suite_index": 68,
16
+ "prompt": "A serene African savanna landscape featuring a single zebra calmly walking through tall grass. The zebra has distinctive black and white stripes, with a gentle, relaxed expression as it moves gracefully. The environment is lush and vibrant, with scattered acacia trees and distant rolling hills visible in the background. The sun casts a warm glow over the scene, creating soft shadows and highlighting the zebra's fluid movement. Wide shot, capturing the expansive savanna and the zebra's peaceful stroll.",
17
+ "original_prompt": "a zebra taking a peaceful walk",
18
+ "seed": 0,
19
+ "video": "/local/zoubin/cz/projects/comparison/eval_out/generated_videos/sf_taylorseer_x1.3_c0FFFF/subject_consistency/068.mp4",
20
+ "num_frames": 81,
21
+ "height": 480,
22
+ "width": 832,
23
+ "fps": 16,
24
+ "policy_latency_ms": 2517.3480277061462,
25
+ "excluded_context_kv_latency_ms": 788.0578918457031,
26
+ "wall_generation_s": 4.932737588882446,
27
+ "reference_strategy": "sf_ffff",
28
+ "pixel_metrics_vs_ffff": {
29
+ "mean_mse": 0.018971959128975868,
30
+ "psnr": 17.218878269195557,
31
+ "ssim": 0.580052375793457,
32
+ "lpips": 0.1521267592906952,
33
+ "num_frames": 81
34
+ },
35
+ "cache_diagnostics": {
36
+ "denoise_forwards": 28,
37
+ "compute_equivalent_forwards": 22.0,
38
+ "middle_steps": 12,
39
+ "middle_compute_equivalent": 6.0,
40
+ "active_step_ratio": 0.0,
41
+ "empty_step_ratio": 0.5,
42
+ "full_step_ratio": 0.5,
43
+ "mean_selected_fraction_active": 0.0,
44
+ "flops_speedup_estimate": 1.2727272727272727
45
+ },
46
+ "generation_run_latency": {
47
+ "policy_latency_ms": 2526.090066432953,
48
+ "excluded_context_kv_latency_ms": 792.1304321289062
49
+ },
50
+ "latency_retimed": "strategy-only denoise rerun on an idle GPU (FFFF not re-run)"
51
+ }
eval_out/per_prompt/sf_taylorseer_x1.3_c0FFFF/subject_consistency_069.json ADDED
@@ -0,0 +1,51 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "status": "complete",
3
+ "protocol": "Self-Forcing Extended-251 Full Evaluation",
4
+ "strategy": "sf_taylorseer_x1.3_c0FFFF",
5
+ "base_model": "self_forcing",
6
+ "method": "taylorseer",
7
+ "param": "interval",
8
+ "param_value": 2.0833333333333335,
9
+ "target_speedup": 1.3,
10
+ "schedule": "FxxF",
11
+ "num_inference_steps": 4,
12
+ "first_chunk_schedule": "FFFF",
13
+ "global_index": 326,
14
+ "prompt_suite": "subject_consistency",
15
+ "suite_index": 69,
16
+ "prompt": "A serene African savanna scene with tall grasses and scattered trees. A tall giraffe, with distinctive brown spots on its creamy white coat, bends its long neck gracefully to drink from a calm river. The giraffe's gaze is focused intently on the water as it lowers its head, revealing its long eyelashes and gentle expression. The river reflects the golden hues of the late afternoon sun, casting a warm glow over the scene. The background shows the vast expanse of the savanna with distant hills. The video is a medium close-up, capturing the giraffe's elegant movement and the tranquil environment.",
17
+ "original_prompt": "a giraffe bending down to drink water from a river",
18
+ "seed": 0,
19
+ "video": "/local/zoubin/cz/projects/comparison/eval_out/generated_videos/sf_taylorseer_x1.3_c0FFFF/subject_consistency/069.mp4",
20
+ "num_frames": 81,
21
+ "height": 480,
22
+ "width": 832,
23
+ "fps": 16,
24
+ "policy_latency_ms": 2518.801233291626,
25
+ "excluded_context_kv_latency_ms": 788.4330749511719,
26
+ "wall_generation_s": 4.91252589225769,
27
+ "reference_strategy": "sf_ffff",
28
+ "pixel_metrics_vs_ffff": {
29
+ "mean_mse": 0.019169025123119354,
30
+ "psnr": 17.173999547958374,
31
+ "ssim": 0.5805978775024414,
32
+ "lpips": 0.1380726397037506,
33
+ "num_frames": 81
34
+ },
35
+ "cache_diagnostics": {
36
+ "denoise_forwards": 28,
37
+ "compute_equivalent_forwards": 22.0,
38
+ "middle_steps": 12,
39
+ "middle_compute_equivalent": 6.0,
40
+ "active_step_ratio": 0.0,
41
+ "empty_step_ratio": 0.5,
42
+ "full_step_ratio": 0.5,
43
+ "mean_selected_fraction_active": 0.0,
44
+ "flops_speedup_estimate": 1.2727272727272727
45
+ },
46
+ "generation_run_latency": {
47
+ "policy_latency_ms": 2513.4681458473206,
48
+ "excluded_context_kv_latency_ms": 785.2755813598633
49
+ },
50
+ "latency_retimed": "strategy-only denoise rerun on an idle GPU (FFFF not re-run)"
51
+ }
eval_out/per_prompt/sf_taylorseer_x1.75_c0FFFF/overall_consistency_003.json ADDED
@@ -0,0 +1,51 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "status": "complete",
3
+ "protocol": "Self-Forcing Extended-251 Full Evaluation",
4
+ "strategy": "sf_taylorseer_x1.75_c0FFFF",
5
+ "base_model": "self_forcing",
6
+ "method": "taylorseer",
7
+ "param": "interval",
8
+ "param_value": 2.75,
9
+ "target_speedup": 1.75,
10
+ "schedule": "FxxF",
11
+ "num_inference_steps": 4,
12
+ "first_chunk_schedule": "FFFF",
13
+ "global_index": 686,
14
+ "prompt_suite": "overall_consistency",
15
+ "suite_index": 3,
16
+ "prompt": "A large, black-and-white panda stands confidently on a surfboard in the golden hour of sunset over the ocean. The panda has a playful expression, with its front paws resting lightly on the board. The waves gently roll around the surfboard, and the sky is painted with warm hues of orange and pink. The water is calm, reflecting the vibrant colors of the setting sun. The scene captures the serene beauty of the ocean at twilight. Medium shot focusing on the panda and the surfboard, with the horizon and sunset visible in the background.",
17
+ "original_prompt": "A panda standing on a surfboard in the ocean in sunset.",
18
+ "seed": 0,
19
+ "video": "/local/zoubin/cz/projects/comparison/eval_out/generated_videos/sf_taylorseer_x1.75_c0FFFF/overall_consistency/003.mp4",
20
+ "num_frames": 81,
21
+ "height": 480,
22
+ "width": 832,
23
+ "fps": 16,
24
+ "policy_latency_ms": 1935.5102429389954,
25
+ "excluded_context_kv_latency_ms": 789.9933624267578,
26
+ "wall_generation_s": 4.307509899139404,
27
+ "reference_strategy": "sf_ffff",
28
+ "pixel_metrics_vs_ffff": {
29
+ "mean_mse": 0.014303394593298435,
30
+ "psnr": 18.445608615875244,
31
+ "ssim": 0.7397565245628357,
32
+ "lpips": 0.17297254502773285,
33
+ "num_frames": 81
34
+ },
35
+ "cache_diagnostics": {
36
+ "denoise_forwards": 28,
37
+ "compute_equivalent_forwards": 17.0,
38
+ "middle_steps": 12,
39
+ "middle_compute_equivalent": 1.0,
40
+ "active_step_ratio": 0.0,
41
+ "empty_step_ratio": 0.9166666666666666,
42
+ "full_step_ratio": 0.08333333333333333,
43
+ "mean_selected_fraction_active": 0.0,
44
+ "flops_speedup_estimate": 1.6470588235294117
45
+ },
46
+ "generation_run_latency": {
47
+ "policy_latency_ms": 1919.4326066970825,
48
+ "excluded_context_kv_latency_ms": 781.3470306396484
49
+ },
50
+ "latency_retimed": "strategy-only denoise rerun on an idle GPU (FFFF not re-run)"
51
+ }
eval_out/per_prompt/sf_taylorseer_x1.75_c0FFFF/overall_consistency_011.json ADDED
@@ -0,0 +1,51 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "status": "complete",
3
+ "protocol": "Self-Forcing Extended-251 Full Evaluation",
4
+ "strategy": "sf_taylorseer_x1.75_c0FFFF",
5
+ "base_model": "self_forcing",
6
+ "method": "taylorseer",
7
+ "param": "interval",
8
+ "param_value": 2.75,
9
+ "target_speedup": 1.75,
10
+ "schedule": "FxxF",
11
+ "num_inference_steps": 4,
12
+ "first_chunk_schedule": "FFFF",
13
+ "global_index": 694,
14
+ "prompt_suite": "overall_consistency",
15
+ "suite_index": 11,
16
+ "prompt": "Animated watercolor painting, soft and fluffy white clouds drifting across a bright blue sky. The clouds move gracefully, changing shape as they float, creating a serene and tranquil atmosphere. The sky is clear with gentle sunlight filtering through, casting a warm glow over the scene. The clouds are rendered with delicate brush strokes, giving them a dreamy, almost ethereal quality. Close-up view focusing on the movement and transformation of the clouds.",
17
+ "original_prompt": "An animated painting of fluffy white clouds moving in sky.",
18
+ "seed": 0,
19
+ "video": "/local/zoubin/cz/projects/comparison/eval_out/generated_videos/sf_taylorseer_x1.75_c0FFFF/overall_consistency/011.mp4",
20
+ "num_frames": 81,
21
+ "height": 480,
22
+ "width": 832,
23
+ "fps": 16,
24
+ "policy_latency_ms": 1932.539038181305,
25
+ "excluded_context_kv_latency_ms": 787.9435119628906,
26
+ "wall_generation_s": 4.293490171432495,
27
+ "reference_strategy": "sf_ffff",
28
+ "pixel_metrics_vs_ffff": {
29
+ "mean_mse": 0.014061737805604935,
30
+ "psnr": 18.519610166549683,
31
+ "ssim": 0.8379062414169312,
32
+ "lpips": 0.24970683455467224,
33
+ "num_frames": 81
34
+ },
35
+ "cache_diagnostics": {
36
+ "denoise_forwards": 28,
37
+ "compute_equivalent_forwards": 17.0,
38
+ "middle_steps": 12,
39
+ "middle_compute_equivalent": 1.0,
40
+ "active_step_ratio": 0.0,
41
+ "empty_step_ratio": 0.9166666666666666,
42
+ "full_step_ratio": 0.08333333333333333,
43
+ "mean_selected_fraction_active": 0.0,
44
+ "flops_speedup_estimate": 1.6470588235294117
45
+ },
46
+ "generation_run_latency": {
47
+ "policy_latency_ms": 1909.6084938049316,
48
+ "excluded_context_kv_latency_ms": 778.5233535766602
49
+ },
50
+ "latency_retimed": "strategy-only denoise rerun on an idle GPU (FFFF not re-run)"
51
+ }
eval_out/per_prompt/sf_taylorseer_x1.75_c0FFFF/subject_consistency_000.json ADDED
@@ -0,0 +1,51 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "status": "complete",
3
+ "protocol": "Self-Forcing Extended-251 Full Evaluation",
4
+ "strategy": "sf_taylorseer_x1.75_c0FFFF",
5
+ "base_model": "self_forcing",
6
+ "method": "taylorseer",
7
+ "param": "interval",
8
+ "param_value": 2.75,
9
+ "target_speedup": 1.75,
10
+ "schedule": "FxxF",
11
+ "num_inference_steps": 4,
12
+ "first_chunk_schedule": "FFFF",
13
+ "global_index": 257,
14
+ "prompt_suite": "subject_consistency",
15
+ "suite_index": 0,
16
+ "prompt": "A person swimming in the ocean, with waves gently crashing around them. The swimmer is fully submerged, with only their head breaking the water's surface, taking deep breaths between strokes. They are wearing a black swimsuit and have wavy, shoulder-length brown hair that floats around their face. The sun is setting, casting a warm golden glow over the water, and the sky is filled with hues of orange and pink. The ocean is vast and serene, with a few seagulls flying overhead. The scene is captured in a medium close-up, focusing on the swimmer's determined facial expressions and fluid arm movements.",
17
+ "original_prompt": "a person swimming in ocean",
18
+ "seed": 0,
19
+ "video": "/local/zoubin/cz/projects/comparison/eval_out/generated_videos/sf_taylorseer_x1.75_c0FFFF/subject_consistency/000.mp4",
20
+ "num_frames": 81,
21
+ "height": 480,
22
+ "width": 832,
23
+ "fps": 16,
24
+ "policy_latency_ms": 1934.2329759597778,
25
+ "excluded_context_kv_latency_ms": 788.8861389160156,
26
+ "wall_generation_s": 10.742273092269897,
27
+ "reference_strategy": "sf_ffff",
28
+ "pixel_metrics_vs_ffff": {
29
+ "mean_mse": 0.010830765590071678,
30
+ "psnr": 19.653408527374268,
31
+ "ssim": 0.7647764682769775,
32
+ "lpips": 0.22866012156009674,
33
+ "num_frames": 81
34
+ },
35
+ "cache_diagnostics": {
36
+ "denoise_forwards": 28,
37
+ "compute_equivalent_forwards": 17.0,
38
+ "middle_steps": 12,
39
+ "middle_compute_equivalent": 1.0,
40
+ "active_step_ratio": 0.0,
41
+ "empty_step_ratio": 0.9166666666666666,
42
+ "full_step_ratio": 0.08333333333333333,
43
+ "mean_selected_fraction_active": 0.0,
44
+ "flops_speedup_estimate": 1.6470588235294117
45
+ },
46
+ "generation_run_latency": {
47
+ "policy_latency_ms": 4432.367091655731,
48
+ "excluded_context_kv_latency_ms": 1860.0807189941406
49
+ },
50
+ "latency_retimed": "strategy-only denoise rerun on an idle GPU (FFFF not re-run)"
51
+ }
eval_out/per_prompt/sf_taylorseer_x1.75_c0FFFF/subject_consistency_001.json ADDED
@@ -0,0 +1,51 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "status": "complete",
3
+ "protocol": "Self-Forcing Extended-251 Full Evaluation",
4
+ "strategy": "sf_taylorseer_x1.75_c0FFFF",
5
+ "base_model": "self_forcing",
6
+ "method": "taylorseer",
7
+ "param": "interval",
8
+ "param_value": 2.75,
9
+ "target_speedup": 1.75,
10
+ "schedule": "FxxF",
11
+ "num_inference_steps": 4,
12
+ "first_chunk_schedule": "FFFF",
13
+ "global_index": 258,
14
+ "prompt_suite": "subject_consistency",
15
+ "suite_index": 1,
16
+ "prompt": "A professional setting where a person is giving a presentation to a room full of colleagues. The presenter stands confidently at the front of the room, holding a laser pointer and gesturing towards a large projection screen displaying slides. They wear a well-fitted suit with a neat tie, exuding professionalism. The audience sits attentively in rows of chairs, some taking notes, others engaged in thoughtful expressions. The room has a modern feel with clean lines, ample lighting, and a professional atmosphere. The camera captures the scene from a medium shot, focusing on the presenter and the audience reaction.",
17
+ "original_prompt": "a person giving a presentation to a room full of colleagues",
18
+ "seed": 0,
19
+ "video": "/local/zoubin/cz/projects/comparison/eval_out/generated_videos/sf_taylorseer_x1.75_c0FFFF/subject_consistency/001.mp4",
20
+ "num_frames": 81,
21
+ "height": 480,
22
+ "width": 832,
23
+ "fps": 16,
24
+ "policy_latency_ms": 1932.2271304130554,
25
+ "excluded_context_kv_latency_ms": 788.8802490234375,
26
+ "wall_generation_s": 12.003735303878784,
27
+ "reference_strategy": "sf_ffff",
28
+ "pixel_metrics_vs_ffff": {
29
+ "mean_mse": 0.012887143529951572,
30
+ "psnr": 18.898433446884155,
31
+ "ssim": 0.7303454875946045,
32
+ "lpips": 0.13901956379413605,
33
+ "num_frames": 81
34
+ },
35
+ "cache_diagnostics": {
36
+ "denoise_forwards": 28,
37
+ "compute_equivalent_forwards": 17.0,
38
+ "middle_steps": 12,
39
+ "middle_compute_equivalent": 1.0,
40
+ "active_step_ratio": 0.0,
41
+ "empty_step_ratio": 0.9166666666666666,
42
+ "full_step_ratio": 0.08333333333333333,
43
+ "mean_selected_fraction_active": 0.0,
44
+ "flops_speedup_estimate": 1.6470588235294117
45
+ },
46
+ "generation_run_latency": {
47
+ "policy_latency_ms": 4850.324035167694,
48
+ "excluded_context_kv_latency_ms": 2031.6282958984375
49
+ },
50
+ "latency_retimed": "strategy-only denoise rerun on an idle GPU (FFFF not re-run)"
51
+ }
eval_out/per_prompt/sf_taylorseer_x1.75_c0FFFF/subject_consistency_002.json ADDED
@@ -0,0 +1,51 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "status": "complete",
3
+ "protocol": "Self-Forcing Extended-251 Full Evaluation",
4
+ "strategy": "sf_taylorseer_x1.75_c0FFFF",
5
+ "base_model": "self_forcing",
6
+ "method": "taylorseer",
7
+ "param": "interval",
8
+ "param_value": 2.75,
9
+ "target_speedup": 1.75,
10
+ "schedule": "FxxF",
11
+ "num_inference_steps": 4,
12
+ "first_chunk_schedule": "FFFF",
13
+ "global_index": 259,
14
+ "prompt_suite": "subject_consistency",
15
+ "suite_index": 2,
16
+ "prompt": "A person in a cozy kitchen setting is washing dishes at a sink filled with sudsy water. The individual, who appears to be middle-aged with casual attire, is scrubbing a plate with a sponge. Water droplets splash and cling to the dishes as they are washed. The kitchen is warmly lit, with sunlight streaming in from a window nearby. The background shows other kitchen appliances and cabinets. The person is standing with a focused expression, moving the sponge in circular motions across the dish. Medium close-up shot, static scene.",
17
+ "original_prompt": "a person washing the dishes",
18
+ "seed": 0,
19
+ "video": "/local/zoubin/cz/projects/comparison/eval_out/generated_videos/sf_taylorseer_x1.75_c0FFFF/subject_consistency/002.mp4",
20
+ "num_frames": 81,
21
+ "height": 480,
22
+ "width": 832,
23
+ "fps": 16,
24
+ "policy_latency_ms": 1930.2893710136414,
25
+ "excluded_context_kv_latency_ms": 787.4390411376953,
26
+ "wall_generation_s": 12.394057750701904,
27
+ "reference_strategy": "sf_ffff",
28
+ "pixel_metrics_vs_ffff": {
29
+ "mean_mse": 0.008774010464549065,
30
+ "psnr": 20.568017959594727,
31
+ "ssim": 0.7685304284095764,
32
+ "lpips": 0.15209968388080597,
33
+ "num_frames": 81
34
+ },
35
+ "cache_diagnostics": {
36
+ "denoise_forwards": 28,
37
+ "compute_equivalent_forwards": 17.0,
38
+ "middle_steps": 12,
39
+ "middle_compute_equivalent": 1.0,
40
+ "active_step_ratio": 0.0,
41
+ "empty_step_ratio": 0.9166666666666666,
42
+ "full_step_ratio": 0.08333333333333333,
43
+ "mean_selected_fraction_active": 0.0,
44
+ "flops_speedup_estimate": 1.6470588235294117
45
+ },
46
+ "generation_run_latency": {
47
+ "policy_latency_ms": 5208.213967323303,
48
+ "excluded_context_kv_latency_ms": 2131.9473266601562
49
+ },
50
+ "latency_retimed": "strategy-only denoise rerun on an idle GPU (FFFF not re-run)"
51
+ }
eval_out/per_prompt/sf_taylorseer_x1.75_c0FFFF/subject_consistency_003.json ADDED
@@ -0,0 +1,51 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "status": "complete",
3
+ "protocol": "Self-Forcing Extended-251 Full Evaluation",
4
+ "strategy": "sf_taylorseer_x1.75_c0FFFF",
5
+ "base_model": "self_forcing",
6
+ "method": "taylorseer",
7
+ "param": "interval",
8
+ "param_value": 2.75,
9
+ "target_speedup": 1.75,
10
+ "schedule": "FxxF",
11
+ "num_inference_steps": 4,
12
+ "first_chunk_schedule": "FFFF",
13
+ "global_index": 260,
14
+ "prompt_suite": "subject_consistency",
15
+ "suite_index": 3,
16
+ "prompt": "A person enjoying a juicy burger, with a satisfied smile on their face. They are seated at a casual dining table, surrounded by napkins and a drink. The burger is topped with lettuce, tomato, and cheese, and the person is taking a bite, showcasing the delicious layers inside. The scene has a warm, inviting atmosphere with soft lighting and a cozy background. Medium close-up shot focusing on the person's hand holding the burger and their facial expressions as they savor each bite.",
17
+ "original_prompt": "a person eating a burger",
18
+ "seed": 0,
19
+ "video": "/local/zoubin/cz/projects/comparison/eval_out/generated_videos/sf_taylorseer_x1.75_c0FFFF/subject_consistency/003.mp4",
20
+ "num_frames": 81,
21
+ "height": 480,
22
+ "width": 832,
23
+ "fps": 16,
24
+ "policy_latency_ms": 1933.4926114082336,
25
+ "excluded_context_kv_latency_ms": 788.2716445922852,
26
+ "wall_generation_s": 4.379638910293579,
27
+ "reference_strategy": "sf_ffff",
28
+ "pixel_metrics_vs_ffff": {
29
+ "mean_mse": 0.012730286456644535,
30
+ "psnr": 18.951618671417236,
31
+ "ssim": 0.7470136880874634,
32
+ "lpips": 0.13545936346054077,
33
+ "num_frames": 81
34
+ },
35
+ "cache_diagnostics": {
36
+ "denoise_forwards": 28,
37
+ "compute_equivalent_forwards": 17.0,
38
+ "middle_steps": 12,
39
+ "middle_compute_equivalent": 1.0,
40
+ "active_step_ratio": 0.0,
41
+ "empty_step_ratio": 0.9166666666666666,
42
+ "full_step_ratio": 0.08333333333333333,
43
+ "mean_selected_fraction_active": 0.0,
44
+ "flops_speedup_estimate": 1.6470588235294117
45
+ },
46
+ "generation_run_latency": {
47
+ "policy_latency_ms": 1963.0230612754822,
48
+ "excluded_context_kv_latency_ms": 797.9525375366211
49
+ },
50
+ "latency_retimed": "strategy-only denoise rerun on an idle GPU (FFFF not re-run)"
51
+ }
eval_out/per_prompt/sf_taylorseer_x1.75_c0FFFF/subject_consistency_007.json ADDED
@@ -0,0 +1,51 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "status": "complete",
3
+ "protocol": "Self-Forcing Extended-251 Full Evaluation",
4
+ "strategy": "sf_taylorseer_x1.75_c0FFFF",
5
+ "base_model": "self_forcing",
6
+ "method": "taylorseer",
7
+ "param": "interval",
8
+ "param_value": 2.75,
9
+ "target_speedup": 1.75,
10
+ "schedule": "FxxF",
11
+ "num_inference_steps": 4,
12
+ "first_chunk_schedule": "FFFF",
13
+ "global_index": 264,
14
+ "prompt_suite": "subject_consistency",
15
+ "suite_index": 7,
16
+ "prompt": "A vintage black bicycle leaning gently against a large, gnarled oak tree in a serene forest clearing. The bicycle has a classic frame with a round handlebar and a single rear mudguard. Its tires are slightly deflated, giving it a slightly neglected yet charming appearance. The tree has lush green leaves and several low branches that gracefully arch over the bike. The forest floor is covered with soft, brown autumn leaves and patches of moss. The scene is bathed in warm, natural sunlight filtering through the canopy, casting dappled shadows. Medium shot capturing the bicycle and the lower part of the tree trunk.",
17
+ "original_prompt": "a bicycle leaning against a tree",
18
+ "seed": 0,
19
+ "video": "/local/zoubin/cz/projects/comparison/eval_out/generated_videos/sf_taylorseer_x1.75_c0FFFF/subject_consistency/007.mp4",
20
+ "num_frames": 81,
21
+ "height": 480,
22
+ "width": 832,
23
+ "fps": 16,
24
+ "policy_latency_ms": 1926.1989979743958,
25
+ "excluded_context_kv_latency_ms": 786.2388229370117,
26
+ "wall_generation_s": 4.355149030685425,
27
+ "reference_strategy": "sf_ffff",
28
+ "pixel_metrics_vs_ffff": {
29
+ "mean_mse": 0.03095785155892372,
30
+ "psnr": 15.092291831970215,
31
+ "ssim": 0.3703557848930359,
32
+ "lpips": 0.1282046139240265,
33
+ "num_frames": 81
34
+ },
35
+ "cache_diagnostics": {
36
+ "denoise_forwards": 28,
37
+ "compute_equivalent_forwards": 17.0,
38
+ "middle_steps": 12,
39
+ "middle_compute_equivalent": 1.0,
40
+ "active_step_ratio": 0.0,
41
+ "empty_step_ratio": 0.9166666666666666,
42
+ "full_step_ratio": 0.08333333333333333,
43
+ "mean_selected_fraction_active": 0.0,
44
+ "flops_speedup_estimate": 1.6470588235294117
45
+ },
46
+ "generation_run_latency": {
47
+ "policy_latency_ms": 1943.4774327278137,
48
+ "excluded_context_kv_latency_ms": 793.9109497070312
49
+ },
50
+ "latency_retimed": "strategy-only denoise rerun on an idle GPU (FFFF not re-run)"
51
+ }
eval_out/per_prompt/sf_taylorseer_x1.75_c0FFFF/subject_consistency_011.json ADDED
@@ -0,0 +1,51 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "status": "complete",
3
+ "protocol": "Self-Forcing Extended-251 Full Evaluation",
4
+ "strategy": "sf_taylorseer_x1.75_c0FFFF",
5
+ "base_model": "self_forcing",
6
+ "method": "taylorseer",
7
+ "param": "interval",
8
+ "param_value": 2.75,
9
+ "target_speedup": 1.75,
10
+ "schedule": "FxxF",
11
+ "num_inference_steps": 4,
12
+ "first_chunk_schedule": "FFFF",
13
+ "global_index": 268,
14
+ "prompt_suite": "subject_consistency",
15
+ "suite_index": 11,
16
+ "prompt": "A realistic depiction of a car stuck in heavy traffic during rush hour. The scene shows multiple cars lined up on a busy urban highway, each vehicle emitting exhaust fumes. The driver in the front car, a middle-aged man with a stressed expression, is leaning forward in his seat, peering ahead through the windshield. The sun is setting, casting a warm golden glow over the scene. The background features tall buildings and billboards. The camera focuses on the car from a medium shot, capturing the interior and exterior details, emphasizing the stationary nature of the vehicles.",
17
+ "original_prompt": "a car stuck in traffic during rush hour",
18
+ "seed": 0,
19
+ "video": "/local/zoubin/cz/projects/comparison/eval_out/generated_videos/sf_taylorseer_x1.75_c0FFFF/subject_consistency/011.mp4",
20
+ "num_frames": 81,
21
+ "height": 480,
22
+ "width": 832,
23
+ "fps": 16,
24
+ "policy_latency_ms": 1932.8375010490417,
25
+ "excluded_context_kv_latency_ms": 788.4101104736328,
26
+ "wall_generation_s": 4.374089956283569,
27
+ "reference_strategy": "sf_ffff",
28
+ "pixel_metrics_vs_ffff": {
29
+ "mean_mse": 0.011499532498419285,
30
+ "psnr": 19.393198490142822,
31
+ "ssim": 0.6780709028244019,
32
+ "lpips": 0.1714649498462677,
33
+ "num_frames": 81
34
+ },
35
+ "cache_diagnostics": {
36
+ "denoise_forwards": 28,
37
+ "compute_equivalent_forwards": 17.0,
38
+ "middle_steps": 12,
39
+ "middle_compute_equivalent": 1.0,
40
+ "active_step_ratio": 0.0,
41
+ "empty_step_ratio": 0.9166666666666666,
42
+ "full_step_ratio": 0.08333333333333333,
43
+ "mean_selected_fraction_active": 0.0,
44
+ "flops_speedup_estimate": 1.6470588235294117
45
+ },
46
+ "generation_run_latency": {
47
+ "policy_latency_ms": 1954.0681409835815,
48
+ "excluded_context_kv_latency_ms": 802.4025115966797
49
+ },
50
+ "latency_retimed": "strategy-only denoise rerun on an idle GPU (FFFF not re-run)"
51
+ }
eval_out/per_prompt/sf_taylorseer_x1.75_c0FFFF/subject_consistency_030.json ADDED
@@ -0,0 +1,51 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "status": "complete",
3
+ "protocol": "Self-Forcing Extended-251 Full Evaluation",
4
+ "strategy": "sf_taylorseer_x1.75_c0FFFF",
5
+ "base_model": "self_forcing",
6
+ "method": "taylorseer",
7
+ "param": "interval",
8
+ "param_value": 2.75,
9
+ "target_speedup": 1.75,
10
+ "schedule": "FxxF",
11
+ "num_inference_steps": 4,
12
+ "first_chunk_schedule": "FFFF",
13
+ "global_index": 287,
14
+ "prompt_suite": "subject_consistency",
15
+ "suite_index": 30,
16
+ "prompt": "A large semi-truck turning a corner on a busy city street during rush hour. The truck has a typical design with multiple mirrors, a distinctive grill, and large tires. It slowly navigates the turn, signaling with its blinker. The truck driver sits attentively at the wheel, looking focused. Surrounding vehicles and pedestrians move around the truck, maintaining their own paths. The environment includes tall buildings and road signs. The scene captures the truck from a mid-shot perspective, emphasizing the truck's size and the dynamics of the turn.",
17
+ "original_prompt": "a truck turning a corner",
18
+ "seed": 0,
19
+ "video": "/local/zoubin/cz/projects/comparison/eval_out/generated_videos/sf_taylorseer_x1.75_c0FFFF/subject_consistency/030.mp4",
20
+ "num_frames": 81,
21
+ "height": 480,
22
+ "width": 832,
23
+ "fps": 16,
24
+ "policy_latency_ms": 1933.1017107963562,
25
+ "excluded_context_kv_latency_ms": 788.1639251708984,
26
+ "wall_generation_s": 4.329467535018921,
27
+ "reference_strategy": "sf_ffff",
28
+ "pixel_metrics_vs_ffff": {
29
+ "mean_mse": 0.03205769881606102,
30
+ "psnr": 14.94067668914795,
31
+ "ssim": 0.5364270806312561,
32
+ "lpips": 0.3494844436645508,
33
+ "num_frames": 81
34
+ },
35
+ "cache_diagnostics": {
36
+ "denoise_forwards": 28,
37
+ "compute_equivalent_forwards": 17.0,
38
+ "middle_steps": 12,
39
+ "middle_compute_equivalent": 1.0,
40
+ "active_step_ratio": 0.0,
41
+ "empty_step_ratio": 0.9166666666666666,
42
+ "full_step_ratio": 0.08333333333333333,
43
+ "mean_selected_fraction_active": 0.0,
44
+ "flops_speedup_estimate": 1.6470588235294117
45
+ },
46
+ "generation_run_latency": {
47
+ "policy_latency_ms": 1930.3374614715576,
48
+ "excluded_context_kv_latency_ms": 786.8281707763672
49
+ },
50
+ "latency_retimed": "strategy-only denoise rerun on an idle GPU (FFFF not re-run)"
51
+ }
eval_out/per_prompt/sf_taylorseer_x1.75_c0FFFF/subject_consistency_032.json ADDED
@@ -0,0 +1,51 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "status": "complete",
3
+ "protocol": "Self-Forcing Extended-251 Full Evaluation",
4
+ "strategy": "sf_taylorseer_x1.75_c0FFFF",
5
+ "base_model": "self_forcing",
6
+ "method": "taylorseer",
7
+ "param": "interval",
8
+ "param_value": 2.75,
9
+ "target_speedup": 1.75,
10
+ "schedule": "FxxF",
11
+ "num_inference_steps": 4,
12
+ "first_chunk_schedule": "FFFF",
13
+ "global_index": 289,
14
+ "prompt_suite": "subject_consistency",
15
+ "suite_index": 32,
16
+ "prompt": "A realistic daytime scene of a large semi-truck stuck in heavy rush hour traffic on a busy highway. The truck is surrounded by other vehicles, including cars and smaller trucks, all moving slowly. The road is congested with multiple lanes of traffic, and there are tall buildings and skyscrapers visible in the background. The sun is bright, casting shadows on the road and vehicles. The truck driver is visible through the window, looking slightly frustrated but patient. The camera captures a mid-shot view of the truck and surrounding vehicles, emphasizing the slow-moving traffic and the congested environment.",
17
+ "original_prompt": "a truck stuck in traffic during rush hour",
18
+ "seed": 0,
19
+ "video": "/local/zoubin/cz/projects/comparison/eval_out/generated_videos/sf_taylorseer_x1.75_c0FFFF/subject_consistency/032.mp4",
20
+ "num_frames": 81,
21
+ "height": 480,
22
+ "width": 832,
23
+ "fps": 16,
24
+ "policy_latency_ms": 1930.8554768562317,
25
+ "excluded_context_kv_latency_ms": 787.7389221191406,
26
+ "wall_generation_s": 4.339115619659424,
27
+ "reference_strategy": "sf_ffff",
28
+ "pixel_metrics_vs_ffff": {
29
+ "mean_mse": 0.025700945407152176,
30
+ "psnr": 15.900509357452393,
31
+ "ssim": 0.5630548000335693,
32
+ "lpips": 0.21798592805862427,
33
+ "num_frames": 81
34
+ },
35
+ "cache_diagnostics": {
36
+ "denoise_forwards": 28,
37
+ "compute_equivalent_forwards": 17.0,
38
+ "middle_steps": 12,
39
+ "middle_compute_equivalent": 1.0,
40
+ "active_step_ratio": 0.0,
41
+ "empty_step_ratio": 0.9166666666666666,
42
+ "full_step_ratio": 0.08333333333333333,
43
+ "mean_selected_fraction_active": 0.0,
44
+ "flops_speedup_estimate": 1.6470588235294117
45
+ },
46
+ "generation_run_latency": {
47
+ "policy_latency_ms": 1934.064393043518,
48
+ "excluded_context_kv_latency_ms": 788.0738220214844
49
+ },
50
+ "latency_retimed": "strategy-only denoise rerun on an idle GPU (FFFF not re-run)"
51
+ }
eval_out/per_prompt/sf_taylorseer_x1.75_c0FFFF/subject_consistency_033.json ADDED
@@ -0,0 +1,51 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "status": "complete",
3
+ "protocol": "Self-Forcing Extended-251 Full Evaluation",
4
+ "strategy": "sf_taylorseer_x1.75_c0FFFF",
5
+ "base_model": "self_forcing",
6
+ "method": "taylorseer",
7
+ "param": "interval",
8
+ "param_value": 2.75,
9
+ "target_speedup": 1.75,
10
+ "schedule": "FxxF",
11
+ "num_inference_steps": 4,
12
+ "first_chunk_schedule": "FFFF",
13
+ "global_index": 290,
14
+ "prompt_suite": "subject_consistency",
15
+ "suite_index": 33,
16
+ "prompt": "A large semi-truck gradually slows down as it approaches a stop sign. The truck has a classic red color with a silver undercarriage and large tires. The driver is visible through the window, maintaining control of the vehicle with a focused expression. The truck's lights and mirrors reflect the surroundings, indicating an urban street environment with other vehicles and pedestrians nearby. The scene captures the smooth deceleration of the truck, emphasizing the gradual reduction in speed until it comes to a complete stop. Wide shot, static camera, focusing on the truck and its interaction with the surrounding traffic.",
17
+ "original_prompt": "a truck slowing down to stop",
18
+ "seed": 0,
19
+ "video": "/local/zoubin/cz/projects/comparison/eval_out/generated_videos/sf_taylorseer_x1.75_c0FFFF/subject_consistency/033.mp4",
20
+ "num_frames": 81,
21
+ "height": 480,
22
+ "width": 832,
23
+ "fps": 16,
24
+ "policy_latency_ms": 1930.9769568443298,
25
+ "excluded_context_kv_latency_ms": 788.3416976928711,
26
+ "wall_generation_s": 4.319477558135986,
27
+ "reference_strategy": "sf_ffff",
28
+ "pixel_metrics_vs_ffff": {
29
+ "mean_mse": 0.03415248170495033,
30
+ "psnr": 14.665777683258057,
31
+ "ssim": 0.4523862600326538,
32
+ "lpips": 0.3183307349681854,
33
+ "num_frames": 81
34
+ },
35
+ "cache_diagnostics": {
36
+ "denoise_forwards": 28,
37
+ "compute_equivalent_forwards": 17.0,
38
+ "middle_steps": 12,
39
+ "middle_compute_equivalent": 1.0,
40
+ "active_step_ratio": 0.0,
41
+ "empty_step_ratio": 0.9166666666666666,
42
+ "full_step_ratio": 0.08333333333333333,
43
+ "mean_selected_fraction_active": 0.0,
44
+ "flops_speedup_estimate": 1.6470588235294117
45
+ },
46
+ "generation_run_latency": {
47
+ "policy_latency_ms": 1921.8145995140076,
48
+ "excluded_context_kv_latency_ms": 785.4107360839844
49
+ },
50
+ "latency_retimed": "strategy-only denoise rerun on an idle GPU (FFFF not re-run)"
51
+ }
eval_out/per_prompt/sf_taylorseer_x1.75_c0FFFF/subject_consistency_038.json ADDED
@@ -0,0 +1,51 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "status": "complete",
3
+ "protocol": "Self-Forcing Extended-251 Full Evaluation",
4
+ "strategy": "sf_taylorseer_x1.75_c0FFFF",
5
+ "base_model": "self_forcing",
6
+ "method": "taylorseer",
7
+ "param": "interval",
8
+ "param_value": 2.75,
9
+ "target_speedup": 1.75,
10
+ "schedule": "FxxF",
11
+ "num_inference_steps": 4,
12
+ "first_chunk_schedule": "FFFF",
13
+ "global_index": 295,
14
+ "prompt_suite": "subject_consistency",
15
+ "suite_index": 38,
16
+ "prompt": "A majestic bird soaring gracefully through the clear blue sky, its wings spread wide as it glides effortlessly on the thermal currents. The bird's feathers shimmer in the sunlight, creating a striking contrast against the bright blue backdrop. The scene is captured from a mid-shot perspective, focusing on the bird's elegant flight path. The camera remains static, allowing viewers to appreciate the bird's fluid movements and the serene beauty of the sky.",
17
+ "original_prompt": "a bird soaring gracefully in the sky",
18
+ "seed": 0,
19
+ "video": "/local/zoubin/cz/projects/comparison/eval_out/generated_videos/sf_taylorseer_x1.75_c0FFFF/subject_consistency/038.mp4",
20
+ "num_frames": 81,
21
+ "height": 480,
22
+ "width": 832,
23
+ "fps": 16,
24
+ "policy_latency_ms": 1927.6942076683044,
25
+ "excluded_context_kv_latency_ms": 788.1643524169922,
26
+ "wall_generation_s": 4.3296380043029785,
27
+ "reference_strategy": "sf_ffff",
28
+ "pixel_metrics_vs_ffff": {
29
+ "mean_mse": 0.006607396993786097,
30
+ "psnr": 21.79969549179077,
31
+ "ssim": 0.9435313940048218,
32
+ "lpips": 0.24029196798801422,
33
+ "num_frames": 81
34
+ },
35
+ "cache_diagnostics": {
36
+ "denoise_forwards": 28,
37
+ "compute_equivalent_forwards": 17.0,
38
+ "middle_steps": 12,
39
+ "middle_compute_equivalent": 1.0,
40
+ "active_step_ratio": 0.0,
41
+ "empty_step_ratio": 0.9166666666666666,
42
+ "full_step_ratio": 0.08333333333333333,
43
+ "mean_selected_fraction_active": 0.0,
44
+ "flops_speedup_estimate": 1.6470588235294117
45
+ },
46
+ "generation_run_latency": {
47
+ "policy_latency_ms": 1932.0487985610962,
48
+ "excluded_context_kv_latency_ms": 786.338996887207
49
+ },
50
+ "latency_retimed": "strategy-only denoise rerun on an idle GPU (FFFF not re-run)"
51
+ }
eval_out/per_prompt/sf_taylorseer_x1.75_c0FFFF/subject_consistency_040.json ADDED
@@ -0,0 +1,51 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "status": "complete",
3
+ "protocol": "Self-Forcing Extended-251 Full Evaluation",
4
+ "strategy": "sf_taylorseer_x1.75_c0FFFF",
5
+ "base_model": "self_forcing",
6
+ "method": "taylorseer",
7
+ "param": "interval",
8
+ "param_value": 2.75,
9
+ "target_speedup": 1.75,
10
+ "schedule": "FxxF",
11
+ "num_inference_steps": 4,
12
+ "first_chunk_schedule": "FFFF",
13
+ "global_index": 297,
14
+ "prompt_suite": "subject_consistency",
15
+ "suite_index": 40,
16
+ "prompt": "A majestic eagle soaring gracefully over a vast, snowy forest. The bird has striking golden feathers with black wingtips, and it is in full flight, wings spread wide as it glides effortlessly through the crisp winter air. The forest below is covered in pristine snow, with tall pine trees reaching towards the sky. The sun casts a soft, warm glow through the branches, creating a serene and tranquil atmosphere. The scene is captured from a medium-high angle, emphasizing the bird's flight path and the expansive landscape beneath.",
17
+ "original_prompt": "a bird flying over a snowy forest",
18
+ "seed": 0,
19
+ "video": "/local/zoubin/cz/projects/comparison/eval_out/generated_videos/sf_taylorseer_x1.75_c0FFFF/subject_consistency/040.mp4",
20
+ "num_frames": 81,
21
+ "height": 480,
22
+ "width": 832,
23
+ "fps": 16,
24
+ "policy_latency_ms": 1931.2557859420776,
25
+ "excluded_context_kv_latency_ms": 789.0855026245117,
26
+ "wall_generation_s": 4.340297698974609,
27
+ "reference_strategy": "sf_ffff",
28
+ "pixel_metrics_vs_ffff": {
29
+ "mean_mse": 0.025094788521528244,
30
+ "psnr": 16.004164218902588,
31
+ "ssim": 0.6292808651924133,
32
+ "lpips": 0.19504499435424805,
33
+ "num_frames": 81
34
+ },
35
+ "cache_diagnostics": {
36
+ "denoise_forwards": 28,
37
+ "compute_equivalent_forwards": 17.0,
38
+ "middle_steps": 12,
39
+ "middle_compute_equivalent": 1.0,
40
+ "active_step_ratio": 0.0,
41
+ "empty_step_ratio": 0.9166666666666666,
42
+ "full_step_ratio": 0.08333333333333333,
43
+ "mean_selected_fraction_active": 0.0,
44
+ "flops_speedup_estimate": 1.6470588235294117
45
+ },
46
+ "generation_run_latency": {
47
+ "policy_latency_ms": 1934.6242656707764,
48
+ "excluded_context_kv_latency_ms": 789.0890502929688
49
+ },
50
+ "latency_retimed": "strategy-only denoise rerun on an idle GPU (FFFF not re-run)"
51
+ }