diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000124_test__val_desert__2__000032/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000124_test__val_desert__2__000032/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..8712b8f0d2a95d8a7fe0c3332dec658599454655 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000124_test__val_desert__2__000032/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "124", + "global_test_index": "124", + "dataset_index": "85", + "relative_path": "test/val_desert/2/000032.mp4", + "clip_id": "test/val_desert/2/000032", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000124, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_desert/2/000032_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.0038343712221831083, + "psnr": 24.4262638092041, + "lpips": 0.1106785237789154 + }, + "frames_100_200": { + "mse": 0.008797272108495235, + "psnr": 20.727378845214844, + "lpips": 0.18920160830020905 + }, + "frames_200_300": { + "mse": 0.008551839739084244, + "psnr": 20.859403610229492, + "lpips": 0.15778397023677826 + }, + "frames_300_400": { + "mse": 0.011435375548899174, + "psnr": 19.606718063354492, + "lpips": 0.21109773218631744 + }, + "frames_400_500": { + "mse": 0.0058588432148098946, + "psnr": 23.32453155517578, + "lpips": 0.13388223946094513 + }, + "overall": { + "mse": 0.0076955403201282024, + "psnr": 21.788862228393555, + "lpips": 0.16052880883216858 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000125_test__val_iceplains__0__000044/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000125_test__val_iceplains__0__000044/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..74670091c2bb9713b6cc5f071ef5498907d44044 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000125_test__val_iceplains__0__000044/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f8a309b97fc4b1451a00268fa5b5dbffba92ecfbfecc82630fc7a4fbceaec712 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000125_test__val_iceplains__0__000044/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000125_test__val_iceplains__0__000044/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..6fe0f83ccc3fd34aa6b7bd1f01d4f5da3434a6f2 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000125_test__val_iceplains__0__000044/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "125", + "global_test_index": "125", + "dataset_index": "358", + "relative_path": "test/val_iceplains/0/000044.mp4", + "clip_id": "test/val_iceplains/0/000044", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000125, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/0/000044_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.0009331777109764516, + "psnr": 30.3817195892334, + "lpips": 0.011008551344275475 + }, + "frames_100_200": { + "mse": 0.000886870373506099, + "psnr": 30.557165145874023, + "lpips": 0.011660434305667877 + }, + "frames_200_300": { + "mse": 0.0008535321103408933, + "psnr": 30.711021423339844, + "lpips": 0.011796060018241405 + }, + "frames_300_400": { + "mse": 0.0009378630202263594, + "psnr": 30.307851791381836, + "lpips": 0.014937634579837322 + }, + "frames_400_500": { + "mse": 0.0008990838541649282, + "psnr": 30.543855667114258, + "lpips": 0.015569162555038929 + }, + "overall": { + "mse": 0.0009021054720506072, + "psnr": 30.500322341918945, + "lpips": 0.012994368560612202 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000126_test__val_desert__1__000020/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000126_test__val_desert__1__000020/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..6c1b0c9eac5ed1617a28762044ef2eef12f86a4e --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000126_test__val_desert__1__000020/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6826f6e85f9dccedbd971b42f473856a77a9a67ec1d0a1bf7319eb6f78a00271 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000126_test__val_desert__1__000020/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000126_test__val_desert__1__000020/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..287316386bf5c1f81724de38598c491fa4c3c9fa --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000126_test__val_desert__1__000020/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "126", + "global_test_index": "126", + "dataset_index": "50", + "relative_path": "test/val_desert/1/000020.mp4", + "clip_id": "test/val_desert/1/000020", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000126, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_desert/1/000020_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.001484862295910716, + "psnr": 29.569835662841797, + "lpips": 0.018319694325327873 + }, + "frames_100_200": { + "mse": 0.0016191304894164205, + "psnr": 30.244709014892578, + "lpips": 0.015003266744315624 + }, + "frames_200_300": { + "mse": 0.0008267026278190315, + "psnr": 30.85266876220703, + "lpips": 0.011609358713030815 + }, + "frames_300_400": { + "mse": 0.0020510931499302387, + "psnr": 28.709508895874023, + "lpips": 0.027202578261494637 + }, + "frames_400_500": { + "mse": 0.006186373997479677, + "psnr": 24.444780349731445, + "lpips": 0.04963857680559158 + }, + "overall": { + "mse": 0.0024336325004696846, + "psnr": 28.764301300048828, + "lpips": 0.02435469441115856 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000127_test__val_savanna__0__000036/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000127_test__val_savanna__0__000036/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..9cc35badb2f19e596c4882589679afa8e14e284f --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000127_test__val_savanna__0__000036/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fdee9e328d93265037da10e8d3e3982086817d9eb2bc17d5cfa8ccfba05c791 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000127_test__val_savanna__0__000036/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000127_test__val_savanna__0__000036/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..aaefecad83953ac6ec6c0c54c91273cbfd8efbec --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000127_test__val_savanna__0__000036/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "127", + "global_test_index": "127", + "dataset_index": "207", + "relative_path": "test/val_savanna/0/000036.mp4", + "clip_id": "test/val_savanna/0/000036", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000127, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_savanna/0/000036_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.006707283202558756, + "psnr": 22.234464645385742, + "lpips": 0.35190626978874207 + }, + "frames_100_200": { + "mse": 0.0072944206185638905, + "psnr": 21.453563690185547, + "lpips": 0.39522361755371094 + }, + "frames_200_300": { + "mse": 0.006893693003803492, + "psnr": 21.749900817871094, + "lpips": 0.3911628723144531 + }, + "frames_300_400": { + "mse": 0.016896530985832214, + "psnr": 19.2712459564209, + "lpips": 0.4398539364337921 + }, + "frames_400_500": { + "mse": 0.024989934638142586, + "psnr": 16.752212524414062, + "lpips": 0.5247270464897156 + }, + "overall": { + "mse": 0.01255637127906084, + "psnr": 20.292278289794922, + "lpips": 0.42057475447654724 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000128_test__val_iceplains__2__000034/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000128_test__val_iceplains__2__000034/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..f638564153c932e403c333e4a71c3cc6ffce418e --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000128_test__val_iceplains__2__000034/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8569d946e4b4e565e3ed244e56acb556e0cef9f006ce0d7e2c2892eec7b32977 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000128_test__val_iceplains__2__000034/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000128_test__val_iceplains__2__000034/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..dfbe8f146cc4e00a1b83436c4c789ac37fbcc214 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000128_test__val_iceplains__2__000034/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "128", + "global_test_index": "128", + "dataset_index": "330", + "relative_path": "test/val_iceplains/2/000034.mp4", + "clip_id": "test/val_iceplains/2/000034", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000128, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/2/000034_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.0009549509268254042, + "psnr": 30.322065353393555, + "lpips": 0.01559457927942276 + }, + "frames_100_200": { + "mse": 0.0011458741500973701, + "psnr": 30.461318969726562, + "lpips": 0.014609893783926964 + }, + "frames_200_300": { + "mse": 0.016090119257569313, + "psnr": 24.57937240600586, + "lpips": 0.0829785168170929 + }, + "frames_300_400": { + "mse": 0.054150525480508804, + "psnr": 13.15584945678711, + "lpips": 0.3769654333591461 + }, + "frames_400_500": { + "mse": 0.09743870049715042, + "psnr": 10.951923370361328, + "lpips": 0.4895250201225281 + }, + "overall": { + "mse": 0.03395603224635124, + "psnr": 21.894105911254883, + "lpips": 0.1959347128868103 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000129_test__val_savanna__0__000001/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000129_test__val_savanna__0__000001/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..c67bf18662c8a030dd16bbe78fd213bd5bf94727 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000129_test__val_savanna__0__000001/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:69faf235a6fcb88dd51eb9ef4e69cba4f0014ffce769971cd130de0c2c191f5f +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000129_test__val_savanna__0__000001/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000129_test__val_savanna__0__000001/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..6f822eda4008ad6ab38d385fbacac09443f7f33e --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000129_test__val_savanna__0__000001/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "129", + "global_test_index": "129", + "dataset_index": "132", + "relative_path": "test/val_savanna/0/000001.mp4", + "clip_id": "test/val_savanna/0/000001", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000129, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_savanna/0/000001_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.0014532122295349836, + "psnr": 28.748018264770508, + "lpips": 0.021940717473626137 + }, + "frames_100_200": { + "mse": 0.0019861303735524416, + "psnr": 28.03094482421875, + "lpips": 0.026773037388920784 + }, + "frames_200_300": { + "mse": 0.011648589745163918, + "psnr": 21.0721378326416, + "lpips": 0.21085555851459503 + }, + "frames_300_400": { + "mse": 0.014550996012985706, + "psnr": 18.718036651611328, + "lpips": 0.2853127717971802 + }, + "frames_400_500": { + "mse": 0.004384123254567385, + "psnr": 24.046506881713867, + "lpips": 0.06934908777475357 + }, + "overall": { + "mse": 0.0068046110682189465, + "psnr": 24.12312889099121, + "lpips": 0.12284623831510544 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000130_test__val_plains__3__000021/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000130_test__val_plains__3__000021/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..fd927726bfc3d98600d921d952cb6f500eb83dd0 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000130_test__val_plains__3__000021/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:71a676e20e92ec2a9d98f7723ed85af85745718040ae6dee84c88cf42ad9e069 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000130_test__val_plains__3__000021/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000130_test__val_plains__3__000021/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..8a8f732285fbdc064c35831f9ca6a346fc33cc92 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000130_test__val_plains__3__000021/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "130", + "global_test_index": "130", + "dataset_index": "423", + "relative_path": "test/val_plains/3/000021.mp4", + "clip_id": "test/val_plains/3/000021", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000130, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_plains/3/000021_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.00328049180097878, + "psnr": 25.106420516967773, + "lpips": 0.2332487553358078 + }, + "frames_100_200": { + "mse": 0.004358980804681778, + "psnr": 23.907419204711914, + "lpips": 0.2951502799987793 + }, + "frames_200_300": { + "mse": 0.004142299760133028, + "psnr": 23.862367630004883, + "lpips": 0.31737029552459717 + }, + "frames_300_400": { + "mse": 0.014077514410018921, + "psnr": 21.26189422607422, + "lpips": 0.36391833424568176 + }, + "frames_400_500": { + "mse": 0.047864340245723724, + "psnr": 14.42151165008545, + "lpips": 0.3694884777069092 + }, + "overall": { + "mse": 0.014744725078344345, + "psnr": 21.71192169189453, + "lpips": 0.31583523750305176 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000131_test__val_desert__2__000025/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000131_test__val_desert__2__000025/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..beb26afce5d5e96c41aa5619d00f74e6ca340127 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000131_test__val_desert__2__000025/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:167a06b9b7fb92e02450c0d5dbe39fd4d91438a0aa65c461a3bea16b2f30ae7b +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000131_test__val_desert__2__000025/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000131_test__val_desert__2__000025/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..1745fbeab78556ce38902f7f7b6e6b8e40289496 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000131_test__val_desert__2__000025/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "131", + "global_test_index": "131", + "dataset_index": "66", + "relative_path": "test/val_desert/2/000025.mp4", + "clip_id": "test/val_desert/2/000025", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000131, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_desert/2/000025_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.009860005229711533, + "psnr": 20.345754623413086, + "lpips": 0.24561290442943573 + }, + "frames_100_200": { + "mse": 0.017666202038526535, + "psnr": 17.62787437438965, + "lpips": 0.2884877026081085 + }, + "frames_200_300": { + "mse": 0.01360994577407837, + "psnr": 18.91973876953125, + "lpips": 0.32690712809562683 + }, + "frames_300_400": { + "mse": 0.01751602254807949, + "psnr": 17.899829864501953, + "lpips": 0.3800671696662903 + }, + "frames_400_500": { + "mse": 0.015232359059154987, + "psnr": 18.476234436035156, + "lpips": 0.3669703006744385 + }, + "overall": { + "mse": 0.014776906929910183, + "psnr": 18.653884887695312, + "lpips": 0.3216090500354767 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000132_test__val_savanna__3__000009/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000132_test__val_savanna__3__000009/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..548a7594c69745f1ec47593472d5d7d3a3759420 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000132_test__val_savanna__3__000009/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77521499d34f2ccb8a59a3528cae6186e007cd490d74ca024c5b59de51bc2ddd +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000132_test__val_savanna__3__000009/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000132_test__val_savanna__3__000009/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..cccb0ff3848f75f2face4fcd69c35233a61479ab --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000132_test__val_savanna__3__000009/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "132", + "global_test_index": "132", + "dataset_index": "153", + "relative_path": "test/val_savanna/3/000009.mp4", + "clip_id": "test/val_savanna/3/000009", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000132, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_savanna/3/000009_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.022195372730493546, + "psnr": 18.302406311035156, + "lpips": 0.22228436172008514 + }, + "frames_100_200": { + "mse": 0.02921193093061447, + "psnr": 16.188037872314453, + "lpips": 0.22970882058143616 + }, + "frames_200_300": { + "mse": 0.016484905034303665, + "psnr": 18.224079132080078, + "lpips": 0.13724012672901154 + }, + "frames_300_400": { + "mse": 0.0289585143327713, + "psnr": 16.264480590820312, + "lpips": 0.21746228635311127 + }, + "frames_400_500": { + "mse": 0.0348900705575943, + "psnr": 14.894621849060059, + "lpips": 0.24207791686058044 + }, + "overall": { + "mse": 0.026348158717155457, + "psnr": 16.77472496032715, + "lpips": 0.20975470542907715 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000133_test__val_plains__0__000004/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000133_test__val_plains__0__000004/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..f9133ff3f826ee02768e145ac2d81e9e5ce2bebb --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000133_test__val_plains__0__000004/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed5273a841f9a78a1e776a1cd11a015294d9426b22d2c8700173dc58e3138e53 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000133_test__val_plains__0__000004/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000133_test__val_plains__0__000004/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..70b630d3c5d82092850206f6cea1694c72fa0f99 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000133_test__val_plains__0__000004/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "133", + "global_test_index": "133", + "dataset_index": "382", + "relative_path": "test/val_plains/0/000004.mp4", + "clip_id": "test/val_plains/0/000004", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000133, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_plains/0/000004_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.0058406745083630085, + "psnr": 22.48453140258789, + "lpips": 0.2485457956790924 + }, + "frames_100_200": { + "mse": 0.0039766086265444756, + "psnr": 24.035066604614258, + "lpips": 0.3241359293460846 + }, + "frames_200_300": { + "mse": 0.004747073166072369, + "psnr": 23.44042205810547, + "lpips": 0.3394511938095093 + }, + "frames_300_400": { + "mse": 0.006929060444235802, + "psnr": 21.747617721557617, + "lpips": 0.4062713384628296 + }, + "frames_400_500": { + "mse": 0.0046330345794558525, + "psnr": 23.454448699951172, + "lpips": 0.365752637386322 + }, + "overall": { + "mse": 0.0052252900786697865, + "psnr": 23.03241729736328, + "lpips": 0.33683139085769653 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000134_test__val_savanna__2__000005/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000134_test__val_savanna__2__000005/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..6a731cf257778d088d99060bfedf38cd2f754dac --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000134_test__val_savanna__2__000005/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9ddb27134e23718ba5dbf1e8c27b279dc56e29a7c582a097add0cad9f279234 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000134_test__val_savanna__2__000005/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000134_test__val_savanna__2__000005/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..d4f92a889d0fa89127a12bc1722264c470eca617 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000134_test__val_savanna__2__000005/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "134", + "global_test_index": "134", + "dataset_index": "143", + "relative_path": "test/val_savanna/2/000005.mp4", + "clip_id": "test/val_savanna/2/000005", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000134, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_savanna/2/000005_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.014485334046185017, + "psnr": 18.5859375, + "lpips": 0.3390156626701355 + }, + "frames_100_200": { + "mse": 0.028073078021407127, + "psnr": 16.182510375976562, + "lpips": 0.41970911622047424 + }, + "frames_200_300": { + "mse": 0.012548437342047691, + "psnr": 19.95612144470215, + "lpips": 0.4414370656013489 + }, + "frames_300_400": { + "mse": 0.026289591565728188, + "psnr": 16.48810577392578, + "lpips": 0.4970936179161072 + }, + "frames_400_500": { + "mse": 0.06847763806581497, + "psnr": 12.170432090759277, + "lpips": 0.547330915927887 + }, + "overall": { + "mse": 0.029974816367030144, + "psnr": 16.676620483398438, + "lpips": 0.4489172697067261 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000135_test__val_iceplains__1__000040/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000135_test__val_iceplains__1__000040/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..251c0914749fba2e42025b67e7c721c76a102768 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000135_test__val_iceplains__1__000040/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a2680df7858b3096a2890248ab32c69f8025ea90fa036d14334fabc1b0ec3725 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000135_test__val_iceplains__1__000040/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000135_test__val_iceplains__1__000040/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..97f196237d8d092a89d561ef739fe894a90bce6f --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000135_test__val_iceplains__1__000040/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "135", + "global_test_index": "135", + "dataset_index": "348", + "relative_path": "test/val_iceplains/1/000040.mp4", + "clip_id": "test/val_iceplains/1/000040", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000135, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/1/000040_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.021408552303910255, + "psnr": 17.150901794433594, + "lpips": 0.20184822380542755 + }, + "frames_100_200": { + "mse": 0.019069790840148926, + "psnr": 17.861360549926758, + "lpips": 0.18908923864364624 + }, + "frames_200_300": { + "mse": 0.02559494972229004, + "psnr": 16.50379180908203, + "lpips": 0.2058320939540863 + }, + "frames_300_400": { + "mse": 0.022684333845973015, + "psnr": 17.9940242767334, + "lpips": 0.26573479175567627 + }, + "frames_400_500": { + "mse": 0.020618490874767303, + "psnr": 17.067407608032227, + "lpips": 0.16639107465744019 + }, + "overall": { + "mse": 0.021875223144888878, + "psnr": 17.31549835205078, + "lpips": 0.20577910542488098 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000136_test__val_plains__2__000005/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000136_test__val_plains__2__000005/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..d7d7a1dedf8e1e0d6dcb9e79527b9fc106eaaab7 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000136_test__val_plains__2__000005/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8083260fcb35c577f1aa1472b6ff3e39f8f6f1814b73fb9bd10f65a2e14d01eb +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000136_test__val_plains__2__000005/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000136_test__val_plains__2__000005/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..244a51430a098a479bbb7be2169aaeeea2d63c50 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000136_test__val_plains__2__000005/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "136", + "global_test_index": "136", + "dataset_index": "384", + "relative_path": "test/val_plains/2/000005.mp4", + "clip_id": "test/val_plains/2/000005", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000136, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_plains/2/000005_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.004267084412276745, + "psnr": 23.781980514526367, + "lpips": 0.3505207896232605 + }, + "frames_100_200": { + "mse": 0.003726803930476308, + "psnr": 24.324541091918945, + "lpips": 0.36018940806388855 + }, + "frames_200_300": { + "mse": 0.06266765296459198, + "psnr": 14.357873916625977, + "lpips": 0.45021724700927734 + }, + "frames_300_400": { + "mse": 0.06701135635375977, + "psnr": 12.021452903747559, + "lpips": 0.38467979431152344 + }, + "frames_400_500": { + "mse": 0.0655532106757164, + "psnr": 11.978496551513672, + "lpips": 0.13128697872161865 + }, + "overall": { + "mse": 0.04064522683620453, + "psnr": 17.29286766052246, + "lpips": 0.33537882566452026 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000137_test__val_desert__0__000012/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000137_test__val_desert__0__000012/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..570f7d8f9281a6cbfd6d7c97e9ca23acfec72efd --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000137_test__val_desert__0__000012/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3bff6aca606d89cd8d1dced7f87dc91cb48682d93457c57bc43f07e1ccdd9752 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000137_test__val_desert__0__000012/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000137_test__val_desert__0__000012/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..666275de2fa02179816f717f29bdb8994b9296cf --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000137_test__val_desert__0__000012/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "137", + "global_test_index": "137", + "dataset_index": "30", + "relative_path": "test/val_desert/0/000012.mp4", + "clip_id": "test/val_desert/0/000012", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000137, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_desert/0/000012_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.0007693073130212724, + "psnr": 31.15523910522461, + "lpips": 0.009894480928778648 + }, + "frames_100_200": { + "mse": 0.0007668848847970366, + "psnr": 31.1663875579834, + "lpips": 0.010341694578528404 + }, + "frames_200_300": { + "mse": 0.0012762021506205201, + "psnr": 29.499380111694336, + "lpips": 0.022373294457793236 + }, + "frames_300_400": { + "mse": 0.0011693239212036133, + "psnr": 30.59243392944336, + "lpips": 0.013964185491204262 + }, + "frames_400_500": { + "mse": 0.0008222288452088833, + "psnr": 30.890939712524414, + "lpips": 0.011983408592641354 + }, + "overall": { + "mse": 0.0009607894462533295, + "psnr": 30.66087532043457, + "lpips": 0.013711412437260151 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000138_test__val_savanna__2__000034/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000138_test__val_savanna__2__000034/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..d15c38cea02f778f283c454718fdbe2db1bff1a9 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000138_test__val_savanna__2__000034/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b95615973f714b899d0e3ab551925051eee910e5ea9cbd73b06538e6015ff970 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000138_test__val_savanna__2__000034/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000138_test__val_savanna__2__000034/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..4b96a52c5072556155386a76b9843d9f9c4ae26b --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000138_test__val_savanna__2__000034/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "138", + "global_test_index": "138", + "dataset_index": "203", + "relative_path": "test/val_savanna/2/000034.mp4", + "clip_id": "test/val_savanna/2/000034", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000138, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_savanna/2/000034_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.0019806530326604843, + "psnr": 27.982154846191406, + "lpips": 0.019615111872553825 + }, + "frames_100_200": { + "mse": 0.0014540873235091567, + "psnr": 29.562135696411133, + "lpips": 0.013446008786559105 + }, + "frames_200_300": { + "mse": 0.0020869376603513956, + "psnr": 28.869915008544922, + "lpips": 0.018795687705278397 + }, + "frames_300_400": { + "mse": 0.003144179005175829, + "psnr": 27.136390686035156, + "lpips": 0.02872839942574501 + }, + "frames_400_500": { + "mse": 0.010119260288774967, + "psnr": 23.284276962280273, + "lpips": 0.05465582758188248 + }, + "overall": { + "mse": 0.0037570232525467873, + "psnr": 27.366973876953125, + "lpips": 0.027048209682106972 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000139_test__val_savanna__0__000022/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000139_test__val_savanna__0__000022/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..7cc3e0b9a64fdaecfb8e49e4b62d06d625f8ef08 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000139_test__val_savanna__0__000022/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c52531befb8c9ccd99079b3149f2c120e57340e813da97b82860f01332e2095 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000139_test__val_savanna__0__000022/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000139_test__val_savanna__0__000022/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..b4554c2818fc446c1a46165b95d5e5a83fde77be --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000139_test__val_savanna__0__000022/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "139", + "global_test_index": "139", + "dataset_index": "179", + "relative_path": "test/val_savanna/0/000022.mp4", + "clip_id": "test/val_savanna/0/000022", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000139, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_savanna/0/000022_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.0013348344946280122, + "psnr": 28.83977699279785, + "lpips": 0.13142722845077515 + }, + "frames_100_200": { + "mse": 0.002187905367463827, + "psnr": 26.7196044921875, + "lpips": 0.14662164449691772 + }, + "frames_200_300": { + "mse": 0.005070777144283056, + "psnr": 23.106117248535156, + "lpips": 0.12211448699235916 + }, + "frames_300_400": { + "mse": 0.007739753928035498, + "psnr": 21.19198989868164, + "lpips": 0.11173173040151596 + }, + "frames_400_500": { + "mse": 0.008362587541341782, + "psnr": 20.861509323120117, + "lpips": 0.12095114588737488 + }, + "overall": { + "mse": 0.00493917241692543, + "psnr": 24.143798828125, + "lpips": 0.1265692412853241 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000140_test__val_iceplains__2__000003/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000140_test__val_iceplains__2__000003/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..33759952dadbbe7a5bec8be98d471319cee072a2 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000140_test__val_iceplains__2__000003/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6783169bf6064121234c12670e0b0870421d905b33d2674a3a6c34c7c6e1be2d +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000140_test__val_iceplains__2__000003/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000140_test__val_iceplains__2__000003/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..63fc9948382255a14fa0d789ead7da1a6acc441e --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000140_test__val_iceplains__2__000003/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "140", + "global_test_index": "140", + "dataset_index": "253", + "relative_path": "test/val_iceplains/2/000003.mp4", + "clip_id": "test/val_iceplains/2/000003", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000140, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/2/000003_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.008791802451014519, + "psnr": 21.091079711914062, + "lpips": 0.16836988925933838 + }, + "frames_100_200": { + "mse": 0.024468516930937767, + "psnr": 16.746850967407227, + "lpips": 0.2349162995815277 + }, + "frames_200_300": { + "mse": 0.07874393463134766, + "psnr": 11.419788360595703, + "lpips": 0.48818641901016235 + }, + "frames_300_400": { + "mse": 0.04718368873000145, + "psnr": 14.619848251342773, + "lpips": 0.3406621813774109 + }, + "frames_400_500": { + "mse": 0.04312962666153908, + "psnr": 14.540613174438477, + "lpips": 0.37893441319465637 + }, + "overall": { + "mse": 0.040463514626026154, + "psnr": 15.683636665344238, + "lpips": 0.3222138583660126 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000141_test__val_iceplains__2__000016/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000141_test__val_iceplains__2__000016/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..4057c701e14aeacc9e1521be355c0423a6a9f37b --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000141_test__val_iceplains__2__000016/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:450fe56fc077c2a14687b5e6c6cfe2c3f2835c92d5cde394a1c57958c910d804 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000141_test__val_iceplains__2__000016/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000141_test__val_iceplains__2__000016/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..13c8735fc88861d6f040d6ad482fb2e33bcbafcb --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000141_test__val_iceplains__2__000016/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "141", + "global_test_index": "141", + "dataset_index": "289", + "relative_path": "test/val_iceplains/2/000016.mp4", + "clip_id": "test/val_iceplains/2/000016", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000141, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/2/000016_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.0007456709281541407, + "psnr": 31.282577514648438, + "lpips": 0.009861068800091743 + }, + "frames_100_200": { + "mse": 0.004965067841112614, + "psnr": 27.782520294189453, + "lpips": 0.040066178888082504 + }, + "frames_200_300": { + "mse": 0.003117287065833807, + "psnr": 26.664409637451172, + "lpips": 0.03511936590075493 + }, + "frames_300_400": { + "mse": 0.002085950458422303, + "psnr": 27.268447875976562, + "lpips": 0.02791164070367813 + }, + "frames_400_500": { + "mse": 0.00514613464474678, + "psnr": 24.119783401489258, + "lpips": 0.060876477509737015 + }, + "overall": { + "mse": 0.0032120223622769117, + "psnr": 27.42354965209961, + "lpips": 0.03476694971323013 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000142_test__val_plains__2__000020/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000142_test__val_plains__2__000020/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..12dbe412b5ec2209c9e16a7724f12e12cbd3a976 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000142_test__val_plains__2__000020/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:90517f0283146d055bf1d91e5d082954ae0c9426acbadb6f8ea33a4beb26f496 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000142_test__val_plains__2__000020/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000142_test__val_plains__2__000020/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..a9f52c4e0f82d149e15ef3637f3e0ebcdbe91097 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000142_test__val_plains__2__000020/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "142", + "global_test_index": "142", + "dataset_index": "422", + "relative_path": "test/val_plains/2/000020.mp4", + "clip_id": "test/val_plains/2/000020", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000142, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_plains/2/000020_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.004075692035257816, + "psnr": 24.302370071411133, + "lpips": 0.05757622420787811 + }, + "frames_100_200": { + "mse": 0.008997473865747452, + "psnr": 20.75374984741211, + "lpips": 0.22283805906772614 + }, + "frames_200_300": { + "mse": 0.007085732650011778, + "psnr": 22.535398483276367, + "lpips": 0.15993793308734894 + }, + "frames_300_400": { + "mse": 0.0048640938475728035, + "psnr": 23.634279251098633, + "lpips": 0.09337618947029114 + }, + "frames_400_500": { + "mse": 0.004075458738952875, + "psnr": 25.187114715576172, + "lpips": 0.05961039662361145 + }, + "overall": { + "mse": 0.005819690879434347, + "psnr": 23.282583236694336, + "lpips": 0.11866774410009384 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000143_test__val_desert__0__000008/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000143_test__val_desert__0__000008/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..c7defcb55e9e7b3886755374df23eac139d56dc0 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000143_test__val_desert__0__000008/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f529641f2d466855a6942680d721821c54491640031a8ee659fcfe756c484665 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000143_test__val_desert__0__000008/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000143_test__val_desert__0__000008/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..90d54551cfe98a1b9d4c2993b0d0c10946df2efa --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000143_test__val_desert__0__000008/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "143", + "global_test_index": "143", + "dataset_index": "20", + "relative_path": "test/val_desert/0/000008.mp4", + "clip_id": "test/val_desert/0/000008", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000143, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_desert/0/000008_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.0020590005442500114, + "psnr": 29.03612518310547, + "lpips": 0.030739037320017815 + }, + "frames_100_200": { + "mse": 0.00984747800976038, + "psnr": 20.688880920410156, + "lpips": 0.17833548784255981 + }, + "frames_200_300": { + "mse": 0.009776131249964237, + "psnr": 20.306804656982422, + "lpips": 0.2711389660835266 + }, + "frames_300_400": { + "mse": 0.011499784886837006, + "psnr": 19.532052993774414, + "lpips": 0.17814262211322784 + }, + "frames_400_500": { + "mse": 0.0026700920425355434, + "psnr": 28.21206283569336, + "lpips": 0.02954581193625927 + }, + "overall": { + "mse": 0.00717049790546298, + "psnr": 23.555185317993164, + "lpips": 0.13758039474487305 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000144_test__val_savanna__0__000000/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000144_test__val_savanna__0__000000/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..0608b6f3c2875f521f67548272ee6a90a712e495 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000144_test__val_savanna__0__000000/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e5c5d13766bf22f2f2970711e67ad72ee951b42d2e7012037e5ebc93b1d795fa +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000144_test__val_savanna__0__000000/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000144_test__val_savanna__0__000000/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..0bde0705e3f414f4f7199f9f76bea552caaba88c --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000144_test__val_savanna__0__000000/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "144", + "global_test_index": "144", + "dataset_index": "130", + "relative_path": "test/val_savanna/0/000000.mp4", + "clip_id": "test/val_savanna/0/000000", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000144, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_savanna/0/000000_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.011638644151389599, + "psnr": 19.769290924072266, + "lpips": 0.16591542959213257 + }, + "frames_100_200": { + "mse": 0.019459480419754982, + "psnr": 17.360668182373047, + "lpips": 0.1535501331090927 + }, + "frames_200_300": { + "mse": 0.018692554906010628, + "psnr": 18.75762176513672, + "lpips": 0.13838034868240356 + }, + "frames_300_400": { + "mse": 0.031162414699792862, + "psnr": 16.123083114624023, + "lpips": 0.17314361035823822 + }, + "frames_400_500": { + "mse": 0.045554712414741516, + "psnr": 13.491724014282227, + "lpips": 0.23765979707241058 + }, + "overall": { + "mse": 0.025301562622189522, + "psnr": 17.10047721862793, + "lpips": 0.17372985184192657 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000145_test__val_iceplains__1__000020/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000145_test__val_iceplains__1__000020/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..994b44439e70e603979a79707268d00681b2bd6e --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000145_test__val_iceplains__1__000020/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6820643762011af5b6a06a11eea8b1365a5c25b611cc99e84cb14c24928a1b61 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000145_test__val_iceplains__1__000020/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000145_test__val_iceplains__1__000020/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..adc89c101c0326025b8364ffca651415ff3112a8 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000145_test__val_iceplains__1__000020/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "145", + "global_test_index": "145", + "dataset_index": "296", + "relative_path": "test/val_iceplains/1/000020.mp4", + "clip_id": "test/val_iceplains/1/000020", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000145, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/1/000020_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.016957759857177734, + "psnr": 17.81981658935547, + "lpips": 0.18485485017299652 + }, + "frames_100_200": { + "mse": 0.019970489665865898, + "psnr": 17.228933334350586, + "lpips": 0.1870379000902176 + }, + "frames_200_300": { + "mse": 0.03969457000494003, + "psnr": 14.1196928024292, + "lpips": 0.29053473472595215 + }, + "frames_300_400": { + "mse": 0.0758398100733757, + "psnr": 11.391704559326172, + "lpips": 0.43325960636138916 + }, + "frames_400_500": { + "mse": 0.07288557291030884, + "psnr": 11.589159965515137, + "lpips": 0.3946675956249237 + }, + "overall": { + "mse": 0.04506963863968849, + "psnr": 14.429861068725586, + "lpips": 0.2980709671974182 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000146_test__val_iceplains__3__000013/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000146_test__val_iceplains__3__000013/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..2b304dd24a1d3a77122db564b20769feaeb453cc --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000146_test__val_iceplains__3__000013/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8de28dc9d9c824d249456abdc0dec47326b992e095aef05b5552f9ce5a55b0e6 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000146_test__val_iceplains__3__000013/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000146_test__val_iceplains__3__000013/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..4598f1f99b0024cd8a7e5f46e9f0ad60a76484d8 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000146_test__val_iceplains__3__000013/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "146", + "global_test_index": "146", + "dataset_index": "278", + "relative_path": "test/val_iceplains/3/000013.mp4", + "clip_id": "test/val_iceplains/3/000013", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000146, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/3/000013_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.014167653396725655, + "psnr": 20.767545700073242, + "lpips": 0.08324377238750458 + }, + "frames_100_200": { + "mse": 0.04269548878073692, + "psnr": 14.481380462646484, + "lpips": 0.3006751537322998 + }, + "frames_200_300": { + "mse": 0.09141496568918228, + "psnr": 10.570194244384766, + "lpips": 0.4685143530368805 + }, + "frames_300_400": { + "mse": 0.03654433414340019, + "psnr": 22.187768936157227, + "lpips": 0.2141980677843094 + }, + "frames_400_500": { + "mse": 0.004716005176305771, + "psnr": 25.456411361694336, + "lpips": 0.0629841685295105 + }, + "overall": { + "mse": 0.037907689809799194, + "psnr": 18.692659378051758, + "lpips": 0.2259231060743332 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000147_test__val_iceplains__0__000009/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000147_test__val_iceplains__0__000009/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..c0769876382d990662e0285d1ab41d7144335174 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000147_test__val_iceplains__0__000009/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a18ade880dcca3c327c548507b42238f9374de186f684193d09cb3392f8daf0b +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000147_test__val_iceplains__0__000009/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000147_test__val_iceplains__0__000009/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..eb0f48897d303b6a8253bc3973673bbac83e3e13 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000147_test__val_iceplains__0__000009/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "147", + "global_test_index": "147", + "dataset_index": "269", + "relative_path": "test/val_iceplains/0/000009.mp4", + "clip_id": "test/val_iceplains/0/000009", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000147, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/0/000009_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.007628425490111113, + "psnr": 21.40215492248535, + "lpips": 0.09683271497488022 + }, + "frames_100_200": { + "mse": 0.01973353885114193, + "psnr": 17.855030059814453, + "lpips": 0.19308441877365112 + }, + "frames_200_300": { + "mse": 0.014538093470036983, + "psnr": 19.753524780273438, + "lpips": 0.10630655288696289 + }, + "frames_300_400": { + "mse": 0.008386390283703804, + "psnr": 22.098949432373047, + "lpips": 0.059912487864494324 + }, + "frames_400_500": { + "mse": 0.006873622070997953, + "psnr": 22.614782333374023, + "lpips": 0.06482533365488052 + }, + "overall": { + "mse": 0.011432014405727386, + "psnr": 20.744888305664062, + "lpips": 0.10419230908155441 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000148_test__val_iceplains__0__000028/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000148_test__val_iceplains__0__000028/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..552d67e7d4f726b07165b8a4d46be58380f3b179 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000148_test__val_iceplains__0__000028/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b4e6d1b176a838199f3c95424da1e63bc848416fc7235a6f2bec963243de652 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000148_test__val_iceplains__0__000028/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000148_test__val_iceplains__0__000028/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..bdceff8e9065be929b7ece85e64e7275a27cba02 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000148_test__val_iceplains__0__000028/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "148", + "global_test_index": "148", + "dataset_index": "314", + "relative_path": "test/val_iceplains/0/000028.mp4", + "clip_id": "test/val_iceplains/0/000028", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000148, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/0/000028_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.0007975163171067834, + "psnr": 31.004505157470703, + "lpips": 0.010168538428843021 + }, + "frames_100_200": { + "mse": 0.0007841441547498107, + "psnr": 31.065732955932617, + "lpips": 0.009760734625160694 + }, + "frames_200_300": { + "mse": 0.0022047495003789663, + "psnr": 29.552980422973633, + "lpips": 0.021267982199788094 + }, + "frames_300_400": { + "mse": 0.037726834416389465, + "psnr": 14.477999687194824, + "lpips": 0.2727294862270355 + }, + "frames_400_500": { + "mse": 0.056888509541749954, + "psnr": 12.888333320617676, + "lpips": 0.3455289602279663 + }, + "overall": { + "mse": 0.01968034729361534, + "psnr": 23.797908782958984, + "lpips": 0.131891131401062 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000149_test__val_iceplains__0__000014/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000149_test__val_iceplains__0__000014/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..3ec1e18be560e4ad845c9f7b4660162e70068c35 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000149_test__val_iceplains__0__000014/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:48c5742d19eb37a96a6590ed2aacd1777f70d5c589003b3edca7ba362e7716ca +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000149_test__val_iceplains__0__000014/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000149_test__val_iceplains__0__000014/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..d81baea9a089a6175aa0216954cb94db583efdbf --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000149_test__val_iceplains__0__000014/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "149", + "global_test_index": "149", + "dataset_index": "281", + "relative_path": "test/val_iceplains/0/000014.mp4", + "clip_id": "test/val_iceplains/0/000014", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000149, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/0/000014_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.023883752524852753, + "psnr": 17.522186279296875, + "lpips": 0.16422584652900696 + }, + "frames_100_200": { + "mse": 0.029447581619024277, + "psnr": 15.506709098815918, + "lpips": 0.21963629126548767 + }, + "frames_200_300": { + "mse": 0.03531176224350929, + "psnr": 14.745416641235352, + "lpips": 0.26565808057785034 + }, + "frames_300_400": { + "mse": 0.05872911587357521, + "psnr": 12.767934799194336, + "lpips": 0.29920199513435364 + }, + "frames_400_500": { + "mse": 0.020222291350364685, + "psnr": 18.453027725219727, + "lpips": 0.1348257213830948 + }, + "overall": { + "mse": 0.033518899232149124, + "psnr": 15.799056053161621, + "lpips": 0.21670958399772644 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000150_test__val_savanna__0__000008/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000150_test__val_savanna__0__000008/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..cfc7e657b9f8722d9996893fc489a336e4e7692f --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000150_test__val_savanna__0__000008/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7a4cf880b8ba3cc424fe11989ee53ebd76228649f4bf4c9c7fd86cdcd8c2686 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000150_test__val_savanna__0__000008/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000150_test__val_savanna__0__000008/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..8e411e285a5f8bc5c17bb3b229f0290b1176829d --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000150_test__val_savanna__0__000008/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "150", + "global_test_index": "150", + "dataset_index": "149", + "relative_path": "test/val_savanna/0/000008.mp4", + "clip_id": "test/val_savanna/0/000008", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000150, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_savanna/0/000008_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.018335938453674316, + "psnr": 17.62586212158203, + "lpips": 0.1703483909368515 + }, + "frames_100_200": { + "mse": 0.02625371143221855, + "psnr": 16.113306045532227, + "lpips": 0.1860198974609375 + }, + "frames_200_300": { + "mse": 0.025263993069529533, + "psnr": 16.484256744384766, + "lpips": 0.3680298626422882 + }, + "frames_300_400": { + "mse": 0.007058681920170784, + "psnr": 21.638063430786133, + "lpips": 0.3760094940662384 + }, + "frames_400_500": { + "mse": 0.03159111365675926, + "psnr": 15.810437202453613, + "lpips": 0.4584665596485138 + }, + "overall": { + "mse": 0.02170068956911564, + "psnr": 17.534385681152344, + "lpips": 0.3117748498916626 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000151_test__val_savanna__0__000024/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000151_test__val_savanna__0__000024/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..78fd346c9195803677bc2ff8e4fab5d4b87bd518 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000151_test__val_savanna__0__000024/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ddae9afef5f52c97c3641e33c78d6efcde2306f5d8b71c972955b1d4e3f9f25 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000151_test__val_savanna__0__000024/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000151_test__val_savanna__0__000024/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..0fc4bbfa58f64a51b5cc74c338d61a831959a715 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000151_test__val_savanna__0__000024/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "151", + "global_test_index": "151", + "dataset_index": "184", + "relative_path": "test/val_savanna/0/000024.mp4", + "clip_id": "test/val_savanna/0/000024", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000151, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_savanna/0/000024_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.016466675326228142, + "psnr": 19.868392944335938, + "lpips": 0.2962701916694641 + }, + "frames_100_200": { + "mse": 0.028967196121811867, + "psnr": 16.891727447509766, + "lpips": 0.34376922249794006 + }, + "frames_200_300": { + "mse": 0.03453047201037407, + "psnr": 16.369192123413086, + "lpips": 0.37675419449806213 + }, + "frames_300_400": { + "mse": 0.010897750034928322, + "psnr": 20.18256187438965, + "lpips": 0.3122343420982361 + }, + "frames_400_500": { + "mse": 0.010017815046012402, + "psnr": 20.54564666748047, + "lpips": 0.3992960453033447 + }, + "overall": { + "mse": 0.020175984129309654, + "psnr": 18.771503448486328, + "lpips": 0.3456648290157318 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000152_test__val_plains__1__000014/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000152_test__val_plains__1__000014/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..8e2973f2ab605def3af17411d42e08dc50cb1a5e --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000152_test__val_plains__1__000014/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:697e2da883a5d44a3b77da9f6d0eb11a42800c5b3e83ad0e4350de6418c37762 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000152_test__val_plains__1__000014/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000152_test__val_plains__1__000014/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..94ca43ef0efb738d07e79a308ab0588e96e9115a --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000152_test__val_plains__1__000014/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "152", + "global_test_index": "152", + "dataset_index": "407", + "relative_path": "test/val_plains/1/000014.mp4", + "clip_id": "test/val_plains/1/000014", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000152, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_plains/1/000014_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.0040666996501386166, + "psnr": 25.87518310546875, + "lpips": 0.03378797695040703 + }, + "frames_100_200": { + "mse": 0.0007979627698659897, + "psnr": 30.988080978393555, + "lpips": 0.012420453131198883 + }, + "frames_200_300": { + "mse": 0.0009024773025885224, + "psnr": 30.451061248779297, + "lpips": 0.015680775046348572 + }, + "frames_300_400": { + "mse": 0.02168256789445877, + "psnr": 17.40382194519043, + "lpips": 0.22938621044158936 + }, + "frames_400_500": { + "mse": 0.028884852305054665, + "psnr": 15.737605094909668, + "lpips": 0.24534060060977936 + }, + "overall": { + "mse": 0.011266912333667278, + "psnr": 24.091150283813477, + "lpips": 0.10732319951057434 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000153_test__val_iceplains__2__000023/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000153_test__val_iceplains__2__000023/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..6abc81a07d313744eefb739c6968e9604b8d0d86 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000153_test__val_iceplains__2__000023/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed3b602f18a6e84aa8fd72017b67e71ed1124d6c01fbf8288ee08b7c091dcc6a +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000153_test__val_iceplains__2__000023/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000153_test__val_iceplains__2__000023/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..02d9bd8aac1b55af6c6ed546dd7be313c8fbf3af --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000153_test__val_iceplains__2__000023/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "153", + "global_test_index": "153", + "dataset_index": "304", + "relative_path": "test/val_iceplains/2/000023.mp4", + "clip_id": "test/val_iceplains/2/000023", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000153, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/2/000023_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.001148126320913434, + "psnr": 30.05938720703125, + "lpips": 0.013447903096675873 + }, + "frames_100_200": { + "mse": 0.006507666315883398, + "psnr": 22.879121780395508, + "lpips": 0.0612560473382473 + }, + "frames_200_300": { + "mse": 0.023379234597086906, + "psnr": 17.031513214111328, + "lpips": 0.20149727165699005 + }, + "frames_300_400": { + "mse": 0.03390571102499962, + "psnr": 15.009591102600098, + "lpips": 0.3150857985019684 + }, + "frames_400_500": { + "mse": 0.0427173376083374, + "psnr": 14.233219146728516, + "lpips": 0.42213043570518494 + }, + "overall": { + "mse": 0.021531615406274796, + "psnr": 19.842565536499023, + "lpips": 0.2026834934949875 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000154_test__val_savanna__3__000037/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000154_test__val_savanna__3__000037/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..865d65cdd4cf1d965ddd0cbb85bcfc2e52b02565 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000154_test__val_savanna__3__000037/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:92a8d03960ee453f23f8aa81a1fff7f80975bb4197d02cb8e66fa7ec4c732da7 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000154_test__val_savanna__3__000037/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000154_test__val_savanna__3__000037/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..68e465c2b8f42f0b5a7a9b37aafb918c4bf0be53 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000154_test__val_savanna__3__000037/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "154", + "global_test_index": "154", + "dataset_index": "208", + "relative_path": "test/val_savanna/3/000037.mp4", + "clip_id": "test/val_savanna/3/000037", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000154, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_savanna/3/000037_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.008436300791800022, + "psnr": 21.147096633911133, + "lpips": 0.2701997756958008 + }, + "frames_100_200": { + "mse": 0.006488245911896229, + "psnr": 22.373741149902344, + "lpips": 0.43420833349227905 + }, + "frames_200_300": { + "mse": 0.005220653954893351, + "psnr": 22.87225341796875, + "lpips": 0.40014025568962097 + }, + "frames_300_400": { + "mse": 0.006085474509745836, + "psnr": 22.242225646972656, + "lpips": 0.41002750396728516 + }, + "frames_400_500": { + "mse": 0.008271731436252594, + "psnr": 21.04086685180664, + "lpips": 0.4503702223300934 + }, + "overall": { + "mse": 0.006900481879711151, + "psnr": 21.93523597717285, + "lpips": 0.39298921823501587 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000155_test__val_savanna__3__000023/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000155_test__val_savanna__3__000023/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..ff5b6c1c643865a9fc74b54f3e092ecfd2cffcbe --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000155_test__val_savanna__3__000023/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb94b7c0287b352aaed649ebf48ce0ac8b4eb4cdf9151e96f2d51aca00d97bcf +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000155_test__val_savanna__3__000023/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000155_test__val_savanna__3__000023/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..320fb8f61c1cea9a602a04d0212c19cf4402d406 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000155_test__val_savanna__3__000023/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "155", + "global_test_index": "155", + "dataset_index": "182", + "relative_path": "test/val_savanna/3/000023.mp4", + "clip_id": "test/val_savanna/3/000023", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000155, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_savanna/3/000023_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.004601343534886837, + "psnr": 23.61968421936035, + "lpips": 0.28783154487609863 + }, + "frames_100_200": { + "mse": 0.009233054704964161, + "psnr": 20.508230209350586, + "lpips": 0.4461623430252075 + }, + "frames_200_300": { + "mse": 0.007206617388874292, + "psnr": 21.514421463012695, + "lpips": 0.4233952462673187 + }, + "frames_300_400": { + "mse": 0.005916452966630459, + "psnr": 22.381385803222656, + "lpips": 0.37659674882888794 + }, + "frames_400_500": { + "mse": 0.009055540896952152, + "psnr": 20.59933090209961, + "lpips": 0.4244733452796936 + }, + "overall": { + "mse": 0.007202601991593838, + "psnr": 21.724611282348633, + "lpips": 0.39169183373451233 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000156_test__val_plains__1__000009/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000156_test__val_plains__1__000009/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..753bbc37f8d5caf0529a4e8e6103dcdc69405381 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000156_test__val_plains__1__000009/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd1bd32df389c0104e7142077cb64e31a82d158ae0537e7e9adf5e17d36ec0f0 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000156_test__val_plains__1__000009/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000156_test__val_plains__1__000009/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..39929c2a3a5302d407e942ef5a5f52ae948514d7 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000156_test__val_plains__1__000009/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "156", + "global_test_index": "156", + "dataset_index": "394", + "relative_path": "test/val_plains/1/000009.mp4", + "clip_id": "test/val_plains/1/000009", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000156, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_plains/1/000009_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.0019135064212605357, + "psnr": 27.3510799407959, + "lpips": 0.17545129358768463 + }, + "frames_100_200": { + "mse": 0.0025787418708205223, + "psnr": 26.09893035888672, + "lpips": 0.23881995677947998 + }, + "frames_200_300": { + "mse": 0.0034431603271514177, + "psnr": 24.785184860229492, + "lpips": 0.2778231203556061 + }, + "frames_300_400": { + "mse": 0.0030521429143846035, + "psnr": 25.488601684570312, + "lpips": 0.258738249540329 + }, + "frames_400_500": { + "mse": 0.0024901889264583588, + "psnr": 26.071331024169922, + "lpips": 0.2504788935184479 + }, + "overall": { + "mse": 0.002695548115298152, + "psnr": 25.959022521972656, + "lpips": 0.24026231467723846 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000157_test__val_desert__0__000006/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000157_test__val_desert__0__000006/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..71f3db0bc2792dcb489d217d4e816e39e1dbf55a --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000157_test__val_desert__0__000006/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c74ce1b0c758736467d7d5e248db1f7721442638ee986f397ab9f70385b06ea +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000157_test__val_desert__0__000006/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000157_test__val_desert__0__000006/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..46a2b2e5cf7222b3b7c52d0a10b5f2369b5c3a66 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000157_test__val_desert__0__000006/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "157", + "global_test_index": "157", + "dataset_index": "14", + "relative_path": "test/val_desert/0/000006.mp4", + "clip_id": "test/val_desert/0/000006", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000157, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_desert/0/000006_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.0030328985303640366, + "psnr": 25.33806610107422, + "lpips": 0.06870916485786438 + }, + "frames_100_200": { + "mse": 0.0047010742127895355, + "psnr": 23.389039993286133, + "lpips": 0.10831869393587112 + }, + "frames_200_300": { + "mse": 0.004259927198290825, + "psnr": 23.79898452758789, + "lpips": 0.12782776355743408 + }, + "frames_300_400": { + "mse": 0.005339461378753185, + "psnr": 23.399124145507812, + "lpips": 0.15097340941429138 + }, + "frames_400_500": { + "mse": 0.006761496886610985, + "psnr": 21.768545150756836, + "lpips": 0.16796141862869263 + }, + "overall": { + "mse": 0.004818972200155258, + "psnr": 23.53875160217285, + "lpips": 0.12475808709859848 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000158_test__val_desert__3__000021/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000158_test__val_desert__3__000021/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..ee5324b3f1bf776b48d2866ed66cf08cd6d57302 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000158_test__val_desert__3__000021/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:622c4cfb07411d990d78f89b92c921ddf191929e44f8cbb442c075c843dd57af +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000158_test__val_desert__3__000021/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000158_test__val_desert__3__000021/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..24a22140da9149186b30ef56bf3ba59108f2199b --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000158_test__val_desert__3__000021/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "158", + "global_test_index": "158", + "dataset_index": "55", + "relative_path": "test/val_desert/3/000021.mp4", + "clip_id": "test/val_desert/3/000021", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000158, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_desert/3/000021_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.0011863659601658583, + "psnr": 29.4697322845459, + "lpips": 0.017685137689113617 + }, + "frames_100_200": { + "mse": 0.0008813371532596648, + "psnr": 30.63031768798828, + "lpips": 0.011446112766861916 + }, + "frames_200_300": { + "mse": 0.0022399432491511106, + "psnr": 28.48546600341797, + "lpips": 0.02506728656589985 + }, + "frames_300_400": { + "mse": 0.000842131907120347, + "psnr": 30.758211135864258, + "lpips": 0.011004899628460407 + }, + "frames_400_500": { + "mse": 0.0015221763169392943, + "psnr": 30.107847213745117, + "lpips": 0.015454377047717571 + }, + "overall": { + "mse": 0.001334390719421208, + "psnr": 29.890316009521484, + "lpips": 0.01613156497478485 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000159_test__val_desert__2__000028/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000159_test__val_desert__2__000028/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..e35e518ec66d1bd28149462896616269f422fdc3 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000159_test__val_desert__2__000028/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9560ad20d67cb0621950f9d2e5d48ffecfa11ddc708a119b88a5c7ee3e319ba +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000159_test__val_desert__2__000028/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000159_test__val_desert__2__000028/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..078c7f6a38aace1674b79b88b6e4a4eaa3be0427 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000159_test__val_desert__2__000028/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "159", + "global_test_index": "159", + "dataset_index": "75", + "relative_path": "test/val_desert/2/000028.mp4", + "clip_id": "test/val_desert/2/000028", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000159, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_desert/2/000028_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.00861777551472187, + "psnr": 20.753211975097656, + "lpips": 0.3230913281440735 + }, + "frames_100_200": { + "mse": 0.012023381888866425, + "psnr": 19.250080108642578, + "lpips": 0.3721536695957184 + }, + "frames_200_300": { + "mse": 0.02672380954027176, + "psnr": 15.888542175292969, + "lpips": 0.38897988200187683 + }, + "frames_300_400": { + "mse": 0.02340417169034481, + "psnr": 16.470413208007812, + "lpips": 0.356011301279068 + }, + "frames_400_500": { + "mse": 0.01864653080701828, + "psnr": 17.77153205871582, + "lpips": 0.2054591327905655 + }, + "overall": { + "mse": 0.01788313500583172, + "psnr": 18.026758193969727, + "lpips": 0.32913902401924133 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000160_test__val_iceplains__1__000031/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000160_test__val_iceplains__1__000031/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..e9aaf849518aae7bb4046857e85ab8d14e8e911f --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000160_test__val_iceplains__1__000031/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:48def79410fbcc49ca64bd35e071cba85d678c8994597ba31ddb8ef55891691b +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000160_test__val_iceplains__1__000031/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000160_test__val_iceplains__1__000031/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..3fe61d113c1c81b247b832a2b3cba0b5fb479c7b --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000160_test__val_iceplains__1__000031/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "160", + "global_test_index": "160", + "dataset_index": "319", + "relative_path": "test/val_iceplains/1/000031.mp4", + "clip_id": "test/val_iceplains/1/000031", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000160, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/1/000031_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.013216010294854641, + "psnr": 20.29324722290039, + "lpips": 0.18976189196109772 + }, + "frames_100_200": { + "mse": 0.015917379409074783, + "psnr": 18.896255493164062, + "lpips": 0.26695916056632996 + }, + "frames_200_300": { + "mse": 0.035895537585020065, + "psnr": 15.485490798950195, + "lpips": 0.3507291376590729 + }, + "frames_300_400": { + "mse": 0.06121327355504036, + "psnr": 12.241976737976074, + "lpips": 0.41813477873802185 + }, + "frames_400_500": { + "mse": 0.06329131871461868, + "psnr": 12.164624214172363, + "lpips": 0.3918444812297821 + }, + "overall": { + "mse": 0.0379067026078701, + "psnr": 15.81631851196289, + "lpips": 0.3234858810901642 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000161_test__val_desert__1__000016/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000161_test__val_desert__1__000016/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..87d259648d6ca1cd6c60ea1797c2cb2c10f78c1d --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000161_test__val_desert__1__000016/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e683c35c8a2832d27324c8233c96003e526cbef7c1ba881527beb749cfa2146 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000161_test__val_desert__1__000016/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000161_test__val_desert__1__000016/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..54747c75242d866a4289cdd1e017a99f1c9e7078 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000161_test__val_desert__1__000016/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "161", + "global_test_index": "161", + "dataset_index": "39", + "relative_path": "test/val_desert/1/000016.mp4", + "clip_id": "test/val_desert/1/000016", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000161, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_desert/1/000016_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.03337273746728897, + "psnr": 17.21125602722168, + "lpips": 0.2853128910064697 + }, + "frames_100_200": { + "mse": 0.02672571875154972, + "psnr": 17.837406158447266, + "lpips": 0.3781731128692627 + }, + "frames_200_300": { + "mse": 0.00817984901368618, + "psnr": 20.907276153564453, + "lpips": 0.4553486704826355 + }, + "frames_300_400": { + "mse": 0.008637163788080215, + "psnr": 20.684856414794922, + "lpips": 0.391526997089386 + }, + "frames_400_500": { + "mse": 0.008418791927397251, + "psnr": 20.778242111206055, + "lpips": 0.38677284121513367 + }, + "overall": { + "mse": 0.017066853120923042, + "psnr": 19.483806610107422, + "lpips": 0.37942686676979065 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000162_test__val_plains__1__000019/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000162_test__val_plains__1__000019/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..7a7035cc6a4f8dc3ad4fc1c342a0de998c098aad --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000162_test__val_plains__1__000019/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c1a5a350fda026ae74dcf1fef0b1c1eae4fa043452a9a39cada035f4472a72ee +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000162_test__val_plains__1__000019/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000162_test__val_plains__1__000019/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..dcd885041a35724a7c4ca9db54cd509c581926f1 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000162_test__val_plains__1__000019/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "162", + "global_test_index": "162", + "dataset_index": "418", + "relative_path": "test/val_plains/1/000019.mp4", + "clip_id": "test/val_plains/1/000019", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000162, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_plains/1/000019_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.004958984907716513, + "psnr": 25.3686580657959, + "lpips": 0.06579401344060898 + }, + "frames_100_200": { + "mse": 0.005287064705044031, + "psnr": 24.399145126342773, + "lpips": 0.0510416217148304 + }, + "frames_200_300": { + "mse": 0.00822988711297512, + "psnr": 23.49044418334961, + "lpips": 0.10127352923154831 + }, + "frames_300_400": { + "mse": 0.018165223300457, + "psnr": 18.199378967285156, + "lpips": 0.3088909089565277 + }, + "frames_400_500": { + "mse": 0.010704751126468182, + "psnr": 20.704689025878906, + "lpips": 0.19470588862895966 + }, + "overall": { + "mse": 0.00946918223053217, + "psnr": 22.432462692260742, + "lpips": 0.14434118568897247 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000163_test__val_plains__3__000007/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000163_test__val_plains__3__000007/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..73aee37aba7487a9a3ff3e1b930728c5526596ee --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000163_test__val_plains__3__000007/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:033d6007cd8a64fb21d3fcd1d216f6cdea4d9bb72febb0b2040d1768df4ee360 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000163_test__val_plains__3__000007/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000163_test__val_plains__3__000007/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..ad949abcfd3949531b93a78b060a7114115669e9 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000163_test__val_plains__3__000007/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "163", + "global_test_index": "163", + "dataset_index": "389", + "relative_path": "test/val_plains/3/000007.mp4", + "clip_id": "test/val_plains/3/000007", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000163, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_plains/3/000007_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.0067608593963086605, + "psnr": 22.378944396972656, + "lpips": 0.2535848319530487 + }, + "frames_100_200": { + "mse": 0.006602516397833824, + "psnr": 21.908130645751953, + "lpips": 0.37573954463005066 + }, + "frames_200_300": { + "mse": 0.013278543017804623, + "psnr": 19.71245574951172, + "lpips": 0.4489229619503021 + }, + "frames_300_400": { + "mse": 0.0469784289598465, + "psnr": 13.918294906616211, + "lpips": 0.5028839111328125 + }, + "frames_400_500": { + "mse": 0.014906643889844418, + "psnr": 19.515756607055664, + "lpips": 0.4435732960700989 + }, + "overall": { + "mse": 0.017705395817756653, + "psnr": 19.486717224121094, + "lpips": 0.4049409031867981 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000164_test__val_plains__2__000014/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000164_test__val_plains__2__000014/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..59e68c92634d8101f2691eff752820caabcc23c1 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000164_test__val_plains__2__000014/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9db439fa050aeac610ecf01990bd489e02ea0aefcc8d4268ac9a66f4d4c62b3 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000164_test__val_plains__2__000014/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000164_test__val_plains__2__000014/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..3d2ab105aca24575e94206067b9ecfe64971ebcd --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000164_test__val_plains__2__000014/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "164", + "global_test_index": "164", + "dataset_index": "409", + "relative_path": "test/val_plains/2/000014.mp4", + "clip_id": "test/val_plains/2/000014", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000164, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_plains/2/000014_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.02061712183058262, + "psnr": 18.830791473388672, + "lpips": 0.3262525200843811 + }, + "frames_100_200": { + "mse": 0.017073027789592743, + "psnr": 20.49120330810547, + "lpips": 0.25251609086990356 + }, + "frames_200_300": { + "mse": 0.003949395846575499, + "psnr": 24.244094848632812, + "lpips": 0.10229390859603882 + }, + "frames_300_400": { + "mse": 0.003506295382976532, + "psnr": 24.76494789123535, + "lpips": 0.12561939656734467 + }, + "frames_400_500": { + "mse": 0.004189122002571821, + "psnr": 23.857593536376953, + "lpips": 0.1888420432806015 + }, + "overall": { + "mse": 0.009866992942988873, + "psnr": 22.437726974487305, + "lpips": 0.19910478591918945 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000165_test__val_iceplains__2__000038/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000165_test__val_iceplains__2__000038/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..808a10d958783594449073e2f9dfab1631badd74 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000165_test__val_iceplains__2__000038/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cced29b6fdc1f3052f73cf62bad01e394bb00d5140d798b29212eb38fbae8797 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000165_test__val_iceplains__2__000038/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000165_test__val_iceplains__2__000038/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..a1cf4cd2a82ddae0e4e61567d82e8e8e36f18121 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000165_test__val_iceplains__2__000038/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "165", + "global_test_index": "165", + "dataset_index": "344", + "relative_path": "test/val_iceplains/2/000038.mp4", + "clip_id": "test/val_iceplains/2/000038", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000165, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/2/000038_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.024305257946252823, + "psnr": 16.632219314575195, + "lpips": 0.2200830578804016 + }, + "frames_100_200": { + "mse": 0.02431330271065235, + "psnr": 17.43303871154785, + "lpips": 0.22740697860717773 + }, + "frames_200_300": { + "mse": 0.028741855174303055, + "psnr": 18.117813110351562, + "lpips": 0.2535088062286377 + }, + "frames_300_400": { + "mse": 0.005986477248370647, + "psnr": 23.121145248413086, + "lpips": 0.19238214194774628 + }, + "frames_400_500": { + "mse": 0.024030275642871857, + "psnr": 17.492454528808594, + "lpips": 0.1784035712480545 + }, + "overall": { + "mse": 0.02147543616592884, + "psnr": 18.55933380126953, + "lpips": 0.2143568992614746 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000166_test__val_plains__2__000009/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000166_test__val_plains__2__000009/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..549965b75d696839e951455d4dc762befe3422f7 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000166_test__val_plains__2__000009/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b1411e3ba38e12ff3d4471ab36b6cf453ae55c5a166afaae82c4fcc3118f5c6 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000166_test__val_plains__2__000009/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000166_test__val_plains__2__000009/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..62f97ab7ecd1348cc92e07576951b78b45914d69 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000166_test__val_plains__2__000009/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "166", + "global_test_index": "166", + "dataset_index": "396", + "relative_path": "test/val_plains/2/000009.mp4", + "clip_id": "test/val_plains/2/000009", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000166, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_plains/2/000009_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.002504517324268818, + "psnr": 26.660385131835938, + "lpips": 0.023145528510212898 + }, + "frames_100_200": { + "mse": 0.0026692687533795834, + "psnr": 27.205537796020508, + "lpips": 0.02398563362658024 + }, + "frames_200_300": { + "mse": 0.01931886374950409, + "psnr": 19.108135223388672, + "lpips": 0.15518909692764282 + }, + "frames_300_400": { + "mse": 0.024598242715001106, + "psnr": 17.6981258392334, + "lpips": 0.28806188702583313 + }, + "frames_400_500": { + "mse": 0.02199440449476242, + "psnr": 17.333641052246094, + "lpips": 0.28984370827674866 + }, + "overall": { + "mse": 0.01421706099063158, + "psnr": 21.60116195678711, + "lpips": 0.15604519844055176 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000167_test__val_desert__1__000026/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000167_test__val_desert__1__000026/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..1a895222921f971e1e87204e4bdd916d9d9ee12d --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000167_test__val_desert__1__000026/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff2a606b3015b35b32e85c7fadc05103584299b7bf502de7f469e3adc9c1aee5 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000167_test__val_desert__1__000026/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000167_test__val_desert__1__000026/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..44884a8f22b6a7d97f768a7d6bfe69613e8b727a --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000167_test__val_desert__1__000026/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "167", + "global_test_index": "167", + "dataset_index": "67", + "relative_path": "test/val_desert/1/000026.mp4", + "clip_id": "test/val_desert/1/000026", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000167, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_desert/1/000026_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.008384788408875465, + "psnr": 20.868324279785156, + "lpips": 0.29444360733032227 + }, + "frames_100_200": { + "mse": 0.009811273775994778, + "psnr": 20.16431427001953, + "lpips": 0.3577231168746948 + }, + "frames_200_300": { + "mse": 0.022331833839416504, + "psnr": 17.60488510131836, + "lpips": 0.4072752892971039 + }, + "frames_300_400": { + "mse": 0.041492536664009094, + "psnr": 14.326738357543945, + "lpips": 0.42112481594085693 + }, + "frames_400_500": { + "mse": 0.042830511927604675, + "psnr": 14.115516662597656, + "lpips": 0.3790923058986664 + }, + "overall": { + "mse": 0.02497018873691559, + "psnr": 17.41595458984375, + "lpips": 0.37193188071250916 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000168_test__val_desert__3__000038/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000168_test__val_desert__3__000038/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..4fa6b46c183e1af8feb67a3f2fc2d6b089d00f3e --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000168_test__val_desert__3__000038/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:04f61efe142dd08b85aeae808b3783def95e42d1a048715f184b00feb3ba0dae +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000168_test__val_desert__3__000038/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000168_test__val_desert__3__000038/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..bc968ca58bb54a78c939a697311de22bec7d49c6 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000168_test__val_desert__3__000038/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "168", + "global_test_index": "168", + "dataset_index": "99", + "relative_path": "test/val_desert/3/000038.mp4", + "clip_id": "test/val_desert/3/000038", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000168, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_desert/3/000038_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.006072105839848518, + "psnr": 22.4818172454834, + "lpips": 0.2734426259994507 + }, + "frames_100_200": { + "mse": 0.00901903584599495, + "psnr": 20.49126434326172, + "lpips": 0.36735451221466064 + }, + "frames_200_300": { + "mse": 0.007984484545886517, + "psnr": 21.02817153930664, + "lpips": 0.33448418974876404 + }, + "frames_300_400": { + "mse": 0.007901420816779137, + "psnr": 21.048383712768555, + "lpips": 0.3518477976322174 + }, + "frames_400_500": { + "mse": 0.01089744083583355, + "psnr": 19.807231903076172, + "lpips": 0.40733227133750916 + }, + "overall": { + "mse": 0.008374896831810474, + "psnr": 20.97137451171875, + "lpips": 0.3468922972679138 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000169_test__val_iceplains__3__000016/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000169_test__val_iceplains__3__000016/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..18aa1ac08f360121c9eb099f8ac9861061176a02 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000169_test__val_iceplains__3__000016/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2792314c06efd8d9495e762b581f7c61dc485b983b90d6860d36b304b1bc6e0a +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000169_test__val_iceplains__3__000016/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000169_test__val_iceplains__3__000016/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..a6772d0852fcfdf9c4900d76e42c078f0922e12f --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000169_test__val_iceplains__3__000016/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "169", + "global_test_index": "169", + "dataset_index": "288", + "relative_path": "test/val_iceplains/3/000016.mp4", + "clip_id": "test/val_iceplains/3/000016", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000169, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/3/000016_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.005700275767594576, + "psnr": 22.986194610595703, + "lpips": 0.15050020813941956 + }, + "frames_100_200": { + "mse": 0.028563298285007477, + "psnr": 18.191184997558594, + "lpips": 0.215707927942276 + }, + "frames_200_300": { + "mse": 0.011174995452165604, + "psnr": 20.9381046295166, + "lpips": 0.11793463677167892 + }, + "frames_300_400": { + "mse": 0.012661230750381947, + "psnr": 19.55938148498535, + "lpips": 0.16487634181976318 + }, + "frames_400_500": { + "mse": 0.008908840827643871, + "psnr": 20.823200225830078, + "lpips": 0.13761185109615326 + }, + "overall": { + "mse": 0.013401729054749012, + "psnr": 20.499614715576172, + "lpips": 0.15732619166374207 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000170_test__val_desert__1__000038/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000170_test__val_desert__1__000038/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..186f9a53df27a4a4f7565c5a4bbc92d4f3c6fabc --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000170_test__val_desert__1__000038/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e558083a3e25f2c0224ad18a9ba60c1fa6e4f53d23a951684e0a589b276d54f +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000170_test__val_desert__1__000038/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000170_test__val_desert__1__000038/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..3e56c402703bea049609cda8779b32888596aa37 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000170_test__val_desert__1__000038/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "170", + "global_test_index": "170", + "dataset_index": "97", + "relative_path": "test/val_desert/1/000038.mp4", + "clip_id": "test/val_desert/1/000038", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000170, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_desert/1/000038_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.006756256800144911, + "psnr": 21.896207809448242, + "lpips": 0.17588751018047333 + }, + "frames_100_200": { + "mse": 0.010069843381643295, + "psnr": 20.09862518310547, + "lpips": 0.22832392156124115 + }, + "frames_200_300": { + "mse": 0.009973987005650997, + "psnr": 20.222824096679688, + "lpips": 0.25258564949035645 + }, + "frames_300_400": { + "mse": 0.006847521755844355, + "psnr": 22.459619522094727, + "lpips": 0.13278479874134064 + }, + "frames_400_500": { + "mse": 0.004233323968946934, + "psnr": 25.873701095581055, + "lpips": 0.07174797356128693 + }, + "overall": { + "mse": 0.007576187141239643, + "psnr": 22.110193252563477, + "lpips": 0.17226597666740417 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000171_test__val_savanna__2__000021/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000171_test__val_savanna__2__000021/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..5afe2de8d1801e406170e005016caa9a53a8ee3b --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000171_test__val_savanna__2__000021/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a4cc82d6f6b9643fccfcd07263d4069d8f0de15276c017eb318f9aea9958093e +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000171_test__val_savanna__2__000021/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000171_test__val_savanna__2__000021/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..dd257e7c1d284c7bd58823f6942a4bff764dd8de --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000171_test__val_savanna__2__000021/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "171", + "global_test_index": "171", + "dataset_index": "178", + "relative_path": "test/val_savanna/2/000021.mp4", + "clip_id": "test/val_savanna/2/000021", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000171, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_savanna/2/000021_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.0009056064300239086, + "psnr": 30.653188705444336, + "lpips": 0.01086969580501318 + }, + "frames_100_200": { + "mse": 0.001570807071402669, + "psnr": 29.30063247680664, + "lpips": 0.014689737930893898 + }, + "frames_200_300": { + "mse": 0.013771194033324718, + "psnr": 20.771520614624023, + "lpips": 0.0686127319931984 + }, + "frames_300_400": { + "mse": 0.04364873841404915, + "psnr": 13.979633331298828, + "lpips": 0.20293819904327393 + }, + "frames_400_500": { + "mse": 0.058054376393556595, + "psnr": 12.683548927307129, + "lpips": 0.2773451507091522 + }, + "overall": { + "mse": 0.023590143769979477, + "psnr": 21.477705001831055, + "lpips": 0.11489109694957733 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000172_test__val_iceplains__1__000007/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000172_test__val_iceplains__1__000007/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..b05eed8b2dc970984532b3c4d38bffb229df6932 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000172_test__val_iceplains__1__000007/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c533e75adf06b246d2d4483afe0a8434ce36e561b9777118df85d2cc7759804f +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000172_test__val_iceplains__1__000007/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000172_test__val_iceplains__1__000007/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..4b8f59d27ac1d2c8ab83a3f0c4da9f7d2b92bd3e --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000172_test__val_iceplains__1__000007/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "172", + "global_test_index": "172", + "dataset_index": "264", + "relative_path": "test/val_iceplains/1/000007.mp4", + "clip_id": "test/val_iceplains/1/000007", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000172, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/1/000007_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.0007611377513967454, + "psnr": 31.192686080932617, + "lpips": 0.010074412450194359 + }, + "frames_100_200": { + "mse": 0.0009530007955618203, + "psnr": 30.670570373535156, + "lpips": 0.012988266535103321 + }, + "frames_200_300": { + "mse": 0.0009126753429882228, + "psnr": 30.746801376342773, + "lpips": 0.011590463109314442 + }, + "frames_300_400": { + "mse": 0.011206031776964664, + "psnr": 22.258790969848633, + "lpips": 0.07743635773658752 + }, + "frames_400_500": { + "mse": 0.0033524271566420794, + "psnr": 26.912487030029297, + "lpips": 0.03579474240541458 + }, + "overall": { + "mse": 0.003437054343521595, + "psnr": 28.35626983642578, + "lpips": 0.029576849192380905 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000173_test__val_iceplains__0__000008/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000173_test__val_iceplains__0__000008/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..c289c6b99712d6d940dc5ebe3197e4ed99be08cf --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000173_test__val_iceplains__0__000008/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:55adf8846fb09176c10660a9b967962c3df2239b8ee975d60591f7ab2347330b +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000173_test__val_iceplains__0__000008/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000173_test__val_iceplains__0__000008/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..08be6328b04675f2e29ab3908c8aa05db5495f5f --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000173_test__val_iceplains__0__000008/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "173", + "global_test_index": "173", + "dataset_index": "266", + "relative_path": "test/val_iceplains/0/000008.mp4", + "clip_id": "test/val_iceplains/0/000008", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000173, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/0/000008_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.011204220354557037, + "psnr": 19.827375411987305, + "lpips": 0.20233877003192902 + }, + "frames_100_200": { + "mse": 0.016282033175230026, + "psnr": 18.206602096557617, + "lpips": 0.226753368973732 + }, + "frames_200_300": { + "mse": 0.02727099135518074, + "psnr": 16.230934143066406, + "lpips": 0.33380553126335144 + }, + "frames_300_400": { + "mse": 0.04089928790926933, + "psnr": 14.258227348327637, + "lpips": 0.4041479825973511 + }, + "frames_400_500": { + "mse": 0.06929445266723633, + "psnr": 11.804277420043945, + "lpips": 0.43157440423965454 + }, + "overall": { + "mse": 0.03299019858241081, + "psnr": 16.06548309326172, + "lpips": 0.3197239935398102 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000174_test__val_iceplains__1__000023/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000174_test__val_iceplains__1__000023/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..699426ade1e6504e8a6b65ef51187702b618f4af --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000174_test__val_iceplains__1__000023/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a601fa9a79157818bec223738aea29bf5c3549071800052f97552269b12b9952 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000174_test__val_iceplains__1__000023/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000174_test__val_iceplains__1__000023/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..95db9829277e44d17b7498d054b88c01d9f56bd5 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000174_test__val_iceplains__1__000023/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "174", + "global_test_index": "174", + "dataset_index": "303", + "relative_path": "test/val_iceplains/1/000023.mp4", + "clip_id": "test/val_iceplains/1/000023", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000174, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/1/000023_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.0007730148499831557, + "psnr": 31.131147384643555, + "lpips": 0.01013339776545763 + }, + "frames_100_200": { + "mse": 0.0009766501607373357, + "psnr": 30.398099899291992, + "lpips": 0.010745929554104805 + }, + "frames_200_300": { + "mse": 0.011618996039032936, + "psnr": 21.336214065551758, + "lpips": 0.08972083777189255 + }, + "frames_300_400": { + "mse": 0.009259659796953201, + "psnr": 22.695913314819336, + "lpips": 0.06891310214996338 + }, + "frames_400_500": { + "mse": 0.0056936671026051044, + "psnr": 23.744949340820312, + "lpips": 0.048195503652095795 + }, + "overall": { + "mse": 0.005664397031068802, + "psnr": 25.86126136779785, + "lpips": 0.045541755855083466 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000175_test__val_plains__0__000031/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000175_test__val_plains__0__000031/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..2eb7fc0e53fe7cb6dae4366efeed706ab1b93a39 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000175_test__val_plains__0__000031/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3909f922047aa71448446d8edea116fbafd9b55d71cc9e3a4140a443aa423cee +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000175_test__val_plains__0__000031/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000175_test__val_plains__0__000031/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..08f7e06ef68db255c1ba0bd5c1887eba99800612 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000175_test__val_plains__0__000031/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "175", + "global_test_index": "175", + "dataset_index": "442", + "relative_path": "test/val_plains/0/000031.mp4", + "clip_id": "test/val_plains/0/000031", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000175, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_plains/0/000031_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.0008690181421115994, + "psnr": 30.816457748413086, + "lpips": 0.010681483894586563 + }, + "frames_100_200": { + "mse": 0.003997800871729851, + "psnr": 27.843759536743164, + "lpips": 0.024640873074531555 + }, + "frames_200_300": { + "mse": 0.02376832813024521, + "psnr": 18.010292053222656, + "lpips": 0.20085690915584564 + }, + "frames_300_400": { + "mse": 0.029983170330524445, + "psnr": 17.7353458404541, + "lpips": 0.18706893920898438 + }, + "frames_400_500": { + "mse": 0.07255242764949799, + "psnr": 12.707385063171387, + "lpips": 0.5046285390853882 + }, + "overall": { + "mse": 0.02623414807021618, + "psnr": 21.422645568847656, + "lpips": 0.18557536602020264 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000176_test__val_savanna__1__000047/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000176_test__val_savanna__1__000047/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..8bee956568e7cd8e2a554fb6e6cf484523367b25 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000176_test__val_savanna__1__000047/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d5673554997e081079834bc5f72ddca820f423d12e5eca6d13fd7bddb548fcc3 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000176_test__val_savanna__1__000047/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000176_test__val_savanna__1__000047/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..47e44a4ef04d18aeccd37640ac4514d7842b62f3 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000176_test__val_savanna__1__000047/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "176", + "global_test_index": "176", + "dataset_index": "233", + "relative_path": "test/val_savanna/1/000047.mp4", + "clip_id": "test/val_savanna/1/000047", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000176, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_savanna/1/000047_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.0082321148365736, + "psnr": 21.664175033569336, + "lpips": 0.17230872809886932 + }, + "frames_100_200": { + "mse": 0.007272200658917427, + "psnr": 23.437681198120117, + "lpips": 0.08719999343156815 + }, + "frames_200_300": { + "mse": 0.021267710253596306, + "psnr": 18.401050567626953, + "lpips": 0.19415618479251862 + }, + "frames_300_400": { + "mse": 0.010768269188702106, + "psnr": 20.244674682617188, + "lpips": 0.13632579147815704 + }, + "frames_400_500": { + "mse": 0.012590575031936169, + "psnr": 19.250198364257812, + "lpips": 0.18945449590682983 + }, + "overall": { + "mse": 0.012026173993945122, + "psnr": 20.59955406188965, + "lpips": 0.15588904917240143 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000177_test__val_savanna__1__000049/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000177_test__val_savanna__1__000049/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..4b99081d2396068ebf376292d11603b2c3b087fa --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000177_test__val_savanna__1__000049/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7692bd75feb8fbba2af9a52095ceea4bd029436b71af0f12346b566c3c011da1 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000177_test__val_savanna__1__000049/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000177_test__val_savanna__1__000049/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..d3e6948e5430dbcbb39a70b1bc31e701b5988a81 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000177_test__val_savanna__1__000049/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "177", + "global_test_index": "177", + "dataset_index": "240", + "relative_path": "test/val_savanna/1/000049.mp4", + "clip_id": "test/val_savanna/1/000049", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000177, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_savanna/1/000049_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.012435311451554298, + "psnr": 19.636119842529297, + "lpips": 0.1152130588889122 + }, + "frames_100_200": { + "mse": 0.011587615124881268, + "psnr": 19.637483596801758, + "lpips": 0.12109953910112381 + }, + "frames_200_300": { + "mse": 0.014801080338656902, + "psnr": 19.145540237426758, + "lpips": 0.08930855989456177 + }, + "frames_300_400": { + "mse": 0.018762126564979553, + "psnr": 18.11812400817871, + "lpips": 0.11621366441249847 + }, + "frames_400_500": { + "mse": 0.02950987033545971, + "psnr": 15.607787132263184, + "lpips": 0.2726207375526428 + }, + "overall": { + "mse": 0.017419200390577316, + "psnr": 18.42901039123535, + "lpips": 0.14289110898971558 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000178_test__val_desert__0__000017/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000178_test__val_desert__0__000017/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..58a455e1d013a6a5f7912f936795854c43876444 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000178_test__val_desert__0__000017/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9def537c505216cfabc67b2c2c16304c55bf2b3201c4d420aebc1212e129a6a1 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000178_test__val_desert__0__000017/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000178_test__val_desert__0__000017/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..d61d1bc0aae2a79444bde43214082b9a6bf9f800 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000178_test__val_desert__0__000017/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "178", + "global_test_index": "178", + "dataset_index": "42", + "relative_path": "test/val_desert/0/000017.mp4", + "clip_id": "test/val_desert/0/000017", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000178, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_desert/0/000017_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.0014847246930003166, + "psnr": 28.625368118286133, + "lpips": 0.024557000026106834 + }, + "frames_100_200": { + "mse": 0.003245070343837142, + "psnr": 25.81885528564453, + "lpips": 0.05121065676212311 + }, + "frames_200_300": { + "mse": 0.0021754938643425703, + "psnr": 27.523637771606445, + "lpips": 0.025184357538819313 + }, + "frames_300_400": { + "mse": 0.0007930751889944077, + "psnr": 31.01394271850586, + "lpips": 0.010461853817105293 + }, + "frames_400_500": { + "mse": 0.005357474088668823, + "psnr": 24.09789276123047, + "lpips": 0.1225956603884697 + }, + "overall": { + "mse": 0.002611167961731553, + "psnr": 27.415937423706055, + "lpips": 0.04680190607905388 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000179_test__val_iceplains__3__000048/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000179_test__val_iceplains__3__000048/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..bfe9f6942024781bf46f43c61269fafbba765214 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000179_test__val_iceplains__3__000048/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:696746e126375628d42412f8efa12e5102b0076359ca56f6ce63e6278ebe70c2 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000179_test__val_iceplains__3__000048/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000179_test__val_iceplains__3__000048/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..e0e10b560ef23bc160a4ac4ef35c8f307958bc88 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000179_test__val_iceplains__3__000048/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "179", + "global_test_index": "179", + "dataset_index": "369", + "relative_path": "test/val_iceplains/3/000048.mp4", + "clip_id": "test/val_iceplains/3/000048", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000179, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/3/000048_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.01586739718914032, + "psnr": 18.110639572143555, + "lpips": 0.1293105185031891 + }, + "frames_100_200": { + "mse": 0.01046802755445242, + "psnr": 21.936840057373047, + "lpips": 0.0977083370089531 + }, + "frames_200_300": { + "mse": 0.0034791468642652035, + "psnr": 25.78677749633789, + "lpips": 0.02914733625948429 + }, + "frames_300_400": { + "mse": 0.016719527542591095, + "psnr": 21.83034896850586, + "lpips": 0.11990199238061905 + }, + "frames_400_500": { + "mse": 0.06834784150123596, + "psnr": 11.901159286499023, + "lpips": 0.3306373953819275 + }, + "overall": { + "mse": 0.022976389154791832, + "psnr": 19.91315269470215, + "lpips": 0.14134110510349274 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000180_test__val_plains__3__000010/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000180_test__val_plains__3__000010/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..9503c33a03849065d8a95af68b969cae276e33f8 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000180_test__val_plains__3__000010/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4575763b2f190cb9f5e71418db8dab41edb722a6b2c545ae3b1f7973a1e1db2e +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000180_test__val_plains__3__000010/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000180_test__val_plains__3__000010/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..1d1b774afaa82e81235460f2d75a0f55971ae489 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000180_test__val_plains__3__000010/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "180", + "global_test_index": "180", + "dataset_index": "398", + "relative_path": "test/val_plains/3/000010.mp4", + "clip_id": "test/val_plains/3/000010", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000180, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_plains/3/000010_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.0020898678340017796, + "psnr": 28.580869674682617, + "lpips": 0.023748695850372314 + }, + "frames_100_200": { + "mse": 0.0008797381888143718, + "psnr": 30.563047409057617, + "lpips": 0.014668681658804417 + }, + "frames_200_300": { + "mse": 0.0009553296840749681, + "psnr": 30.219270706176758, + "lpips": 0.017239870503544807 + }, + "frames_300_400": { + "mse": 0.001547525404021144, + "psnr": 29.346763610839844, + "lpips": 0.029184691607952118 + }, + "frames_400_500": { + "mse": 0.014241380617022514, + "psnr": 20.015640258789062, + "lpips": 0.18104667961597443 + }, + "overall": { + "mse": 0.003942768555134535, + "psnr": 27.7451171875, + "lpips": 0.053177718073129654 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000181_test__val_plains__2__000027/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000181_test__val_plains__2__000027/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..0ae344536d106ef751ba1ddeb84f9d4962e2d213 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000181_test__val_plains__2__000027/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9dfb2f6aa52683827f372b129e60ea1d916393bf3a44a5db9189e150891e7681 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000181_test__val_plains__2__000027/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000181_test__val_plains__2__000027/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..5949c0d87fb4f97b5fec61294cea6101b5f0ed61 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000181_test__val_plains__2__000027/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "181", + "global_test_index": "181", + "dataset_index": "435", + "relative_path": "test/val_plains/2/000027.mp4", + "clip_id": "test/val_plains/2/000027", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000181, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_plains/2/000027_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.002095781499519944, + "psnr": 28.05923080444336, + "lpips": 0.01824798434972763 + }, + "frames_100_200": { + "mse": 0.0009113795240409672, + "psnr": 30.43268585205078, + "lpips": 0.015096311457455158 + }, + "frames_200_300": { + "mse": 0.003931194078177214, + "psnr": 25.245399475097656, + "lpips": 0.02979814074933529 + }, + "frames_300_400": { + "mse": 0.0017778058536350727, + "psnr": 28.71277618408203, + "lpips": 0.021965565159916878 + }, + "frames_400_500": { + "mse": 0.006078690756112337, + "psnr": 23.942028045654297, + "lpips": 0.13340957462787628 + }, + "overall": { + "mse": 0.0029589703772217035, + "psnr": 27.278425216674805, + "lpips": 0.04370351508259773 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000182_test__val_savanna__2__000049/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000182_test__val_savanna__2__000049/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..bda6e872caa714cb39525f2efe004899777f71cb --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000182_test__val_savanna__2__000049/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4d13c05e599e58917d31f5f67fb7f1b914ee758d118e1e290136572cea58aab2 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000182_test__val_savanna__2__000049/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000182_test__val_savanna__2__000049/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..6618f37413ee061461d2ebdd2f050877a0173016 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000182_test__val_savanna__2__000049/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "182", + "global_test_index": "182", + "dataset_index": "242", + "relative_path": "test/val_savanna/2/000049.mp4", + "clip_id": "test/val_savanna/2/000049", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000182, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_savanna/2/000049_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.003224204294383526, + "psnr": 26.52105140686035, + "lpips": 0.025331558659672737 + }, + "frames_100_200": { + "mse": 0.03035971336066723, + "psnr": 16.903127670288086, + "lpips": 0.1790418177843094 + }, + "frames_200_300": { + "mse": 0.0614386610686779, + "psnr": 12.184653282165527, + "lpips": 0.34111300110816956 + }, + "frames_300_400": { + "mse": 0.06666376441717148, + "psnr": 11.917845726013184, + "lpips": 0.32782626152038574 + }, + "frames_400_500": { + "mse": 0.05848073959350586, + "psnr": 12.86009407043457, + "lpips": 0.2872786223888397 + }, + "overall": { + "mse": 0.044033415615558624, + "psnr": 16.077356338500977, + "lpips": 0.23211823403835297 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000183_test__val_iceplains__2__000045/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000183_test__val_iceplains__2__000045/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..eb8b6245ebb77671187fd8033d75421a4e8af4ea --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000183_test__val_iceplains__2__000045/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80b70fd7b6eaddd041c78e1253fb15cc2fecef780ae141f209c666dba12213cb +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000183_test__val_iceplains__2__000045/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000183_test__val_iceplains__2__000045/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..8943d2ab9a95d64ce5c75da4145f075cfd781a0d --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000183_test__val_iceplains__2__000045/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "183", + "global_test_index": "183", + "dataset_index": "361", + "relative_path": "test/val_iceplains/2/000045.mp4", + "clip_id": "test/val_iceplains/2/000045", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000183, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/2/000045_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.015030554495751858, + "psnr": 21.577838897705078, + "lpips": 0.09701094031333923 + }, + "frames_100_200": { + "mse": 0.02224985510110855, + "psnr": 18.930713653564453, + "lpips": 0.15153315663337708 + }, + "frames_200_300": { + "mse": 0.017942842096090317, + "psnr": 21.839906692504883, + "lpips": 0.09122315049171448 + }, + "frames_300_400": { + "mse": 0.013749548234045506, + "psnr": 20.723779678344727, + "lpips": 0.08030437678098679 + }, + "frames_400_500": { + "mse": 0.005298234056681395, + "psnr": 26.568828582763672, + "lpips": 0.036887217313051224 + }, + "overall": { + "mse": 0.014854205772280693, + "psnr": 21.928213119506836, + "lpips": 0.09139177203178406 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000184_test__val_plains__2__000047/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000184_test__val_plains__2__000047/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..af5b71dc010be0c02b81cf1ca7e836bb0a1b4be7 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000184_test__val_plains__2__000047/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b091a5d9d36cef3461b104d5d2c952bcf44cc0c0511f1e96e471846d57b9519c +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000184_test__val_plains__2__000047/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000184_test__val_plains__2__000047/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..158a850f856a6ea9d5b9b03c1d047d85258ecb2a --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000184_test__val_plains__2__000047/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "184", + "global_test_index": "184", + "dataset_index": "470", + "relative_path": "test/val_plains/2/000047.mp4", + "clip_id": "test/val_plains/2/000047", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000184, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_plains/2/000047_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.010801620781421661, + "psnr": 19.801908493041992, + "lpips": 0.2835107147693634 + }, + "frames_100_200": { + "mse": 0.007741251029074192, + "psnr": 21.292314529418945, + "lpips": 0.36144500970840454 + }, + "frames_200_300": { + "mse": 0.005745041184127331, + "psnr": 22.683143615722656, + "lpips": 0.3717186748981476 + }, + "frames_300_400": { + "mse": 0.005481937900185585, + "psnr": 22.786720275878906, + "lpips": 0.39716145396232605 + }, + "frames_400_500": { + "mse": 0.0073119825683534145, + "psnr": 21.746564865112305, + "lpips": 0.462901771068573 + }, + "overall": { + "mse": 0.007416366599500179, + "psnr": 21.66213035583496, + "lpips": 0.3753474950790405 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000185_test__val_iceplains__2__000042/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000185_test__val_iceplains__2__000042/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..f5af56f851e887ffd0d7cfa7371a5a987ca9816f --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000185_test__val_iceplains__2__000042/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ce83043ff842b8526348e802db96be0311bb0c2709ddc421da73a16a5fef87b +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000185_test__val_iceplains__2__000042/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000185_test__val_iceplains__2__000042/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..12b6def4f076cbbcefb4624e660a34127a3ce798 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000185_test__val_iceplains__2__000042/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "185", + "global_test_index": "185", + "dataset_index": "356", + "relative_path": "test/val_iceplains/2/000042.mp4", + "clip_id": "test/val_iceplains/2/000042", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000185, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/2/000042_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.023637382313609123, + "psnr": 16.777170181274414, + "lpips": 0.212004616856575 + }, + "frames_100_200": { + "mse": 0.05217193439602852, + "psnr": 15.531290054321289, + "lpips": 0.2879217267036438 + }, + "frames_200_300": { + "mse": 0.031966082751750946, + "psnr": 18.66866111755371, + "lpips": 0.11516910791397095 + }, + "frames_300_400": { + "mse": 0.00794298853725195, + "psnr": 25.75560760498047, + "lpips": 0.040060821920633316 + }, + "frames_400_500": { + "mse": 0.0023539895191788673, + "psnr": 29.827486038208008, + "lpips": 0.01762685365974903 + }, + "overall": { + "mse": 0.02361447550356388, + "psnr": 21.312044143676758, + "lpips": 0.1345566213130951 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000186_test__val_plains__2__000046/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000186_test__val_plains__2__000046/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..cb53fbe0028c3dfe2de51a761437123baea24c99 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000186_test__val_plains__2__000046/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:49329a461b2d6b0260ae6ce990f183c112aea1473a928b4937044477b904551f +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000186_test__val_plains__2__000046/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000186_test__val_plains__2__000046/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..f09a5be582e83cffd0ac4aa95c7960edce7ee1a8 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000186_test__val_plains__2__000046/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "186", + "global_test_index": "186", + "dataset_index": "468", + "relative_path": "test/val_plains/2/000046.mp4", + "clip_id": "test/val_plains/2/000046", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000186, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_plains/2/000046_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.006870870478451252, + "psnr": 22.39883041381836, + "lpips": 0.26317694783210754 + }, + "frames_100_200": { + "mse": 0.005591460503637791, + "psnr": 22.778650283813477, + "lpips": 0.3190120458602905 + }, + "frames_200_300": { + "mse": 0.08326177299022675, + "psnr": 11.745837211608887, + "lpips": 0.4909980893135071 + }, + "frames_300_400": { + "mse": 0.09751467406749725, + "psnr": 10.153943061828613, + "lpips": 0.6252343058586121 + }, + "frames_400_500": { + "mse": 0.02074352651834488, + "psnr": 18.048921585083008, + "lpips": 0.5304876565933228 + }, + "overall": { + "mse": 0.04279646277427673, + "psnr": 17.025236129760742, + "lpips": 0.4457818269729614 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000187_test__val_desert__0__000043/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000187_test__val_desert__0__000043/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..a75d7a4ea329233b808fb3ef91f761e41898e5de --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000187_test__val_desert__0__000043/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:36dac3e56d251942d7e20a75db52de2cdbef75d6a12092e578a4b72c90eb9bde +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000187_test__val_desert__0__000043/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000187_test__val_desert__0__000043/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..81d707aad79376dff1c72644d5b3b8d4a7e09f67 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000187_test__val_desert__0__000043/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "187", + "global_test_index": "187", + "dataset_index": "113", + "relative_path": "test/val_desert/0/000043.mp4", + "clip_id": "test/val_desert/0/000043", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000187, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_desert/0/000043_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.00719994492828846, + "psnr": 21.65642547607422, + "lpips": 0.30119776725769043 + }, + "frames_100_200": { + "mse": 0.02000521682202816, + "psnr": 17.817256927490234, + "lpips": 0.44503432512283325 + }, + "frames_200_300": { + "mse": 0.01291284803301096, + "psnr": 19.050596237182617, + "lpips": 0.4598923623561859 + }, + "frames_300_400": { + "mse": 0.020653920248150826, + "psnr": 17.088417053222656, + "lpips": 0.3898875117301941 + }, + "frames_400_500": { + "mse": 0.027299776673316956, + "psnr": 15.790645599365234, + "lpips": 0.420536607503891 + }, + "overall": { + "mse": 0.017614342272281647, + "psnr": 18.280668258666992, + "lpips": 0.40330973267555237 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000188_test__val_desert__1__000008/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000188_test__val_desert__1__000008/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..c673e018238c477ec531bedaaa120bb738d94629 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000188_test__val_desert__1__000008/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:897cb86875681ed8c429d77f4bd1cc4523098868d2b24951f0790ae312f638fb +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000188_test__val_desert__1__000008/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000188_test__val_desert__1__000008/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..e5dc9eed33102bcaee08ac7d077f9f832b76349c --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000188_test__val_desert__1__000008/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "188", + "global_test_index": "188", + "dataset_index": "19", + "relative_path": "test/val_desert/1/000008.mp4", + "clip_id": "test/val_desert/1/000008", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000188, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_desert/1/000008_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.00788421556353569, + "psnr": 21.21518325805664, + "lpips": 0.28902748227119446 + }, + "frames_100_200": { + "mse": 0.010320177301764488, + "psnr": 20.023286819458008, + "lpips": 0.3948096036911011 + }, + "frames_200_300": { + "mse": 0.012198465876281261, + "psnr": 19.35126304626465, + "lpips": 0.4014281928539276 + }, + "frames_300_400": { + "mse": 0.010768579319119453, + "psnr": 19.815410614013672, + "lpips": 0.38748592138290405 + }, + "frames_400_500": { + "mse": 0.016303187236189842, + "psnr": 18.047475814819336, + "lpips": 0.3581221103668213 + }, + "overall": { + "mse": 0.011494924314320087, + "psnr": 19.690523147583008, + "lpips": 0.3661746382713318 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000189_test__val_savanna__0__000033/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000189_test__val_savanna__0__000033/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..9c96a293da0d7a27970d62522bdd4ba5ebd50bff --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000189_test__val_savanna__0__000033/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b2b26f95a201b22597c20aa4e4fecfc3b477cf625849b294545768af2e709142 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000189_test__val_savanna__0__000033/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000189_test__val_savanna__0__000033/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..67e1d80875d7e5d09e6e6ba6c6a0fa0334846fa7 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000189_test__val_savanna__0__000033/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "189", + "global_test_index": "189", + "dataset_index": "199", + "relative_path": "test/val_savanna/0/000033.mp4", + "clip_id": "test/val_savanna/0/000033", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000189, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_savanna/0/000033_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.01655678078532219, + "psnr": 18.79707145690918, + "lpips": 0.19351078569889069 + }, + "frames_100_200": { + "mse": 0.012185025960206985, + "psnr": 19.729427337646484, + "lpips": 0.1190560832619667 + }, + "frames_200_300": { + "mse": 0.02508222870528698, + "psnr": 16.101085662841797, + "lpips": 0.326070636510849 + }, + "frames_300_400": { + "mse": 0.02609718032181263, + "psnr": 15.889033317565918, + "lpips": 0.3663289546966553 + }, + "frames_400_500": { + "mse": 0.035437725484371185, + "psnr": 14.613367080688477, + "lpips": 0.3929700553417206 + }, + "overall": { + "mse": 0.023071788251399994, + "psnr": 17.025997161865234, + "lpips": 0.2795872986316681 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000190_test__val_desert__2__000045/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000190_test__val_desert__2__000045/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..040a533235c48821213600ad04eac36fc64cdbb4 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000190_test__val_desert__2__000045/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c812ad74cdb16c0a1bf80f4f3e4a2daa0c193c6bc152ac7c94c5755faa6745a1 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000190_test__val_desert__2__000045/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000190_test__val_desert__2__000045/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..26784fd6de5418a6dd0fa2da863e9685d20736ad --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000190_test__val_desert__2__000045/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "190", + "global_test_index": "190", + "dataset_index": "118", + "relative_path": "test/val_desert/2/000045.mp4", + "clip_id": "test/val_desert/2/000045", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000190, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_desert/2/000045_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.0007766224443912506, + "psnr": 31.287193298339844, + "lpips": 0.020316846668720245 + }, + "frames_100_200": { + "mse": 0.0008049819152802229, + "psnr": 31.011259078979492, + "lpips": 0.015407714061439037 + }, + "frames_200_300": { + "mse": 0.0009951036190614104, + "psnr": 30.814716339111328, + "lpips": 0.017657237127423286 + }, + "frames_300_400": { + "mse": 0.0008009701850824058, + "psnr": 30.990379333496094, + "lpips": 0.013888373039662838 + }, + "frames_400_500": { + "mse": 0.0012929870281368494, + "psnr": 29.21282196044922, + "lpips": 0.02465774305164814 + }, + "overall": { + "mse": 0.0009341330733150244, + "psnr": 30.66327476501465, + "lpips": 0.01838558167219162 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000191_test__val_plains__0__000027/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000191_test__val_plains__0__000027/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..4b1406b77cc6b3d3e2ff41e0daf54f21d01158cf --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000191_test__val_plains__0__000027/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:865709226912d60ba6cbcc8ca23be93cfd841b45a8d8ff0d3ed874c2ae73ec28 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000191_test__val_plains__0__000027/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000191_test__val_plains__0__000027/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..a3a376f1193d20ad66251f5f53fa22af70baf5ed --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000191_test__val_plains__0__000027/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "191", + "global_test_index": "191", + "dataset_index": "433", + "relative_path": "test/val_plains/0/000027.mp4", + "clip_id": "test/val_plains/0/000027", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000191, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_plains/0/000027_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.003270205343142152, + "psnr": 26.74852752685547, + "lpips": 0.05904204398393631 + }, + "frames_100_200": { + "mse": 0.003855867078527808, + "psnr": 24.69672203063965, + "lpips": 0.05760841444134712 + }, + "frames_200_300": { + "mse": 0.0016988589195534587, + "psnr": 28.993989944458008, + "lpips": 0.02774740196764469 + }, + "frames_300_400": { + "mse": 0.000969638058450073, + "psnr": 30.173255920410156, + "lpips": 0.01708030328154564 + }, + "frames_400_500": { + "mse": 0.0009163655340671539, + "psnr": 30.388097763061523, + "lpips": 0.017831552773714066 + }, + "overall": { + "mse": 0.002142186975106597, + "psnr": 28.200117111206055, + "lpips": 0.035861946642398834 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000192_test__val_iceplains__0__000034/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000192_test__val_iceplains__0__000034/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..4a5152b74435febe40e825fc6434f2bd95197132 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000192_test__val_iceplains__0__000034/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:530d3f4c6a0228f70b271ef3fd061301dd19c192d7afaa5c93a976b2d563a574 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000192_test__val_iceplains__0__000034/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000192_test__val_iceplains__0__000034/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..01d40541732161f539ee923e9f17f9200da47166 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000192_test__val_iceplains__0__000034/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "192", + "global_test_index": "192", + "dataset_index": "328", + "relative_path": "test/val_iceplains/0/000034.mp4", + "clip_id": "test/val_iceplains/0/000034", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000192, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/0/000034_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.01165390107780695, + "psnr": 19.555118560791016, + "lpips": 0.1573842614889145 + }, + "frames_100_200": { + "mse": 0.014602111652493477, + "psnr": 18.384843826293945, + "lpips": 0.14256253838539124 + }, + "frames_200_300": { + "mse": 0.015119016170501709, + "psnr": 18.27315330505371, + "lpips": 0.17968639731407166 + }, + "frames_300_400": { + "mse": 0.015781378373503685, + "psnr": 18.23710060119629, + "lpips": 0.18516665697097778 + }, + "frames_400_500": { + "mse": 0.018125396221876144, + "psnr": 17.523866653442383, + "lpips": 0.19929946959018707 + }, + "overall": { + "mse": 0.015056359581649303, + "psnr": 18.394817352294922, + "lpips": 0.17281986773014069 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000193_test__val_savanna__0__000039/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000193_test__val_savanna__0__000039/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..ce54ff484f70b2a4bc261a0aa38180f22b98d9b5 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000193_test__val_savanna__0__000039/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:441c9acbccddd269f507842819c8934150896a26b3c8f717ab3e3f9e3174ce2b +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000193_test__val_savanna__0__000039/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000193_test__val_savanna__0__000039/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..780bf5dc40e8fc0c8a9c58015db0dc3820b68c35 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000193_test__val_savanna__0__000039/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "193", + "global_test_index": "193", + "dataset_index": "213", + "relative_path": "test/val_savanna/0/000039.mp4", + "clip_id": "test/val_savanna/0/000039", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000193, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_savanna/0/000039_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.003838899778202176, + "psnr": 24.419382095336914, + "lpips": 0.260092556476593 + }, + "frames_100_200": { + "mse": 0.006949201226234436, + "psnr": 21.89242172241211, + "lpips": 0.3672601282596588 + }, + "frames_200_300": { + "mse": 0.023926658555865288, + "psnr": 18.779008865356445, + "lpips": 0.4429510533809662 + }, + "frames_300_400": { + "mse": 0.06636171042919159, + "psnr": 12.394372940063477, + "lpips": 0.5573611259460449 + }, + "frames_400_500": { + "mse": 0.010141860693693161, + "psnr": 20.595809936523438, + "lpips": 0.4881654977798462 + }, + "overall": { + "mse": 0.022243667393922806, + "psnr": 19.616199493408203, + "lpips": 0.42316606640815735 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000194_test__val_iceplains__0__000037/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000194_test__val_iceplains__0__000037/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..301a888004cbfa733a168df4fef6747fcb01f466 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000194_test__val_iceplains__0__000037/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cae90a8439fef61a083df9e83189c42b1d550e2cd23d154fa0abc8b13c29e8e2 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000194_test__val_iceplains__0__000037/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000194_test__val_iceplains__0__000037/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..2518803d05de3d990bbe8715e23272e74dcb4020 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000194_test__val_iceplains__0__000037/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "194", + "global_test_index": "194", + "dataset_index": "339", + "relative_path": "test/val_iceplains/0/000037.mp4", + "clip_id": "test/val_iceplains/0/000037", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000194, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/0/000037_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.012973365373909473, + "psnr": 19.425655364990234, + "lpips": 0.17243905365467072 + }, + "frames_100_200": { + "mse": 0.010421017184853554, + "psnr": 20.879562377929688, + "lpips": 0.16470319032669067 + }, + "frames_200_300": { + "mse": 0.09428119659423828, + "psnr": 12.012603759765625, + "lpips": 0.3835826814174652 + }, + "frames_300_400": { + "mse": 0.06859192252159119, + "psnr": 13.03342056274414, + "lpips": 0.3486291170120239 + }, + "frames_400_500": { + "mse": 0.043806981295347214, + "psnr": 16.236713409423828, + "lpips": 0.42702141404151917 + }, + "overall": { + "mse": 0.04601489380002022, + "psnr": 16.31759262084961, + "lpips": 0.29927513003349304 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000195_test__val_plains__1__000046/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000195_test__val_plains__1__000046/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..366f26d58f513273e2283589830b36994ddb8f70 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000195_test__val_plains__1__000046/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78c898c8df0a5d8d9a55236df5197ddf58cf7cea4c0687260ceb904b7e25e580 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000195_test__val_plains__1__000046/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000195_test__val_plains__1__000046/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..15844f4b3da4d57f18acae405036de6d9c3d3ac6 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000195_test__val_plains__1__000046/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "195", + "global_test_index": "195", + "dataset_index": "466", + "relative_path": "test/val_plains/1/000046.mp4", + "clip_id": "test/val_plains/1/000046", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000195, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_plains/1/000046_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.011781610548496246, + "psnr": 19.535154342651367, + "lpips": 0.2680381238460541 + }, + "frames_100_200": { + "mse": 0.012321479618549347, + "psnr": 19.46162223815918, + "lpips": 0.40182316303253174 + }, + "frames_200_300": { + "mse": 0.010255955159664154, + "psnr": 20.114517211914062, + "lpips": 0.24734020233154297 + }, + "frames_300_400": { + "mse": 0.01087535172700882, + "psnr": 19.8126220703125, + "lpips": 0.22720588743686676 + }, + "frames_400_500": { + "mse": 0.022489307448267937, + "psnr": 16.820215225219727, + "lpips": 0.44997909665107727 + }, + "overall": { + "mse": 0.013544741086661816, + "psnr": 19.148828506469727, + "lpips": 0.31887727975845337 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000196_test__val_iceplains__3__000042/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000196_test__val_iceplains__3__000042/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..742eddaa21bcf0c02fc9a490859dc7ec00c47033 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000196_test__val_iceplains__3__000042/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:07c33357718eea4efcbd25c8f2c22e944f348d822a5acdd22e736fe6d066a30f +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000196_test__val_iceplains__3__000042/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000196_test__val_iceplains__3__000042/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..04ad46047a0721d04b28f13dba04fa5914e6ac3a --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000196_test__val_iceplains__3__000042/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "196", + "global_test_index": "196", + "dataset_index": "355", + "relative_path": "test/val_iceplains/3/000042.mp4", + "clip_id": "test/val_iceplains/3/000042", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000196, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/3/000042_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.026119031012058258, + "psnr": 18.71758270263672, + "lpips": 0.13722263276576996 + }, + "frames_100_200": { + "mse": 0.05864124745130539, + "psnr": 13.031752586364746, + "lpips": 0.31890666484832764 + }, + "frames_200_300": { + "mse": 0.0315854549407959, + "psnr": 16.716197967529297, + "lpips": 0.19045230746269226 + }, + "frames_300_400": { + "mse": 0.05077892169356346, + "psnr": 13.370013236999512, + "lpips": 0.31607821583747864 + }, + "frames_400_500": { + "mse": 0.05971290171146393, + "psnr": 13.088178634643555, + "lpips": 0.307039350271225 + }, + "overall": { + "mse": 0.045367512851953506, + "psnr": 14.984743118286133, + "lpips": 0.25393983721733093 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000197_test__val_iceplains__3__000036/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000197_test__val_iceplains__3__000036/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..d638bcf7e76a3d03c828666383fd52b7adc9216c --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000197_test__val_iceplains__3__000036/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9199c75257b2c85e01ad9ff5ae6c37eb43bb6de78b92718f22efcd5a7bee860c +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000197_test__val_iceplains__3__000036/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000197_test__val_iceplains__3__000036/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..30a1c5e903564d755816d1d500079d9df5e5364f --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000197_test__val_iceplains__3__000036/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "197", + "global_test_index": "197", + "dataset_index": "336", + "relative_path": "test/val_iceplains/3/000036.mp4", + "clip_id": "test/val_iceplains/3/000036", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000197, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/3/000036_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.0037536723539233208, + "psnr": 25.008386611938477, + "lpips": 0.04598537087440491 + }, + "frames_100_200": { + "mse": 0.013280708342790604, + "psnr": 19.72237777709961, + "lpips": 0.11649289727210999 + }, + "frames_200_300": { + "mse": 0.04187723249197006, + "psnr": 14.714927673339844, + "lpips": 0.3121563792228699 + }, + "frames_300_400": { + "mse": 0.039005063474178314, + "psnr": 14.838973045349121, + "lpips": 0.2829457223415375 + }, + "frames_400_500": { + "mse": 0.02279549278318882, + "psnr": 16.86123275756836, + "lpips": 0.18742865324020386 + }, + "overall": { + "mse": 0.0241424348205328, + "psnr": 18.22917938232422, + "lpips": 0.18900181353092194 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000198_test__val_plains__0__000020/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000198_test__val_plains__0__000020/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..5721f291b5d8d09deac752436ec27a862bda88bd --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000198_test__val_plains__0__000020/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:48d47348095da7883b54867f42106f48c50daefe8c849fb2640c4a02176d6461 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000198_test__val_plains__0__000020/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000198_test__val_plains__0__000020/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..8fefd2a16396bf668682ea9b000ff27d0e819b25 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000198_test__val_plains__0__000020/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "198", + "global_test_index": "198", + "dataset_index": "420", + "relative_path": "test/val_plains/0/000020.mp4", + "clip_id": "test/val_plains/0/000020", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000198, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_plains/0/000020_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.01265726424753666, + "psnr": 19.483806610107422, + "lpips": 0.37695562839508057 + }, + "frames_100_200": { + "mse": 0.005077116657048464, + "psnr": 23.185571670532227, + "lpips": 0.3253048360347748 + }, + "frames_200_300": { + "mse": 0.012996340170502663, + "psnr": 19.913772583007812, + "lpips": 0.4360426366329193 + }, + "frames_300_400": { + "mse": 0.005133774131536484, + "psnr": 22.950769424438477, + "lpips": 0.4277568757534027 + }, + "frames_400_500": { + "mse": 0.005058693699538708, + "psnr": 22.981420516967773, + "lpips": 0.4221839904785156 + }, + "overall": { + "mse": 0.008184637874364853, + "psnr": 21.70306968688965, + "lpips": 0.3976488411426544 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000199_test__val_plains__1__000015/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000199_test__val_plains__1__000015/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..082e89716167680a025e5bffd75d18490545ff7f --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000199_test__val_plains__1__000015/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:846af00806cffeff391c8399c98ad29ee48a5daa11f7836e198faf5245812cd6 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000199_test__val_plains__1__000015/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000199_test__val_plains__1__000015/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..7e3d00b9fd4edf4bbf2979885358c295247aefa5 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000199_test__val_plains__1__000015/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "199", + "global_test_index": "199", + "dataset_index": "410", + "relative_path": "test/val_plains/1/000015.mp4", + "clip_id": "test/val_plains/1/000015", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000199, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_plains/1/000015_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.0012669621501117945, + "psnr": 29.621719360351562, + "lpips": 0.01482558622956276 + }, + "frames_100_200": { + "mse": 0.0017576708924025297, + "psnr": 29.280961990356445, + "lpips": 0.01793042942881584 + }, + "frames_200_300": { + "mse": 0.0056952000595629215, + "psnr": 23.840984344482422, + "lpips": 0.04253862053155899 + }, + "frames_300_400": { + "mse": 0.0022799947764724493, + "psnr": 28.119909286499023, + "lpips": 0.022561902180314064 + }, + "frames_400_500": { + "mse": 0.0065550487488508224, + "psnr": 23.879493713378906, + "lpips": 0.049152862280607224 + }, + "overall": { + "mse": 0.0035109752789139748, + "psnr": 26.9486141204834, + "lpips": 0.029401879757642746 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000200_test__val_desert__1__000011/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000200_test__val_desert__1__000011/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..836841bcde4e170806cff7d5fa645a7f65b21c97 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000200_test__val_desert__1__000011/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f1228931502b0e90f7af8736519d8e9fe615e38b21b6c6db038b705690005325 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000200_test__val_desert__1__000011/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000200_test__val_desert__1__000011/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..cca1b6f6d82d545092198b07a1087cccd76d810d --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000200_test__val_desert__1__000011/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "200", + "global_test_index": "200", + "dataset_index": "27", + "relative_path": "test/val_desert/1/000011.mp4", + "clip_id": "test/val_desert/1/000011", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000200, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_desert/1/000011_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.003122991416603327, + "psnr": 25.839265823364258, + "lpips": 0.08238042891025543 + }, + "frames_100_200": { + "mse": 0.010606072843074799, + "psnr": 19.9818172454834, + "lpips": 0.21633514761924744 + }, + "frames_200_300": { + "mse": 0.009290444664657116, + "psnr": 20.841083526611328, + "lpips": 0.1652889996767044 + }, + "frames_300_400": { + "mse": 0.006274795159697533, + "psnr": 22.43781089782715, + "lpips": 0.12929022312164307 + }, + "frames_400_500": { + "mse": 0.00790464784950018, + "psnr": 22.544452667236328, + "lpips": 0.09742235392332077 + }, + "overall": { + "mse": 0.007439790293574333, + "psnr": 22.328887939453125, + "lpips": 0.13814343512058258 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000201_test__val_desert__1__000033/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000201_test__val_desert__1__000033/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..5688a36c4fa49efa3a4fe61b4de5c9dfd63d261e --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000201_test__val_desert__1__000033/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:614875c0a2ab7bbabbbc50a7fe992432c32738ce6a0c7b41c2d3d48455b1f88c +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000201_test__val_desert__1__000033/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000201_test__val_desert__1__000033/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..e090873b1bb3dcc1b26c3547280fc59b7a2ccbd6 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000201_test__val_desert__1__000033/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "201", + "global_test_index": "201", + "dataset_index": "86", + "relative_path": "test/val_desert/1/000033.mp4", + "clip_id": "test/val_desert/1/000033", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000201, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_desert/1/000033_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.000989078776910901, + "psnr": 30.354869842529297, + "lpips": 0.01267954334616661 + }, + "frames_100_200": { + "mse": 0.0007947044214233756, + "psnr": 31.007173538208008, + "lpips": 0.010580032132565975 + }, + "frames_200_300": { + "mse": 0.0008363889064639807, + "psnr": 30.796274185180664, + "lpips": 0.012280055321753025 + }, + "frames_300_400": { + "mse": 0.005243869964033365, + "psnr": 24.293424606323242, + "lpips": 0.07968548685312271 + }, + "frames_400_500": { + "mse": 0.008624348789453506, + "psnr": 21.011791229248047, + "lpips": 0.14952769875526428 + }, + "overall": { + "mse": 0.00329767819494009, + "psnr": 27.492706298828125, + "lpips": 0.05295056477189064 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000202_test__val_savanna__3__000038/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000202_test__val_savanna__3__000038/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..f67646f3eec83c62cf9270cdc46724549ae3c3a1 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000202_test__val_savanna__3__000038/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:09edb423cbac090dda6a03160ccb8fb46cb92718d46fa848b11427bdc0fdd8c2 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000202_test__val_savanna__3__000038/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000202_test__val_savanna__3__000038/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..129c765deb573a5fd644c6b7b0f6d5728880a84b --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000202_test__val_savanna__3__000038/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "202", + "global_test_index": "202", + "dataset_index": "210", + "relative_path": "test/val_savanna/3/000038.mp4", + "clip_id": "test/val_savanna/3/000038", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000202, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_savanna/3/000038_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.004722938872873783, + "psnr": 23.430063247680664, + "lpips": 0.3404156565666199 + }, + "frames_100_200": { + "mse": 0.00544694485142827, + "psnr": 22.69683265686035, + "lpips": 0.36381199955940247 + }, + "frames_200_300": { + "mse": 0.008468350395560265, + "psnr": 20.860485076904297, + "lpips": 0.44137001037597656 + }, + "frames_300_400": { + "mse": 0.009081288240849972, + "psnr": 20.573104858398438, + "lpips": 0.46553757786750793 + }, + "frames_400_500": { + "mse": 0.009622226469218731, + "psnr": 20.333450317382812, + "lpips": 0.482937216758728 + }, + "overall": { + "mse": 0.007468349765986204, + "psnr": 21.578786849975586, + "lpips": 0.4188145101070404 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000203_test__val_desert__3__000039/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000203_test__val_desert__3__000039/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..dd66659c36e867658b154eec9370c21757d9fdc0 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000203_test__val_desert__3__000039/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4fb4e6e35d868df88f735fd979bb5d5a825c39cdc48a7d34aedded6da1683be8 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000203_test__val_desert__3__000039/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000203_test__val_desert__3__000039/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..c7585cbe459bb778c13b3e7131568e5dcbf03afb --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000203_test__val_desert__3__000039/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "203", + "global_test_index": "203", + "dataset_index": "103", + "relative_path": "test/val_desert/3/000039.mp4", + "clip_id": "test/val_desert/3/000039", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000203, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_desert/3/000039_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.007547653745859861, + "psnr": 21.478185653686523, + "lpips": 0.1814785599708557 + }, + "frames_100_200": { + "mse": 0.010597921907901764, + "psnr": 19.994930267333984, + "lpips": 0.21055151522159576 + }, + "frames_200_300": { + "mse": 0.013795408420264721, + "psnr": 21.650541305541992, + "lpips": 0.15725216269493103 + }, + "frames_300_400": { + "mse": 0.006460053846240044, + "psnr": 23.711156845092773, + "lpips": 0.10992742329835892 + }, + "frames_400_500": { + "mse": 0.011621372774243355, + "psnr": 20.654382705688477, + "lpips": 0.21127617359161377 + }, + "overall": { + "mse": 0.010004482232034206, + "psnr": 21.497838973999023, + "lpips": 0.17409716546535492 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000204_test__val_iceplains__3__000000/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000204_test__val_iceplains__3__000000/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..d472126ac34aed50ff072af199ec480b56bfe97b --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000204_test__val_iceplains__3__000000/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6355d1d6a06817db3d89dbc37fb0eaabb55fe4096af68a9a9f7dc6b31baea001 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000204_test__val_iceplains__3__000000/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000204_test__val_iceplains__3__000000/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..5811fce4237c78a542353ac82ee0c6be6fe21190 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000204_test__val_iceplains__3__000000/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "204", + "global_test_index": "204", + "dataset_index": "245", + "relative_path": "test/val_iceplains/3/000000.mp4", + "clip_id": "test/val_iceplains/3/000000", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000204, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/3/000000_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.0007456380990333855, + "psnr": 31.298994064331055, + "lpips": 0.00990358553826809 + }, + "frames_100_200": { + "mse": 0.0007754020625725389, + "psnr": 31.121530532836914, + "lpips": 0.01045573502779007 + }, + "frames_200_300": { + "mse": 0.0007840616744942963, + "psnr": 31.086933135986328, + "lpips": 0.010714685544371605 + }, + "frames_300_400": { + "mse": 0.0007511821459047496, + "psnr": 31.2590274810791, + "lpips": 0.010537814348936081 + }, + "frames_400_500": { + "mse": 0.0011526207672432065, + "psnr": 30.464214324951172, + "lpips": 0.015020805411040783 + }, + "overall": { + "mse": 0.0008417809731326997, + "psnr": 31.046142578125, + "lpips": 0.011326524429023266 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000205_test__val_desert__2__000009/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000205_test__val_desert__2__000009/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..6a7ccd1223f25267f05728a6636040ad81987099 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000205_test__val_desert__2__000009/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5da200628acb5e9f48adb4e46f1ee58172bfd4dbbd2508c095c92a4b7b75c03 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000205_test__val_desert__2__000009/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000205_test__val_desert__2__000009/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..60f058c00e6dceb6f7e4127338a65c3160bb45e3 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000205_test__val_desert__2__000009/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "205", + "global_test_index": "205", + "dataset_index": "25", + "relative_path": "test/val_desert/2/000009.mp4", + "clip_id": "test/val_desert/2/000009", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000205, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_desert/2/000009_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.0014246356440708041, + "psnr": 28.5594539642334, + "lpips": 0.13882257044315338 + }, + "frames_100_200": { + "mse": 0.0016503019724041224, + "psnr": 27.909160614013672, + "lpips": 0.15173126757144928 + }, + "frames_200_300": { + "mse": 0.017557265236973763, + "psnr": 18.081371307373047, + "lpips": 0.28496453166007996 + }, + "frames_300_400": { + "mse": 0.012394178658723831, + "psnr": 20.688003540039062, + "lpips": 0.18890182673931122 + }, + "frames_400_500": { + "mse": 0.004761263262480497, + "psnr": 24.087194442749023, + "lpips": 0.10197010636329651 + }, + "overall": { + "mse": 0.007557529490441084, + "psnr": 23.86503791809082, + "lpips": 0.1732780784368515 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000206_test__val_iceplains__2__000037/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000206_test__val_iceplains__2__000037/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..0288b87adff6000d9005d6c3db749461fad155e9 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000206_test__val_iceplains__2__000037/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:34e22eefc55801753627e797f50a83a6f7eaa6d75af4a5c11790f76ee8ffc4dd +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000206_test__val_iceplains__2__000037/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000206_test__val_iceplains__2__000037/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..94e60fbd5050c1ef32357ae05c6955c65efc1317 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000206_test__val_iceplains__2__000037/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "206", + "global_test_index": "206", + "dataset_index": "340", + "relative_path": "test/val_iceplains/2/000037.mp4", + "clip_id": "test/val_iceplains/2/000037", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000206, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/2/000037_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.05129618942737579, + "psnr": 14.913664817810059, + "lpips": 0.2960737943649292 + }, + "frames_100_200": { + "mse": 0.07810007780790329, + "psnr": 11.388714790344238, + "lpips": 0.3772538900375366 + }, + "frames_200_300": { + "mse": 0.10109966993331909, + "psnr": 10.134106636047363, + "lpips": 0.4902299642562866 + }, + "frames_300_400": { + "mse": 0.08597598969936371, + "psnr": 10.835257530212402, + "lpips": 0.4721630811691284 + }, + "frames_400_500": { + "mse": 0.09798898547887802, + "psnr": 10.704950332641602, + "lpips": 0.4980398416519165 + }, + "overall": { + "mse": 0.08289217948913574, + "psnr": 11.595337867736816, + "lpips": 0.4267520606517792 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000207_test__val_savanna__0__000016/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000207_test__val_savanna__0__000016/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..0568e1e97b6bd8488137e7089a3987f11cf76280 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000207_test__val_savanna__0__000016/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6bd37a7cdeab8cd9deb17f10f42dd7eada7b4ebebc94395224546a5e400e5a2a +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000207_test__val_savanna__0__000016/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000207_test__val_savanna__0__000016/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..e2d5a92c666b9b1a8db8c29c1fd473a4ab414b71 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000207_test__val_savanna__0__000016/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "207", + "global_test_index": "207", + "dataset_index": "167", + "relative_path": "test/val_savanna/0/000016.mp4", + "clip_id": "test/val_savanna/0/000016", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000207, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_savanna/0/000016_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.009269463829696178, + "psnr": 20.793386459350586, + "lpips": 0.17438215017318726 + }, + "frames_100_200": { + "mse": 0.014878853224217892, + "psnr": 18.615251541137695, + "lpips": 0.24145159125328064 + }, + "frames_200_300": { + "mse": 0.026892630383372307, + "psnr": 15.799381256103516, + "lpips": 0.3837350010871887 + }, + "frames_300_400": { + "mse": 0.044748082756996155, + "psnr": 13.714571952819824, + "lpips": 0.5389169454574585 + }, + "frames_400_500": { + "mse": 0.02405548095703125, + "psnr": 16.47006607055664, + "lpips": 0.3582155108451843 + }, + "overall": { + "mse": 0.023968903347849846, + "psnr": 17.078533172607422, + "lpips": 0.3393402397632599 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000208_test__val_plains__0__000022/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000208_test__val_plains__0__000022/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..25bcb9b6e5ce0641e5b18c870634c5f9796dff39 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000208_test__val_plains__0__000022/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:116f05c71a64a6933a96286f54e4c646f725ca1f4542959fabb304b9b204951e +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000208_test__val_plains__0__000022/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000208_test__val_plains__0__000022/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..b68f17daedc8a9940ff033fe8f58d6546e7f6c2a --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000208_test__val_plains__0__000022/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "208", + "global_test_index": "208", + "dataset_index": "424", + "relative_path": "test/val_plains/0/000022.mp4", + "clip_id": "test/val_plains/0/000022", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000208, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_plains/0/000022_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.000832044577691704, + "psnr": 30.8116455078125, + "lpips": 0.011538258753716946 + }, + "frames_100_200": { + "mse": 0.0008968657348304987, + "psnr": 30.519287109375, + "lpips": 0.015591784380376339 + }, + "frames_200_300": { + "mse": 0.001205873559229076, + "psnr": 29.664356231689453, + "lpips": 0.01805671490728855 + }, + "frames_300_400": { + "mse": 0.0009638286428526044, + "psnr": 30.168237686157227, + "lpips": 0.017814965918660164 + }, + "frames_400_500": { + "mse": 0.0012589755933731794, + "psnr": 29.48503875732422, + "lpips": 0.023767225444316864 + }, + "overall": { + "mse": 0.0010315177496522665, + "psnr": 30.129711151123047, + "lpips": 0.017353789880871773 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000209_test__val_plains__2__000033/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000209_test__val_plains__2__000033/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..632926859e5123ec7a07015559aa1d025dcddf65 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000209_test__val_plains__2__000033/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:84e3a2023f3d089cf5519dd82f809c56d58d004677a3aaca6247770f2894085c +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000209_test__val_plains__2__000033/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000209_test__val_plains__2__000033/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..12d5ee53f3f42bc93275e9318018bc4d126796ca --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000209_test__val_plains__2__000033/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "209", + "global_test_index": "209", + "dataset_index": "448", + "relative_path": "test/val_plains/2/000033.mp4", + "clip_id": "test/val_plains/2/000033", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000209, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_plains/2/000033_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.00621925899758935, + "psnr": 22.720619201660156, + "lpips": 0.11256980895996094 + }, + "frames_100_200": { + "mse": 0.006250405218452215, + "psnr": 23.180845260620117, + "lpips": 0.08659686893224716 + }, + "frames_200_300": { + "mse": 0.01633474789559841, + "psnr": 19.26941680908203, + "lpips": 0.20335903763771057 + }, + "frames_300_400": { + "mse": 0.008123981766402721, + "psnr": 23.5685977935791, + "lpips": 0.11010223627090454 + }, + "frames_400_500": { + "mse": 0.0010600705863907933, + "psnr": 30.101951599121094, + "lpips": 0.01847297139465809 + }, + "overall": { + "mse": 0.007597692776471376, + "psnr": 23.768287658691406, + "lpips": 0.10622018575668335 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000210_test__val_plains__0__000006/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000210_test__val_plains__0__000006/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..4f32dce6bfd59f5cebbd8dd729aa61a3906ebd25 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000210_test__val_plains__0__000006/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c76e20a8b0728829884412937349df0498be756dda90faeba1aad508088c38d +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000210_test__val_plains__0__000006/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000210_test__val_plains__0__000006/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..2e25468851b1460ce1879deb873f8c0748acacd2 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000210_test__val_plains__0__000006/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "210", + "global_test_index": "210", + "dataset_index": "386", + "relative_path": "test/val_plains/0/000006.mp4", + "clip_id": "test/val_plains/0/000006", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000210, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_plains/0/000006_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.006581659894436598, + "psnr": 22.206195831298828, + "lpips": 0.22650691866874695 + }, + "frames_100_200": { + "mse": 0.011730985715985298, + "psnr": 19.564531326293945, + "lpips": 0.33458197116851807 + }, + "frames_200_300": { + "mse": 0.018680255860090256, + "psnr": 17.79530143737793, + "lpips": 0.3235115110874176 + }, + "frames_300_400": { + "mse": 0.008956297300755978, + "psnr": 20.645193099975586, + "lpips": 0.44522690773010254 + }, + "frames_400_500": { + "mse": 0.007327401544898748, + "psnr": 21.446725845336914, + "lpips": 0.3899763524532318 + }, + "overall": { + "mse": 0.01065532024949789, + "psnr": 20.33159065246582, + "lpips": 0.3439607322216034 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000211_test__val_plains__3__000008/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000211_test__val_plains__3__000008/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..8faa98a5ee9abdfc46bdb61448c854f42017c808 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000211_test__val_plains__3__000008/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5be2c70b2dd336119bc555a4f070bd0fbd603321ffee58defd81932775e12830 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000211_test__val_plains__3__000008/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000211_test__val_plains__3__000008/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..dc016878549f5b46ca20fe4b3fa5f8b78b93c296 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000211_test__val_plains__3__000008/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "211", + "global_test_index": "211", + "dataset_index": "392", + "relative_path": "test/val_plains/3/000008.mp4", + "clip_id": "test/val_plains/3/000008", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000211, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_plains/3/000008_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.01810871995985508, + "psnr": 17.93561363220215, + "lpips": 0.33405566215515137 + }, + "frames_100_200": { + "mse": 0.013646713458001614, + "psnr": 19.375019073486328, + "lpips": 0.45090949535369873 + }, + "frames_200_300": { + "mse": 0.009611991234123707, + "psnr": 20.279760360717773, + "lpips": 0.5255575776100159 + }, + "frames_300_400": { + "mse": 0.006947372108697891, + "psnr": 22.25067138671875, + "lpips": 0.4279654622077942 + }, + "frames_400_500": { + "mse": 0.011366630904376507, + "psnr": 19.875850677490234, + "lpips": 0.4954170882701874 + }, + "overall": { + "mse": 0.01193628553301096, + "psnr": 19.943382263183594, + "lpips": 0.44678106904029846 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000212_test__val_desert__1__000040/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000212_test__val_desert__1__000040/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..099b39adefdc494e8017b3860b656957836bf00c --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000212_test__val_desert__1__000040/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:000dd69eef14dcecef3d3d35abc47ab7c357b431053997237982a3e5e6120880 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000212_test__val_desert__1__000040/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000212_test__val_desert__1__000040/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..d52c1c21d58b0faab0dc90865b75c13da86ae672 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000212_test__val_desert__1__000040/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "212", + "global_test_index": "212", + "dataset_index": "105", + "relative_path": "test/val_desert/1/000040.mp4", + "clip_id": "test/val_desert/1/000040", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000212, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_desert/1/000040_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.007259534206241369, + "psnr": 21.514493942260742, + "lpips": 0.2166927009820938 + }, + "frames_100_200": { + "mse": 0.009821952320635319, + "psnr": 20.262537002563477, + "lpips": 0.280265212059021 + }, + "frames_200_300": { + "mse": 0.02890259586274624, + "psnr": 15.940967559814453, + "lpips": 0.4257504343986511 + }, + "frames_300_400": { + "mse": 0.03467531502246857, + "psnr": 15.6025972366333, + "lpips": 0.4195886254310608 + }, + "frames_400_500": { + "mse": 0.026516465470194817, + "psnr": 16.863290786743164, + "lpips": 0.2869519889354706 + }, + "overall": { + "mse": 0.021435173228383064, + "psnr": 18.036779403686523, + "lpips": 0.32584983110427856 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000213_test__val_iceplains__2__000008/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000213_test__val_iceplains__2__000008/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..3f9b6a47357c64ddd7919b934faa6a839fa8f542 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000213_test__val_iceplains__2__000008/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c671782d5aea7d332131336f60246e68f81aa831cc1b46238f50edff59d5786 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000213_test__val_iceplains__2__000008/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000213_test__val_iceplains__2__000008/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..6747a37d1f4a0028f5bac51b749a3440fae8a87d --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000213_test__val_iceplains__2__000008/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "213", + "global_test_index": "213", + "dataset_index": "267", + "relative_path": "test/val_iceplains/2/000008.mp4", + "clip_id": "test/val_iceplains/2/000008", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000213, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/2/000008_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.011619916185736656, + "psnr": 19.526647567749023, + "lpips": 0.12673185765743256 + }, + "frames_100_200": { + "mse": 0.010241330601274967, + "psnr": 20.791645050048828, + "lpips": 0.08694499731063843 + }, + "frames_200_300": { + "mse": 0.002712605521082878, + "psnr": 27.93071937561035, + "lpips": 0.020836645737290382 + }, + "frames_300_400": { + "mse": 0.015707289800047874, + "psnr": 19.355125427246094, + "lpips": 0.1134205311536789 + }, + "frames_400_500": { + "mse": 0.0230040792375803, + "psnr": 16.729656219482422, + "lpips": 0.1814320683479309 + }, + "overall": { + "mse": 0.012657043524086475, + "psnr": 20.866758346557617, + "lpips": 0.1058732271194458 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000214_test__val_desert__0__000034/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000214_test__val_desert__0__000034/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..86d2a94f05adfc031108cfc701533b121cb279f1 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000214_test__val_desert__0__000034/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7ee8f2979c1926967d5efddbfc2e9482a2f6acc4742b524911ea0ba1c0d4bf8 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000214_test__val_desert__0__000034/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000214_test__val_desert__0__000034/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..80e27af14f2a7451bcd24019f4d5c8d0c97e7ac4 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000214_test__val_desert__0__000034/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "214", + "global_test_index": "214", + "dataset_index": "90", + "relative_path": "test/val_desert/0/000034.mp4", + "clip_id": "test/val_desert/0/000034", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000214, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_desert/0/000034_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.0009337752126157284, + "psnr": 30.84769058227539, + "lpips": 0.011783388443291187 + }, + "frames_100_200": { + "mse": 0.0012116602156311274, + "psnr": 30.427255630493164, + "lpips": 0.015225457958877087 + }, + "frames_200_300": { + "mse": 0.0007713083759881556, + "psnr": 31.13990020751953, + "lpips": 0.010552528314292431 + }, + "frames_300_400": { + "mse": 0.000892368727363646, + "psnr": 30.770490646362305, + "lpips": 0.012619751505553722 + }, + "frames_400_500": { + "mse": 0.005947737488895655, + "psnr": 23.679237365722656, + "lpips": 0.10557783395051956 + }, + "overall": { + "mse": 0.0019513699226081371, + "psnr": 29.37291717529297, + "lpips": 0.031151792034506798 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000215_test__val_plains__0__000023/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000215_test__val_plains__0__000023/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..efc1d4f37f5d6a73e1de698c18c0c51772b1bafd --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000215_test__val_plains__0__000023/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ac079a04b8636af229c30c6a50bd4b82666397a03abfe56e724b847b33b570b +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000215_test__val_plains__0__000023/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000215_test__val_plains__0__000023/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..d641e57f48c3c2922e8c89712a545011704685f2 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000215_test__val_plains__0__000023/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "215", + "global_test_index": "215", + "dataset_index": "426", + "relative_path": "test/val_plains/0/000023.mp4", + "clip_id": "test/val_plains/0/000023", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000215, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_plains/0/000023_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.00489387521520257, + "psnr": 23.186857223510742, + "lpips": 0.35062336921691895 + }, + "frames_100_200": { + "mse": 0.007710097823292017, + "psnr": 21.350980758666992, + "lpips": 0.4260234534740448 + }, + "frames_200_300": { + "mse": 0.007731682620942593, + "psnr": 21.155113220214844, + "lpips": 0.4427829086780548 + }, + "frames_300_400": { + "mse": 0.006045113317668438, + "psnr": 22.340503692626953, + "lpips": 0.42024052143096924 + }, + "frames_400_500": { + "mse": 0.006546996999531984, + "psnr": 21.94158172607422, + "lpips": 0.44797638058662415 + }, + "overall": { + "mse": 0.006585553288459778, + "psnr": 21.99500846862793, + "lpips": 0.4175293445587158 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000216_test__val_savanna__3__000016/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000216_test__val_savanna__3__000016/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..88da52bd3b2f3973eebb04b348bc9299743eb018 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000216_test__val_savanna__3__000016/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e8142b8d6a22427f9190df212e13cf78dd72e710b8f11c0e03e764af1df3f84a +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000216_test__val_savanna__3__000016/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000216_test__val_savanna__3__000016/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..cac118e4d8d282a17751d156fea132611727174e --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000216_test__val_savanna__3__000016/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "216", + "global_test_index": "216", + "dataset_index": "168", + "relative_path": "test/val_savanna/3/000016.mp4", + "clip_id": "test/val_savanna/3/000016", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000216, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_savanna/3/000016_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.024519432336091995, + "psnr": 16.62133026123047, + "lpips": 0.16429640352725983 + }, + "frames_100_200": { + "mse": 0.049277037382125854, + "psnr": 13.482080459594727, + "lpips": 0.2903948724269867 + }, + "frames_200_300": { + "mse": 0.04812322184443474, + "psnr": 13.459843635559082, + "lpips": 0.3832510709762573 + }, + "frames_300_400": { + "mse": 0.04734478145837784, + "psnr": 13.383462905883789, + "lpips": 0.30156657099723816 + }, + "frames_400_500": { + "mse": 0.05392729863524437, + "psnr": 12.988225936889648, + "lpips": 0.38223645091056824 + }, + "overall": { + "mse": 0.04463835433125496, + "psnr": 13.986988067626953, + "lpips": 0.30434906482696533 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000217_test__val_savanna__3__000040/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000217_test__val_savanna__3__000040/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..5eb3afb15a4f68a026fa05f4843e00bc89fd4585 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000217_test__val_savanna__3__000040/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b3a13288e1a146d49592d857ab57db7e303c53e9934751fa3bd33b5563816378 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000217_test__val_savanna__3__000040/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000217_test__val_savanna__3__000040/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..ecf01ecc2b67db96e87f69a4e0e19365f0dfcadf --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000217_test__val_savanna__3__000040/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "217", + "global_test_index": "217", + "dataset_index": "218", + "relative_path": "test/val_savanna/3/000040.mp4", + "clip_id": "test/val_savanna/3/000040", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000217, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_savanna/3/000040_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.004237588029354811, + "psnr": 24.360883712768555, + "lpips": 0.041114598512649536 + }, + "frames_100_200": { + "mse": 0.006459637079387903, + "psnr": 23.58684730529785, + "lpips": 0.049237318336963654 + }, + "frames_200_300": { + "mse": 0.025219928473234177, + "psnr": 17.582704544067383, + "lpips": 0.13424405455589294 + }, + "frames_300_400": { + "mse": 0.03288602456450462, + "psnr": 15.412240028381348, + "lpips": 0.297998309135437 + }, + "frames_400_500": { + "mse": 0.015393479727208614, + "psnr": 18.864768981933594, + "lpips": 0.356761634349823 + }, + "overall": { + "mse": 0.01683933101594448, + "psnr": 19.961488723754883, + "lpips": 0.17587116360664368 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000218_test__val_plains__0__000032/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000218_test__val_plains__0__000032/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..12ca832360055f8a88a083463efda37cc7bdd0d5 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000218_test__val_plains__0__000032/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ef0474671fb69f4553e8524d3daf49e89880f7cd9119ad342dbb1a383d5a6d8 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000218_test__val_plains__0__000032/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000218_test__val_plains__0__000032/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..21318251cb65363e47f22b2dc1b36d326935399e --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000218_test__val_plains__0__000032/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "218", + "global_test_index": "218", + "dataset_index": "445", + "relative_path": "test/val_plains/0/000032.mp4", + "clip_id": "test/val_plains/0/000032", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000218, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_plains/0/000032_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.0014354180311784148, + "psnr": 29.47576332092285, + "lpips": 0.018947936594486237 + }, + "frames_100_200": { + "mse": 0.0017891519237309694, + "psnr": 29.11267852783203, + "lpips": 0.02219214290380478 + }, + "frames_200_300": { + "mse": 0.000848401861730963, + "psnr": 30.736177444458008, + "lpips": 0.013014128431677818 + }, + "frames_300_400": { + "mse": 0.0010450248373672366, + "psnr": 30.114763259887695, + "lpips": 0.015882553532719612 + }, + "frames_400_500": { + "mse": 0.007345056626945734, + "psnr": 24.748058319091797, + "lpips": 0.0893385112285614 + }, + "overall": { + "mse": 0.0024926105979830027, + "psnr": 28.837488174438477, + "lpips": 0.031875062733888626 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000219_test__val_desert__3__000019/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000219_test__val_desert__3__000019/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..7a58bd0d3d5501797fe095773a9f66186f31c05d --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000219_test__val_desert__3__000019/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:104b05615cdceddc1ad75aac90c33cfcad36d8e006658538e566936b2118503b +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000219_test__val_desert__3__000019/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000219_test__val_desert__3__000019/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..7d7cee844c28396f38b84c47faa140915e62b1f6 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000219_test__val_desert__3__000019/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "219", + "global_test_index": "219", + "dataset_index": "48", + "relative_path": "test/val_desert/3/000019.mp4", + "clip_id": "test/val_desert/3/000019", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000219, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_desert/3/000019_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.0036964653991162777, + "psnr": 26.341245651245117, + "lpips": 0.04687431454658508 + }, + "frames_100_200": { + "mse": 0.007369231432676315, + "psnr": 22.927690505981445, + "lpips": 0.11324251443147659 + }, + "frames_200_300": { + "mse": 0.005152663215994835, + "psnr": 25.772109985351562, + "lpips": 0.0770995020866394 + }, + "frames_300_400": { + "mse": 0.003190748393535614, + "psnr": 26.803869247436523, + "lpips": 0.05312945321202278 + }, + "frames_400_500": { + "mse": 0.015915514901280403, + "psnr": 19.448970794677734, + "lpips": 0.24794988334178925 + }, + "overall": { + "mse": 0.007064924109727144, + "psnr": 24.258779525756836, + "lpips": 0.10765912383794785 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000220_test__val_desert__2__000039/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000220_test__val_desert__2__000039/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..f7fc903de013e77e596b663656fe846133a7ac76 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000220_test__val_desert__2__000039/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:abcbfa9eb7c0962f701d7efe319c509ab072a423d761dbed82f0e8e9e9e9994b +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000220_test__val_desert__2__000039/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000220_test__val_desert__2__000039/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..c3aacdfb20917600fb59c82b02256c21d597e27f --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000220_test__val_desert__2__000039/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "220", + "global_test_index": "220", + "dataset_index": "104", + "relative_path": "test/val_desert/2/000039.mp4", + "clip_id": "test/val_desert/2/000039", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000220, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_desert/2/000039_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.0030203452333807945, + "psnr": 26.387880325317383, + "lpips": 0.09280514717102051 + }, + "frames_100_200": { + "mse": 0.008055887185037136, + "psnr": 21.011987686157227, + "lpips": 0.2699405252933502 + }, + "frames_200_300": { + "mse": 0.013862062245607376, + "psnr": 18.846271514892578, + "lpips": 0.4358113408088684 + }, + "frames_300_400": { + "mse": 0.014866742305457592, + "psnr": 18.434947967529297, + "lpips": 0.4463344216346741 + }, + "frames_400_500": { + "mse": 0.014701093547046185, + "psnr": 18.489749908447266, + "lpips": 0.425394207239151 + }, + "overall": { + "mse": 0.010901226662099361, + "psnr": 20.63416862487793, + "lpips": 0.33405715227127075 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000221_test__val_savanna__2__000035/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000221_test__val_savanna__2__000035/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..5566219f26b6d6a6d34608098e324a66cfbe0ca9 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000221_test__val_savanna__2__000035/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f98ca041ff855763bdf1072f65c3315401f1438be4d055f686e391f73af74c1 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000221_test__val_savanna__2__000035/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000221_test__val_savanna__2__000035/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..0d407c5777d7e9e79a94adf70835ba03b499298e --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000221_test__val_savanna__2__000035/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "221", + "global_test_index": "221", + "dataset_index": "206", + "relative_path": "test/val_savanna/2/000035.mp4", + "clip_id": "test/val_savanna/2/000035", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000221, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_savanna/2/000035_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.010146867483854294, + "psnr": 20.102102279663086, + "lpips": 0.2689058780670166 + }, + "frames_100_200": { + "mse": 0.014401355758309364, + "psnr": 18.69432830810547, + "lpips": 0.26813995838165283 + }, + "frames_200_300": { + "mse": 0.009933455847203732, + "psnr": 21.222667694091797, + "lpips": 0.35345378518104553 + }, + "frames_300_400": { + "mse": 0.005784731358289719, + "psnr": 22.49805450439453, + "lpips": 0.3740805387496948 + }, + "frames_400_500": { + "mse": 0.014195225201547146, + "psnr": 19.036888122558594, + "lpips": 0.46590232849121094 + }, + "overall": { + "mse": 0.01089232787489891, + "psnr": 20.310810089111328, + "lpips": 0.3460964858531952 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000222_test__val_savanna__1__000045/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000222_test__val_savanna__1__000045/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..03e36324a130cabc3d7207f474cbc6f9e995f503 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000222_test__val_savanna__1__000045/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bba642528b3bbe62fa7573c45392aa25b9e2d63b30818f9c5908c60dae0a7f18 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000222_test__val_savanna__1__000045/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000222_test__val_savanna__1__000045/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..61e14d72a30f780ab43284e8e94fd795fcdd108f --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000222_test__val_savanna__1__000045/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "222", + "global_test_index": "222", + "dataset_index": "228", + "relative_path": "test/val_savanna/1/000045.mp4", + "clip_id": "test/val_savanna/1/000045", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000222, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_savanna/1/000045_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.014580752700567245, + "psnr": 19.02422523498535, + "lpips": 0.16638164222240448 + }, + "frames_100_200": { + "mse": 0.03812935948371887, + "psnr": 14.336634635925293, + "lpips": 0.21084724366664886 + }, + "frames_200_300": { + "mse": 0.029722776263952255, + "psnr": 15.7266206741333, + "lpips": 0.22795821726322174 + }, + "frames_300_400": { + "mse": 0.017635684460401535, + "psnr": 18.183141708374023, + "lpips": 0.3094389736652374 + }, + "frames_400_500": { + "mse": 0.014085028320550919, + "psnr": 18.829269409179688, + "lpips": 0.33859238028526306 + }, + "overall": { + "mse": 0.022830720990896225, + "psnr": 17.2199764251709, + "lpips": 0.25064367055892944 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000223_test__val_savanna__2__000033/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000223_test__val_savanna__2__000033/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..c0184803994a0c7180190ebbfd729f215ce0e462 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000223_test__val_savanna__2__000033/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df925ce659b35faa7469d0ef1c7c2c99b4d8ba780902c6c3f21ca74a97fe75ed +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000223_test__val_savanna__2__000033/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000223_test__val_savanna__2__000033/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..108a2f5430d948a97b0c582dc6f2f4b76fa0b833 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000223_test__val_savanna__2__000033/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "223", + "global_test_index": "223", + "dataset_index": "200", + "relative_path": "test/val_savanna/2/000033.mp4", + "clip_id": "test/val_savanna/2/000033", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000223, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_savanna/2/000033_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.003493639174848795, + "psnr": 24.67325210571289, + "lpips": 0.22806449234485626 + }, + "frames_100_200": { + "mse": 0.005864209961146116, + "psnr": 22.448606491088867, + "lpips": 0.35851791501045227 + }, + "frames_200_300": { + "mse": 0.004768610466271639, + "psnr": 23.252891540527344, + "lpips": 0.35443103313446045 + }, + "frames_300_400": { + "mse": 0.022612258791923523, + "psnr": 16.851085662841797, + "lpips": 0.38914695382118225 + }, + "frames_400_500": { + "mse": 0.03704902157187462, + "psnr": 14.667105674743652, + "lpips": 0.3367379903793335 + }, + "overall": { + "mse": 0.014757548458874226, + "psnr": 20.378585815429688, + "lpips": 0.3333796560764313 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000224_test__val_desert__2__000024/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000224_test__val_desert__2__000024/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..a39b5b8cdd4f656836607687f34cefa5a88d923d --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000224_test__val_desert__2__000024/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f9db2f39ce4e9241e0c359c8ab1b2f38ecf15c1252faed971a6a52c47f37f22a +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000224_test__val_desert__2__000024/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000224_test__val_desert__2__000024/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..94a25805a32187c0eea0236c9d2ba4c355032bb9 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000224_test__val_desert__2__000024/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "224", + "global_test_index": "224", + "dataset_index": "62", + "relative_path": "test/val_desert/2/000024.mp4", + "clip_id": "test/val_desert/2/000024", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000224, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_desert/2/000024_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.006876545492559671, + "psnr": 21.70090103149414, + "lpips": 0.21602897346019745 + }, + "frames_100_200": { + "mse": 0.011868787929415703, + "psnr": 19.6616268157959, + "lpips": 0.26174044609069824 + }, + "frames_200_300": { + "mse": 0.011893062852323055, + "psnr": 19.649063110351562, + "lpips": 0.29468587040901184 + }, + "frames_300_400": { + "mse": 0.017499208450317383, + "psnr": 17.955171585083008, + "lpips": 0.39390042424201965 + }, + "frames_400_500": { + "mse": 0.014317640103399754, + "psnr": 18.650039672851562, + "lpips": 0.37355419993400574 + }, + "overall": { + "mse": 0.01249105017632246, + "psnr": 19.523357391357422, + "lpips": 0.3079819977283478 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000225_test__val_desert__1__000047/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000225_test__val_desert__1__000047/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..9dbcde2250dfda2c49f2b8f34e2310e694357c9a --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000225_test__val_desert__1__000047/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0fc68a8ac908078573b112988923841dd198f5b929198663f03f32bbbed3ecc +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000225_test__val_desert__1__000047/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000225_test__val_desert__1__000047/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..892b5d1f8b1b49a0ab755afae61785cb6da3a7a5 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000225_test__val_desert__1__000047/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "225", + "global_test_index": "225", + "dataset_index": "121", + "relative_path": "test/val_desert/1/000047.mp4", + "clip_id": "test/val_desert/1/000047", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000225, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_desert/1/000047_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.006421353667974472, + "psnr": 22.014869689941406, + "lpips": 0.2282811403274536 + }, + "frames_100_200": { + "mse": 0.007262602914124727, + "psnr": 21.415189743041992, + "lpips": 0.2860904335975647 + }, + "frames_200_300": { + "mse": 0.007348658982664347, + "psnr": 21.377300262451172, + "lpips": 0.35206854343414307 + }, + "frames_300_400": { + "mse": 0.013105465099215508, + "psnr": 19.351177215576172, + "lpips": 0.4281613826751709 + }, + "frames_400_500": { + "mse": 0.015741577371954918, + "psnr": 18.72663116455078, + "lpips": 0.3871784210205078 + }, + "overall": { + "mse": 0.009975932538509369, + "psnr": 20.577035903930664, + "lpips": 0.33635595440864563 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000226_test__val_savanna__0__000007/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000226_test__val_savanna__0__000007/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..28c1cc6e613bb404903029d87f4b488041e97ad2 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000226_test__val_savanna__0__000007/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:412f16ba25dea9002e0a800148c57f030023aec65505229f7c4c7ea278f188ef +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000226_test__val_savanna__0__000007/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000226_test__val_savanna__0__000007/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..09938f18de08d7c30bebd97eeec042e255f3973e --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000226_test__val_savanna__0__000007/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "226", + "global_test_index": "226", + "dataset_index": "147", + "relative_path": "test/val_savanna/0/000007.mp4", + "clip_id": "test/val_savanna/0/000007", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000226, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_savanna/0/000007_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.003918759990483522, + "psnr": 24.285415649414062, + "lpips": 0.23890739679336548 + }, + "frames_100_200": { + "mse": 0.011149045079946518, + "psnr": 20.551742553710938, + "lpips": 0.3337075114250183 + }, + "frames_200_300": { + "mse": 0.044299498200416565, + "psnr": 14.56477165222168, + "lpips": 0.3630608022212982 + }, + "frames_300_400": { + "mse": 0.022964471951127052, + "psnr": 16.806787490844727, + "lpips": 0.3323090672492981 + }, + "frames_400_500": { + "mse": 0.04899601638317108, + "psnr": 14.3379545211792, + "lpips": 0.37749478220939636 + }, + "overall": { + "mse": 0.02626555785536766, + "psnr": 18.109336853027344, + "lpips": 0.3290959596633911 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000227_test__val_savanna__2__000006/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000227_test__val_savanna__2__000006/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..379b08a185d3f03c802390c7f72d1a15af82814a --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000227_test__val_savanna__2__000006/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a1bd3c081ee01a419a948b9966e5c39fc862f338192c61bb045d6ebc2891f8ca +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000227_test__val_savanna__2__000006/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000227_test__val_savanna__2__000006/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..80517b070134899436e0d6e5b2a3ff68019e6b2f --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000227_test__val_savanna__2__000006/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "227", + "global_test_index": "227", + "dataset_index": "146", + "relative_path": "test/val_savanna/2/000006.mp4", + "clip_id": "test/val_savanna/2/000006", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000227, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_savanna/2/000006_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.019700516015291214, + "psnr": 19.215652465820312, + "lpips": 0.4279322028160095 + }, + "frames_100_200": { + "mse": 0.031372860074043274, + "psnr": 15.316547393798828, + "lpips": 0.42393848299980164 + }, + "frames_200_300": { + "mse": 0.010345534421503544, + "psnr": 20.504980087280273, + "lpips": 0.44339102506637573 + }, + "frames_300_400": { + "mse": 0.010926314629614353, + "psnr": 20.755266189575195, + "lpips": 0.4324131906032562 + }, + "frames_400_500": { + "mse": 0.03689274191856384, + "psnr": 14.645551681518555, + "lpips": 0.4420328438282013 + }, + "overall": { + "mse": 0.021847592666745186, + "psnr": 18.08759880065918, + "lpips": 0.4339416027069092 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000228_test__val_savanna__1__000043/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000228_test__val_savanna__1__000043/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..4d4c4fe6615895091a95d5a0d97aed65bb8cce54 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000228_test__val_savanna__1__000043/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe0669fd3a1e116db593247aec13298141898ad150bc4a5e959b41f30cd5feac +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000228_test__val_savanna__1__000043/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000228_test__val_savanna__1__000043/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..6610aa660135e36d9d6c821d56e02d9fc7fe8126 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000228_test__val_savanna__1__000043/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "228", + "global_test_index": "228", + "dataset_index": "225", + "relative_path": "test/val_savanna/1/000043.mp4", + "clip_id": "test/val_savanna/1/000043", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000228, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_savanna/1/000043_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.007034569978713989, + "psnr": 22.234689712524414, + "lpips": 0.3033251166343689 + }, + "frames_100_200": { + "mse": 0.008615821599960327, + "psnr": 20.94219970703125, + "lpips": 0.37405508756637573 + }, + "frames_200_300": { + "mse": 0.00888727605342865, + "psnr": 20.59259033203125, + "lpips": 0.44988536834716797 + }, + "frames_300_400": { + "mse": 0.009484831243753433, + "psnr": 20.299983978271484, + "lpips": 0.4412327706813812 + }, + "frames_400_500": { + "mse": 0.007243102416396141, + "psnr": 21.511369705200195, + "lpips": 0.4495011568069458 + }, + "overall": { + "mse": 0.008253120817244053, + "psnr": 21.116165161132812, + "lpips": 0.40359988808631897 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000229_test__val_iceplains__3__000027/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000229_test__val_iceplains__3__000027/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..74f23a7e0b1d6ec82c6260c9fb476fc414b2791b --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000229_test__val_iceplains__3__000027/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2fa2459726693144b8ff7f9419d5701ebc090965d0d636f485f926f19dae4ed4 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000229_test__val_iceplains__3__000027/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000229_test__val_iceplains__3__000027/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..27f8cfd5af052e20adcc5e6d8062093e5b40f9a0 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000229_test__val_iceplains__3__000027/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "229", + "global_test_index": "229", + "dataset_index": "313", + "relative_path": "test/val_iceplains/3/000027.mp4", + "clip_id": "test/val_iceplains/3/000027", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000229, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/3/000027_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.0010228451574221253, + "psnr": 30.777963638305664, + "lpips": 0.01129656657576561 + }, + "frames_100_200": { + "mse": 0.0011125062592327595, + "psnr": 29.841066360473633, + "lpips": 0.015034467913210392 + }, + "frames_200_300": { + "mse": 0.0011964308796450496, + "psnr": 29.414875030517578, + "lpips": 0.017070097848773003 + }, + "frames_300_400": { + "mse": 0.0016202223487198353, + "psnr": 28.581972122192383, + "lpips": 0.021295839920639992 + }, + "frames_400_500": { + "mse": 0.0048498790711164474, + "psnr": 25.818025588989258, + "lpips": 0.042263150215148926 + }, + "overall": { + "mse": 0.0019603767432272434, + "psnr": 28.886777877807617, + "lpips": 0.02139202505350113 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000230_test__val_iceplains__0__000042/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000230_test__val_iceplains__0__000042/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..929fef1c245f71256c31a23bee6d9da3eaeb6b45 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000230_test__val_iceplains__0__000042/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:79f050b785a9aedef8b8a3aa47ea2ab0db27f83a5158a7c1ad4de5e0160faa66 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000230_test__val_iceplains__0__000042/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000230_test__val_iceplains__0__000042/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..a22f4208da6b6a3ac83e50d8d80f67edd634f6e0 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000230_test__val_iceplains__0__000042/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "230", + "global_test_index": "230", + "dataset_index": "354", + "relative_path": "test/val_iceplains/0/000042.mp4", + "clip_id": "test/val_iceplains/0/000042", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000230, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/0/000042_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.02341621182858944, + "psnr": 17.191011428833008, + "lpips": 0.19668422639369965 + }, + "frames_100_200": { + "mse": 0.03223161771893501, + "psnr": 15.169669151306152, + "lpips": 0.25903990864753723 + }, + "frames_200_300": { + "mse": 0.056772999465465546, + "psnr": 12.560439109802246, + "lpips": 0.3321693539619446 + }, + "frames_300_400": { + "mse": 0.04457579180598259, + "psnr": 14.438989639282227, + "lpips": 0.23784175515174866 + }, + "frames_400_500": { + "mse": 0.011740028858184814, + "psnr": 21.138629913330078, + "lpips": 0.07994338870048523 + }, + "overall": { + "mse": 0.03374732658267021, + "psnr": 16.099748611450195, + "lpips": 0.2211357206106186 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000231_test__val_iceplains__1__000048/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000231_test__val_iceplains__1__000048/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..cad69bda34856197a0b42d79ae100ac6e324331b --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000231_test__val_iceplains__1__000048/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0caef9f36b720bd0bc3d41a42083eee7b64779838d12dfd833bd76d2c13f7e2f +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000231_test__val_iceplains__1__000048/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000231_test__val_iceplains__1__000048/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..5b9a9a54c01a7d5f754e07d4a671ba5bc71e3d4e --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000231_test__val_iceplains__1__000048/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "231", + "global_test_index": "231", + "dataset_index": "367", + "relative_path": "test/val_iceplains/1/000048.mp4", + "clip_id": "test/val_iceplains/1/000048", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000231, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/1/000048_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.007495575118809938, + "psnr": 21.604917526245117, + "lpips": 0.12377846986055374 + }, + "frames_100_200": { + "mse": 0.004100011195987463, + "psnr": 23.888446807861328, + "lpips": 0.2036283165216446 + }, + "frames_200_300": { + "mse": 0.02538377232849598, + "psnr": 18.646259307861328, + "lpips": 0.34038084745407104 + }, + "frames_300_400": { + "mse": 0.0043318262323737144, + "psnr": 23.641429901123047, + "lpips": 0.32908570766448975 + }, + "frames_400_500": { + "mse": 0.004400254692882299, + "psnr": 23.577255249023438, + "lpips": 0.34433817863464355 + }, + "overall": { + "mse": 0.009142288006842136, + "psnr": 22.271663665771484, + "lpips": 0.2682422995567322 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000232_test__val_iceplains__3__000020/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000232_test__val_iceplains__3__000020/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..9763f4c8dde19a6eaa634c899121d631b4e02f18 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000232_test__val_iceplains__3__000020/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb090e97379e9dd3c08a7581ca967cd168e6ce9cd1f1fd6f6026760915fb065f +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000232_test__val_iceplains__3__000020/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000232_test__val_iceplains__3__000020/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..9cc5ca5c166e45c2700b98e7dca9406e70362f82 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000232_test__val_iceplains__3__000020/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "232", + "global_test_index": "232", + "dataset_index": "297", + "relative_path": "test/val_iceplains/3/000020.mp4", + "clip_id": "test/val_iceplains/3/000020", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000232, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/3/000020_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.00775052048265934, + "psnr": 21.318252563476562, + "lpips": 0.1690644472837448 + }, + "frames_100_200": { + "mse": 0.009206803515553474, + "psnr": 20.553634643554688, + "lpips": 0.19736240804195404 + }, + "frames_200_300": { + "mse": 0.011746960692107677, + "psnr": 19.36302375793457, + "lpips": 0.22147825360298157 + }, + "frames_300_400": { + "mse": 0.015266832895576954, + "psnr": 18.4599552154541, + "lpips": 0.3515205383300781 + }, + "frames_400_500": { + "mse": 0.04309134557843208, + "psnr": 14.036503791809082, + "lpips": 0.39286744594573975 + }, + "overall": { + "mse": 0.017412494868040085, + "psnr": 18.746273040771484, + "lpips": 0.2664586305618286 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000233_test__val_desert__1__000048/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000233_test__val_desert__1__000048/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..0de4136c8b1f884b11eb02e4fb9b252a9255ac78 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000233_test__val_desert__1__000048/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c70cb04019bba6a936c1bad18e9eb014345f501716755b763675ffdb89ffd064 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000233_test__val_desert__1__000048/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000233_test__val_desert__1__000048/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..679dfad293d2830d37aecee88f28619017ffec28 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000233_test__val_desert__1__000048/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "233", + "global_test_index": "233", + "dataset_index": "124", + "relative_path": "test/val_desert/1/000048.mp4", + "clip_id": "test/val_desert/1/000048", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000233, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_desert/1/000048_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.00612955866381526, + "psnr": 22.206401824951172, + "lpips": 0.19396036863327026 + }, + "frames_100_200": { + "mse": 0.006139801349490881, + "psnr": 22.195737838745117, + "lpips": 0.19713877141475677 + }, + "frames_200_300": { + "mse": 0.0033441828563809395, + "psnr": 25.289306640625, + "lpips": 0.08675925433635712 + }, + "frames_300_400": { + "mse": 0.006677063647657633, + "psnr": 21.894922256469727, + "lpips": 0.1856546849012375 + }, + "frames_400_500": { + "mse": 0.010102961212396622, + "psnr": 20.129701614379883, + "lpips": 0.23885756731033325 + }, + "overall": { + "mse": 0.006478713359683752, + "psnr": 22.343212127685547, + "lpips": 0.18047411739826202 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000234_test__val_savanna__2__000017/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000234_test__val_savanna__2__000017/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..d7149b309f1b15296b08c91bdd421585d6ea425a --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000234_test__val_savanna__2__000017/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:83ee106c525e275f3e3ebcacb993c0b1053beb378f418d55555eb081efc9fcae +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000234_test__val_savanna__2__000017/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000234_test__val_savanna__2__000017/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..02c22101ad56d5339a7f61c96e2ba2d70041c86a --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000234_test__val_savanna__2__000017/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "234", + "global_test_index": "234", + "dataset_index": "171", + "relative_path": "test/val_savanna/2/000017.mp4", + "clip_id": "test/val_savanna/2/000017", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000234, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_savanna/2/000017_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.03695836663246155, + "psnr": 15.636307716369629, + "lpips": 0.32780739665031433 + }, + "frames_100_200": { + "mse": 0.028755266219377518, + "psnr": 17.12102699279785, + "lpips": 0.40321627259254456 + }, + "frames_200_300": { + "mse": 0.0061502354219555855, + "psnr": 22.445032119750977, + "lpips": 0.3773205280303955 + }, + "frames_300_400": { + "mse": 0.005700491834431887, + "psnr": 22.610204696655273, + "lpips": 0.3886846601963043 + }, + "frames_400_500": { + "mse": 0.0047443644143640995, + "psnr": 23.271387100219727, + "lpips": 0.3612760901451111 + }, + "overall": { + "mse": 0.016461746767163277, + "psnr": 20.216793060302734, + "lpips": 0.371660977602005 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000235_test__val_iceplains__0__000038/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000235_test__val_iceplains__0__000038/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..48ae1d604a9ec5389c17f2c9d2c563b578a07b8c --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000235_test__val_iceplains__0__000038/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e04b5b4dcbd3cb4286388156019af2224bdc3369f6558d15ee660c69a178fb1 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000235_test__val_iceplains__0__000038/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000235_test__val_iceplains__0__000038/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..2b8c339e9f9461a7c6e2d4dcbef21629095e9da4 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000235_test__val_iceplains__0__000038/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "235", + "global_test_index": "235", + "dataset_index": "342", + "relative_path": "test/val_iceplains/0/000038.mp4", + "clip_id": "test/val_iceplains/0/000038", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000235, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/0/000038_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.006894542369991541, + "psnr": 21.834199905395508, + "lpips": 0.13142462074756622 + }, + "frames_100_200": { + "mse": 0.007497596088796854, + "psnr": 21.684904098510742, + "lpips": 0.14737233519554138 + }, + "frames_200_300": { + "mse": 0.009259399957954884, + "psnr": 20.443614959716797, + "lpips": 0.13073672354221344 + }, + "frames_300_400": { + "mse": 0.005133780185133219, + "psnr": 23.040122985839844, + "lpips": 0.18723516166210175 + }, + "frames_400_500": { + "mse": 0.005882397759705782, + "psnr": 22.697145462036133, + "lpips": 0.16812202334403992 + }, + "overall": { + "mse": 0.006933543831110001, + "psnr": 21.939998626708984, + "lpips": 0.15297818183898926 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000236_test__val_desert__3__000023/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000236_test__val_desert__3__000023/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..b73ee9e65338fb4fa026760f48b61a2218e5a2c9 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000236_test__val_desert__3__000023/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff63dc68dd588ec7e742c6966150eff434debd34e3503af59256ebf4f8249bdc +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000236_test__val_desert__3__000023/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000236_test__val_desert__3__000023/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..78a8ac055c50604cff1bd43451fe0e8c2b82cabf --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000236_test__val_desert__3__000023/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "236", + "global_test_index": "236", + "dataset_index": "60", + "relative_path": "test/val_desert/3/000023.mp4", + "clip_id": "test/val_desert/3/000023", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000236, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_desert/3/000023_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.007305053994059563, + "psnr": 21.545385360717773, + "lpips": 0.24812400341033936 + }, + "frames_100_200": { + "mse": 0.010411831550300121, + "psnr": 20.037595748901367, + "lpips": 0.4165325462818146 + }, + "frames_200_300": { + "mse": 0.017913397401571274, + "psnr": 17.597997665405273, + "lpips": 0.440107524394989 + }, + "frames_300_400": { + "mse": 0.01386736985296011, + "psnr": 18.813282012939453, + "lpips": 0.3935522735118866 + }, + "frames_400_500": { + "mse": 0.019662855193018913, + "psnr": 17.857421875, + "lpips": 0.4147801995277405 + }, + "overall": { + "mse": 0.013832100667059422, + "psnr": 19.170337677001953, + "lpips": 0.38261929154396057 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000237_test__val_savanna__1__000001/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000237_test__val_savanna__1__000001/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..f439619a9480366fad60dfd75c266e83f9964497 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000237_test__val_savanna__1__000001/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eea74f8f546f7788d4dd3338268de5f238df58b157cb07c5739d012b59c27d9f +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000237_test__val_savanna__1__000001/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000237_test__val_savanna__1__000001/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..e3e3301689c7ddb9c1b4ff386f2932bccb0d42e3 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000237_test__val_savanna__1__000001/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "237", + "global_test_index": "237", + "dataset_index": "131", + "relative_path": "test/val_savanna/1/000001.mp4", + "clip_id": "test/val_savanna/1/000001", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000237, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_savanna/1/000001_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.004401722922921181, + "psnr": 23.68244171142578, + "lpips": 0.27250686287879944 + }, + "frames_100_200": { + "mse": 0.004498502239584923, + "psnr": 23.4962100982666, + "lpips": 0.3318064212799072 + }, + "frames_200_300": { + "mse": 0.005111666861921549, + "psnr": 22.947710037231445, + "lpips": 0.34755054116249084 + }, + "frames_300_400": { + "mse": 0.005247978493571281, + "psnr": 22.825376510620117, + "lpips": 0.33681362867355347 + }, + "frames_400_500": { + "mse": 0.0053060525096952915, + "psnr": 22.80716323852539, + "lpips": 0.352163165807724 + }, + "overall": { + "mse": 0.004913184326142073, + "psnr": 23.15178108215332, + "lpips": 0.3281681537628174 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000238_test__val_iceplains__1__000047/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000238_test__val_iceplains__1__000047/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..867dea31c42cbb2f8fb77c0f91edf1e3571f07db --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000238_test__val_iceplains__1__000047/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15cf859cf7af4a201fef790f23472c3436bcb6e6b7299fc30223eff66040912c +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000238_test__val_iceplains__1__000047/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000238_test__val_iceplains__1__000047/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..cb8f5cc84f2b18d4a093b1c76a36218dfe6d2a5f --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000238_test__val_iceplains__1__000047/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "238", + "global_test_index": "238", + "dataset_index": "365", + "relative_path": "test/val_iceplains/1/000047.mp4", + "clip_id": "test/val_iceplains/1/000047", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000238, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/1/000047_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.005395991262048483, + "psnr": 23.376636505126953, + "lpips": 0.11338348686695099 + }, + "frames_100_200": { + "mse": 0.029078591614961624, + "psnr": 20.431861877441406, + "lpips": 0.2208804488182068 + }, + "frames_200_300": { + "mse": 0.07972981780767441, + "psnr": 11.887426376342773, + "lpips": 0.41528642177581787 + }, + "frames_300_400": { + "mse": 0.17644552886486053, + "psnr": 8.224865913391113, + "lpips": 0.5843903422355652 + }, + "frames_400_500": { + "mse": 0.16525354981422424, + "psnr": 7.946079254150391, + "lpips": 0.5523620843887329 + }, + "overall": { + "mse": 0.09118068963289261, + "psnr": 14.373373031616211, + "lpips": 0.3772605359554291 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000239_test__val_iceplains__2__000028/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000239_test__val_iceplains__2__000028/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..66262f8f8a63763c88f61a14786947aebe7fd6fd --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000239_test__val_iceplains__2__000028/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f46f92df7571ee890b3e50228f3ad1289b7fe9acc734e4efbf77ac96950928de +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000239_test__val_iceplains__2__000028/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000239_test__val_iceplains__2__000028/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..39481993077775eff24b0064231679b9c4e6aa3a --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000239_test__val_iceplains__2__000028/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "239", + "global_test_index": "239", + "dataset_index": "315", + "relative_path": "test/val_iceplains/2/000028.mp4", + "clip_id": "test/val_iceplains/2/000028", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000239, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/2/000028_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.010294110514223576, + "psnr": 20.262205123901367, + "lpips": 0.09520455449819565 + }, + "frames_100_200": { + "mse": 0.04098287969827652, + "psnr": 14.456059455871582, + "lpips": 0.2934500277042389 + }, + "frames_200_300": { + "mse": 0.042673807591199875, + "psnr": 14.120894432067871, + "lpips": 0.28389137983322144 + }, + "frames_300_400": { + "mse": 0.08627994358539581, + "psnr": 10.837449073791504, + "lpips": 0.45699822902679443 + }, + "frames_400_500": { + "mse": 0.11862234771251678, + "psnr": 9.424814224243164, + "lpips": 0.5584912300109863 + }, + "overall": { + "mse": 0.059770617634058, + "psnr": 13.820283889770508, + "lpips": 0.3376070559024811 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000240_test__val_savanna__1__000041/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000240_test__val_savanna__1__000041/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..077ee71356657ea59ed0dbbd6d472ddefd7319ba --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000240_test__val_savanna__1__000041/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:243655d6ffbf1b2844c51789048f4a9e0d3fc48c8e6a9dce96d4905727fbee2d +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000240_test__val_savanna__1__000041/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000240_test__val_savanna__1__000041/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..444ed80856283af3630658872fbc269334faf67c --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000240_test__val_savanna__1__000041/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "240", + "global_test_index": "240", + "dataset_index": "220", + "relative_path": "test/val_savanna/1/000041.mp4", + "clip_id": "test/val_savanna/1/000041", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000240, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_savanna/1/000041_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.008934867568314075, + "psnr": 20.635526657104492, + "lpips": 0.15110749006271362 + }, + "frames_100_200": { + "mse": 0.013156717643141747, + "psnr": 19.230674743652344, + "lpips": 0.17636117339134216 + }, + "frames_200_300": { + "mse": 0.015155438333749771, + "psnr": 18.46820640563965, + "lpips": 0.33629676699638367 + }, + "frames_300_400": { + "mse": 0.012678113766014576, + "psnr": 19.070858001708984, + "lpips": 0.3838249146938324 + }, + "frames_400_500": { + "mse": 0.01614559441804886, + "psnr": 18.395835876464844, + "lpips": 0.40232613682746887 + }, + "overall": { + "mse": 0.013214146718382835, + "psnr": 19.160221099853516, + "lpips": 0.2899833023548126 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000241_test__val_desert__1__000003/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000241_test__val_desert__1__000003/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..f947ca400c34f5d6f5f2abc40f87493e2b6b84d6 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000241_test__val_desert__1__000003/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:65dff349510accee94b97d04359f379d6460123f8d8f0d4a327fed48f8bd2ce4 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000241_test__val_desert__1__000003/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000241_test__val_desert__1__000003/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..4b8d19f086dbc7df8eebcdde35690b7414011f93 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000241_test__val_desert__1__000003/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "241", + "global_test_index": "241", + "dataset_index": "8", + "relative_path": "test/val_desert/1/000003.mp4", + "clip_id": "test/val_desert/1/000003", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000241, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_desert/1/000003_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.0011857941281050444, + "psnr": 30.402700424194336, + "lpips": 0.015097396448254585 + }, + "frames_100_200": { + "mse": 0.0008427728898823261, + "psnr": 30.781259536743164, + "lpips": 0.012150674127042294 + }, + "frames_200_300": { + "mse": 0.0008141730213537812, + "psnr": 30.919832229614258, + "lpips": 0.011515984311699867 + }, + "frames_300_400": { + "mse": 0.0007923657540231943, + "psnr": 31.039081573486328, + "lpips": 0.010770270600914955 + }, + "frames_400_500": { + "mse": 0.0008864761330187321, + "psnr": 30.61347007751465, + "lpips": 0.013910030946135521 + }, + "overall": { + "mse": 0.0009043164900504053, + "psnr": 30.75126838684082, + "lpips": 0.01268887147307396 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000242_test__val_iceplains__0__000006/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000242_test__val_iceplains__0__000006/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..05c770841aeb515fc8b7af91dd38fc2197db1fbc --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000242_test__val_iceplains__0__000006/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:75f87ea161e2e608933bad1e7d568aae743a1c570c5c8a3d9a3a0c81c5dbb975 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000242_test__val_iceplains__0__000006/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000242_test__val_iceplains__0__000006/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..01e14149540774d397d0ae89ed6a234da331e5cb --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000242_test__val_iceplains__0__000006/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "242", + "global_test_index": "242", + "dataset_index": "261", + "relative_path": "test/val_iceplains/0/000006.mp4", + "clip_id": "test/val_iceplains/0/000006", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000242, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/0/000006_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.0046816421672701836, + "psnr": 23.43265151977539, + "lpips": 0.21842342615127563 + }, + "frames_100_200": { + "mse": 0.12228460609912872, + "psnr": 12.085269927978516, + "lpips": 0.4387568533420563 + }, + "frames_200_300": { + "mse": 0.16087408363819122, + "psnr": 8.290518760681152, + "lpips": 0.572498083114624 + }, + "frames_300_400": { + "mse": 0.07240387052297592, + "psnr": 11.773868560791016, + "lpips": 0.4211384654045105 + }, + "frames_400_500": { + "mse": 0.01948985457420349, + "psnr": 20.304027557373047, + "lpips": 0.3625389337539673 + }, + "overall": { + "mse": 0.07594680786132812, + "psnr": 15.177268028259277, + "lpips": 0.40267112851142883 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000243_test__val_desert__1__000007/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000243_test__val_desert__1__000007/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..616f4343b8ed1b5f5d9b269df7ef3e988d1803f3 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000243_test__val_desert__1__000007/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:93fffe05cbe3d198d36bff601bcb0572b1d428ffbda3145405bd8bd0e8278140 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000243_test__val_desert__1__000007/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000243_test__val_desert__1__000007/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..e1948887f93428043fdc61585894db9ac82b50ad --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000243_test__val_desert__1__000007/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "243", + "global_test_index": "243", + "dataset_index": "17", + "relative_path": "test/val_desert/1/000007.mp4", + "clip_id": "test/val_desert/1/000007", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000243, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_desert/1/000007_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.006003387738019228, + "psnr": 24.57956314086914, + "lpips": 0.06440512835979462 + }, + "frames_100_200": { + "mse": 0.007510168012231588, + "psnr": 23.266572952270508, + "lpips": 0.0657050684094429 + }, + "frames_200_300": { + "mse": 0.00864707212895155, + "psnr": 21.840173721313477, + "lpips": 0.08548736572265625 + }, + "frames_300_400": { + "mse": 0.007683320436626673, + "psnr": 24.2474422454834, + "lpips": 0.08921289443969727 + }, + "frames_400_500": { + "mse": 0.00791888777166605, + "psnr": 23.782188415527344, + "lpips": 0.07282619178295135 + }, + "overall": { + "mse": 0.007552567403763533, + "psnr": 23.54318618774414, + "lpips": 0.07552732527256012 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000244_test__val_plains__1__000043/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000244_test__val_plains__1__000043/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..932d40142f890b5575fa34c42c162642bcd6b26f --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000244_test__val_plains__1__000043/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:00f95d642a3a0e1ca1572b3571d736432103e5eced0d162a42244f1fc8e57da6 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000244_test__val_plains__1__000043/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000244_test__val_plains__1__000043/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..16d32a4c0174763cddbe5cf0b91ceefc8a499b7c --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000244_test__val_plains__1__000043/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "244", + "global_test_index": "244", + "dataset_index": "462", + "relative_path": "test/val_plains/1/000043.mp4", + "clip_id": "test/val_plains/1/000043", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000244, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_plains/1/000043_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.003991454839706421, + "psnr": 24.3883113861084, + "lpips": 0.134834423661232 + }, + "frames_100_200": { + "mse": 0.004811493679881096, + "psnr": 23.251073837280273, + "lpips": 0.17274440824985504 + }, + "frames_200_300": { + "mse": 0.004346376284956932, + "psnr": 23.72034454345703, + "lpips": 0.08532891422510147 + }, + "frames_300_400": { + "mse": 0.0032203432638198137, + "psnr": 25.247900009155273, + "lpips": 0.0752578154206276 + }, + "frames_400_500": { + "mse": 0.0017574612284079194, + "psnr": 29.214380264282227, + "lpips": 0.024497294798493385 + }, + "overall": { + "mse": 0.0036254259757697582, + "psnr": 25.164400100708008, + "lpips": 0.09853257238864899 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000245_test__val_plains__3__000038/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000245_test__val_plains__3__000038/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..5a10cabf5744e66a896a263ad7d6b6f608745422 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000245_test__val_plains__3__000038/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6dc93b65e9153498c50926cea8bcef7bc28bd1a894d3f25a21b90aac991674a4 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000245_test__val_plains__3__000038/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000245_test__val_plains__3__000038/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..180004e8181b9f93c0f0fc0e1b7a278c191a7925 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000245_test__val_plains__3__000038/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "245", + "global_test_index": "245", + "dataset_index": "454", + "relative_path": "test/val_plains/3/000038.mp4", + "clip_id": "test/val_plains/3/000038", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000245, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_plains/3/000038_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.004889856092631817, + "psnr": 23.1954345703125, + "lpips": 0.29685643315315247 + }, + "frames_100_200": { + "mse": 0.00583159364759922, + "psnr": 22.53093719482422, + "lpips": 0.37659111618995667 + }, + "frames_200_300": { + "mse": 0.00615149037912488, + "psnr": 22.218793869018555, + "lpips": 0.4030366539955139 + }, + "frames_300_400": { + "mse": 0.006721828132867813, + "psnr": 21.757505416870117, + "lpips": 0.4310856759548187 + }, + "frames_400_500": { + "mse": 0.006379415281116962, + "psnr": 22.003129959106445, + "lpips": 0.42414388060569763 + }, + "overall": { + "mse": 0.005994836799800396, + "psnr": 22.34115982055664, + "lpips": 0.38634273409843445 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000246_test__val_iceplains__1__000041/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000246_test__val_iceplains__1__000041/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..977b05b02a65665f60942314e8d27aa8d4c31c51 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000246_test__val_iceplains__1__000041/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cea0df9992b874c891fdf81622e6b4c40c097ff770a9eb96654c4884b40cf0f4 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000246_test__val_iceplains__1__000041/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000246_test__val_iceplains__1__000041/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..72240c0b9c5986ebac1b2cb4c503f9b6eb39da9b --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000246_test__val_iceplains__1__000041/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "246", + "global_test_index": "246", + "dataset_index": "351", + "relative_path": "test/val_iceplains/1/000041.mp4", + "clip_id": "test/val_iceplains/1/000041", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000246, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/1/000041_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.0007756262784823775, + "psnr": 31.121061325073242, + "lpips": 0.010197252035140991 + }, + "frames_100_200": { + "mse": 0.0011670734966173768, + "psnr": 30.39655113220215, + "lpips": 0.012597210705280304 + }, + "frames_200_300": { + "mse": 0.000851938733831048, + "psnr": 30.731199264526367, + "lpips": 0.012595751322805882 + }, + "frames_300_400": { + "mse": 0.0008106934255920351, + "psnr": 30.962610244750977, + "lpips": 0.01212138868868351 + }, + "frames_400_500": { + "mse": 0.0012973945122212172, + "psnr": 30.453432083129883, + "lpips": 0.01652202196419239 + }, + "overall": { + "mse": 0.000980545300990343, + "psnr": 30.73297119140625, + "lpips": 0.012806723825633526 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000247_test__val_plains__3__000005/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000247_test__val_plains__3__000005/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..0d14802218224c98e75a866f051b057f42730199 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000247_test__val_plains__3__000005/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:755ed3fb4d1124922748ed91f321f1427b76abe0b4e7d450272e31ab02b2b77c +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000247_test__val_plains__3__000005/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000247_test__val_plains__3__000005/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..d1bbba2508d65523ea0900338ae284d375543bf1 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000247_test__val_plains__3__000005/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "247", + "global_test_index": "247", + "dataset_index": "383", + "relative_path": "test/val_plains/3/000005.mp4", + "clip_id": "test/val_plains/3/000005", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000247, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_plains/3/000005_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.0008031539618968964, + "psnr": 30.97528648376465, + "lpips": 0.010354649275541306 + }, + "frames_100_200": { + "mse": 0.00101529061794281, + "psnr": 30.109800338745117, + "lpips": 0.014773373492062092 + }, + "frames_200_300": { + "mse": 0.0010511201107874513, + "psnr": 29.854820251464844, + "lpips": 0.019330676645040512 + }, + "frames_300_400": { + "mse": 0.007890344597399235, + "psnr": 25.802940368652344, + "lpips": 0.06865904480218887 + }, + "frames_400_500": { + "mse": 0.004250612109899521, + "psnr": 27.19109535217285, + "lpips": 0.046216897666454315 + }, + "overall": { + "mse": 0.0030021043494343758, + "psnr": 28.786788940429688, + "lpips": 0.031866930425167084 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000248_test__val_desert__2__000007/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000248_test__val_desert__2__000007/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..dcf253e53f0bdba15d8ff828c8674b43e14f9854 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000248_test__val_desert__2__000007/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:99d602890583950bef91f3615da21b03760a52a8e194437dba0d89d69f345201 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000248_test__val_desert__2__000007/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000248_test__val_desert__2__000007/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..3e970f76f2c9b9ff8141884f4dbeb17ebe0d88d0 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000248_test__val_desert__2__000007/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "248", + "global_test_index": "248", + "dataset_index": "18", + "relative_path": "test/val_desert/2/000007.mp4", + "clip_id": "test/val_desert/2/000007", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 42000248, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_desert/2/000007_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.002716979943215847, + "psnr": 26.119924545288086, + "lpips": 0.05324321612715721 + }, + "frames_100_200": { + "mse": 0.006410086061805487, + "psnr": 22.18276596069336, + "lpips": 0.2260221242904663 + }, + "frames_200_300": { + "mse": 0.009402289055287838, + "psnr": 20.43561363220215, + "lpips": 0.37847501039505005 + }, + "frames_300_400": { + "mse": 0.009198058396577835, + "psnr": 20.454057693481445, + "lpips": 0.4278305470943451 + }, + "frames_400_500": { + "mse": 0.014261170290410519, + "psnr": 18.720226287841797, + "lpips": 0.4215008020401001 + }, + "overall": { + "mse": 0.008397716097533703, + "psnr": 21.582517623901367, + "lpips": 0.3014143705368042 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000249_test__val_plains__1__000040/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000249_test__val_plains__1__000040/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..11f8a9a2962d5e593d235478ebe14519f5a18f47 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed42_job84976/cases/g000249_test__val_plains__1__000040/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc7cffcf667c6ede39f13c3e3a49fdabdef935b33633b7b103bcaeb1c2672bd9 +size 6742