diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000065_test__val_savanna__2__000016/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000065_test__val_savanna__2__000016/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..d77088a58c7d7322965557b07d4b93a79910881c --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000065_test__val_savanna__2__000016/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "65", + "global_test_index": "65", + "dataset_index": "169", + "relative_path": "test/val_savanna/2/000016.mp4", + "clip_id": "test/val_savanna/2/000016", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000065, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_savanna/2/000016_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.004014372825622559, + "psnr": 24.73114585876465, + "lpips": 0.21281827986240387 + }, + "frames_100_200": { + "mse": 0.007330615073442459, + "psnr": 21.654787063598633, + "lpips": 0.33493173122406006 + }, + "frames_200_300": { + "mse": 0.04523978382349014, + "psnr": 14.69363784790039, + "lpips": 0.49253830313682556 + }, + "frames_300_400": { + "mse": 0.0367109440267086, + "psnr": 15.020256042480469, + "lpips": 0.4842808246612549 + }, + "frames_400_500": { + "mse": 0.031806234270334244, + "psnr": 15.92253303527832, + "lpips": 0.5028233528137207 + }, + "overall": { + "mse": 0.025020388886332512, + "psnr": 18.404470443725586, + "lpips": 0.40547850728034973 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000066_test__val_iceplains__1__000027/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000066_test__val_iceplains__1__000027/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..b3d4a50f29cb72f1ceff268f05fd435ca8d5eb2d --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000066_test__val_iceplains__1__000027/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c84f5bec3385c04db85d5256e87fd559bf410e518269283bdd783a404e0da80b +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000066_test__val_iceplains__1__000027/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000066_test__val_iceplains__1__000027/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..2473679b18574cba615680a3374adc9c690e07f7 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000066_test__val_iceplains__1__000027/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "66", + "global_test_index": "66", + "dataset_index": "312", + "relative_path": "test/val_iceplains/1/000027.mp4", + "clip_id": "test/val_iceplains/1/000027", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000066, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/1/000027_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.03341749310493469, + "psnr": 15.692256927490234, + "lpips": 0.20027373731136322 + }, + "frames_100_200": { + "mse": 0.045889515429735184, + "psnr": 13.689079284667969, + "lpips": 0.26659515500068665 + }, + "frames_200_300": { + "mse": 0.03775228187441826, + "psnr": 14.451379776000977, + "lpips": 0.23427170515060425 + }, + "frames_300_400": { + "mse": 0.03829583153128624, + "psnr": 14.485011100769043, + "lpips": 0.24214956164360046 + }, + "frames_400_500": { + "mse": 0.023182572796940804, + "psnr": 16.42613983154297, + "lpips": 0.17161248624324799 + }, + "overall": { + "mse": 0.035707537084817886, + "psnr": 14.948773384094238, + "lpips": 0.2229805290699005 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000067_test__val_iceplains__0__000048/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000067_test__val_iceplains__0__000048/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..246b04ab169f6535b623314ace09db3a4aca06fe --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000067_test__val_iceplains__0__000048/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d92d41fa3aa7cdfb3958038ff18a2ffd856386b902679ba7651b8250dd080425 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000067_test__val_iceplains__0__000048/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000067_test__val_iceplains__0__000048/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..8e8d19ee4c801d4703e291672b8485c40ea7faf3 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000067_test__val_iceplains__0__000048/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "67", + "global_test_index": "67", + "dataset_index": "368", + "relative_path": "test/val_iceplains/0/000048.mp4", + "clip_id": "test/val_iceplains/0/000048", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000067, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/0/000048_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.018511664122343063, + "psnr": 17.558216094970703, + "lpips": 0.1909932643175125 + }, + "frames_100_200": { + "mse": 0.018761316314339638, + "psnr": 17.441848754882812, + "lpips": 0.230560302734375 + }, + "frames_200_300": { + "mse": 0.018833298236131668, + "psnr": 17.360692977905273, + "lpips": 0.18460948765277863 + }, + "frames_300_400": { + "mse": 0.02397480420768261, + "psnr": 16.524255752563477, + "lpips": 0.20898346602916718 + }, + "frames_400_500": { + "mse": 0.029439635574817657, + "psnr": 15.517582893371582, + "lpips": 0.26548025012016296 + }, + "overall": { + "mse": 0.021904144436120987, + "psnr": 16.88051986694336, + "lpips": 0.21612533926963806 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000068_test__val_plains__1__000034/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000068_test__val_plains__1__000034/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..6c886a621887dacadce38fbc24057849db27fa84 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000068_test__val_plains__1__000034/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad72b4939888ed9979fd7a0816cd7d093270ca200e6b13189d17b98c4314c38d +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000068_test__val_plains__1__000034/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000068_test__val_plains__1__000034/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..ec3c697055a1fee14c9143ae91bef9b954f1e692 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000068_test__val_plains__1__000034/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "68", + "global_test_index": "68", + "dataset_index": "449", + "relative_path": "test/val_plains/1/000034.mp4", + "clip_id": "test/val_plains/1/000034", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000068, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_plains/1/000034_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.02620217576622963, + "psnr": 18.96230125427246, + "lpips": 0.1933584213256836 + }, + "frames_100_200": { + "mse": 0.0367947593331337, + "psnr": 15.132673263549805, + "lpips": 0.3913692533969879 + }, + "frames_200_300": { + "mse": 0.020278511568903923, + "psnr": 17.276405334472656, + "lpips": 0.4713697135448456 + }, + "frames_300_400": { + "mse": 0.011657017283141613, + "psnr": 19.586938858032227, + "lpips": 0.4975817799568176 + }, + "frames_400_500": { + "mse": 0.01569548435509205, + "psnr": 18.3051815032959, + "lpips": 0.5181838870048523 + }, + "overall": { + "mse": 0.022125588729977608, + "psnr": 17.852699279785156, + "lpips": 0.41437259316444397 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000069_test__val_desert__1__000004/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000069_test__val_desert__1__000004/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..d3bb736caf00ea7c92a18d715aaaee1c6a4239ec --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000069_test__val_desert__1__000004/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f39fc5bd68c49adfc7156573e6253ca76525ef88972574a4144af8230f84a387 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000069_test__val_desert__1__000004/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000069_test__val_desert__1__000004/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..1df824477520b2702476015dd3cfa71d6b9b5686 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000069_test__val_desert__1__000004/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "69", + "global_test_index": "69", + "dataset_index": "11", + "relative_path": "test/val_desert/1/000004.mp4", + "clip_id": "test/val_desert/1/000004", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000069, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_desert/1/000004_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.007419729605317116, + "psnr": 21.3951473236084, + "lpips": 0.23921996355056763 + }, + "frames_100_200": { + "mse": 0.008625335060060024, + "psnr": 20.78408432006836, + "lpips": 0.367950439453125 + }, + "frames_200_300": { + "mse": 0.016007713973522186, + "psnr": 18.248327255249023, + "lpips": 0.4495605230331421 + }, + "frames_300_400": { + "mse": 0.03778526186943054, + "psnr": 15.024605751037598, + "lpips": 0.4829889237880707 + }, + "frames_400_500": { + "mse": 0.04321294650435448, + "psnr": 14.007286071777344, + "lpips": 0.5312978029251099 + }, + "overall": { + "mse": 0.022610198706388474, + "psnr": 17.891891479492188, + "lpips": 0.4142035245895386 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000070_test__val_desert__2__000001/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000070_test__val_desert__2__000001/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..499ed0ec939dae88dc0f0fc1e54cda00e8ccd85b --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000070_test__val_desert__2__000001/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d24e4c1cb7b736d5fe000494af0a9bbd1c77002798d2064803bd70cc00fe7f43 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000070_test__val_desert__2__000001/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000070_test__val_desert__2__000001/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..c434a2f3b0b26161176efa39e179c0160e426646 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000070_test__val_desert__2__000001/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "70", + "global_test_index": "70", + "dataset_index": "6", + "relative_path": "test/val_desert/2/000001.mp4", + "clip_id": "test/val_desert/2/000001", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000070, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_desert/2/000001_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.004076240584254265, + "psnr": 24.220115661621094, + "lpips": 0.06879425048828125 + }, + "frames_100_200": { + "mse": 0.004011362791061401, + "psnr": 24.842662811279297, + "lpips": 0.05746771767735481 + }, + "frames_200_300": { + "mse": 0.002503311960026622, + "psnr": 27.600914001464844, + "lpips": 0.038191843777894974 + }, + "frames_300_400": { + "mse": 0.0008896967628970742, + "psnr": 30.556379318237305, + "lpips": 0.012969447299838066 + }, + "frames_400_500": { + "mse": 0.0015840891283005476, + "psnr": 30.00088119506836, + "lpips": 0.015592323616147041 + }, + "overall": { + "mse": 0.0026129400357604027, + "psnr": 27.444189071655273, + "lpips": 0.03860311955213547 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000071_test__val_iceplains__0__000005/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000071_test__val_iceplains__0__000005/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..af655746a4b5a7b323d818b395913d50cfe704ae --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000071_test__val_iceplains__0__000005/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8235982b3e0d07337687746116fc0c897d74267181d0ab7b146b647eb6a0ec38 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000071_test__val_iceplains__0__000005/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000071_test__val_iceplains__0__000005/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..74489d6aad7797b5d3be0536ec71da2c919c0f66 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000071_test__val_iceplains__0__000005/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "71", + "global_test_index": "71", + "dataset_index": "257", + "relative_path": "test/val_iceplains/0/000005.mp4", + "clip_id": "test/val_iceplains/0/000005", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000071, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/0/000005_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.02564489282667637, + "psnr": 16.70078468322754, + "lpips": 0.2122913897037506 + }, + "frames_100_200": { + "mse": 0.007004851475358009, + "psnr": 22.290752410888672, + "lpips": 0.23672369122505188 + }, + "frames_200_300": { + "mse": 0.02094113826751709, + "psnr": 17.696422576904297, + "lpips": 0.31616321206092834 + }, + "frames_300_400": { + "mse": 0.005959112662822008, + "psnr": 22.855588912963867, + "lpips": 0.2691815197467804 + }, + "frames_400_500": { + "mse": 0.018363988026976585, + "psnr": 17.680204391479492, + "lpips": 0.38818472623825073 + }, + "overall": { + "mse": 0.0155827971175313, + "psnr": 19.44474983215332, + "lpips": 0.28450891375541687 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000072_test__val_iceplains__3__000006/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000072_test__val_iceplains__3__000006/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..8b8523fe940b22ebc69c506eec8f86b70747bd26 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000072_test__val_iceplains__3__000006/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a831a6f0f3d2dec3d4fe0d7c72f7d9c5eeca344bb4f55c91db949027b35ca6c +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000072_test__val_iceplains__3__000006/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000072_test__val_iceplains__3__000006/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..8dfa54af3937b250e90d2e9f5bac967f5725e831 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000072_test__val_iceplains__3__000006/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "72", + "global_test_index": "72", + "dataset_index": "262", + "relative_path": "test/val_iceplains/3/000006.mp4", + "clip_id": "test/val_iceplains/3/000006", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000072, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/3/000006_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.004939154256135225, + "psnr": 24.171674728393555, + "lpips": 0.15284396708011627 + }, + "frames_100_200": { + "mse": 0.018846845254302025, + "psnr": 17.99581527709961, + "lpips": 0.1997644305229187 + }, + "frames_200_300": { + "mse": 0.029573652893304825, + "psnr": 16.305177688598633, + "lpips": 0.17197054624557495 + }, + "frames_300_400": { + "mse": 0.01824493706226349, + "psnr": 18.500707626342773, + "lpips": 0.1221550852060318 + }, + "frames_400_500": { + "mse": 0.046771831810474396, + "psnr": 13.485700607299805, + "lpips": 0.32107117772102356 + }, + "overall": { + "mse": 0.023675285279750824, + "psnr": 18.091815948486328, + "lpips": 0.19356104731559753 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000073_test__val_savanna__0__000015/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000073_test__val_savanna__0__000015/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..c9e10cf4cd6c92b147be5c0b4bbce8c002eebb2c --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000073_test__val_savanna__0__000015/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c8e7edd3fe0a4af2774e4b0c3b65b36a284288a4d445e10f026a8a328c2b476 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000073_test__val_savanna__0__000015/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000073_test__val_savanna__0__000015/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..a27faa4d7d38932f8af4c2d64db6a8811a1ba4d1 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000073_test__val_savanna__0__000015/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "73", + "global_test_index": "73", + "dataset_index": "164", + "relative_path": "test/val_savanna/0/000015.mp4", + "clip_id": "test/val_savanna/0/000015", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000073, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_savanna/0/000015_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.026894740760326385, + "psnr": 18.513084411621094, + "lpips": 0.1595374345779419 + }, + "frames_100_200": { + "mse": 0.19733141362667084, + "psnr": 8.006949424743652, + "lpips": 0.6579327583312988 + }, + "frames_200_300": { + "mse": 0.29380542039871216, + "psnr": 5.346463203430176, + "lpips": 0.8078112006187439 + }, + "frames_300_400": { + "mse": 0.26710739731788635, + "psnr": 5.820764064788818, + "lpips": 0.7569387555122375 + }, + "frames_400_500": { + "mse": 0.30158257484436035, + "psnr": 5.236715793609619, + "lpips": 0.8018047213554382 + }, + "overall": { + "mse": 0.21734432876110077, + "psnr": 8.584794998168945, + "lpips": 0.63680499792099 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000074_test__val_iceplains__2__000036/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000074_test__val_iceplains__2__000036/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..a17efcabb722eedae46172bd989af1d66fa08cb4 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000074_test__val_iceplains__2__000036/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:98fd882fd142d781c713c2b2130c8bdb0763051b7fa7fce48c03419e14208581 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000074_test__val_iceplains__2__000036/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000074_test__val_iceplains__2__000036/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..0f4cff86dcc82f333c429c26c530e192613fd997 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000074_test__val_iceplains__2__000036/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "74", + "global_test_index": "74", + "dataset_index": "337", + "relative_path": "test/val_iceplains/2/000036.mp4", + "clip_id": "test/val_iceplains/2/000036", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000074, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/2/000036_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.009120816364884377, + "psnr": 20.750158309936523, + "lpips": 0.10336937010288239 + }, + "frames_100_200": { + "mse": 0.010461344383656979, + "psnr": 20.395462036132812, + "lpips": 0.08742618560791016 + }, + "frames_200_300": { + "mse": 0.008670277893543243, + "psnr": 21.52127456665039, + "lpips": 0.08238144218921661 + }, + "frames_300_400": { + "mse": 0.005135504994541407, + "psnr": 23.335512161254883, + "lpips": 0.05467966943979263 + }, + "frames_400_500": { + "mse": 0.0017478882800787687, + "psnr": 29.054672241210938, + "lpips": 0.023778129369020462 + }, + "overall": { + "mse": 0.007027166895568371, + "psnr": 23.011415481567383, + "lpips": 0.07032694667577744 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000075_test__val_desert__0__000036/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000075_test__val_desert__0__000036/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..5351787bd1011f3439686f8de0a8ed6d2510f620 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000075_test__val_desert__0__000036/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0ee32ff9f4a304d0d4e461040acff7646a2ef8710c2039f07d4527570171fc9 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000075_test__val_desert__0__000036/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000075_test__val_desert__0__000036/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..0a6ddad22eedaf2235adb59993c52dbc9bb49635 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000075_test__val_desert__0__000036/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "75", + "global_test_index": "75", + "dataset_index": "95", + "relative_path": "test/val_desert/0/000036.mp4", + "clip_id": "test/val_desert/0/000036", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000075, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_desert/0/000036_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.0016190209425985813, + "psnr": 28.893476486206055, + "lpips": 0.025067996233701706 + }, + "frames_100_200": { + "mse": 0.0007734189857728779, + "psnr": 31.128684997558594, + "lpips": 0.009813263081014156 + }, + "frames_200_300": { + "mse": 0.0018020159332081676, + "psnr": 28.51152992248535, + "lpips": 0.02812228724360466 + }, + "frames_300_400": { + "mse": 0.007088941987603903, + "psnr": 22.067663192749023, + "lpips": 0.13363191485404968 + }, + "frames_400_500": { + "mse": 0.016006767749786377, + "psnr": 18.305557250976562, + "lpips": 0.3150196075439453 + }, + "overall": { + "mse": 0.005458032712340355, + "psnr": 25.781381607055664, + "lpips": 0.1023310124874115 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000076_test__val_iceplains__0__000013/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000076_test__val_iceplains__0__000013/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..2bfb793a1bf811b7f5752140cc9aa5342e1a8449 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000076_test__val_iceplains__0__000013/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c3760451026a39a51ab2c620eea531de56d9118a87c8728bad3045703a035be +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000076_test__val_iceplains__0__000013/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000076_test__val_iceplains__0__000013/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..bf0a3022a88d84491e99f15f2f6c13d0c07b0223 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000076_test__val_iceplains__0__000013/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "76", + "global_test_index": "76", + "dataset_index": "277", + "relative_path": "test/val_iceplains/0/000013.mp4", + "clip_id": "test/val_iceplains/0/000013", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000076, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/0/000013_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.0012489876244217157, + "psnr": 29.882312774658203, + "lpips": 0.014558336697518826 + }, + "frames_100_200": { + "mse": 0.001941458904184401, + "psnr": 28.567697525024414, + "lpips": 0.018520863726735115 + }, + "frames_200_300": { + "mse": 0.002243684371933341, + "psnr": 27.8562068939209, + "lpips": 0.023359812796115875 + }, + "frames_300_400": { + "mse": 0.0007919078343547881, + "psnr": 31.069229125976562, + "lpips": 0.011025523766875267 + }, + "frames_400_500": { + "mse": 0.012342230416834354, + "psnr": 21.896635055541992, + "lpips": 0.12641845643520355 + }, + "overall": { + "mse": 0.0037136536557227373, + "psnr": 27.854413986206055, + "lpips": 0.03877659887075424 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000077_test__val_desert__1__000010/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000077_test__val_desert__1__000010/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..c6c7a17b11168ef6d8fe60c14908c09a92b33329 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000077_test__val_desert__1__000010/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0cb68aacef5dfa7310d653eaec4f5b29e422b4bfe470b0e163aaa416dabd4dd +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000077_test__val_desert__1__000010/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000077_test__val_desert__1__000010/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..adc56d4e233b752d12cac63bff3f2e066d277a51 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000077_test__val_desert__1__000010/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "77", + "global_test_index": "77", + "dataset_index": "26", + "relative_path": "test/val_desert/1/000010.mp4", + "clip_id": "test/val_desert/1/000010", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000077, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_desert/1/000010_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.012197958305478096, + "psnr": 19.62904930114746, + "lpips": 0.28380635380744934 + }, + "frames_100_200": { + "mse": 0.020117277279496193, + "psnr": 17.162813186645508, + "lpips": 0.39623939990997314 + }, + "frames_200_300": { + "mse": 0.01860169880092144, + "psnr": 17.54905128479004, + "lpips": 0.3376738429069519 + }, + "frames_300_400": { + "mse": 0.01646866649389267, + "psnr": 18.835878372192383, + "lpips": 0.3862321376800537 + }, + "frames_400_500": { + "mse": 0.009749620221555233, + "psnr": 20.240331649780273, + "lpips": 0.3306339383125305 + }, + "overall": { + "mse": 0.015427043661475182, + "psnr": 18.683425903320312, + "lpips": 0.34691715240478516 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000078_test__val_desert__0__000029/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000078_test__val_desert__0__000029/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..8ae96484f81c83c36b5febd31a751044cc9255c6 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000078_test__val_desert__0__000029/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:abc3ea52727cd8400ea031dd3b2af7fc8d0b056cd14a9c81f557ddc1524fb2bb +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000078_test__val_desert__0__000029/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000078_test__val_desert__0__000029/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..62a66a4ee6b7948a9f198b53e2b27e13f49d41da --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000078_test__val_desert__0__000029/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "78", + "global_test_index": "78", + "dataset_index": "77", + "relative_path": "test/val_desert/0/000029.mp4", + "clip_id": "test/val_desert/0/000029", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000078, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_desert/0/000029_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.006035820115357637, + "psnr": 22.439062118530273, + "lpips": 0.21610312163829803 + }, + "frames_100_200": { + "mse": 0.008181639015674591, + "psnr": 21.663942337036133, + "lpips": 0.2001114785671234 + }, + "frames_200_300": { + "mse": 0.006328573450446129, + "psnr": 22.289691925048828, + "lpips": 0.21686047315597534 + }, + "frames_300_400": { + "mse": 0.006920985411852598, + "psnr": 21.679929733276367, + "lpips": 0.24497073888778687 + }, + "frames_400_500": { + "mse": 0.004025949630886316, + "psnr": 24.53351593017578, + "lpips": 0.08744464069604874 + }, + "overall": { + "mse": 0.006298593711107969, + "psnr": 22.521230697631836, + "lpips": 0.19309809803962708 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000079_test__val_iceplains__1__000019/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000079_test__val_iceplains__1__000019/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..7b21bac3963957215c4b844ad815d592e7a9faf9 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000079_test__val_iceplains__1__000019/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e644fec1bb259a05c6954aa0e643b88b9ed352ccf86d39dbfe022ddc96ec7e99 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000079_test__val_iceplains__1__000019/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000079_test__val_iceplains__1__000019/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..b9f579c8571dfc1656f992309d74743faac63239 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000079_test__val_iceplains__1__000019/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "79", + "global_test_index": "79", + "dataset_index": "294", + "relative_path": "test/val_iceplains/1/000019.mp4", + "clip_id": "test/val_iceplains/1/000019", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000079, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/1/000019_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.07124108076095581, + "psnr": 13.09792709350586, + "lpips": 0.16915149986743927 + }, + "frames_100_200": { + "mse": 0.07218680530786514, + "psnr": 13.233342170715332, + "lpips": 0.21796123683452606 + }, + "frames_200_300": { + "mse": 0.005865737795829773, + "psnr": 23.287294387817383, + "lpips": 0.04332983121275902 + }, + "frames_300_400": { + "mse": 0.008965272456407547, + "psnr": 20.71026611328125, + "lpips": 0.04578326269984245 + }, + "frames_400_500": { + "mse": 0.010463632643222809, + "psnr": 19.8395938873291, + "lpips": 0.04343164339661598 + }, + "overall": { + "mse": 0.033744506537914276, + "psnr": 18.0336856842041, + "lpips": 0.1039314940571785 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000080_test__val_desert__1__000014/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000080_test__val_desert__1__000014/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..b03bc6a39a4f837c1ec2130bcf8adcec2d62a4f9 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000080_test__val_desert__1__000014/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c051b2609d71093d2a5fa0012421d24ffe17a52607913142d7a8c206e278d4ce +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000080_test__val_desert__1__000014/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000080_test__val_desert__1__000014/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..18477dce6c040ab9c5d4f31bc58dffdbe6f9a97d --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000080_test__val_desert__1__000014/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "80", + "global_test_index": "80", + "dataset_index": "34", + "relative_path": "test/val_desert/1/000014.mp4", + "clip_id": "test/val_desert/1/000014", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000080, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_desert/1/000014_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.0016106539405882359, + "psnr": 30.389219284057617, + "lpips": 0.014143029227852821 + }, + "frames_100_200": { + "mse": 0.0008103664149530232, + "psnr": 30.92504119873047, + "lpips": 0.01078521367162466 + }, + "frames_200_300": { + "mse": 0.0016456968151032925, + "psnr": 30.239500045776367, + "lpips": 0.01528274454176426 + }, + "frames_300_400": { + "mse": 0.0016629217425361276, + "psnr": 29.953857421875, + "lpips": 0.017112256959080696 + }, + "frames_400_500": { + "mse": 0.001559185329824686, + "psnr": 28.758094787597656, + "lpips": 0.02505512163043022 + }, + "overall": { + "mse": 0.0014577648835256696, + "psnr": 30.05314064025879, + "lpips": 0.016475673764944077 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000081_test__val_savanna__1__000029/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000081_test__val_savanna__1__000029/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..46cf3d65f3905df97573813671b027ceb036d5be --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000081_test__val_savanna__1__000029/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:54c6ce387202c82fb19955fa0af563a9e73095b07ab4c55e40f9a0ae88bb2348 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000081_test__val_savanna__1__000029/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000081_test__val_savanna__1__000029/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..28cc99914740127927f13fe2e02f5378cef74171 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000081_test__val_savanna__1__000029/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "81", + "global_test_index": "81", + "dataset_index": "188", + "relative_path": "test/val_savanna/1/000029.mp4", + "clip_id": "test/val_savanna/1/000029", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000081, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_savanna/1/000029_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.006841068156063557, + "psnr": 23.338159561157227, + "lpips": 0.05330326408147812 + }, + "frames_100_200": { + "mse": 0.018118485808372498, + "psnr": 18.39935302734375, + "lpips": 0.15198276937007904 + }, + "frames_200_300": { + "mse": 0.04170526936650276, + "psnr": 13.90273666381836, + "lpips": 0.3556647002696991 + }, + "frames_300_400": { + "mse": 0.025220200419425964, + "psnr": 16.141603469848633, + "lpips": 0.34059828519821167 + }, + "frames_400_500": { + "mse": 0.029699452221393585, + "psnr": 15.861163139343262, + "lpips": 0.4500250220298767 + }, + "overall": { + "mse": 0.024316895753145218, + "psnr": 17.528600692749023, + "lpips": 0.2703147828578949 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000082_test__val_savanna__0__000017/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000082_test__val_savanna__0__000017/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..0ca585f9c26c1fccb439aee3d953ad27372955bf --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000082_test__val_savanna__0__000017/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f49e87763e04233137637045c75099bc521e30828577d5c16c3bffedad115728 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000082_test__val_savanna__0__000017/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000082_test__val_savanna__0__000017/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..fbdc70b67e50ef3157416fe5b6c9d5345b39bedd --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000082_test__val_savanna__0__000017/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "82", + "global_test_index": "82", + "dataset_index": "170", + "relative_path": "test/val_savanna/0/000017.mp4", + "clip_id": "test/val_savanna/0/000017", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000082, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_savanna/0/000017_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.010828632861375809, + "psnr": 19.821306228637695, + "lpips": 0.2727340757846832 + }, + "frames_100_200": { + "mse": 0.027536902576684952, + "psnr": 15.764906883239746, + "lpips": 0.31653282046318054 + }, + "frames_200_300": { + "mse": 0.053111132234334946, + "psnr": 13.122760772705078, + "lpips": 0.41764411330223083 + }, + "frames_300_400": { + "mse": 0.07883300632238388, + "psnr": 11.294658660888672, + "lpips": 0.3880220055580139 + }, + "frames_400_500": { + "mse": 0.08292018622159958, + "psnr": 11.375901222229004, + "lpips": 0.4811946153640747 + }, + "overall": { + "mse": 0.05064597353339195, + "psnr": 14.275906562805176, + "lpips": 0.3752255439758301 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000083_test__val_plains__2__000017/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000083_test__val_plains__2__000017/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..7abbb8a15471a8394fe0e6add542de2d8ebff7ee --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000083_test__val_plains__2__000017/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ea7d2acfbced61dc4adf5ba25af7a01e282a6ec1c580065f507d5a80b314799 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000083_test__val_plains__2__000017/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000083_test__val_plains__2__000017/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..a4e8f8667c3af8b2302c70816307a3f7d5c5fa2d --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000083_test__val_plains__2__000017/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "83", + "global_test_index": "83", + "dataset_index": "415", + "relative_path": "test/val_plains/2/000017.mp4", + "clip_id": "test/val_plains/2/000017", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000083, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_plains/2/000017_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.006784230936318636, + "psnr": 22.10260009765625, + "lpips": 0.11409496515989304 + }, + "frames_100_200": { + "mse": 0.022898998111486435, + "psnr": 17.466732025146484, + "lpips": 0.3183107078075409 + }, + "frames_200_300": { + "mse": 0.020590664818882942, + "psnr": 17.65020179748535, + "lpips": 0.3238554000854492 + }, + "frames_300_400": { + "mse": 0.011044168844819069, + "psnr": 19.74256134033203, + "lpips": 0.2604333162307739 + }, + "frames_400_500": { + "mse": 0.011853259988129139, + "psnr": 19.429931640625, + "lpips": 0.33292847871780396 + }, + "overall": { + "mse": 0.014634265564382076, + "psnr": 19.278406143188477, + "lpips": 0.2699245512485504 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000084_test__val_savanna__0__000006/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000084_test__val_savanna__0__000006/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..a424c711ae3b7268090f03faf1bde9a07e76ea28 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000084_test__val_savanna__0__000006/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fba6131250b389bcbcaaa7f1bbb42e0176a40c2584cd6821b33a70d3a69a5116 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000084_test__val_savanna__0__000006/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000084_test__val_savanna__0__000006/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..30306df406503d0cc3978e850cd404e19ddd6e78 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000084_test__val_savanna__0__000006/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "84", + "global_test_index": "84", + "dataset_index": "145", + "relative_path": "test/val_savanna/0/000006.mp4", + "clip_id": "test/val_savanna/0/000006", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000084, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_savanna/0/000006_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.015611572191119194, + "psnr": 18.780807495117188, + "lpips": 0.12070924788713455 + }, + "frames_100_200": { + "mse": 0.018946601077914238, + "psnr": 17.455018997192383, + "lpips": 0.21950136125087738 + }, + "frames_200_300": { + "mse": 0.014058049768209457, + "psnr": 18.930194854736328, + "lpips": 0.3905223608016968 + }, + "frames_300_400": { + "mse": 0.0074511026032269, + "psnr": 21.52984619140625, + "lpips": 0.4059504270553589 + }, + "frames_400_500": { + "mse": 0.0058480678126215935, + "psnr": 22.511276245117188, + "lpips": 0.3924799859523773 + }, + "overall": { + "mse": 0.01238307822495699, + "psnr": 19.841426849365234, + "lpips": 0.3058326840400696 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000085_test__val_savanna__1__000002/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000085_test__val_savanna__1__000002/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..4615347e718464b09f510820926b20f445321041 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000085_test__val_savanna__1__000002/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:703432f3ebc7590f8cee8f3598d1286c267deb8ee7cc168f0ff412500bebfa42 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000085_test__val_savanna__1__000002/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000085_test__val_savanna__1__000002/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..4bdd0e19878044a4b4b5f89226b6a36a84eaf4a7 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000085_test__val_savanna__1__000002/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "85", + "global_test_index": "85", + "dataset_index": "133", + "relative_path": "test/val_savanna/1/000002.mp4", + "clip_id": "test/val_savanna/1/000002", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000085, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_savanna/1/000002_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.009289972484111786, + "psnr": 20.701269149780273, + "lpips": 0.2666792571544647 + }, + "frames_100_200": { + "mse": 0.011189105920493603, + "psnr": 19.91399383544922, + "lpips": 0.3061886131763458 + }, + "frames_200_300": { + "mse": 0.014358417131006718, + "psnr": 18.747610092163086, + "lpips": 0.3554259240627289 + }, + "frames_300_400": { + "mse": 0.01757115311920643, + "psnr": 18.271543502807617, + "lpips": 0.4052739441394806 + }, + "frames_400_500": { + "mse": 0.02507631853222847, + "psnr": 17.426790237426758, + "lpips": 0.41056838631629944 + }, + "overall": { + "mse": 0.015496992506086826, + "psnr": 19.012239456176758, + "lpips": 0.34882721304893494 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000086_test__val_iceplains__2__000039/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000086_test__val_iceplains__2__000039/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..39a5c397b1117b4ed10fd85ceb4086bc2e82b466 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000086_test__val_iceplains__2__000039/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ee2054e038e25639bee95639ee41ff141bffbf8eb5294e94135c7e12930aedf3 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000086_test__val_iceplains__2__000039/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000086_test__val_iceplains__2__000039/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..c56e723843f8ca3d569964212ff547f023e17b74 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000086_test__val_iceplains__2__000039/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "86", + "global_test_index": "86", + "dataset_index": "347", + "relative_path": "test/val_iceplains/2/000039.mp4", + "clip_id": "test/val_iceplains/2/000039", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000086, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/2/000039_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.017345823347568512, + "psnr": 17.94886589050293, + "lpips": 0.1950908601284027 + }, + "frames_100_200": { + "mse": 0.014885928481817245, + "psnr": 19.007240295410156, + "lpips": 0.2708568871021271 + }, + "frames_200_300": { + "mse": 0.024832768365740776, + "psnr": 17.850805282592773, + "lpips": 0.34176427125930786 + }, + "frames_300_400": { + "mse": 0.08388739824295044, + "psnr": 11.177172660827637, + "lpips": 0.44589027762413025 + }, + "frames_400_500": { + "mse": 0.08137616515159607, + "psnr": 11.372044563293457, + "lpips": 0.4673200845718384 + }, + "overall": { + "mse": 0.04446560889482498, + "psnr": 15.471226692199707, + "lpips": 0.3441844582557678 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000087_test__val_savanna__0__000038/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000087_test__val_savanna__0__000038/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..59db0085247ec62c5b867ffbbae7015a41a2369e --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000087_test__val_savanna__0__000038/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15a3e538ee0bfa3c7bdbb2c436ed9b540c0b26f83ef27693ae9ed356ee91a362 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000087_test__val_savanna__0__000038/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000087_test__val_savanna__0__000038/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..9b9fd7988e79718041c3880e4bbaee4ae867ce77 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000087_test__val_savanna__0__000038/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "87", + "global_test_index": "87", + "dataset_index": "209", + "relative_path": "test/val_savanna/0/000038.mp4", + "clip_id": "test/val_savanna/0/000038", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000087, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_savanna/0/000038_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.012337503023445606, + "psnr": 19.54021644592285, + "lpips": 0.25639790296554565 + }, + "frames_100_200": { + "mse": 0.020619649440050125, + "psnr": 17.437349319458008, + "lpips": 0.36619749665260315 + }, + "frames_200_300": { + "mse": 0.018356572836637497, + "psnr": 17.68050193786621, + "lpips": 0.3532315194606781 + }, + "frames_300_400": { + "mse": 0.006292724050581455, + "psnr": 22.170820236206055, + "lpips": 0.3768339455127716 + }, + "frames_400_500": { + "mse": 0.009856903925538063, + "psnr": 20.439516067504883, + "lpips": 0.4309653341770172 + }, + "overall": { + "mse": 0.013492671772837639, + "psnr": 19.45368194580078, + "lpips": 0.35672521591186523 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000088_test__val_plains__1__000013/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000088_test__val_plains__1__000013/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..fc1716612fa0a65719a81e00ee9e1a63b891e216 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000088_test__val_plains__1__000013/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d340afcc04d0e2bb093f67a0e3ff52ea2894f02200d2bff2dd897c78f69dcc9a +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000088_test__val_plains__1__000013/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000088_test__val_plains__1__000013/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..b13834bf22d0e4b423d6190e3b0897a98fd39778 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000088_test__val_plains__1__000013/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "88", + "global_test_index": "88", + "dataset_index": "404", + "relative_path": "test/val_plains/1/000013.mp4", + "clip_id": "test/val_plains/1/000013", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000088, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_plains/1/000013_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.0014102114364504814, + "psnr": 28.685462951660156, + "lpips": 0.12700296938419342 + }, + "frames_100_200": { + "mse": 0.0037337630055844784, + "psnr": 25.541946411132812, + "lpips": 0.19388142228126526 + }, + "frames_200_300": { + "mse": 0.009554834105074406, + "psnr": 20.841018676757812, + "lpips": 0.2509021461009979 + }, + "frames_300_400": { + "mse": 0.004533831495791674, + "psnr": 24.2772159576416, + "lpips": 0.14964593946933746 + }, + "frames_400_500": { + "mse": 0.011396617628633976, + "psnr": 20.10059356689453, + "lpips": 0.25289323925971985 + }, + "overall": { + "mse": 0.006125851534307003, + "psnr": 23.889245986938477, + "lpips": 0.1948651224374771 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000089_test__val_savanna__2__000038/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000089_test__val_savanna__2__000038/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..9bbca9a718ee71d8d1dd7b40e6c1a212bccae6a5 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000089_test__val_savanna__2__000038/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:98aac07b6e1fb7aae27756d876926781932352f884b44a6df7cd7f778e6e40a3 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000089_test__val_savanna__2__000038/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000089_test__val_savanna__2__000038/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..cb71f04903aafbec4f2d2c19b702c4aaeb9da3c4 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000089_test__val_savanna__2__000038/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "89", + "global_test_index": "89", + "dataset_index": "211", + "relative_path": "test/val_savanna/2/000038.mp4", + "clip_id": "test/val_savanna/2/000038", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000089, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_savanna/2/000038_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.013012063689529896, + "psnr": 19.066543579101562, + "lpips": 0.3802356719970703 + }, + "frames_100_200": { + "mse": 0.006951131857931614, + "psnr": 21.698877334594727, + "lpips": 0.41791361570358276 + }, + "frames_200_300": { + "mse": 0.006270250305533409, + "psnr": 22.11797523498535, + "lpips": 0.37911689281463623 + }, + "frames_300_400": { + "mse": 0.010513953864574432, + "psnr": 20.22835922241211, + "lpips": 0.4178915023803711 + }, + "frames_400_500": { + "mse": 0.009852658957242966, + "psnr": 20.3792667388916, + "lpips": 0.458982914686203 + }, + "overall": { + "mse": 0.009320011362433434, + "psnr": 20.698205947875977, + "lpips": 0.4108281433582306 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000090_test__val_plains__2__000044/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000090_test__val_plains__2__000044/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..ebb106660914b69bbe6d83acc19e5600af5ccf10 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000090_test__val_plains__2__000044/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4dda458c94d1191ecda05e2ba70bb4806e957d58c25a13c6c3ff02450b199987 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000090_test__val_plains__2__000044/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000090_test__val_plains__2__000044/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..b150cbbe4f8fc12ffa999a7b29cbc452354ea4a6 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000090_test__val_plains__2__000044/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "90", + "global_test_index": "90", + "dataset_index": "464", + "relative_path": "test/val_plains/2/000044.mp4", + "clip_id": "test/val_plains/2/000044", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000090, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_plains/2/000044_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.009817150421440601, + "psnr": 22.06405258178711, + "lpips": 0.29228678345680237 + }, + "frames_100_200": { + "mse": 0.029143381863832474, + "psnr": 15.699549674987793, + "lpips": 0.37973254919052124 + }, + "frames_200_300": { + "mse": 0.046205148100852966, + "psnr": 14.526034355163574, + "lpips": 0.41873085498809814 + }, + "frames_300_400": { + "mse": 0.021684104576706886, + "psnr": 17.12316131591797, + "lpips": 0.37332871556282043 + }, + "frames_400_500": { + "mse": 0.015798768028616905, + "psnr": 18.654827117919922, + "lpips": 0.4441998302936554 + }, + "overall": { + "mse": 0.0245297122746706, + "psnr": 17.613527297973633, + "lpips": 0.381655752658844 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000091_test__val_savanna__1__000014/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000091_test__val_savanna__1__000014/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..411d16dc82f8c0d2865bc1d7080936d0c729f85f --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000091_test__val_savanna__1__000014/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:50aaf1817c59d789b06d0ff4fc7666b06f2727c8ede8dcedbe6c7f25d29a8b02 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000091_test__val_savanna__1__000014/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000091_test__val_savanna__1__000014/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..adfd62837644af8766cb97a10fb869e086efa44e --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000091_test__val_savanna__1__000014/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "91", + "global_test_index": "91", + "dataset_index": "162", + "relative_path": "test/val_savanna/1/000014.mp4", + "clip_id": "test/val_savanna/1/000014", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000091, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_savanna/1/000014_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.021948056295514107, + "psnr": 17.39195442199707, + "lpips": 0.3390779495239258 + }, + "frames_100_200": { + "mse": 0.05297599732875824, + "psnr": 13.103137016296387, + "lpips": 0.5804449319839478 + }, + "frames_200_300": { + "mse": 0.011137744411826134, + "psnr": 20.890840530395508, + "lpips": 0.4129979610443115 + }, + "frames_300_400": { + "mse": 0.00559851061552763, + "psnr": 22.624773025512695, + "lpips": 0.3991730511188507 + }, + "frames_400_500": { + "mse": 0.00659726420417428, + "psnr": 21.874483108520508, + "lpips": 0.39846137166023254 + }, + "overall": { + "mse": 0.019651513546705246, + "psnr": 19.177034378051758, + "lpips": 0.42603105306625366 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000092_test__val_plains__2__000019/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000092_test__val_plains__2__000019/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..fe7865c782607e0ba9e59689c3caf327815704da --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000092_test__val_plains__2__000019/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0edab1b7f623e8fad04b28bb91fb51be1a69adfe18e6f7521b0a699a9448a64 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000092_test__val_plains__2__000019/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000092_test__val_plains__2__000019/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..bd124dfa6ae5398db52622441f175d204697298c --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000092_test__val_plains__2__000019/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "92", + "global_test_index": "92", + "dataset_index": "419", + "relative_path": "test/val_plains/2/000019.mp4", + "clip_id": "test/val_plains/2/000019", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000092, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_plains/2/000019_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.011212405748665333, + "psnr": 20.218000411987305, + "lpips": 0.23572330176830292 + }, + "frames_100_200": { + "mse": 0.012113552540540695, + "psnr": 19.8581485748291, + "lpips": 0.36476126313209534 + }, + "frames_200_300": { + "mse": 0.009208614006638527, + "psnr": 20.905685424804688, + "lpips": 0.3487642705440521 + }, + "frames_300_400": { + "mse": 0.01245319377630949, + "psnr": 21.253446578979492, + "lpips": 0.17385883629322052 + }, + "frames_400_500": { + "mse": 0.00612592464312911, + "psnr": 24.201536178588867, + "lpips": 0.05630883201956749 + }, + "overall": { + "mse": 0.010222737677395344, + "psnr": 21.287363052368164, + "lpips": 0.23588328063488007 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000093_test__val_desert__3__000049/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000093_test__val_desert__3__000049/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..43adba62757172f207ea9ef0cd825469648f3ccb --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000093_test__val_desert__3__000049/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7210b5d2e99d917ea037282d3e3279fd0e538308ee1c4efdde8cb82fbeaad4db +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000093_test__val_desert__3__000049/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000093_test__val_desert__3__000049/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..022ab3808eb92e4a3af2128897175a42b61f1bbd --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000093_test__val_desert__3__000049/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "93", + "global_test_index": "93", + "dataset_index": "127", + "relative_path": "test/val_desert/3/000049.mp4", + "clip_id": "test/val_desert/3/000049", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000093, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_desert/3/000049_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.005194228142499924, + "psnr": 23.93198013305664, + "lpips": 0.12471587210893631 + }, + "frames_100_200": { + "mse": 0.0187770314514637, + "psnr": 19.002410888671875, + "lpips": 0.299960732460022 + }, + "frames_200_300": { + "mse": 0.02056986093521118, + "psnr": 17.85968589782715, + "lpips": 0.34530410170555115 + }, + "frames_300_400": { + "mse": 0.007845601066946983, + "psnr": 22.9403133392334, + "lpips": 0.15100079774856567 + }, + "frames_400_500": { + "mse": 0.017220541834831238, + "psnr": 18.432876586914062, + "lpips": 0.2834799885749817 + }, + "overall": { + "mse": 0.013921452686190605, + "psnr": 20.433452606201172, + "lpips": 0.24089232087135315 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000094_test__val_iceplains__2__000048/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000094_test__val_iceplains__2__000048/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..e28be97433f8b41b8b8d68fba5059e56c824025c --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000094_test__val_iceplains__2__000048/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fefc6a1b6064f4a9a55e905968ed4896df5826b94d7427362ff02c9b97163385 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000094_test__val_iceplains__2__000048/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000094_test__val_iceplains__2__000048/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..82dc0fc9b33041fb4c527b9d778ba64b31fb744d --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000094_test__val_iceplains__2__000048/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "94", + "global_test_index": "94", + "dataset_index": "370", + "relative_path": "test/val_iceplains/2/000048.mp4", + "clip_id": "test/val_iceplains/2/000048", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000094, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/2/000048_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.005655695218592882, + "psnr": 22.622861862182617, + "lpips": 0.07972772419452667 + }, + "frames_100_200": { + "mse": 0.012437796220183372, + "psnr": 19.787277221679688, + "lpips": 0.09408221393823624 + }, + "frames_200_300": { + "mse": 0.007247153669595718, + "psnr": 22.46295738220215, + "lpips": 0.0557902492582798 + }, + "frames_300_400": { + "mse": 0.0104783084243536, + "psnr": 22.28655433654785, + "lpips": 0.09320426732301712 + }, + "frames_400_500": { + "mse": 0.0739329531788826, + "psnr": 11.663463592529297, + "lpips": 0.4359401762485504 + }, + "overall": { + "mse": 0.021950382739305496, + "psnr": 19.76462173461914, + "lpips": 0.1517489105463028 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000095_test__val_iceplains__2__000018/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000095_test__val_iceplains__2__000018/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..e2fffe950f5bdee7d18594fc38dad9a16a7ad380 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000095_test__val_iceplains__2__000018/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fafe3668e10784983ee3fe8cef9504380981419defb0a76c73e31589ce2f8b6f +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000095_test__val_iceplains__2__000018/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000095_test__val_iceplains__2__000018/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..f86069786a8a6c9e02dd7ab9e5c02a714345d531 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000095_test__val_iceplains__2__000018/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "95", + "global_test_index": "95", + "dataset_index": "293", + "relative_path": "test/val_iceplains/2/000018.mp4", + "clip_id": "test/val_iceplains/2/000018", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000095, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/2/000018_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.003243616782128811, + "psnr": 24.948070526123047, + "lpips": 0.12859097123146057 + }, + "frames_100_200": { + "mse": 0.00499358307570219, + "psnr": 23.814502716064453, + "lpips": 0.13516956567764282 + }, + "frames_200_300": { + "mse": 0.02217828668653965, + "psnr": 17.99000358581543, + "lpips": 0.19591036438941956 + }, + "frames_300_400": { + "mse": 0.007897485047578812, + "psnr": 21.686599731445312, + "lpips": 0.16854523122310638 + }, + "frames_400_500": { + "mse": 0.016262942925095558, + "psnr": 19.204303741455078, + "lpips": 0.14065513014793396 + }, + "overall": { + "mse": 0.010915183462202549, + "psnr": 21.52869415283203, + "lpips": 0.153774231672287 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000096_test__val_iceplains__1__000003/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000096_test__val_iceplains__1__000003/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..cf9388775278cd25208eae1f792c445a958b1b9f --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000096_test__val_iceplains__1__000003/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:24cede5ffbf87180da7322bcef0885a2e55ae30c9f07a4f27cd6c2dfc1b81034 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000096_test__val_iceplains__1__000003/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000096_test__val_iceplains__1__000003/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..eb7530fa9ecd8b605b1a7301633aa57185bbc7b7 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000096_test__val_iceplains__1__000003/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "96", + "global_test_index": "96", + "dataset_index": "251", + "relative_path": "test/val_iceplains/1/000003.mp4", + "clip_id": "test/val_iceplains/1/000003", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000096, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/1/000003_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.007593045011162758, + "psnr": 23.811017990112305, + "lpips": 0.05476979911327362 + }, + "frames_100_200": { + "mse": 0.0053995344787836075, + "psnr": 24.991769790649414, + "lpips": 0.043969959020614624 + }, + "frames_200_300": { + "mse": 0.004130628891289234, + "psnr": 25.645275115966797, + "lpips": 0.0380912721157074 + }, + "frames_300_400": { + "mse": 0.0008555282838642597, + "psnr": 30.70444107055664, + "lpips": 0.011225852183997631 + }, + "frames_400_500": { + "mse": 0.0017621313454583287, + "psnr": 28.64368438720703, + "lpips": 0.01873643323779106 + }, + "overall": { + "mse": 0.003948173485696316, + "psnr": 26.759235382080078, + "lpips": 0.03335866332054138 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000097_test__val_plains__1__000031/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000097_test__val_plains__1__000031/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..1caa90b88c9ae17e9c8346dd67ebba94386e02d9 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000097_test__val_plains__1__000031/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a82520b239574f89a3e76492a2274f95f8cf996530cc3cd84beccc554ca36fbb +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000097_test__val_plains__1__000031/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000097_test__val_plains__1__000031/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..f63432473d66ad1826047b958027d565ae01e949 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000097_test__val_plains__1__000031/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "97", + "global_test_index": "97", + "dataset_index": "441", + "relative_path": "test/val_plains/1/000031.mp4", + "clip_id": "test/val_plains/1/000031", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000097, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_plains/1/000031_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.008965798653662205, + "psnr": 23.556255340576172, + "lpips": 0.07299555093050003 + }, + "frames_100_200": { + "mse": 0.017639504745602608, + "psnr": 19.019407272338867, + "lpips": 0.15700699388980865 + }, + "frames_200_300": { + "mse": 0.06846167892217636, + "psnr": 11.906994819641113, + "lpips": 0.4319377541542053 + }, + "frames_300_400": { + "mse": 0.060255054384469986, + "psnr": 12.914912223815918, + "lpips": 0.4085858464241028 + }, + "frames_400_500": { + "mse": 0.04101429134607315, + "psnr": 14.234013557434082, + "lpips": 0.48871147632598877 + }, + "overall": { + "mse": 0.039267268031835556, + "psnr": 16.326316833496094, + "lpips": 0.3118475079536438 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000098_test__val_desert__2__000026/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000098_test__val_desert__2__000026/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..de09ae9a46a7c58bfd23d149948470af8e92abf0 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000098_test__val_desert__2__000026/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:85f27cc43366de94d4f1d29120086bd5e792413ad6f4119674daca238422b9bf +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000098_test__val_desert__2__000026/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000098_test__val_desert__2__000026/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..7de325b40f81466f4cc9b35ce47ef6c3e84a8d14 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000098_test__val_desert__2__000026/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "98", + "global_test_index": "98", + "dataset_index": "69", + "relative_path": "test/val_desert/2/000026.mp4", + "clip_id": "test/val_desert/2/000026", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000098, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_desert/2/000026_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.006428284104913473, + "psnr": 22.080848693847656, + "lpips": 0.2143724411725998 + }, + "frames_100_200": { + "mse": 0.01179407350718975, + "psnr": 19.616230010986328, + "lpips": 0.3371715843677521 + }, + "frames_200_300": { + "mse": 0.021862776950001717, + "psnr": 16.958494186401367, + "lpips": 0.4291427731513977 + }, + "frames_300_400": { + "mse": 0.02105175517499447, + "psnr": 17.215858459472656, + "lpips": 0.4563262164592743 + }, + "frames_400_500": { + "mse": 0.01642347127199173, + "psnr": 18.57614517211914, + "lpips": 0.4756382703781128 + }, + "overall": { + "mse": 0.015512072481215, + "psnr": 18.889514923095703, + "lpips": 0.3825302720069885 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000099_test__val_iceplains__2__000015/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000099_test__val_iceplains__2__000015/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..922836a3faa13bc9787f3a1f790610f0dd394c08 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000099_test__val_iceplains__2__000015/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:915840f975e74886cb42f57a1057f6fb1cb6e432293f84cc1ad080c16ee7e27c +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000099_test__val_iceplains__2__000015/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000099_test__val_iceplains__2__000015/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..17693a5ca8bc16216598015e4bba3f3624684821 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000099_test__val_iceplains__2__000015/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "99", + "global_test_index": "99", + "dataset_index": "286", + "relative_path": "test/val_iceplains/2/000015.mp4", + "clip_id": "test/val_iceplains/2/000015", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000099, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/2/000015_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.02050715498626232, + "psnr": 17.537811279296875, + "lpips": 0.20922434329986572 + }, + "frames_100_200": { + "mse": 0.024869613349437714, + "psnr": 17.380695343017578, + "lpips": 0.30019885301589966 + }, + "frames_200_300": { + "mse": 0.04512585699558258, + "psnr": 14.169766426086426, + "lpips": 0.361023873090744 + }, + "frames_300_400": { + "mse": 0.03638758510351181, + "psnr": 15.023761749267578, + "lpips": 0.3937431573867798 + }, + "frames_400_500": { + "mse": 0.03352150321006775, + "psnr": 15.78067398071289, + "lpips": 0.3551176190376282 + }, + "overall": { + "mse": 0.032082341611385345, + "psnr": 15.97854232788086, + "lpips": 0.32386156916618347 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000100_test__val_plains__0__000029/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000100_test__val_plains__0__000029/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..63f3a25b063b15397de59bca8391a9705f6d97f0 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000100_test__val_plains__0__000029/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ab0082d8a50c41764eedb7a88ac780a6f3db64d946aa02545681b8ea171cc18 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000100_test__val_plains__0__000029/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000100_test__val_plains__0__000029/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..526746141375a7133eaa46ad54238f1aee8382ee --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000100_test__val_plains__0__000029/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "100", + "global_test_index": "100", + "dataset_index": "438", + "relative_path": "test/val_plains/0/000029.mp4", + "clip_id": "test/val_plains/0/000029", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000100, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_plains/0/000029_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.011014662683010101, + "psnr": 21.728374481201172, + "lpips": 0.16010741889476776 + }, + "frames_100_200": { + "mse": 0.0021475288085639477, + "psnr": 28.426755905151367, + "lpips": 0.020974790677428246 + }, + "frames_200_300": { + "mse": 0.002341275569051504, + "psnr": 28.17304229736328, + "lpips": 0.023683743551373482 + }, + "frames_300_400": { + "mse": 0.011910539120435715, + "psnr": 21.984891891479492, + "lpips": 0.06129106134176254 + }, + "frames_400_500": { + "mse": 0.02277098409831524, + "psnr": 17.273557662963867, + "lpips": 0.2626326382160187 + }, + "overall": { + "mse": 0.010036997497081757, + "psnr": 23.517324447631836, + "lpips": 0.10573793947696686 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000101_test__val_plains__0__000002/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000101_test__val_plains__0__000002/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..4bfe7c49ea14fb1ac40826a9b3a94680aa198c4d --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000101_test__val_plains__0__000002/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f9eb3449fcd0b97469845346f58972b1ddab98be9c084106d2940ae84107e9c +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000101_test__val_plains__0__000002/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000101_test__val_plains__0__000002/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..7b814aadaf71c974bfc3b5a179d5b3bc971a1a88 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000101_test__val_plains__0__000002/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "101", + "global_test_index": "101", + "dataset_index": "376", + "relative_path": "test/val_plains/0/000002.mp4", + "clip_id": "test/val_plains/0/000002", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000101, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_plains/0/000002_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.005657189060002565, + "psnr": 22.785341262817383, + "lpips": 0.16283711791038513 + }, + "frames_100_200": { + "mse": 0.006132856011390686, + "psnr": 22.317068099975586, + "lpips": 0.20743869245052338 + }, + "frames_200_300": { + "mse": 0.0069464026018977165, + "psnr": 21.724994659423828, + "lpips": 0.2599196135997772 + }, + "frames_300_400": { + "mse": 0.01257234439253807, + "psnr": 20.4638614654541, + "lpips": 0.28282082080841064 + }, + "frames_400_500": { + "mse": 0.015081522054970264, + "psnr": 20.25960922241211, + "lpips": 0.2459738552570343 + }, + "overall": { + "mse": 0.009278062731027603, + "psnr": 21.510175704956055, + "lpips": 0.23179803788661957 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000102_test__val_savanna__2__000014/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000102_test__val_savanna__2__000014/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..fafed1130065367d4bd402c70b4bb419b7ebf61d --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000102_test__val_savanna__2__000014/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:353a0e93baa48b0ab171a22efb24045671d9045291e388e76c90bc1f3ab72257 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000102_test__val_savanna__2__000014/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000102_test__val_savanna__2__000014/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..70029387a407e07d71020e2f264403455e08bead --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000102_test__val_savanna__2__000014/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "102", + "global_test_index": "102", + "dataset_index": "163", + "relative_path": "test/val_savanna/2/000014.mp4", + "clip_id": "test/val_savanna/2/000014", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000102, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_savanna/2/000014_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.006613884586840868, + "psnr": 21.905410766601562, + "lpips": 0.33228740096092224 + }, + "frames_100_200": { + "mse": 0.008516473695635796, + "psnr": 20.822582244873047, + "lpips": 0.37318146228790283 + }, + "frames_200_300": { + "mse": 0.009367463178932667, + "psnr": 20.375268936157227, + "lpips": 0.44472476840019226 + }, + "frames_300_400": { + "mse": 0.008299416862428188, + "psnr": 21.303194046020508, + "lpips": 0.45336654782295227 + }, + "frames_400_500": { + "mse": 0.007341804914176464, + "psnr": 21.481761932373047, + "lpips": 0.4516735374927521 + }, + "overall": { + "mse": 0.008027808740735054, + "psnr": 21.177644729614258, + "lpips": 0.41104671359062195 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000103_test__val_desert__1__000013/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000103_test__val_desert__1__000013/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..2d059805b7a8c030107c3366987d2e3a0a7ac10b --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000103_test__val_desert__1__000013/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:11d4ee1ad603acb5ddc59d3cbe3d510abe6ec853af3b4da0dd121d6ebc91cfd3 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000103_test__val_desert__1__000013/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000103_test__val_desert__1__000013/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..a04f1fcdcf132a0a4717e86d0750abd95a394efb --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000103_test__val_desert__1__000013/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "103", + "global_test_index": "103", + "dataset_index": "31", + "relative_path": "test/val_desert/1/000013.mp4", + "clip_id": "test/val_desert/1/000013", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000103, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_desert/1/000013_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.0008271471597254276, + "psnr": 30.837238311767578, + "lpips": 0.010782756842672825 + }, + "frames_100_200": { + "mse": 0.0008063640561886132, + "psnr": 30.943986892700195, + "lpips": 0.010190390050411224 + }, + "frames_200_300": { + "mse": 0.00197942485101521, + "psnr": 28.25726318359375, + "lpips": 0.03420465067028999 + }, + "frames_300_400": { + "mse": 0.002908748108893633, + "psnr": 25.7596435546875, + "lpips": 0.08334004133939743 + }, + "frames_400_500": { + "mse": 0.0015406726161018014, + "psnr": 28.552221298217773, + "lpips": 0.02982919290661812 + }, + "overall": { + "mse": 0.001612471416592598, + "psnr": 28.87006950378418, + "lpips": 0.03366940841078758 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000104_test__val_savanna__0__000049/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000104_test__val_savanna__0__000049/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..57cebd68766bb8e996fdbd1460f82b93e96c6c92 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000104_test__val_savanna__0__000049/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8bfde06189b6439d25391643048b5a5ff30da1255f70bf0f8e4a0591727d789b +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000104_test__val_savanna__0__000049/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000104_test__val_savanna__0__000049/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..e73d348fb991b14b08f5971f81a2ea81639b8c1e --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000104_test__val_savanna__0__000049/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "104", + "global_test_index": "104", + "dataset_index": "241", + "relative_path": "test/val_savanna/0/000049.mp4", + "clip_id": "test/val_savanna/0/000049", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000104, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_savanna/0/000049_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.020324913784861565, + "psnr": 17.362258911132812, + "lpips": 0.16288205981254578 + }, + "frames_100_200": { + "mse": 0.027738843113183975, + "psnr": 16.407987594604492, + "lpips": 0.1880587935447693 + }, + "frames_200_300": { + "mse": 0.030680956318974495, + "psnr": 15.937946319580078, + "lpips": 0.2229706346988678 + }, + "frames_300_400": { + "mse": 0.017881521955132484, + "psnr": 18.153881072998047, + "lpips": 0.1344011127948761 + }, + "frames_400_500": { + "mse": 0.01928546465933323, + "psnr": 18.459732055664062, + "lpips": 0.1163644790649414 + }, + "overall": { + "mse": 0.0231823418289423, + "psnr": 17.264362335205078, + "lpips": 0.1649354249238968 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000105_test__val_plains__0__000040/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000105_test__val_plains__0__000040/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..15a2ef25a7adb45400283035874913dc7abefffe --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000105_test__val_plains__0__000040/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0e7d142880319735fdcf4eb841541f5814ad0c50fa37dc7b527641871cb9e2b +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000105_test__val_plains__0__000040/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000105_test__val_plains__0__000040/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..3cb6646e57b79f3f4c71a3950196a256a9f5fcac --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000105_test__val_plains__0__000040/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "105", + "global_test_index": "105", + "dataset_index": "458", + "relative_path": "test/val_plains/0/000040.mp4", + "clip_id": "test/val_plains/0/000040", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000105, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_plains/0/000040_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.0042113554663956165, + "psnr": 23.88638687133789, + "lpips": 0.27419570088386536 + }, + "frames_100_200": { + "mse": 0.004814589861780405, + "psnr": 23.314720153808594, + "lpips": 0.3101295828819275 + }, + "frames_200_300": { + "mse": 0.004617148544639349, + "psnr": 23.464624404907227, + "lpips": 0.3574104309082031 + }, + "frames_300_400": { + "mse": 0.004914138000458479, + "psnr": 23.200000762939453, + "lpips": 0.3616783022880554 + }, + "frames_400_500": { + "mse": 0.0046851979568600655, + "psnr": 23.525541305541992, + "lpips": 0.39512306451797485 + }, + "overall": { + "mse": 0.004648486152291298, + "psnr": 23.478256225585938, + "lpips": 0.3397074341773987 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000106_test__val_iceplains__1__000049/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000106_test__val_iceplains__1__000049/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..86534789e660ccba136cded097fdfc08b2e3728a --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000106_test__val_iceplains__1__000049/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c545bc3b373b2e9d05151d7e1a1f433e27e59f222aaa45c5dae9112528318ae5 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000106_test__val_iceplains__1__000049/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000106_test__val_iceplains__1__000049/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..409dae4851e30f5163a07b38a85cf2b6478f25f6 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000106_test__val_iceplains__1__000049/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "106", + "global_test_index": "106", + "dataset_index": "371", + "relative_path": "test/val_iceplains/1/000049.mp4", + "clip_id": "test/val_iceplains/1/000049", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000106, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/1/000049_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.005954848602414131, + "psnr": 23.91751480102539, + "lpips": 0.05312316492199898 + }, + "frames_100_200": { + "mse": 0.0008228293736465275, + "psnr": 30.86727523803711, + "lpips": 0.011452382430434227 + }, + "frames_200_300": { + "mse": 0.0007802118198014796, + "psnr": 31.087785720825195, + "lpips": 0.010380148887634277 + }, + "frames_300_400": { + "mse": 0.000787888013292104, + "psnr": 31.055835723876953, + "lpips": 0.011167233809828758 + }, + "frames_400_500": { + "mse": 0.0011472906917333603, + "psnr": 30.1364688873291, + "lpips": 0.013949623331427574 + }, + "overall": { + "mse": 0.001898613409139216, + "psnr": 29.41297721862793, + "lpips": 0.020014511421322823 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000107_test__val_savanna__3__000045/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000107_test__val_savanna__3__000045/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..33febca469b8b76ce10ce43332e0dd6b0b6f3e93 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000107_test__val_savanna__3__000045/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97fef810c962022c0398deaf4e084817e2654b9bb32d61706f7c87d6dc029fe2 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000107_test__val_savanna__3__000045/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000107_test__val_savanna__3__000045/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..a15a3efe4319f2a6330ee38c9cd1354bb12f9c6c --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000107_test__val_savanna__3__000045/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "107", + "global_test_index": "107", + "dataset_index": "230", + "relative_path": "test/val_savanna/3/000045.mp4", + "clip_id": "test/val_savanna/3/000045", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000107, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_savanna/3/000045_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.013274523429572582, + "psnr": 22.249982833862305, + "lpips": 0.10211960971355438 + }, + "frames_100_200": { + "mse": 0.004188560415059328, + "psnr": 25.888505935668945, + "lpips": 0.031040530651807785 + }, + "frames_200_300": { + "mse": 0.004064552951604128, + "psnr": 25.461318969726562, + "lpips": 0.033622562885284424 + }, + "frames_300_400": { + "mse": 0.00789472833275795, + "psnr": 25.529067993164062, + "lpips": 0.05475180596113205 + }, + "frames_400_500": { + "mse": 0.07968359440565109, + "psnr": 11.117253303527832, + "lpips": 0.45081859827041626 + }, + "overall": { + "mse": 0.021821191534399986, + "psnr": 22.049224853515625, + "lpips": 0.1344706267118454 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000108_test__val_iceplains__2__000049/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000108_test__val_iceplains__2__000049/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..2173c9fae1d24e71d162214d2ce0b118d70da0ad --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000108_test__val_iceplains__2__000049/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:69e406b7372cc959f948423c0cc7fe0f8ce4ecfa933842a70a94de81b23c61b6 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000108_test__val_iceplains__2__000049/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000108_test__val_iceplains__2__000049/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..16040e9254ee9dc9a4e7313667f9287d011b4c3e --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000108_test__val_iceplains__2__000049/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "108", + "global_test_index": "108", + "dataset_index": "372", + "relative_path": "test/val_iceplains/2/000049.mp4", + "clip_id": "test/val_iceplains/2/000049", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000108, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/2/000049_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.009993534535169601, + "psnr": 20.676050186157227, + "lpips": 0.10470864921808243 + }, + "frames_100_200": { + "mse": 0.016400853171944618, + "psnr": 18.049097061157227, + "lpips": 0.12444226443767548 + }, + "frames_200_300": { + "mse": 0.06907922029495239, + "psnr": 13.122562408447266, + "lpips": 0.30240002274513245 + }, + "frames_300_400": { + "mse": 0.14121758937835693, + "psnr": 8.564942359924316, + "lpips": 0.5768263936042786 + }, + "frames_400_500": { + "mse": 0.1770337074995041, + "psnr": 7.540010929107666, + "lpips": 0.6297520995140076 + }, + "overall": { + "mse": 0.08274497836828232, + "psnr": 13.590532302856445, + "lpips": 0.34762585163116455 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000109_test__val_savanna__1__000042/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000109_test__val_savanna__1__000042/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..876f8e1376f706c792569124ea543afcd7dd0088 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000109_test__val_savanna__1__000042/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:07b67761b07b0ce4daf97a43c91d9c2470151761c9cca9d884ef97a6ae52767f +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000109_test__val_savanna__1__000042/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000109_test__val_savanna__1__000042/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..f81a0b88ce0bc6806ef2c0cda7f5682c95231c38 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000109_test__val_savanna__1__000042/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "109", + "global_test_index": "109", + "dataset_index": "223", + "relative_path": "test/val_savanna/1/000042.mp4", + "clip_id": "test/val_savanna/1/000042", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000109, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_savanna/1/000042_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.009758352302014828, + "psnr": 20.17902946472168, + "lpips": 0.22845648229122162 + }, + "frames_100_200": { + "mse": 0.020841173827648163, + "psnr": 17.175466537475586, + "lpips": 0.24680112302303314 + }, + "frames_200_300": { + "mse": 0.019035110250115395, + "psnr": 18.28268051147461, + "lpips": 0.22324661910533905 + }, + "frames_300_400": { + "mse": 0.030362024903297424, + "psnr": 16.439983367919922, + "lpips": 0.34375086426734924 + }, + "frames_400_500": { + "mse": 0.02337292581796646, + "psnr": 16.781652450561523, + "lpips": 0.3778188228607178 + }, + "overall": { + "mse": 0.02067391574382782, + "psnr": 17.771760940551758, + "lpips": 0.2840147614479065 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000110_test__val_desert__0__000026/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000110_test__val_desert__0__000026/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..0dc48b72f98042d62adde84d770f0252dad3c17e --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000110_test__val_desert__0__000026/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a29dd859929e3f0d0db171fb6aebbbe11ddd61c591ef64e39eccc4bef282cbbc +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000110_test__val_desert__0__000026/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000110_test__val_desert__0__000026/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..1dad7c56d2a653f62063bf3a4af1118ba3913843 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000110_test__val_desert__0__000026/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "110", + "global_test_index": "110", + "dataset_index": "68", + "relative_path": "test/val_desert/0/000026.mp4", + "clip_id": "test/val_desert/0/000026", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000110, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_desert/0/000026_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.0020722835324704647, + "psnr": 27.372793197631836, + "lpips": 0.13417677581310272 + }, + "frames_100_200": { + "mse": 0.0033772303722798824, + "psnr": 24.728620529174805, + "lpips": 0.18128973245620728 + }, + "frames_200_300": { + "mse": 0.0039338706992566586, + "psnr": 24.204692840576172, + "lpips": 0.15604791045188904 + }, + "frames_300_400": { + "mse": 0.0028995764441788197, + "psnr": 26.036293029785156, + "lpips": 0.16487178206443787 + }, + "frames_400_500": { + "mse": 0.002150716492906213, + "psnr": 26.98138427734375, + "lpips": 0.14095686376094818 + }, + "overall": { + "mse": 0.002886735135689378, + "psnr": 25.864757537841797, + "lpips": 0.15546861290931702 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000111_test__val_plains__2__000042/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000111_test__val_plains__2__000042/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..230c37433b838f5f586210102badd875c2bff551 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000111_test__val_plains__2__000042/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d602bf6608ae40dc95fce87449e169432fd5d2fc20084d9614a74231f6fbd597 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000111_test__val_plains__2__000042/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000111_test__val_plains__2__000042/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..e6e3c2bb947489ccdfe589700691cd3a7be90269 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000111_test__val_plains__2__000042/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "111", + "global_test_index": "111", + "dataset_index": "461", + "relative_path": "test/val_plains/2/000042.mp4", + "clip_id": "test/val_plains/2/000042", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000111, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_plains/2/000042_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.002780497306957841, + "psnr": 27.02068328857422, + "lpips": 0.032105136662721634 + }, + "frames_100_200": { + "mse": 0.0009116565342992544, + "psnr": 30.41555404663086, + "lpips": 0.014904463663697243 + }, + "frames_200_300": { + "mse": 0.0010534230386838317, + "psnr": 29.8951416015625, + "lpips": 0.018714668229222298 + }, + "frames_300_400": { + "mse": 0.003015038324519992, + "psnr": 27.74016761779785, + "lpips": 0.03382595628499985 + }, + "frames_400_500": { + "mse": 0.018017204478383064, + "psnr": 23.384536743164062, + "lpips": 0.10752303898334503 + }, + "overall": { + "mse": 0.005155563820153475, + "psnr": 27.691219329833984, + "lpips": 0.04141465574502945 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000112_test__val_savanna__1__000009/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000112_test__val_savanna__1__000009/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..c6d713c017966ebff697d38ca0623c5d5c3eb9c6 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000112_test__val_savanna__1__000009/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d16e46e7403026949263ea6a9dd859aa68eaa53a652e86e2f20ac83717d1cab +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000112_test__val_savanna__1__000009/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000112_test__val_savanna__1__000009/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..87c7aa859c6de92ae035b2e374fff6d500fd0b7c --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000112_test__val_savanna__1__000009/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "112", + "global_test_index": "112", + "dataset_index": "151", + "relative_path": "test/val_savanna/1/000009.mp4", + "clip_id": "test/val_savanna/1/000009", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000112, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_savanna/1/000009_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.006916448473930359, + "psnr": 22.05770492553711, + "lpips": 0.2589899003505707 + }, + "frames_100_200": { + "mse": 0.007257770746946335, + "psnr": 21.592456817626953, + "lpips": 0.28402242064476013 + }, + "frames_200_300": { + "mse": 0.005781051702797413, + "psnr": 22.436067581176758, + "lpips": 0.320378839969635 + }, + "frames_300_400": { + "mse": 0.006351894233375788, + "psnr": 22.050546646118164, + "lpips": 0.3575570583343506 + }, + "frames_400_500": { + "mse": 0.004997661337256432, + "psnr": 23.089662551879883, + "lpips": 0.3375605344772339 + }, + "overall": { + "mse": 0.00626096548512578, + "psnr": 22.245288848876953, + "lpips": 0.3117017149925232 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000113_test__val_desert__3__000027/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000113_test__val_desert__3__000027/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..18ca5a0a59e284aa858f1d9ba0bf22c531abf5b8 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000113_test__val_desert__3__000027/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9192f3dab39993b5547f9a80e57764fa26dd7668ba85ee18b52f0881f902c5ae +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000113_test__val_desert__3__000027/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000113_test__val_desert__3__000027/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..dc80a75ba3c8ffcdf1bce7f37b6505c9d9dbeb44 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000113_test__val_desert__3__000027/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "113", + "global_test_index": "113", + "dataset_index": "72", + "relative_path": "test/val_desert/3/000027.mp4", + "clip_id": "test/val_desert/3/000027", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000113, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_desert/3/000027_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.002932193223387003, + "psnr": 26.128915786743164, + "lpips": 0.06866689026355743 + }, + "frames_100_200": { + "mse": 0.0008520427509211004, + "psnr": 30.72229766845703, + "lpips": 0.012959020212292671 + }, + "frames_200_300": { + "mse": 0.0008300771005451679, + "psnr": 30.822004318237305, + "lpips": 0.010996797122061253 + }, + "frames_300_400": { + "mse": 0.0008403641986660659, + "psnr": 30.7871036529541, + "lpips": 0.011467983946204185 + }, + "frames_400_500": { + "mse": 0.0008816251065582037, + "psnr": 30.6480655670166, + "lpips": 0.013023905456066132 + }, + "overall": { + "mse": 0.001267260522581637, + "psnr": 29.821680068969727, + "lpips": 0.02342292107641697 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000114_test__val_plains__0__000024/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000114_test__val_plains__0__000024/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..cc119105e6705a8e00e7b9a51df024cd713532cb --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000114_test__val_plains__0__000024/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:898a5f765edad66c388912b25b05ef0e95efe2bbbfff14dad36390edd4b5e977 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000114_test__val_plains__0__000024/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000114_test__val_plains__0__000024/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..21008065c8932b684558a13c8f7046c9c8448d0f --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000114_test__val_plains__0__000024/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "114", + "global_test_index": "114", + "dataset_index": "427", + "relative_path": "test/val_plains/0/000024.mp4", + "clip_id": "test/val_plains/0/000024", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000114, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_plains/0/000024_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.0032091245520859957, + "psnr": 25.21215057373047, + "lpips": 0.23371464014053345 + }, + "frames_100_200": { + "mse": 0.0057793413288891315, + "psnr": 22.553361892700195, + "lpips": 0.26570165157318115 + }, + "frames_200_300": { + "mse": 0.005742449779063463, + "psnr": 22.4958553314209, + "lpips": 0.3105796277523041 + }, + "frames_300_400": { + "mse": 0.006608020048588514, + "psnr": 21.948650360107422, + "lpips": 0.30599716305732727 + }, + "frames_400_500": { + "mse": 0.02050611563026905, + "psnr": 18.185579299926758, + "lpips": 0.413435161113739 + }, + "overall": { + "mse": 0.008369010873138905, + "psnr": 22.079118728637695, + "lpips": 0.3058856129646301 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000115_test__val_iceplains__3__000029/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000115_test__val_iceplains__3__000029/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..63e584bad9895f34461075f17e80cdd0ca783fd8 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000115_test__val_iceplains__3__000029/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:01b17d8190e3f2d34f11b9fd2fbba27f48243276f6325d6291eaf5a0aa7f28a8 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000115_test__val_iceplains__3__000029/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000115_test__val_iceplains__3__000029/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..9962504708e63eec1de1750b4d2597f6516ef2bf --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000115_test__val_iceplains__3__000029/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "115", + "global_test_index": "115", + "dataset_index": "317", + "relative_path": "test/val_iceplains/3/000029.mp4", + "clip_id": "test/val_iceplains/3/000029", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000115, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/3/000029_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.008364398963749409, + "psnr": 22.18102264404297, + "lpips": 0.1155116930603981 + }, + "frames_100_200": { + "mse": 0.005731170065701008, + "psnr": 23.052753448486328, + "lpips": 0.07710318267345428 + }, + "frames_200_300": { + "mse": 0.0104069197550416, + "psnr": 21.13109588623047, + "lpips": 0.07954317331314087 + }, + "frames_300_400": { + "mse": 0.019691966474056244, + "psnr": 18.394132614135742, + "lpips": 0.148819699883461 + }, + "frames_400_500": { + "mse": 0.009589237160980701, + "psnr": 21.949270248413086, + "lpips": 0.08370378613471985 + }, + "overall": { + "mse": 0.010756738483905792, + "psnr": 21.34165382385254, + "lpips": 0.10093631595373154 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000116_test__val_iceplains__2__000001/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000116_test__val_iceplains__2__000001/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..98c6600105ad85f221782ca56a640753fc066576 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000116_test__val_iceplains__2__000001/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c5aa2d35446c96cc6ccc47fd6c8529256b4d8b21ed757d1552787d1f6b3147b +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000116_test__val_iceplains__2__000001/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000116_test__val_iceplains__2__000001/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..b26557f7bd5cfdf62a326731c1995a9c0272d91f --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000116_test__val_iceplains__2__000001/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "116", + "global_test_index": "116", + "dataset_index": "249", + "relative_path": "test/val_iceplains/2/000001.mp4", + "clip_id": "test/val_iceplains/2/000001", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000116, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/2/000001_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.11967460811138153, + "psnr": 15.642480850219727, + "lpips": 0.2731485366821289 + }, + "frames_100_200": { + "mse": 0.15964087843894958, + "psnr": 8.147207260131836, + "lpips": 0.572786808013916 + }, + "frames_200_300": { + "mse": 0.14875037968158722, + "psnr": 8.360014915466309, + "lpips": 0.5919588208198547 + }, + "frames_300_400": { + "mse": 0.2218797653913498, + "psnr": 6.720223426818848, + "lpips": 0.6742709279060364 + }, + "frames_400_500": { + "mse": 0.19434215128421783, + "psnr": 7.65548038482666, + "lpips": 0.6874052286148071 + }, + "overall": { + "mse": 0.16885755956172943, + "psnr": 9.305082321166992, + "lpips": 0.5599140524864197 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000117_test__val_iceplains__2__000046/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000117_test__val_iceplains__2__000046/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..95a6ff20e136fc92f5468c9c6e0e4f1ffcf5b73c --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000117_test__val_iceplains__2__000046/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5746936c24866b133a05c8baa4b6bdec24f7459b30f21c3183af86fdb1f97faa +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000117_test__val_iceplains__2__000046/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000117_test__val_iceplains__2__000046/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..164b228bcab616d2eaac264bdf1e3b850a2fc686 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000117_test__val_iceplains__2__000046/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "117", + "global_test_index": "117", + "dataset_index": "364", + "relative_path": "test/val_iceplains/2/000046.mp4", + "clip_id": "test/val_iceplains/2/000046", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000117, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/2/000046_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.0163605697453022, + "psnr": 18.410846710205078, + "lpips": 0.15298470854759216 + }, + "frames_100_200": { + "mse": 0.028568662703037262, + "psnr": 16.133726119995117, + "lpips": 0.2295544296503067 + }, + "frames_200_300": { + "mse": 0.008767427876591682, + "psnr": 21.819612503051758, + "lpips": 0.185492604970932 + }, + "frames_300_400": { + "mse": 0.02227250300347805, + "psnr": 17.055625915527344, + "lpips": 0.19594165682792664 + }, + "frames_400_500": { + "mse": 0.01709243655204773, + "psnr": 18.67211151123047, + "lpips": 0.12843607366085052 + }, + "overall": { + "mse": 0.018612319603562355, + "psnr": 18.41838264465332, + "lpips": 0.17848187685012817 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000118_test__val_desert__2__000037/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000118_test__val_desert__2__000037/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..f8b13652bd009319986361ac6c73eca17fb6cbf3 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000118_test__val_desert__2__000037/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2bc1437ff56ae0f2d36f3863907e5533202068ed4842d65cd841b340181ead00 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000118_test__val_desert__2__000037/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000118_test__val_desert__2__000037/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..87bb76c0b11c00b7e90e5c7b26a2a0a1e50f5b29 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000118_test__val_desert__2__000037/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "118", + "global_test_index": "118", + "dataset_index": "96", + "relative_path": "test/val_desert/2/000037.mp4", + "clip_id": "test/val_desert/2/000037", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000118, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_desert/2/000037_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.0007630030158907175, + "psnr": 31.186067581176758, + "lpips": 0.010031122714281082 + }, + "frames_100_200": { + "mse": 0.0008650166564621031, + "psnr": 30.715059280395508, + "lpips": 0.014076989144086838 + }, + "frames_200_300": { + "mse": 0.0007862616912461817, + "psnr": 31.0632266998291, + "lpips": 0.010557170026004314 + }, + "frames_300_400": { + "mse": 0.0010534569155424833, + "psnr": 30.635427474975586, + "lpips": 0.013852422125637531 + }, + "frames_400_500": { + "mse": 0.0008052626508288085, + "psnr": 30.961442947387695, + "lpips": 0.011108790524303913 + }, + "overall": { + "mse": 0.0008546001627109945, + "psnr": 30.912246704101562, + "lpips": 0.01192529872059822 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000119_test__val_savanna__3__000029/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000119_test__val_savanna__3__000029/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..d73d3a66116cc1ec38e3ff5b3cb30f401decfa5e --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000119_test__val_savanna__3__000029/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9bae3fdd8437e58fbd31b0a21c77b847d8944271b656b970816562903260118c +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000119_test__val_savanna__3__000029/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000119_test__val_savanna__3__000029/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..2989c797759f30456ac1b1975eb32a56995c2d39 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000119_test__val_savanna__3__000029/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "119", + "global_test_index": "119", + "dataset_index": "190", + "relative_path": "test/val_savanna/3/000029.mp4", + "clip_id": "test/val_savanna/3/000029", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000119, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_savanna/3/000029_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.006352095399051905, + "psnr": 22.150135040283203, + "lpips": 0.3322334587574005 + }, + "frames_100_200": { + "mse": 0.008392971940338612, + "psnr": 20.850313186645508, + "lpips": 0.40120887756347656 + }, + "frames_200_300": { + "mse": 0.009415778331458569, + "psnr": 20.634069442749023, + "lpips": 0.4430408179759979 + }, + "frames_300_400": { + "mse": 0.008644289337098598, + "psnr": 20.91852378845215, + "lpips": 0.4345099925994873 + }, + "frames_400_500": { + "mse": 0.01581471972167492, + "psnr": 18.52924156188965, + "lpips": 0.479642778635025 + }, + "overall": { + "mse": 0.009723971597850323, + "psnr": 20.616456985473633, + "lpips": 0.4181272089481354 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000120_test__val_savanna__2__000044/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000120_test__val_savanna__2__000044/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..5c2bdd2dc319bc1397fae0b9988b9228e3725763 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000120_test__val_savanna__2__000044/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7fbb9681998afcbe4f56ee56fbfceb64122b25f5b9221fa6810f11f9fdaf1ae7 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000120_test__val_savanna__2__000044/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000120_test__val_savanna__2__000044/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..1d945d832353ca09d0c4854c30f114b0e621f1e2 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000120_test__val_savanna__2__000044/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "120", + "global_test_index": "120", + "dataset_index": "227", + "relative_path": "test/val_savanna/2/000044.mp4", + "clip_id": "test/val_savanna/2/000044", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000120, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_savanna/2/000044_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.008775853551924229, + "psnr": 21.127370834350586, + "lpips": 0.23448213934898376 + }, + "frames_100_200": { + "mse": 0.022997751832008362, + "psnr": 16.865135192871094, + "lpips": 0.2743813693523407 + }, + "frames_200_300": { + "mse": 0.024198420345783234, + "psnr": 16.506929397583008, + "lpips": 0.3092748522758484 + }, + "frames_300_400": { + "mse": 0.041988302022218704, + "psnr": 13.924062728881836, + "lpips": 0.2860987186431885 + }, + "frames_400_500": { + "mse": 0.05083233490586281, + "psnr": 12.989921569824219, + "lpips": 0.30969372391700745 + }, + "overall": { + "mse": 0.029758533462882042, + "psnr": 16.282686233520508, + "lpips": 0.28278616070747375 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000121_test__val_iceplains__0__000000/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000121_test__val_iceplains__0__000000/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..f0cad481eeb543359995cff913fc7361dd41689b --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000121_test__val_iceplains__0__000000/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ecc27e5f659a5c12da810dee5a57a9a6c432bf0a7b68455630ea398052fc421 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000121_test__val_iceplains__0__000000/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000121_test__val_iceplains__0__000000/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..22786e01597a2f76c8587192339120954dd1dcb3 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000121_test__val_iceplains__0__000000/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "121", + "global_test_index": "121", + "dataset_index": "244", + "relative_path": "test/val_iceplains/0/000000.mp4", + "clip_id": "test/val_iceplains/0/000000", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000121, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/0/000000_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.0056976573541760445, + "psnr": 24.517860412597656, + "lpips": 0.04380323365330696 + }, + "frames_100_200": { + "mse": 0.016921186819672585, + "psnr": 19.877336502075195, + "lpips": 0.11825570464134216 + }, + "frames_200_300": { + "mse": 0.009243734180927277, + "psnr": 24.374408721923828, + "lpips": 0.0569198839366436 + }, + "frames_300_400": { + "mse": 0.0030493461526930332, + "psnr": 27.629018783569336, + "lpips": 0.02626420557498932 + }, + "frames_400_500": { + "mse": 0.0034458136651664972, + "psnr": 25.977283477783203, + "lpips": 0.03606557846069336 + }, + "overall": { + "mse": 0.007671547122299671, + "psnr": 24.475183486938477, + "lpips": 0.05626171827316284 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000122_test__val_desert__3__000015/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000122_test__val_desert__3__000015/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..b610b8efad14612944296d03c01551ae786d8328 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000122_test__val_desert__3__000015/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:abe9ae27e370542393605cc7963281ef26fca980ba1c32ce0ceb92aadbfbebd8 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000122_test__val_desert__3__000015/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000122_test__val_desert__3__000015/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..5cb433505d95f88a1dc7701ec145efbb7d305805 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000122_test__val_desert__3__000015/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "122", + "global_test_index": "122", + "dataset_index": "38", + "relative_path": "test/val_desert/3/000015.mp4", + "clip_id": "test/val_desert/3/000015", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000122, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_desert/3/000015_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.021145325154066086, + "psnr": 16.94322395324707, + "lpips": 0.400046706199646 + }, + "frames_100_200": { + "mse": 0.030510758981108665, + "psnr": 15.285277366638184, + "lpips": 0.44226086139678955 + }, + "frames_200_300": { + "mse": 0.026783067733049393, + "psnr": 15.966036796569824, + "lpips": 0.43109130859375 + }, + "frames_300_400": { + "mse": 0.03701387718319893, + "psnr": 16.130037307739258, + "lpips": 0.3567872643470764 + }, + "frames_400_500": { + "mse": 0.06399945169687271, + "psnr": 12.516956329345703, + "lpips": 0.6133360266685486 + }, + "overall": { + "mse": 0.03589049354195595, + "psnr": 15.368305206298828, + "lpips": 0.44870442152023315 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000123_test__val_iceplains__2__000017/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000123_test__val_iceplains__2__000017/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..14143a424dbb60c2779ea07a6cdf96f46bb7b4e2 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000123_test__val_iceplains__2__000017/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0329540dfd2d6183bd772104ae0ee0cb8b460d7fef4c396d3b18e30fada0ccff +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000123_test__val_iceplains__2__000017/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000123_test__val_iceplains__2__000017/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..f5fbf31463d8889c30e03ace2f6fcf1fd5086701 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000123_test__val_iceplains__2__000017/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "123", + "global_test_index": "123", + "dataset_index": "291", + "relative_path": "test/val_iceplains/2/000017.mp4", + "clip_id": "test/val_iceplains/2/000017", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000123, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/2/000017_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.005648059770464897, + "psnr": 25.132734298706055, + "lpips": 0.04401273652911186 + }, + "frames_100_200": { + "mse": 0.03479870408773422, + "psnr": 15.57012939453125, + "lpips": 0.2625121772289276 + }, + "frames_200_300": { + "mse": 0.0208117738366127, + "psnr": 17.262744903564453, + "lpips": 0.17174643278121948 + }, + "frames_300_400": { + "mse": 0.021855700761079788, + "psnr": 16.917757034301758, + "lpips": 0.15892009437084198 + }, + "frames_400_500": { + "mse": 0.027956724166870117, + "psnr": 17.119016647338867, + "lpips": 0.1983371376991272 + }, + "overall": { + "mse": 0.022214191034436226, + "psnr": 18.400476455688477, + "lpips": 0.16710571944713593 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000124_test__val_desert__2__000032/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000124_test__val_desert__2__000032/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..cad4459dfaef20a7ee89b84cf3859d63c35917b9 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000124_test__val_desert__2__000032/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:69446ac98fb015695943efba05ed5614cdea4af7579d4ab0c777e8d476ec7801 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000124_test__val_desert__2__000032/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000124_test__val_desert__2__000032/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..19c65ba408be4fe8fe81adbf43fda1710df72f93 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000124_test__val_desert__2__000032/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "124", + "global_test_index": "124", + "dataset_index": "85", + "relative_path": "test/val_desert/2/000032.mp4", + "clip_id": "test/val_desert/2/000032", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000124, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_desert/2/000032_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.004331690259277821, + "psnr": 23.97394561767578, + "lpips": 0.11815916001796722 + }, + "frames_100_200": { + "mse": 0.010476481169462204, + "psnr": 20.030044555664062, + "lpips": 0.19860833883285522 + }, + "frames_200_300": { + "mse": 0.00905869621783495, + "psnr": 20.602577209472656, + "lpips": 0.16354501247406006 + }, + "frames_300_400": { + "mse": 0.013608221895992756, + "psnr": 18.859127044677734, + "lpips": 0.2369501143693924 + }, + "frames_400_500": { + "mse": 0.009792959317564964, + "psnr": 21.632484436035156, + "lpips": 0.15315750241279602 + }, + "overall": { + "mse": 0.009453609585762024, + "psnr": 21.019636154174805, + "lpips": 0.17408403754234314 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000125_test__val_iceplains__0__000044/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000125_test__val_iceplains__0__000044/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..09d10cc130c9d63a12f04fdd835aa593a9c4cf18 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000125_test__val_iceplains__0__000044/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:819e66e90abc6825f4525bf4565292fd038811ba415fd211417ba9438e5db657 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000125_test__val_iceplains__0__000044/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000125_test__val_iceplains__0__000044/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..33220d52980294a20b657f2c293159e0c9898d65 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000125_test__val_iceplains__0__000044/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "125", + "global_test_index": "125", + "dataset_index": "358", + "relative_path": "test/val_iceplains/0/000044.mp4", + "clip_id": "test/val_iceplains/0/000044", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000125, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/0/000044_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.0009121017646975815, + "psnr": 30.448078155517578, + "lpips": 0.011078937910497189 + }, + "frames_100_200": { + "mse": 0.0008803029195405543, + "psnr": 30.5762996673584, + "lpips": 0.011476310901343822 + }, + "frames_200_300": { + "mse": 0.0008539684349671006, + "psnr": 30.705490112304688, + "lpips": 0.011835695244371891 + }, + "frames_300_400": { + "mse": 0.0011749888071790338, + "psnr": 30.000391006469727, + "lpips": 0.01733907125890255 + }, + "frames_400_500": { + "mse": 0.0008278899476863444, + "psnr": 30.844362258911133, + "lpips": 0.01336725801229477 + }, + "overall": { + "mse": 0.0009298503864556551, + "psnr": 30.514923095703125, + "lpips": 0.013019452802836895 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000126_test__val_desert__1__000020/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000126_test__val_desert__1__000020/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..5705b5a615ba8a3ed06b057dfe4caa58b8f4ac0a --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000126_test__val_desert__1__000020/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae75625d6b7dc1fe7334ada6a3ed55752665c39b0c4530ed000a81c53027a1dd +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000126_test__val_desert__1__000020/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000126_test__val_desert__1__000020/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..f05a0fb4bdb38e795ebc5c72814e6f16541a9606 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000126_test__val_desert__1__000020/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "126", + "global_test_index": "126", + "dataset_index": "50", + "relative_path": "test/val_desert/1/000020.mp4", + "clip_id": "test/val_desert/1/000020", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000126, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_desert/1/000020_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.0015453249216079712, + "psnr": 29.552017211914062, + "lpips": 0.018544360995292664 + }, + "frames_100_200": { + "mse": 0.000784842821303755, + "psnr": 31.063325881958008, + "lpips": 0.010349755175411701 + }, + "frames_200_300": { + "mse": 0.0008128781337291002, + "psnr": 30.918550491333008, + "lpips": 0.010957338847219944 + }, + "frames_300_400": { + "mse": 0.0020316855516284704, + "psnr": 28.971208572387695, + "lpips": 0.02472817711532116 + }, + "frames_400_500": { + "mse": 0.00611824169754982, + "psnr": 24.93108558654785, + "lpips": 0.04952217638492584 + }, + "overall": { + "mse": 0.002258594613522291, + "psnr": 29.087236404418945, + "lpips": 0.02282036282122135 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000127_test__val_savanna__0__000036/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000127_test__val_savanna__0__000036/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..e15829482ed3a337d49fc747beabc46376e5fc08 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000127_test__val_savanna__0__000036/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb2a4003eeb9bc76a5cb559fb2afff583df1bdcc0c61944d964029628a99b967 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000127_test__val_savanna__0__000036/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000127_test__val_savanna__0__000036/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..eca01c922f06367329db32d89801a4412609b334 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000127_test__val_savanna__0__000036/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "127", + "global_test_index": "127", + "dataset_index": "207", + "relative_path": "test/val_savanna/0/000036.mp4", + "clip_id": "test/val_savanna/0/000036", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000127, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_savanna/0/000036_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.00934706162661314, + "psnr": 21.11994171142578, + "lpips": 0.3539647161960602 + }, + "frames_100_200": { + "mse": 0.008250241167843342, + "psnr": 20.954423904418945, + "lpips": 0.4209368824958801 + }, + "frames_200_300": { + "mse": 0.008380095474421978, + "psnr": 20.93848419189453, + "lpips": 0.43296489119529724 + }, + "frames_300_400": { + "mse": 0.019491130486130714, + "psnr": 18.368589401245117, + "lpips": 0.5332638025283813 + }, + "frames_400_500": { + "mse": 0.01947379671037197, + "psnr": 17.571882247924805, + "lpips": 0.5891820788383484 + }, + "overall": { + "mse": 0.01298846397548914, + "psnr": 19.790664672851562, + "lpips": 0.4660624861717224 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000128_test__val_iceplains__2__000034/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000128_test__val_iceplains__2__000034/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..2c56f37b04c4c4ff46cadf4c1c94b9b56e1615c9 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000128_test__val_iceplains__2__000034/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e5e6e12be967a92f446bd523d4e0da9acf767c0a7bb32309ed345aa5d7a9e8ed +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000128_test__val_iceplains__2__000034/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000128_test__val_iceplains__2__000034/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..2635efc1acf47985bdbe1ec24018eba4724476e0 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000128_test__val_iceplains__2__000034/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "128", + "global_test_index": "128", + "dataset_index": "330", + "relative_path": "test/val_iceplains/2/000034.mp4", + "clip_id": "test/val_iceplains/2/000034", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000128, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/2/000034_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.0009940838208422065, + "psnr": 30.175966262817383, + "lpips": 0.016340218484401703 + }, + "frames_100_200": { + "mse": 0.001152851851657033, + "psnr": 30.4147891998291, + "lpips": 0.014554125256836414 + }, + "frames_200_300": { + "mse": 0.013380881398916245, + "psnr": 24.874099731445312, + "lpips": 0.07959706336259842 + }, + "frames_300_400": { + "mse": 0.09359496831893921, + "psnr": 10.645700454711914, + "lpips": 0.4439235031604767 + }, + "frames_400_500": { + "mse": 0.0837288349866867, + "psnr": 11.043944358825684, + "lpips": 0.4600609540939331 + }, + "overall": { + "mse": 0.038570329546928406, + "psnr": 21.430898666381836, + "lpips": 0.2028951793909073 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000129_test__val_savanna__0__000001/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000129_test__val_savanna__0__000001/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..59ee8f0643ccb2c1ebc54aa185be896cdf0d7d3f --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000129_test__val_savanna__0__000001/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c73ce49a016439e96603490e01e2edba4a1c90f42ce205e9762f9e7a2f63e86 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000129_test__val_savanna__0__000001/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000129_test__val_savanna__0__000001/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..f592e6cac3b0294a69152a531879606aaa1ed883 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000129_test__val_savanna__0__000001/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "129", + "global_test_index": "129", + "dataset_index": "132", + "relative_path": "test/val_savanna/0/000001.mp4", + "clip_id": "test/val_savanna/0/000001", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000129, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_savanna/0/000001_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.0014387064147740602, + "psnr": 28.693891525268555, + "lpips": 0.021446244791150093 + }, + "frames_100_200": { + "mse": 0.0018437565304338932, + "psnr": 28.190732955932617, + "lpips": 0.024640575051307678 + }, + "frames_200_300": { + "mse": 0.006858705077320337, + "psnr": 22.237876892089844, + "lpips": 0.16996392607688904 + }, + "frames_300_400": { + "mse": 0.012084349058568478, + "psnr": 19.346473693847656, + "lpips": 0.25897935032844543 + }, + "frames_400_500": { + "mse": 0.00478698406368494, + "psnr": 23.55335235595703, + "lpips": 0.07106269150972366 + }, + "overall": { + "mse": 0.005402500741183758, + "psnr": 24.404462814331055, + "lpips": 0.1092185527086258 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000130_test__val_plains__3__000021/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000130_test__val_plains__3__000021/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..32c437e96e06eac55c659455e031402cd5f59162 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000130_test__val_plains__3__000021/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f08b58d9a23dfdc296d56eee7aaa47d3347e28912d6be54bb59f7971895b73fd +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000130_test__val_plains__3__000021/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000130_test__val_plains__3__000021/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..7c53bdf9a11e0feaf1eca3f96bce3a072b7394d3 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000130_test__val_plains__3__000021/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "130", + "global_test_index": "130", + "dataset_index": "423", + "relative_path": "test/val_plains/3/000021.mp4", + "clip_id": "test/val_plains/3/000021", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000130, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_plains/3/000021_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.0034952580463141203, + "psnr": 24.717187881469727, + "lpips": 0.23438766598701477 + }, + "frames_100_200": { + "mse": 0.004916871897876263, + "psnr": 23.409868240356445, + "lpips": 0.3208896517753601 + }, + "frames_200_300": { + "mse": 0.0044639743864536285, + "psnr": 23.681018829345703, + "lpips": 0.3653680682182312 + }, + "frames_300_400": { + "mse": 0.012522201985120773, + "psnr": 21.1378116607666, + "lpips": 0.40361639857292175 + }, + "frames_400_500": { + "mse": 0.05724256485700607, + "psnr": 12.943222999572754, + "lpips": 0.4502119719982147 + }, + "overall": { + "mse": 0.01652817241847515, + "psnr": 21.177820205688477, + "lpips": 0.354894757270813 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000131_test__val_desert__2__000025/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000131_test__val_desert__2__000025/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..486c53c95bc025953590865e6e71f0c8a37c9579 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000131_test__val_desert__2__000025/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:54e938f7303044c45f953ec32a40a0738e238320d33455eee6c904d6c5c8543a +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000131_test__val_desert__2__000025/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000131_test__val_desert__2__000025/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..d0825a94d0b4028cbf68985d18f0cbabc37864ea --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000131_test__val_desert__2__000025/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "131", + "global_test_index": "131", + "dataset_index": "66", + "relative_path": "test/val_desert/2/000025.mp4", + "clip_id": "test/val_desert/2/000025", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000131, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_desert/2/000025_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.010205404832959175, + "psnr": 20.287734985351562, + "lpips": 0.24120420217514038 + }, + "frames_100_200": { + "mse": 0.014158777892589569, + "psnr": 18.63282012939453, + "lpips": 0.26702290773391724 + }, + "frames_200_300": { + "mse": 0.027739841490983963, + "psnr": 16.456689834594727, + "lpips": 0.3387245535850525 + }, + "frames_300_400": { + "mse": 0.05560064688324928, + "psnr": 12.877233505249023, + "lpips": 0.44360265135765076 + }, + "frames_400_500": { + "mse": 0.03110302984714508, + "psnr": 15.574995994567871, + "lpips": 0.391200989484787 + }, + "overall": { + "mse": 0.027761541306972504, + "psnr": 16.765893936157227, + "lpips": 0.33635103702545166 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000132_test__val_savanna__3__000009/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000132_test__val_savanna__3__000009/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..a35d7628043a5ed4bb436f9f6b001aedbbe40cce --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000132_test__val_savanna__3__000009/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e20e094a9e94d9ce1eaa3411758b5add08a7f375321c48c17791c86dee2f9bf +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000132_test__val_savanna__3__000009/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000132_test__val_savanna__3__000009/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..ebe069ceab16b5060d0784d179f798dbab50baea --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000132_test__val_savanna__3__000009/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "132", + "global_test_index": "132", + "dataset_index": "153", + "relative_path": "test/val_savanna/3/000009.mp4", + "clip_id": "test/val_savanna/3/000009", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000132, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_savanna/3/000009_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.01654445007443428, + "psnr": 19.029138565063477, + "lpips": 0.15810824930667877 + }, + "frames_100_200": { + "mse": 0.016569197177886963, + "psnr": 18.72527313232422, + "lpips": 0.12477918714284897 + }, + "frames_200_300": { + "mse": 0.015135022811591625, + "psnr": 18.4718074798584, + "lpips": 0.13277095556259155 + }, + "frames_300_400": { + "mse": 0.02913081645965576, + "psnr": 16.724702835083008, + "lpips": 0.14161890745162964 + }, + "frames_400_500": { + "mse": 0.14923836290836334, + "psnr": 8.58316421508789, + "lpips": 0.3343929946422577 + }, + "overall": { + "mse": 0.04532356560230255, + "psnr": 16.30681610107422, + "lpips": 0.178334042429924 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000133_test__val_plains__0__000004/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000133_test__val_plains__0__000004/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..c398e172c9ab4eead33aee94995bf018403be9a1 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000133_test__val_plains__0__000004/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56d49648ef227edd8bd595e1cf759b840b3be8eda9a3f2847f841f357b75728e +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000133_test__val_plains__0__000004/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000133_test__val_plains__0__000004/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..0726692101ab2da4f88025b7f68cf846b1641cf1 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000133_test__val_plains__0__000004/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "133", + "global_test_index": "133", + "dataset_index": "382", + "relative_path": "test/val_plains/0/000004.mp4", + "clip_id": "test/val_plains/0/000004", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000133, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_plains/0/000004_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.005991317797452211, + "psnr": 22.39032554626465, + "lpips": 0.250559538602829 + }, + "frames_100_200": { + "mse": 0.003943488467484713, + "psnr": 24.075307846069336, + "lpips": 0.32272613048553467 + }, + "frames_200_300": { + "mse": 0.00420202873647213, + "psnr": 23.908235549926758, + "lpips": 0.319621741771698 + }, + "frames_300_400": { + "mse": 0.006108183413743973, + "psnr": 22.359298706054688, + "lpips": 0.3688417673110962 + }, + "frames_400_500": { + "mse": 0.004429328721016645, + "psnr": 23.64690589904785, + "lpips": 0.3704599440097809 + }, + "overall": { + "mse": 0.0049348692409694195, + "psnr": 23.276016235351562, + "lpips": 0.32644182443618774 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000134_test__val_savanna__2__000005/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000134_test__val_savanna__2__000005/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..28ef480d2017328c33e521465ada5fc4493684b7 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000134_test__val_savanna__2__000005/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:605769f06b78304f94ab6b34f476e933b2b16d4a047525c8ae0ad6af75bed3e0 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000134_test__val_savanna__2__000005/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000134_test__val_savanna__2__000005/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..f34b1a873ef55888dd0d84fa1c1e918156d6de12 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000134_test__val_savanna__2__000005/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "134", + "global_test_index": "134", + "dataset_index": "143", + "relative_path": "test/val_savanna/2/000005.mp4", + "clip_id": "test/val_savanna/2/000005", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000134, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_savanna/2/000005_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.01974167674779892, + "psnr": 17.754274368286133, + "lpips": 0.34650132060050964 + }, + "frames_100_200": { + "mse": 0.02998770773410797, + "psnr": 15.897770881652832, + "lpips": 0.40583544969558716 + }, + "frames_200_300": { + "mse": 0.012863597832620144, + "psnr": 19.866682052612305, + "lpips": 0.40820327401161194 + }, + "frames_300_400": { + "mse": 0.022547312080860138, + "psnr": 17.378515243530273, + "lpips": 0.4979960322380066 + }, + "frames_400_500": { + "mse": 0.0702403262257576, + "psnr": 11.788537979125977, + "lpips": 0.5287275910377502 + }, + "overall": { + "mse": 0.03107612393796444, + "psnr": 16.53715705871582, + "lpips": 0.4374527335166931 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000135_test__val_iceplains__1__000040/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000135_test__val_iceplains__1__000040/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..b7b360721a400b75680a7104fc5ef8c52bb33473 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000135_test__val_iceplains__1__000040/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5b4ab32e738f5cfd624efd30cb43ba11ad55b94612f0a0ad4ab05085370eb727 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000135_test__val_iceplains__1__000040/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000135_test__val_iceplains__1__000040/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..60736905067ec4d3ce875c24565d48f5c967cd48 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000135_test__val_iceplains__1__000040/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "135", + "global_test_index": "135", + "dataset_index": "348", + "relative_path": "test/val_iceplains/1/000040.mp4", + "clip_id": "test/val_iceplains/1/000040", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000135, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/1/000040_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.024649320170283318, + "psnr": 16.78228759765625, + "lpips": 0.21245817840099335 + }, + "frames_100_200": { + "mse": 0.022290397435426712, + "psnr": 17.625572204589844, + "lpips": 0.19443005323410034 + }, + "frames_200_300": { + "mse": 0.023904450237751007, + "psnr": 16.78777503967285, + "lpips": 0.1947242021560669 + }, + "frames_300_400": { + "mse": 0.023318922147154808, + "psnr": 18.358909606933594, + "lpips": 0.23628747463226318 + }, + "frames_400_500": { + "mse": 0.021162495017051697, + "psnr": 17.299556732177734, + "lpips": 0.1690971553325653 + }, + "overall": { + "mse": 0.023065118119120598, + "psnr": 17.370819091796875, + "lpips": 0.20139941573143005 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000136_test__val_plains__2__000005/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000136_test__val_plains__2__000005/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..5717b7a27b2017f46dfe61998a8bd43b420947c5 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000136_test__val_plains__2__000005/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:091c24a4f3d977e5a015a416f32e63fbc6fd56c68eb0a5842b59f39b339d5d6d +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000136_test__val_plains__2__000005/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000136_test__val_plains__2__000005/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..423cbe56399c78e6524d4c3339cbd7ffbe2c1938 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000136_test__val_plains__2__000005/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "136", + "global_test_index": "136", + "dataset_index": "384", + "relative_path": "test/val_plains/2/000005.mp4", + "clip_id": "test/val_plains/2/000005", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000136, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_plains/2/000005_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.004062649328261614, + "psnr": 23.954421997070312, + "lpips": 0.32999157905578613 + }, + "frames_100_200": { + "mse": 0.004088507499545813, + "psnr": 23.908784866333008, + "lpips": 0.3262858986854553 + }, + "frames_200_300": { + "mse": 0.014656939543783665, + "psnr": 18.995656967163086, + "lpips": 0.4316607713699341 + }, + "frames_300_400": { + "mse": 0.01462460309267044, + "psnr": 18.6549015045166, + "lpips": 0.3486877381801605 + }, + "frames_400_500": { + "mse": 0.003924056421965361, + "psnr": 25.588773727416992, + "lpips": 0.03851732611656189 + }, + "overall": { + "mse": 0.008271350525319576, + "psnr": 22.22050666809082, + "lpips": 0.2950286865234375 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000137_test__val_desert__0__000012/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000137_test__val_desert__0__000012/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..f7a75fec90bea2a35626c196801f117dff7bb5bc --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000137_test__val_desert__0__000012/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f94cedd2a90fbda9291879015bd292c88c7b3d25853c7dde8814159f25b2f43 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000137_test__val_desert__0__000012/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000137_test__val_desert__0__000012/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..9467586d1814226b97f865e2e8f816b7c32293d3 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000137_test__val_desert__0__000012/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "137", + "global_test_index": "137", + "dataset_index": "30", + "relative_path": "test/val_desert/0/000012.mp4", + "clip_id": "test/val_desert/0/000012", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000137, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_desert/0/000012_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.0007719050627201796, + "psnr": 31.1411190032959, + "lpips": 0.009894508868455887 + }, + "frames_100_200": { + "mse": 0.0007748998468741775, + "psnr": 31.125513076782227, + "lpips": 0.010625000111758709 + }, + "frames_200_300": { + "mse": 0.0012482637539505959, + "psnr": 29.591825485229492, + "lpips": 0.021802593022584915 + }, + "frames_300_400": { + "mse": 0.0007879508193582296, + "psnr": 31.054079055786133, + "lpips": 0.010389416478574276 + }, + "frames_400_500": { + "mse": 0.0008022466790862381, + "psnr": 30.968128204345703, + "lpips": 0.011472775600850582 + }, + "overall": { + "mse": 0.0008770531858317554, + "psnr": 30.7761287689209, + "lpips": 0.012836858630180359 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000138_test__val_savanna__2__000034/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000138_test__val_savanna__2__000034/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..09a2d3afb27cf63cd4ab358da710045cdf8bbf37 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000138_test__val_savanna__2__000034/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:05b5543d4c8576f78d7c4430b5b6fe08d4b76984254bb271044e3cc70e8453bf +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000138_test__val_savanna__2__000034/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000138_test__val_savanna__2__000034/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..de59d2f44249305eef490c1c31af99bf3a92890f --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000138_test__val_savanna__2__000034/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "138", + "global_test_index": "138", + "dataset_index": "203", + "relative_path": "test/val_savanna/2/000034.mp4", + "clip_id": "test/val_savanna/2/000034", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000138, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_savanna/2/000034_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.002222759649157524, + "psnr": 27.98806381225586, + "lpips": 0.01969381794333458 + }, + "frames_100_200": { + "mse": 0.0015432279324159026, + "psnr": 29.43335723876953, + "lpips": 0.013780159875750542 + }, + "frames_200_300": { + "mse": 0.002052953466773033, + "psnr": 28.7656192779541, + "lpips": 0.01826636493206024 + }, + "frames_300_400": { + "mse": 0.0032546070870012045, + "psnr": 26.99090576171875, + "lpips": 0.030374800786376 + }, + "frames_400_500": { + "mse": 0.012069626711308956, + "psnr": 23.199552536010742, + "lpips": 0.05726233124732971 + }, + "overall": { + "mse": 0.004228635225445032, + "psnr": 27.27549934387207, + "lpips": 0.027875494211912155 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000139_test__val_savanna__0__000022/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000139_test__val_savanna__0__000022/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..c05e9721a98751648917aeaa79084023df60be8c --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000139_test__val_savanna__0__000022/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b914752f655dcb381a80f01dbb5fa13138842b865b79e639e89c8b6df9eaba7 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000139_test__val_savanna__0__000022/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000139_test__val_savanna__0__000022/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..14c7acbe30b45fd29c7e52197a6b0126a823079e --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000139_test__val_savanna__0__000022/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "139", + "global_test_index": "139", + "dataset_index": "179", + "relative_path": "test/val_savanna/0/000022.mp4", + "clip_id": "test/val_savanna/0/000022", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000139, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_savanna/0/000022_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.001304142759181559, + "psnr": 28.92938995361328, + "lpips": 0.12705115973949432 + }, + "frames_100_200": { + "mse": 0.0020905453711748123, + "psnr": 26.878023147583008, + "lpips": 0.14873452484607697 + }, + "frames_200_300": { + "mse": 0.0028520100750029087, + "psnr": 25.615097045898438, + "lpips": 0.11871013790369034 + }, + "frames_300_400": { + "mse": 0.005283931270241737, + "psnr": 22.979887008666992, + "lpips": 0.1048877164721489 + }, + "frames_400_500": { + "mse": 0.006808391306549311, + "psnr": 21.790315628051758, + "lpips": 0.11640611290931702 + }, + "overall": { + "mse": 0.0036678037140518427, + "psnr": 25.238542556762695, + "lpips": 0.12315792590379715 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000140_test__val_iceplains__2__000003/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000140_test__val_iceplains__2__000003/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..441cec3467ad5f7f89e6edf746c23d79b00dd539 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000140_test__val_iceplains__2__000003/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eca1086cd717af842a099b527de19b0822a103b5e7f70f20a98147fb2db93e32 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000140_test__val_iceplains__2__000003/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000140_test__val_iceplains__2__000003/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..a0cc0b33602e357e507b676ee4518a765c5d0c0d --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000140_test__val_iceplains__2__000003/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "140", + "global_test_index": "140", + "dataset_index": "253", + "relative_path": "test/val_iceplains/2/000003.mp4", + "clip_id": "test/val_iceplains/2/000003", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000140, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/2/000003_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.00913443323224783, + "psnr": 20.837167739868164, + "lpips": 0.15265963971614838 + }, + "frames_100_200": { + "mse": 0.017077041789889336, + "psnr": 18.087236404418945, + "lpips": 0.19725391268730164 + }, + "frames_200_300": { + "mse": 0.027854908257722855, + "psnr": 17.341846466064453, + "lpips": 0.28225815296173096 + }, + "frames_300_400": { + "mse": 0.02478175237774849, + "psnr": 17.55271339416504, + "lpips": 0.2806892991065979 + }, + "frames_400_500": { + "mse": 0.020468389615416527, + "psnr": 18.03195571899414, + "lpips": 0.349531888961792 + }, + "overall": { + "mse": 0.019863303750753403, + "psnr": 18.37018394470215, + "lpips": 0.25247859954833984 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000141_test__val_iceplains__2__000016/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000141_test__val_iceplains__2__000016/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..e2ca7467fa07d7bf9894593da803166bb3a97e15 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000141_test__val_iceplains__2__000016/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf75ba629fe93cfd35b22b78196d85f684a17130ff39a5a25499f1bb820b7b15 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000141_test__val_iceplains__2__000016/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000141_test__val_iceplains__2__000016/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..d18e84eac6a18a12b635989c3fca278535b4d684 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000141_test__val_iceplains__2__000016/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "141", + "global_test_index": "141", + "dataset_index": "289", + "relative_path": "test/val_iceplains/2/000016.mp4", + "clip_id": "test/val_iceplains/2/000016", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000141, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/2/000016_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.0007425774820148945, + "psnr": 31.299795150756836, + "lpips": 0.00986927468329668 + }, + "frames_100_200": { + "mse": 0.0037242628168314695, + "psnr": 28.017667770385742, + "lpips": 0.03413931280374527 + }, + "frames_200_300": { + "mse": 0.0024608566891402006, + "psnr": 27.717504501342773, + "lpips": 0.02930237539112568 + }, + "frames_300_400": { + "mse": 0.0014874145854264498, + "psnr": 28.451997756958008, + "lpips": 0.026257500052452087 + }, + "frames_400_500": { + "mse": 0.004511452279984951, + "psnr": 24.351301193237305, + "lpips": 0.05438784509897232 + }, + "overall": { + "mse": 0.002585312817245722, + "psnr": 27.967655181884766, + "lpips": 0.030791260302066803 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000142_test__val_plains__2__000020/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000142_test__val_plains__2__000020/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..81a673826ee25bb9fb4c745e5c2cb81fa489ba22 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000142_test__val_plains__2__000020/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4da1b3f513c3b3233c7d9b555ea696313344f33e840a73fc060e1fca7ca0a7aa +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000142_test__val_plains__2__000020/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000142_test__val_plains__2__000020/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..4ed123f9c6ff4fe806a33102365e29edfcd289a7 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000142_test__val_plains__2__000020/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "142", + "global_test_index": "142", + "dataset_index": "422", + "relative_path": "test/val_plains/2/000020.mp4", + "clip_id": "test/val_plains/2/000020", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000142, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_plains/2/000020_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.0048900931142270565, + "psnr": 23.64590835571289, + "lpips": 0.06069273129105568 + }, + "frames_100_200": { + "mse": 0.009247745387256145, + "psnr": 20.74612808227539, + "lpips": 0.21660655736923218 + }, + "frames_200_300": { + "mse": 0.014709634706377983, + "psnr": 18.847129821777344, + "lpips": 0.19656328856945038 + }, + "frames_300_400": { + "mse": 0.020013777539134026, + "psnr": 17.78667640686035, + "lpips": 0.17603296041488647 + }, + "frames_400_500": { + "mse": 0.014958977699279785, + "psnr": 20.473955154418945, + "lpips": 0.10282346606254578 + }, + "overall": { + "mse": 0.012764046899974346, + "psnr": 20.299959182739258, + "lpips": 0.15054380893707275 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000143_test__val_desert__0__000008/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000143_test__val_desert__0__000008/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..948fbbae1a15301fde0bfb18dc9beec746b523f6 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000143_test__val_desert__0__000008/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:626083995bae2a4d9455b296a3306c161479daa0abffa61437c0f920cb1bda8e +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000143_test__val_desert__0__000008/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000143_test__val_desert__0__000008/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..c51dd527acfbe0205332576169f931b6f460f68d --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000143_test__val_desert__0__000008/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "143", + "global_test_index": "143", + "dataset_index": "20", + "relative_path": "test/val_desert/0/000008.mp4", + "clip_id": "test/val_desert/0/000008", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000143, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_desert/0/000008_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.0020095231011509895, + "psnr": 29.04540252685547, + "lpips": 0.03063368797302246 + }, + "frames_100_200": { + "mse": 0.008682151325047016, + "psnr": 21.169387817382812, + "lpips": 0.16521528363227844 + }, + "frames_200_300": { + "mse": 0.009504999965429306, + "psnr": 20.45068359375, + "lpips": 0.2609337568283081 + }, + "frames_300_400": { + "mse": 0.011991730891168118, + "psnr": 19.367080688476562, + "lpips": 0.18150188028812408 + }, + "frames_400_500": { + "mse": 0.002889221301302314, + "psnr": 28.034852981567383, + "lpips": 0.03135009855031967 + }, + "overall": { + "mse": 0.00701552489772439, + "psnr": 23.613479614257812, + "lpips": 0.13392692804336548 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000144_test__val_savanna__0__000000/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000144_test__val_savanna__0__000000/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..da852cd3ac10e0fcafa44c2ace381314af178651 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000144_test__val_savanna__0__000000/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e4607e5eca2b95a071b5ac0f2d4132190ec383fd2ff1dd82e882070bfa660d9 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000144_test__val_savanna__0__000000/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000144_test__val_savanna__0__000000/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..b2afcf2da02e7bd1b430f000aea527addceeb36a --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000144_test__val_savanna__0__000000/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "144", + "global_test_index": "144", + "dataset_index": "130", + "relative_path": "test/val_savanna/0/000000.mp4", + "clip_id": "test/val_savanna/0/000000", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000144, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_savanna/0/000000_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.011036898009479046, + "psnr": 19.92107582092285, + "lpips": 0.16690991818904877 + }, + "frames_100_200": { + "mse": 0.02122030034661293, + "psnr": 16.897611618041992, + "lpips": 0.16124369204044342 + }, + "frames_200_300": { + "mse": 0.025194847956299782, + "psnr": 17.472145080566406, + "lpips": 0.1710321605205536 + }, + "frames_300_400": { + "mse": 0.037652336061000824, + "psnr": 15.202138900756836, + "lpips": 0.20272643864154816 + }, + "frames_400_500": { + "mse": 0.059820517897605896, + "psnr": 12.310754776000977, + "lpips": 0.2851780951023102 + }, + "overall": { + "mse": 0.03098498098552227, + "psnr": 16.360746383666992, + "lpips": 0.19741806387901306 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000145_test__val_iceplains__1__000020/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000145_test__val_iceplains__1__000020/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..c00f69bfd400ab45d67e418a9673e2ad57b81b1d --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000145_test__val_iceplains__1__000020/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e3f8e778efe736b02e76059d51c09245a158a35fbbc18f2534a146428228ae72 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000145_test__val_iceplains__1__000020/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000145_test__val_iceplains__1__000020/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..bac98b355f38e728e40ea6b2982a763f8dfc9244 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000145_test__val_iceplains__1__000020/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "145", + "global_test_index": "145", + "dataset_index": "296", + "relative_path": "test/val_iceplains/1/000020.mp4", + "clip_id": "test/val_iceplains/1/000020", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000145, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/1/000020_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.015748627483844757, + "psnr": 18.12831687927246, + "lpips": 0.18181243538856506 + }, + "frames_100_200": { + "mse": 0.020074909552931786, + "psnr": 17.250537872314453, + "lpips": 0.18514741957187653 + }, + "frames_200_300": { + "mse": 0.07494097948074341, + "psnr": 11.766812324523926, + "lpips": 0.3444903492927551 + }, + "frames_300_400": { + "mse": 0.11422456800937653, + "psnr": 9.7221040725708, + "lpips": 0.49721306562423706 + }, + "frames_400_500": { + "mse": 0.10514868050813675, + "psnr": 9.978658676147461, + "lpips": 0.46637260913848877 + }, + "overall": { + "mse": 0.06602755188941956, + "psnr": 13.369287490844727, + "lpips": 0.3350071609020233 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000146_test__val_iceplains__3__000013/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000146_test__val_iceplains__3__000013/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..595aa454605622230261ab0722c47714e14617a4 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000146_test__val_iceplains__3__000013/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f395280f5b0a097a0738fd6ef40d7c49c83c82182d7bed234a521576fc1a472 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000146_test__val_iceplains__3__000013/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000146_test__val_iceplains__3__000013/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..beef36b777481228dc57e64d531c4d86abbee115 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000146_test__val_iceplains__3__000013/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "146", + "global_test_index": "146", + "dataset_index": "278", + "relative_path": "test/val_iceplains/3/000013.mp4", + "clip_id": "test/val_iceplains/3/000013", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000146, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/3/000013_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.009828705340623856, + "psnr": 21.659238815307617, + "lpips": 0.06777357310056686 + }, + "frames_100_200": { + "mse": 0.024030810222029686, + "psnr": 16.761898040771484, + "lpips": 0.17529119551181793 + }, + "frames_200_300": { + "mse": 0.07107331603765488, + "psnr": 11.919683456420898, + "lpips": 0.3747313320636749 + }, + "frames_300_400": { + "mse": 0.044847797602415085, + "psnr": 22.470502853393555, + "lpips": 0.21533961594104767 + }, + "frames_400_500": { + "mse": 0.0008884036797098815, + "psnr": 30.602449417114258, + "lpips": 0.013425111770629883 + }, + "overall": { + "mse": 0.0301338080316782, + "psnr": 20.682754516601562, + "lpips": 0.16931214928627014 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000147_test__val_iceplains__0__000009/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000147_test__val_iceplains__0__000009/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..b6c3e5c5a6db594b88e708d77a551dd3cb30778e --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000147_test__val_iceplains__0__000009/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:13ab9ed96b7466a99e93d6388cad6b84bb551efcc83a51162c294b0d871a8bb5 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000147_test__val_iceplains__0__000009/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000147_test__val_iceplains__0__000009/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..48f21401b2ad3ee0cc53c98eb4d2ac7c367c5234 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000147_test__val_iceplains__0__000009/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "147", + "global_test_index": "147", + "dataset_index": "269", + "relative_path": "test/val_iceplains/0/000009.mp4", + "clip_id": "test/val_iceplains/0/000009", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000147, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/0/000009_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.007971120066940784, + "psnr": 21.27252960205078, + "lpips": 0.09886342287063599 + }, + "frames_100_200": { + "mse": 0.02013007551431656, + "psnr": 17.738849639892578, + "lpips": 0.2087218314409256 + }, + "frames_200_300": { + "mse": 0.015147763304412365, + "psnr": 19.678150177001953, + "lpips": 0.10820939391851425 + }, + "frames_300_400": { + "mse": 0.008053692057728767, + "psnr": 22.469533920288086, + "lpips": 0.05709761008620262 + }, + "frames_400_500": { + "mse": 0.005609289277344942, + "psnr": 23.4450626373291, + "lpips": 0.05907566472887993 + }, + "overall": { + "mse": 0.011382387951016426, + "psnr": 20.92082405090332, + "lpips": 0.10639357566833496 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000148_test__val_iceplains__0__000028/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000148_test__val_iceplains__0__000028/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..1250e6389b3b7e770aab040298c55f22b4d0d816 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000148_test__val_iceplains__0__000028/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f36b482606315fe8ce277e15e898f1cd22c73f0e030a81a473e9bf9774a65c2d +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000148_test__val_iceplains__0__000028/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000148_test__val_iceplains__0__000028/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..3bf64ad49b7f2a4887e3e8c8a047c4ca55ca73be --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000148_test__val_iceplains__0__000028/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "148", + "global_test_index": "148", + "dataset_index": "314", + "relative_path": "test/val_iceplains/0/000028.mp4", + "clip_id": "test/val_iceplains/0/000028", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000148, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/0/000028_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.0007933095330372453, + "psnr": 31.030569076538086, + "lpips": 0.010161982849240303 + }, + "frames_100_200": { + "mse": 0.0007813509437255561, + "psnr": 31.078716278076172, + "lpips": 0.009602279402315617 + }, + "frames_200_300": { + "mse": 0.002337879268452525, + "psnr": 29.591665267944336, + "lpips": 0.022025400772690773 + }, + "frames_300_400": { + "mse": 0.058630045503377914, + "psnr": 12.691878318786621, + "lpips": 0.3518124520778656 + }, + "frames_400_500": { + "mse": 0.07119105011224747, + "psnr": 11.840930938720703, + "lpips": 0.39768868684768677 + }, + "overall": { + "mse": 0.026746729388833046, + "psnr": 23.24675178527832, + "lpips": 0.15825816988945007 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000149_test__val_iceplains__0__000014/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000149_test__val_iceplains__0__000014/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..ff3aa27f20f90364f2ea9f566a7cb2118a9e45e8 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000149_test__val_iceplains__0__000014/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:578ad474d8680942404bb3ea909369fc6142437ab665005e7ecefe1ae069acf9 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000149_test__val_iceplains__0__000014/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000149_test__val_iceplains__0__000014/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..ea60baf7e0ef4328fa752421d9e5c906d480fe55 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000149_test__val_iceplains__0__000014/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "149", + "global_test_index": "149", + "dataset_index": "281", + "relative_path": "test/val_iceplains/0/000014.mp4", + "clip_id": "test/val_iceplains/0/000014", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000149, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/0/000014_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.027644148096442223, + "psnr": 16.740089416503906, + "lpips": 0.16828277707099915 + }, + "frames_100_200": { + "mse": 0.07163389027118683, + "psnr": 11.96069622039795, + "lpips": 0.3122710883617401 + }, + "frames_200_300": { + "mse": 0.09339779615402222, + "psnr": 10.618962287902832, + "lpips": 0.3779897689819336 + }, + "frames_300_400": { + "mse": 0.05829279497265816, + "psnr": 12.778314590454102, + "lpips": 0.31545186042785645 + }, + "frames_400_500": { + "mse": 0.01641893945634365, + "psnr": 18.955018997192383, + "lpips": 0.1340496987104416 + }, + "overall": { + "mse": 0.053477510809898376, + "psnr": 14.210617065429688, + "lpips": 0.2616090476512909 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000150_test__val_savanna__0__000008/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000150_test__val_savanna__0__000008/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..77d75df19c4b68cf67becc3ccf30367ca71e640a --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000150_test__val_savanna__0__000008/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:efee778143d51074d2249bee01d3e4286a6a819ffa38db9f3e31d816dea82f07 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000150_test__val_savanna__0__000008/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000150_test__val_savanna__0__000008/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..bab477cbe5861900af25e290340b33ca86c3bdd6 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000150_test__val_savanna__0__000008/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "150", + "global_test_index": "150", + "dataset_index": "149", + "relative_path": "test/val_savanna/0/000008.mp4", + "clip_id": "test/val_savanna/0/000008", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000150, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_savanna/0/000008_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.01739102602005005, + "psnr": 17.704994201660156, + "lpips": 0.1644931584596634 + }, + "frames_100_200": { + "mse": 0.02047029510140419, + "psnr": 17.156269073486328, + "lpips": 0.15995636582374573 + }, + "frames_200_300": { + "mse": 0.02135060541331768, + "psnr": 17.10740852355957, + "lpips": 0.347759485244751 + }, + "frames_300_400": { + "mse": 0.006974186282604933, + "psnr": 21.697908401489258, + "lpips": 0.35648113489151 + }, + "frames_400_500": { + "mse": 0.02356274053454399, + "psnr": 16.70166778564453, + "lpips": 0.42533156275749207 + }, + "overall": { + "mse": 0.017949769273400307, + "psnr": 18.073650360107422, + "lpips": 0.29080435633659363 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000151_test__val_savanna__0__000024/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000151_test__val_savanna__0__000024/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..0ecf07866f5ac1b5eda0a9fad298966b8b02ba40 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000151_test__val_savanna__0__000024/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb75cf135d185c5fa9f8d88ce4d8460b2110fb1f8b26b59068179aecece450ba +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000151_test__val_savanna__0__000024/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000151_test__val_savanna__0__000024/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..c82a04dc78af2ef6f91f28e56796f985278d1aac --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000151_test__val_savanna__0__000024/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "151", + "global_test_index": "151", + "dataset_index": "184", + "relative_path": "test/val_savanna/0/000024.mp4", + "clip_id": "test/val_savanna/0/000024", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000151, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_savanna/0/000024_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.01797068864107132, + "psnr": 19.096403121948242, + "lpips": 0.3035520315170288 + }, + "frames_100_200": { + "mse": 0.02538558468222618, + "psnr": 17.31404685974121, + "lpips": 0.3632836937904358 + }, + "frames_200_300": { + "mse": 0.03508196026086807, + "psnr": 16.473325729370117, + "lpips": 0.377555251121521 + }, + "frames_300_400": { + "mse": 0.011878635734319687, + "psnr": 19.872243881225586, + "lpips": 0.34987393021583557 + }, + "frames_400_500": { + "mse": 0.00942204985767603, + "psnr": 20.65838050842285, + "lpips": 0.41628792881965637 + }, + "overall": { + "mse": 0.019947784021496773, + "psnr": 18.682880401611328, + "lpips": 0.36211052536964417 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000152_test__val_plains__1__000014/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000152_test__val_plains__1__000014/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..c8a74977f2f1b77601fc617d6a656d5215c8be97 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000152_test__val_plains__1__000014/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:08fd143043dc279b86adde25f9a02a8bbeb5e942667021db672215a969db2399 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000152_test__val_plains__1__000014/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000152_test__val_plains__1__000014/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..e4670d6c3c6273da8060c9a3e18e6defa6f363ea --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000152_test__val_plains__1__000014/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "152", + "global_test_index": "152", + "dataset_index": "407", + "relative_path": "test/val_plains/1/000014.mp4", + "clip_id": "test/val_plains/1/000014", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000152, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_plains/1/000014_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.0045885732397437096, + "psnr": 26.039417266845703, + "lpips": 0.033945146948099136 + }, + "frames_100_200": { + "mse": 0.0012465534964576364, + "psnr": 30.284738540649414, + "lpips": 0.01935230940580368 + }, + "frames_200_300": { + "mse": 0.04182104021310806, + "psnr": 16.36471939086914, + "lpips": 0.25117257237434387 + }, + "frames_300_400": { + "mse": 0.09604170173406601, + "psnr": 11.736322402954102, + "lpips": 0.46794813871383667 + }, + "frames_400_500": { + "mse": 0.15848904848098755, + "psnr": 8.601259231567383, + "lpips": 0.6501309275627136 + }, + "overall": { + "mse": 0.06043738126754761, + "psnr": 18.60529136657715, + "lpips": 0.2845098078250885 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000153_test__val_iceplains__2__000023/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000153_test__val_iceplains__2__000023/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..e080c8de64402eda3efa6ec2697437c96a509bd9 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000153_test__val_iceplains__2__000023/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:07965431563821eee1ed045b033a4c79548f56b619602a134a0c094b046f70c9 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000153_test__val_iceplains__2__000023/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000153_test__val_iceplains__2__000023/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..698a16c13fee14e93dfc98f6308a7c3aaf7dd3e4 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000153_test__val_iceplains__2__000023/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "153", + "global_test_index": "153", + "dataset_index": "304", + "relative_path": "test/val_iceplains/2/000023.mp4", + "clip_id": "test/val_iceplains/2/000023", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000153, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/2/000023_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.0011202353052794933, + "psnr": 30.09531021118164, + "lpips": 0.013136771507561207 + }, + "frames_100_200": { + "mse": 0.005820955615490675, + "psnr": 23.257753372192383, + "lpips": 0.059344567358493805 + }, + "frames_200_300": { + "mse": 0.023989036679267883, + "psnr": 16.964008331298828, + "lpips": 0.19605325162410736 + }, + "frames_300_400": { + "mse": 0.039803605526685715, + "psnr": 14.574650764465332, + "lpips": 0.31739312410354614 + }, + "frames_400_500": { + "mse": 0.052260514348745346, + "psnr": 13.899173736572266, + "lpips": 0.427410364151001 + }, + "overall": { + "mse": 0.0245988667011261, + "psnr": 19.758180618286133, + "lpips": 0.20266762375831604 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000154_test__val_savanna__3__000037/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000154_test__val_savanna__3__000037/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..df5b7db04f1b3059f9126a8cff05dcd3d6cea0a6 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000154_test__val_savanna__3__000037/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9549d0ca641745668704f4dbaf7c4ab0785f78c16e1e8b5447d4276e5d6902c +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000154_test__val_savanna__3__000037/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000154_test__val_savanna__3__000037/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..44129af84995e5f1541bee28b6f682386b1971fe --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000154_test__val_savanna__3__000037/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "154", + "global_test_index": "154", + "dataset_index": "208", + "relative_path": "test/val_savanna/3/000037.mp4", + "clip_id": "test/val_savanna/3/000037", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000154, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_savanna/3/000037_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.009729288518428802, + "psnr": 20.6905517578125, + "lpips": 0.2798389494419098 + }, + "frames_100_200": { + "mse": 0.0061994874849915504, + "psnr": 22.554889678955078, + "lpips": 0.43718621134757996 + }, + "frames_200_300": { + "mse": 0.005600739270448685, + "psnr": 22.580585479736328, + "lpips": 0.4016195237636566 + }, + "frames_300_400": { + "mse": 0.006176857743412256, + "psnr": 22.194475173950195, + "lpips": 0.3802674412727356 + }, + "frames_400_500": { + "mse": 0.014383042231202126, + "psnr": 19.061248779296875, + "lpips": 0.459075391292572 + }, + "overall": { + "mse": 0.008417882956564426, + "psnr": 21.416349411010742, + "lpips": 0.39159753918647766 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000155_test__val_savanna__3__000023/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000155_test__val_savanna__3__000023/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..451ed95379d53c8c52f96a20e89f54e9eae33250 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000155_test__val_savanna__3__000023/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bcd2663657973ad4bfafc9abe3cb608706d8ec0be0777a78f7abb92a5fdebc66 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000155_test__val_savanna__3__000023/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000155_test__val_savanna__3__000023/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..053cc7f1e73e3fbc1103d66e322c1eef77504265 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000155_test__val_savanna__3__000023/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "155", + "global_test_index": "155", + "dataset_index": "182", + "relative_path": "test/val_savanna/3/000023.mp4", + "clip_id": "test/val_savanna/3/000023", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000155, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_savanna/3/000023_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.003987411968410015, + "psnr": 24.146350860595703, + "lpips": 0.28780925273895264 + }, + "frames_100_200": { + "mse": 0.005324963480234146, + "psnr": 22.770431518554688, + "lpips": 0.3286348581314087 + }, + "frames_200_300": { + "mse": 0.006300166714936495, + "psnr": 22.07854652404785, + "lpips": 0.3890300691127777 + }, + "frames_300_400": { + "mse": 0.006131622940301895, + "psnr": 22.242286682128906, + "lpips": 0.3978605568408966 + }, + "frames_400_500": { + "mse": 0.007141302339732647, + "psnr": 21.529722213745117, + "lpips": 0.4210602641105652 + }, + "overall": { + "mse": 0.005777094047516584, + "psnr": 22.553468704223633, + "lpips": 0.3648790419101715 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000156_test__val_plains__1__000009/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000156_test__val_plains__1__000009/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..74c3e7fb1ae3eddf0f460439b4f35ddcc39302a9 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000156_test__val_plains__1__000009/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:285fe5eac1522a6c244f9ac7a843c8e2f178120a7c8afe529160e7b6a0fe7c0f +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000156_test__val_plains__1__000009/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000156_test__val_plains__1__000009/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..25bc6f809a1fbca35c1e2e2bf2af66f5f07a2e77 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000156_test__val_plains__1__000009/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "156", + "global_test_index": "156", + "dataset_index": "394", + "relative_path": "test/val_plains/1/000009.mp4", + "clip_id": "test/val_plains/1/000009", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000156, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_plains/1/000009_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.0020266936626285315, + "psnr": 27.227500915527344, + "lpips": 0.16900375485420227 + }, + "frames_100_200": { + "mse": 0.0029361960478127003, + "psnr": 25.64105796813965, + "lpips": 0.23620185256004333 + }, + "frames_200_300": { + "mse": 0.0036728226114064455, + "psnr": 24.503711700439453, + "lpips": 0.28628966212272644 + }, + "frames_300_400": { + "mse": 0.0035146332811564207, + "psnr": 24.8013916015625, + "lpips": 0.28565144538879395 + }, + "frames_400_500": { + "mse": 0.003045293502509594, + "psnr": 25.210426330566406, + "lpips": 0.2722412645816803 + }, + "overall": { + "mse": 0.003039127914234996, + "psnr": 25.476818084716797, + "lpips": 0.24987760186195374 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000157_test__val_desert__0__000006/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000157_test__val_desert__0__000006/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..58412df1d52720576fba1644ba0cce183e1cbc1e --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000157_test__val_desert__0__000006/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7fc522d41c6ed063a98642f85a91ab2a12bc85bfa16cd52984ede4da668532a8 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000157_test__val_desert__0__000006/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000157_test__val_desert__0__000006/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..3fa5f89d874c5985cc9d0bad1c9ca3b9cc7ef460 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000157_test__val_desert__0__000006/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "157", + "global_test_index": "157", + "dataset_index": "14", + "relative_path": "test/val_desert/0/000006.mp4", + "clip_id": "test/val_desert/0/000006", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000157, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_desert/0/000006_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.003168567083775997, + "psnr": 25.201658248901367, + "lpips": 0.06832889467477798 + }, + "frames_100_200": { + "mse": 0.004970119800418615, + "psnr": 23.136377334594727, + "lpips": 0.1115088015794754 + }, + "frames_200_300": { + "mse": 0.004485507495701313, + "psnr": 23.680580139160156, + "lpips": 0.12413804978132248 + }, + "frames_300_400": { + "mse": 0.006102403625845909, + "psnr": 23.040050506591797, + "lpips": 0.15169116854667664 + }, + "frames_400_500": { + "mse": 0.007500550709664822, + "psnr": 21.35930633544922, + "lpips": 0.16764864325523376 + }, + "overall": { + "mse": 0.005245429929345846, + "psnr": 23.283594131469727, + "lpips": 0.1246631070971489 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000158_test__val_desert__3__000021/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000158_test__val_desert__3__000021/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..d09a6f1c1c8132ae67402d3faa78a39a64b1209c --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000158_test__val_desert__3__000021/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:30c726228e6b3b42d0b11fc088f345b76469fd302d3dc0158ba8323c7bdf27d0 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000158_test__val_desert__3__000021/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000158_test__val_desert__3__000021/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..b6c0a1260cf1368f105c325bace8cdcbcb3e1aac --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000158_test__val_desert__3__000021/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "158", + "global_test_index": "158", + "dataset_index": "55", + "relative_path": "test/val_desert/3/000021.mp4", + "clip_id": "test/val_desert/3/000021", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000158, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_desert/3/000021_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.0012695230543613434, + "psnr": 29.307384490966797, + "lpips": 0.01902073062956333 + }, + "frames_100_200": { + "mse": 0.000905252993106842, + "psnr": 30.5664005279541, + "lpips": 0.011564943939447403 + }, + "frames_200_300": { + "mse": 0.0023010647855699062, + "psnr": 28.403432846069336, + "lpips": 0.027533162385225296 + }, + "frames_300_400": { + "mse": 0.00083332130452618, + "psnr": 30.805208206176758, + "lpips": 0.011033103801310062 + }, + "frames_400_500": { + "mse": 0.001650504651479423, + "psnr": 30.050363540649414, + "lpips": 0.01596832647919655 + }, + "overall": { + "mse": 0.001391933299601078, + "psnr": 29.826562881469727, + "lpips": 0.017024053260684013 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000159_test__val_desert__2__000028/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000159_test__val_desert__2__000028/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..d5aa2c5a220eb2125c0aa9bfdfb5ae76609edfb0 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000159_test__val_desert__2__000028/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f4dd7b2aa8960a799f858e4aad396cc6b8f42a3bbaeb0264aff2ec56efafd3f +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000159_test__val_desert__2__000028/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000159_test__val_desert__2__000028/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..c8ba62ac5b8a1fc2d17da415eb05221d40dfe8f9 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000159_test__val_desert__2__000028/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "159", + "global_test_index": "159", + "dataset_index": "75", + "relative_path": "test/val_desert/2/000028.mp4", + "clip_id": "test/val_desert/2/000028", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000159, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_desert/2/000028_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.009223424829542637, + "psnr": 20.488374710083008, + "lpips": 0.3096087574958801 + }, + "frames_100_200": { + "mse": 0.01215670257806778, + "psnr": 19.23905372619629, + "lpips": 0.3557189702987671 + }, + "frames_200_300": { + "mse": 0.017088942229747772, + "psnr": 17.969018936157227, + "lpips": 0.36213669180870056 + }, + "frames_300_400": { + "mse": 0.015536177903413773, + "psnr": 18.16918182373047, + "lpips": 0.31894513964653015 + }, + "frames_400_500": { + "mse": 0.011723948642611504, + "psnr": 20.082902908325195, + "lpips": 0.19843333959579468 + }, + "overall": { + "mse": 0.013145838864147663, + "psnr": 19.189706802368164, + "lpips": 0.30896857380867004 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000160_test__val_iceplains__1__000031/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000160_test__val_iceplains__1__000031/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..f4c37c38956ef8fe1924c35abbb38f43f40324c0 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000160_test__val_iceplains__1__000031/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:65395f0c88cc1baae9c48f24b0969f575dfa17ea0b2a4d13b225479d50491013 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000160_test__val_iceplains__1__000031/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000160_test__val_iceplains__1__000031/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..2d9bbcdbf275c843fbb1d5577fc6a91c624e9ad9 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000160_test__val_iceplains__1__000031/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "160", + "global_test_index": "160", + "dataset_index": "319", + "relative_path": "test/val_iceplains/1/000031.mp4", + "clip_id": "test/val_iceplains/1/000031", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000160, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/1/000031_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.01416734792292118, + "psnr": 19.998319625854492, + "lpips": 0.19866938889026642 + }, + "frames_100_200": { + "mse": 0.012743833474814892, + "psnr": 19.389881134033203, + "lpips": 0.24251656234264374 + }, + "frames_200_300": { + "mse": 0.04585330933332443, + "psnr": 15.587272644042969, + "lpips": 0.3690023422241211 + }, + "frames_300_400": { + "mse": 0.08927451819181442, + "psnr": 11.496914863586426, + "lpips": 0.41860148310661316 + }, + "frames_400_500": { + "mse": 0.04384661093354225, + "psnr": 13.775019645690918, + "lpips": 0.32559284567832947 + }, + "overall": { + "mse": 0.04117712378501892, + "psnr": 16.049482345581055, + "lpips": 0.3108765184879303 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000161_test__val_desert__1__000016/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000161_test__val_desert__1__000016/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..71144eecbdffa62d7ac3df5482e9375ccd0ebc7a --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000161_test__val_desert__1__000016/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:02d7a2a0851db0beea5acfa7632bb8d45421b89856d00f730d03c34f8f686d32 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000161_test__val_desert__1__000016/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000161_test__val_desert__1__000016/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..d816181cec816705d0372bdcb74aa6bc93b0f98c --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000161_test__val_desert__1__000016/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "161", + "global_test_index": "161", + "dataset_index": "39", + "relative_path": "test/val_desert/1/000016.mp4", + "clip_id": "test/val_desert/1/000016", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000161, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_desert/1/000016_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.029931478202342987, + "psnr": 17.60317611694336, + "lpips": 0.2922813594341278 + }, + "frames_100_200": { + "mse": 0.027039840817451477, + "psnr": 17.74761962890625, + "lpips": 0.41203343868255615 + }, + "frames_200_300": { + "mse": 0.007638003211468458, + "psnr": 21.190567016601562, + "lpips": 0.45305344462394714 + }, + "frames_300_400": { + "mse": 0.008152521215379238, + "psnr": 20.94780731201172, + "lpips": 0.42503440380096436 + }, + "frames_400_500": { + "mse": 0.007692254148423672, + "psnr": 21.174100875854492, + "lpips": 0.42529192566871643 + }, + "overall": { + "mse": 0.016090819612145424, + "psnr": 19.732656478881836, + "lpips": 0.4015389382839203 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000162_test__val_plains__1__000019/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000162_test__val_plains__1__000019/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..c3b0aa8057cb4f3d33766a25bdbf327b74eb971b --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000162_test__val_plains__1__000019/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1df4b0f81581ef58b058bfbaa39fe9e4bb36ae6e0adbae64a8d90a78464310d +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000162_test__val_plains__1__000019/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000162_test__val_plains__1__000019/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..a190a5d070d7149e50b5178e36c399359a8a2d18 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000162_test__val_plains__1__000019/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "162", + "global_test_index": "162", + "dataset_index": "418", + "relative_path": "test/val_plains/1/000019.mp4", + "clip_id": "test/val_plains/1/000019", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000162, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_plains/1/000019_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.006841608323156834, + "psnr": 24.817293167114258, + "lpips": 0.0780685544013977 + }, + "frames_100_200": { + "mse": 0.005909836385399103, + "psnr": 23.44506072998047, + "lpips": 0.0622999481856823 + }, + "frames_200_300": { + "mse": 0.006808399688452482, + "psnr": 23.452774047851562, + "lpips": 0.0995871052145958 + }, + "frames_300_400": { + "mse": 0.013029413297772408, + "psnr": 18.909202575683594, + "lpips": 0.28315821290016174 + }, + "frames_400_500": { + "mse": 0.013101894408464432, + "psnr": 19.644489288330078, + "lpips": 0.2173905223608017 + }, + "overall": { + "mse": 0.009138231165707111, + "psnr": 22.05376625061035, + "lpips": 0.14810088276863098 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000163_test__val_plains__3__000007/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000163_test__val_plains__3__000007/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..89b73493bba26d7261e1abc5a56eefdfd74788ef --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000163_test__val_plains__3__000007/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:87e5faee886e7fa21b737d6871074935edbbd44124bb3eff9c155e5843081252 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000163_test__val_plains__3__000007/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000163_test__val_plains__3__000007/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..037ad3c35ffa5e8419f7eb0083e8cf702522f594 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000163_test__val_plains__3__000007/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "163", + "global_test_index": "163", + "dataset_index": "389", + "relative_path": "test/val_plains/3/000007.mp4", + "clip_id": "test/val_plains/3/000007", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000163, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_plains/3/000007_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.0056228782050311565, + "psnr": 23.064462661743164, + "lpips": 0.2394961565732956 + }, + "frames_100_200": { + "mse": 0.006225147284567356, + "psnr": 22.265148162841797, + "lpips": 0.34935128688812256 + }, + "frames_200_300": { + "mse": 0.01250223908573389, + "psnr": 19.908893585205078, + "lpips": 0.4109385013580322 + }, + "frames_300_400": { + "mse": 0.024202141910791397, + "psnr": 16.805862426757812, + "lpips": 0.43312835693359375 + }, + "frames_400_500": { + "mse": 0.01115934457629919, + "psnr": 20.210725784301758, + "lpips": 0.4037443995475769 + }, + "overall": { + "mse": 0.011942350305616856, + "psnr": 20.451019287109375, + "lpips": 0.36733174324035645 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000164_test__val_plains__2__000014/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000164_test__val_plains__2__000014/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..f8caf7879070c76375abd484127be3e4f00b114c --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000164_test__val_plains__2__000014/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:09505802f85a7153a209664e39bd50632810df405119d48037b966f9dea4cfd6 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000164_test__val_plains__2__000014/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000164_test__val_plains__2__000014/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..acdfc06628de128391abe6148ea54fcfb0c93b26 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000164_test__val_plains__2__000014/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "164", + "global_test_index": "164", + "dataset_index": "409", + "relative_path": "test/val_plains/2/000014.mp4", + "clip_id": "test/val_plains/2/000014", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000164, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_plains/2/000014_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.021304314956068993, + "psnr": 18.6710262298584, + "lpips": 0.31772395968437195 + }, + "frames_100_200": { + "mse": 0.009380041621625423, + "psnr": 21.494373321533203, + "lpips": 0.1982826590538025 + }, + "frames_200_300": { + "mse": 0.004344922490417957, + "psnr": 23.870866775512695, + "lpips": 0.10456950217485428 + }, + "frames_300_400": { + "mse": 0.0037371707148849964, + "psnr": 24.609783172607422, + "lpips": 0.12950998544692993 + }, + "frames_400_500": { + "mse": 0.004339042119681835, + "psnr": 23.736621856689453, + "lpips": 0.21334080398082733 + }, + "overall": { + "mse": 0.008621098473668098, + "psnr": 22.476533889770508, + "lpips": 0.19268538057804108 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000165_test__val_iceplains__2__000038/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000165_test__val_iceplains__2__000038/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..1b0dd7713347ab17139baf9812829016059ac964 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000165_test__val_iceplains__2__000038/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d24f4625bfdb7cc859a383b0d24c19f14aa9595cf31c3665cc48807ba1165b0 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000165_test__val_iceplains__2__000038/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000165_test__val_iceplains__2__000038/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..da30e25544389fa350567f8dd50bc2406b39057d --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000165_test__val_iceplains__2__000038/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "165", + "global_test_index": "165", + "dataset_index": "344", + "relative_path": "test/val_iceplains/2/000038.mp4", + "clip_id": "test/val_iceplains/2/000038", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000165, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/2/000038_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.027353057637810707, + "psnr": 16.115779876708984, + "lpips": 0.23633433878421783 + }, + "frames_100_200": { + "mse": 0.02595016174018383, + "psnr": 17.101472854614258, + "lpips": 0.2558610737323761 + }, + "frames_200_300": { + "mse": 0.04149561747908592, + "psnr": 15.649199485778809, + "lpips": 0.3007337749004364 + }, + "frames_300_400": { + "mse": 0.006000750232487917, + "psnr": 22.999469757080078, + "lpips": 0.19849318265914917 + }, + "frames_400_500": { + "mse": 0.024627957493066788, + "psnr": 17.530244827270508, + "lpips": 0.18025542795658112 + }, + "overall": { + "mse": 0.025085508823394775, + "psnr": 17.87923240661621, + "lpips": 0.23433557152748108 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000166_test__val_plains__2__000009/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000166_test__val_plains__2__000009/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..e7c9e4bded5169f7b128acb4e5f71426e5637521 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000166_test__val_plains__2__000009/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fbfb2045e3377467265d0bc8e6084d8a02e58bb7a694e3e3e9f03f871c0ff52e +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000166_test__val_plains__2__000009/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000166_test__val_plains__2__000009/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..4adf95136bb726ddebb34507e14d8d16143065fe --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000166_test__val_plains__2__000009/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "166", + "global_test_index": "166", + "dataset_index": "396", + "relative_path": "test/val_plains/2/000009.mp4", + "clip_id": "test/val_plains/2/000009", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000166, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_plains/2/000009_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.002183197531849146, + "psnr": 26.895946502685547, + "lpips": 0.021853122860193253 + }, + "frames_100_200": { + "mse": 0.00289347255602479, + "psnr": 26.886930465698242, + "lpips": 0.02493460103869438 + }, + "frames_200_300": { + "mse": 0.017570655792951584, + "psnr": 19.565011978149414, + "lpips": 0.15172646939754486 + }, + "frames_300_400": { + "mse": 0.01806255616247654, + "psnr": 18.12986946105957, + "lpips": 0.2663692831993103 + }, + "frames_400_500": { + "mse": 0.018940800800919533, + "psnr": 17.78221893310547, + "lpips": 0.29151037335395813 + }, + "overall": { + "mse": 0.011930136941373348, + "psnr": 21.851993560791016, + "lpips": 0.1512787640094757 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000167_test__val_desert__1__000026/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000167_test__val_desert__1__000026/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..20403b50864450fce9adbd541c303bca30393c5d --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000167_test__val_desert__1__000026/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce6282d05b308b9b304bd7fca1cbeef8ef8e4e28bb1b649c9f7627a5cf864c34 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000167_test__val_desert__1__000026/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000167_test__val_desert__1__000026/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..5086eb4e4b1f4515729626cd220bc7e5fa1a363c --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000167_test__val_desert__1__000026/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "167", + "global_test_index": "167", + "dataset_index": "67", + "relative_path": "test/val_desert/1/000026.mp4", + "clip_id": "test/val_desert/1/000026", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000167, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_desert/1/000026_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.008605454117059708, + "psnr": 20.78237533569336, + "lpips": 0.2948433756828308 + }, + "frames_100_200": { + "mse": 0.010317591018974781, + "psnr": 20.01201820373535, + "lpips": 0.3537874221801758 + }, + "frames_200_300": { + "mse": 0.02081919275224209, + "psnr": 18.292490005493164, + "lpips": 0.37390056252479553 + }, + "frames_300_400": { + "mse": 0.053999610245227814, + "psnr": 13.349135398864746, + "lpips": 0.39932018518447876 + }, + "frames_400_500": { + "mse": 0.022989731281995773, + "psnr": 16.70436668395996, + "lpips": 0.31416478753089905 + }, + "overall": { + "mse": 0.023346316069364548, + "psnr": 17.82807731628418, + "lpips": 0.34720325469970703 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000168_test__val_desert__3__000038/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000168_test__val_desert__3__000038/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..f8a9f4625914daee240553ab0471d8373c4e727a --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000168_test__val_desert__3__000038/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:725c2406cdb9abc6318ba95d2364280d399f34f9fcd65cec98d9ccdb907c0d00 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000168_test__val_desert__3__000038/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000168_test__val_desert__3__000038/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..aa2e529997b40be8f691430860c037069197199b --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000168_test__val_desert__3__000038/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "168", + "global_test_index": "168", + "dataset_index": "99", + "relative_path": "test/val_desert/3/000038.mp4", + "clip_id": "test/val_desert/3/000038", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000168, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_desert/3/000038_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.006059327628463507, + "psnr": 22.490520477294922, + "lpips": 0.2777692675590515 + }, + "frames_100_200": { + "mse": 0.008254190906882286, + "psnr": 20.8654842376709, + "lpips": 0.3664596676826477 + }, + "frames_200_300": { + "mse": 0.007112429011613131, + "psnr": 21.552160263061523, + "lpips": 0.32050368189811707 + }, + "frames_300_400": { + "mse": 0.0082048699259758, + "psnr": 20.886335372924805, + "lpips": 0.34027549624443054 + }, + "frames_400_500": { + "mse": 0.008532486855983734, + "psnr": 20.71047592163086, + "lpips": 0.3892980217933655 + }, + "overall": { + "mse": 0.0076326606795191765, + "psnr": 21.300994873046875, + "lpips": 0.3388611674308777 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000169_test__val_iceplains__3__000016/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000169_test__val_iceplains__3__000016/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..fd89d97f6387c573870acdf9d759177d46eccc08 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000169_test__val_iceplains__3__000016/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6085b7e89a17fa484021f530f2c2a7dbf11653a89d3f87336d9166ff7412f0e0 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000169_test__val_iceplains__3__000016/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000169_test__val_iceplains__3__000016/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..654a64f788d09093ac17e877fd35440189f00f68 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000169_test__val_iceplains__3__000016/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "169", + "global_test_index": "169", + "dataset_index": "288", + "relative_path": "test/val_iceplains/3/000016.mp4", + "clip_id": "test/val_iceplains/3/000016", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000169, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/3/000016_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.006862635724246502, + "psnr": 22.51287078857422, + "lpips": 0.1594824194908142 + }, + "frames_100_200": { + "mse": 0.0521806925535202, + "psnr": 14.637690544128418, + "lpips": 0.28311964869499207 + }, + "frames_200_300": { + "mse": 0.023229192942380905, + "psnr": 18.511211395263672, + "lpips": 0.16353516280651093 + }, + "frames_300_400": { + "mse": 0.011655701324343681, + "psnr": 19.762344360351562, + "lpips": 0.15894895792007446 + }, + "frames_400_500": { + "mse": 0.007871586829423904, + "psnr": 21.21441078186035, + "lpips": 0.1282794326543808 + }, + "overall": { + "mse": 0.020359963178634644, + "psnr": 19.32770538330078, + "lpips": 0.1786731481552124 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000170_test__val_desert__1__000038/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000170_test__val_desert__1__000038/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..6d4da3782bf7411369f6979e532d65daaaa5a130 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000170_test__val_desert__1__000038/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c6d616a338d6a156b7e30993172fc9e3fccfb531c30fbd2347ec9f75363d4ee +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000170_test__val_desert__1__000038/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000170_test__val_desert__1__000038/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..ab49d92b35cef105f35ef3aa10d5396eeaa1d759 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000170_test__val_desert__1__000038/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "170", + "global_test_index": "170", + "dataset_index": "97", + "relative_path": "test/val_desert/1/000038.mp4", + "clip_id": "test/val_desert/1/000038", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000170, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_desert/1/000038_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.007246223744004965, + "psnr": 21.62635040283203, + "lpips": 0.1884198784828186 + }, + "frames_100_200": { + "mse": 0.011003631167113781, + "psnr": 19.7232608795166, + "lpips": 0.26027247309684753 + }, + "frames_200_300": { + "mse": 0.009765270166099072, + "psnr": 20.2574405670166, + "lpips": 0.28183281421661377 + }, + "frames_300_400": { + "mse": 0.006184824276715517, + "psnr": 22.6976375579834, + "lpips": 0.13438506424427032 + }, + "frames_400_500": { + "mse": 0.004725713282823563, + "psnr": 25.582948684692383, + "lpips": 0.07669081538915634 + }, + "overall": { + "mse": 0.007785132620483637, + "psnr": 21.97753143310547, + "lpips": 0.18832018971443176 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000171_test__val_savanna__2__000021/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000171_test__val_savanna__2__000021/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..79b4e2a6ae5d135f4e809b689050eb8e01efcd4d --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000171_test__val_savanna__2__000021/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c215a0cc27d806469106895b77c863161f4e131360218587722b1fd99cf40144 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000171_test__val_savanna__2__000021/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000171_test__val_savanna__2__000021/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..473c887a8b3fa3f61e72486db0c3f2d7aff1e765 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000171_test__val_savanna__2__000021/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "171", + "global_test_index": "171", + "dataset_index": "178", + "relative_path": "test/val_savanna/2/000021.mp4", + "clip_id": "test/val_savanna/2/000021", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000171, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_savanna/2/000021_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.0009216491016559303, + "psnr": 30.61834144592285, + "lpips": 0.011011561378836632 + }, + "frames_100_200": { + "mse": 0.0014890682650730014, + "psnr": 29.391592025756836, + "lpips": 0.01478344202041626 + }, + "frames_200_300": { + "mse": 0.00667536398395896, + "psnr": 23.613140106201172, + "lpips": 0.04113201051950455 + }, + "frames_300_400": { + "mse": 0.014935910701751709, + "psnr": 19.600177764892578, + "lpips": 0.09082912653684616 + }, + "frames_400_500": { + "mse": 0.026862721890211105, + "psnr": 17.537128448486328, + "lpips": 0.1743985414505005 + }, + "overall": { + "mse": 0.010176942683756351, + "psnr": 24.152076721191406, + "lpips": 0.06643093377351761 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000172_test__val_iceplains__1__000007/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000172_test__val_iceplains__1__000007/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..896e929f3735367b921f33974f6fba7a7dc519ce --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000172_test__val_iceplains__1__000007/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:069fffc035af81e8ae10fdf9dde58b1813a621635bcee53129cb04e8a0ad4cd6 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000172_test__val_iceplains__1__000007/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000172_test__val_iceplains__1__000007/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..a04a6ddaa8686cdafd0237454bd47f955450d948 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000172_test__val_iceplains__1__000007/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "172", + "global_test_index": "172", + "dataset_index": "264", + "relative_path": "test/val_iceplains/1/000007.mp4", + "clip_id": "test/val_iceplains/1/000007", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000172, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/1/000007_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.0007620938122272491, + "psnr": 31.188636779785156, + "lpips": 0.009863962419331074 + }, + "frames_100_200": { + "mse": 0.0008377427002415061, + "psnr": 30.866804122924805, + "lpips": 0.012332957237958908 + }, + "frames_200_300": { + "mse": 0.0009243691456504166, + "psnr": 30.727563858032227, + "lpips": 0.011637999676167965 + }, + "frames_300_400": { + "mse": 0.01028765831142664, + "psnr": 22.19556427001953, + "lpips": 0.07794191688299179 + }, + "frames_400_500": { + "mse": 0.0037824970204383135, + "psnr": 26.7729549407959, + "lpips": 0.03896884620189667 + }, + "overall": { + "mse": 0.003318872069939971, + "psnr": 28.350305557250977, + "lpips": 0.03014913573861122 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000173_test__val_iceplains__0__000008/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000173_test__val_iceplains__0__000008/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..a00dcdccaf9a911f2206e2263e81e74e992a53ec --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000173_test__val_iceplains__0__000008/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:783a94a8ae94732a6426679d1193f47202f2539ed0f8c2bddaed26c5a573e189 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000173_test__val_iceplains__0__000008/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000173_test__val_iceplains__0__000008/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..86a20b591997a6530a2889a98ae83baf2464349c --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000173_test__val_iceplains__0__000008/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "173", + "global_test_index": "173", + "dataset_index": "266", + "relative_path": "test/val_iceplains/0/000008.mp4", + "clip_id": "test/val_iceplains/0/000008", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000173, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/0/000008_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.022284066304564476, + "psnr": 17.391197204589844, + "lpips": 0.22247806191444397 + }, + "frames_100_200": { + "mse": 0.017892984673380852, + "psnr": 17.92555046081543, + "lpips": 0.22155243158340454 + }, + "frames_200_300": { + "mse": 0.03417552635073662, + "psnr": 14.969188690185547, + "lpips": 0.3293423056602478 + }, + "frames_300_400": { + "mse": 0.03436149284243584, + "psnr": 14.934359550476074, + "lpips": 0.35960322618484497 + }, + "frames_400_500": { + "mse": 0.08087073266506195, + "psnr": 11.208952903747559, + "lpips": 0.48298197984695435 + }, + "overall": { + "mse": 0.037916962057352066, + "psnr": 15.285850524902344, + "lpips": 0.3231916129589081 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000174_test__val_iceplains__1__000023/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000174_test__val_iceplains__1__000023/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..61482bc812234e5d9010b6f47ab4800a1241f494 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000174_test__val_iceplains__1__000023/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b74f4384ba0fae3ce73a1aa87758f43b33a1e323b3c7f0fdb215934b76c97723 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000174_test__val_iceplains__1__000023/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000174_test__val_iceplains__1__000023/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..27c873d2ac7cb2020f8db938ad4da65f16a75678 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000174_test__val_iceplains__1__000023/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "174", + "global_test_index": "174", + "dataset_index": "303", + "relative_path": "test/val_iceplains/1/000023.mp4", + "clip_id": "test/val_iceplains/1/000023", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000174, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/1/000023_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.000780428817961365, + "psnr": 31.091346740722656, + "lpips": 0.010223796591162682 + }, + "frames_100_200": { + "mse": 0.0009895756375044584, + "psnr": 30.365219116210938, + "lpips": 0.011054263450205326 + }, + "frames_200_300": { + "mse": 0.011277339421212673, + "psnr": 21.19949722290039, + "lpips": 0.09287837147712708 + }, + "frames_300_400": { + "mse": 0.009490354917943478, + "psnr": 22.756013870239258, + "lpips": 0.07235828042030334 + }, + "frames_400_500": { + "mse": 0.005386077333241701, + "psnr": 23.930526733398438, + "lpips": 0.0450359545648098 + }, + "overall": { + "mse": 0.005584754981100559, + "psnr": 25.868518829345703, + "lpips": 0.04631013423204422 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000175_test__val_plains__0__000031/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000175_test__val_plains__0__000031/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..4abd9c3cdcd4d7d82101c70d48aab1171e1f16a3 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000175_test__val_plains__0__000031/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d7d54be2b53f617e15b1314a077a94a7d5ee52feecf3fc78279c91ba075d21e0 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000175_test__val_plains__0__000031/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000175_test__val_plains__0__000031/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..42cf172962793426cd1fdb66d2d123e0efd41b67 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000175_test__val_plains__0__000031/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "175", + "global_test_index": "175", + "dataset_index": "442", + "relative_path": "test/val_plains/0/000031.mp4", + "clip_id": "test/val_plains/0/000031", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000175, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_plains/0/000031_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.0008595269755460322, + "psnr": 30.8408203125, + "lpips": 0.010930771008133888 + }, + "frames_100_200": { + "mse": 0.004353676922619343, + "psnr": 27.40094757080078, + "lpips": 0.02527894452214241 + }, + "frames_200_300": { + "mse": 0.020736727863550186, + "psnr": 17.763710021972656, + "lpips": 0.21288971602916718 + }, + "frames_300_400": { + "mse": 0.015404491685330868, + "psnr": 19.452083587646484, + "lpips": 0.17402149736881256 + }, + "frames_400_500": { + "mse": 0.07116113603115082, + "psnr": 13.13608169555664, + "lpips": 0.48793166875839233 + }, + "overall": { + "mse": 0.022503111511468887, + "psnr": 21.718727111816406, + "lpips": 0.1822105199098587 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000176_test__val_savanna__1__000047/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000176_test__val_savanna__1__000047/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..6bf65f1978049dcb23cd5ffee6e8e3ad57ab6231 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000176_test__val_savanna__1__000047/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3be5cb0abfcbf6538ca1cfc37fe10d8dbd6013158990f7e4956c9f872b7bbc2 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000176_test__val_savanna__1__000047/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000176_test__val_savanna__1__000047/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..fc7a6cc24af44546d980a0a0ef10e86c036e4828 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000176_test__val_savanna__1__000047/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "176", + "global_test_index": "176", + "dataset_index": "233", + "relative_path": "test/val_savanna/1/000047.mp4", + "clip_id": "test/val_savanna/1/000047", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000176, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_savanna/1/000047_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.008358673192560673, + "psnr": 21.630327224731445, + "lpips": 0.17067556083202362 + }, + "frames_100_200": { + "mse": 0.008570044301450253, + "psnr": 23.110898971557617, + "lpips": 0.09225184470415115 + }, + "frames_200_300": { + "mse": 0.02207282930612564, + "psnr": 17.866056442260742, + "lpips": 0.18821300566196442 + }, + "frames_300_400": { + "mse": 0.01187162660062313, + "psnr": 19.844547271728516, + "lpips": 0.1457580029964447 + }, + "frames_400_500": { + "mse": 0.012656111270189285, + "psnr": 19.259069442749023, + "lpips": 0.18428264558315277 + }, + "overall": { + "mse": 0.012705856934189796, + "psnr": 20.342180252075195, + "lpips": 0.1562362015247345 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000177_test__val_savanna__1__000049/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000177_test__val_savanna__1__000049/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..4851472a451ee15f1c44a646f87971d654e18917 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000177_test__val_savanna__1__000049/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61a0f2fdbd2a855a1a264b0518901d2ff66fca1254f435327ab16a9f4427360f +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000177_test__val_savanna__1__000049/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000177_test__val_savanna__1__000049/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..57f80a070825cf9a0f98b0e315990aafa0063773 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000177_test__val_savanna__1__000049/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "177", + "global_test_index": "177", + "dataset_index": "240", + "relative_path": "test/val_savanna/1/000049.mp4", + "clip_id": "test/val_savanna/1/000049", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000177, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_savanna/1/000049_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.009057569317519665, + "psnr": 20.820165634155273, + "lpips": 0.09904490411281586 + }, + "frames_100_200": { + "mse": 0.01068631187081337, + "psnr": 19.94817543029785, + "lpips": 0.11220523715019226 + }, + "frames_200_300": { + "mse": 0.007162266876548529, + "psnr": 22.098207473754883, + "lpips": 0.05844632163643837 + }, + "frames_300_400": { + "mse": 0.016740847378969193, + "psnr": 18.38831329345703, + "lpips": 0.10926606506109238 + }, + "frames_400_500": { + "mse": 0.024965891614556313, + "psnr": 16.517841339111328, + "lpips": 0.2551072835922241 + }, + "overall": { + "mse": 0.013722578063607216, + "psnr": 19.554540634155273, + "lpips": 0.12681396305561066 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000178_test__val_desert__0__000017/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000178_test__val_desert__0__000017/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..e566d8df79e17215f2f69a6f3a26c5f98447ed81 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000178_test__val_desert__0__000017/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:856d6087441248f682d10a2afa2a3c80327937ada9411fd21d2b03be0a4965dd +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000178_test__val_desert__0__000017/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000178_test__val_desert__0__000017/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..f6c892550350d8b2a943de50bdcd98de52ff86b4 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000178_test__val_desert__0__000017/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "178", + "global_test_index": "178", + "dataset_index": "42", + "relative_path": "test/val_desert/0/000017.mp4", + "clip_id": "test/val_desert/0/000017", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000178, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_desert/0/000017_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.0015406779712066054, + "psnr": 28.581571578979492, + "lpips": 0.02414448745548725 + }, + "frames_100_200": { + "mse": 0.003072392428293824, + "psnr": 26.107688903808594, + "lpips": 0.04957672208547592 + }, + "frames_200_300": { + "mse": 0.0021255966275930405, + "psnr": 27.598875045776367, + "lpips": 0.02567768096923828 + }, + "frames_300_400": { + "mse": 0.000783158466219902, + "psnr": 31.068187713623047, + "lpips": 0.010371793061494827 + }, + "frames_400_500": { + "mse": 0.005013496149331331, + "psnr": 24.416793823242188, + "lpips": 0.11982809007167816 + }, + "overall": { + "mse": 0.0025070642586797476, + "psnr": 27.554622650146484, + "lpips": 0.0459197573363781 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000179_test__val_iceplains__3__000048/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000179_test__val_iceplains__3__000048/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..bcb6086c8a911bd29f6bee1824b77d0f9a3eff80 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000179_test__val_iceplains__3__000048/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:787c9ee3240872bfab9c1116af2a675406a43fb6684aba8d600e48a2b07021c8 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000179_test__val_iceplains__3__000048/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000179_test__val_iceplains__3__000048/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..0e7b14f8c20afbb12264e70391e62db8d79d8d1b --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000179_test__val_iceplains__3__000048/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "179", + "global_test_index": "179", + "dataset_index": "369", + "relative_path": "test/val_iceplains/3/000048.mp4", + "clip_id": "test/val_iceplains/3/000048", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000179, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/3/000048_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.016763247549533844, + "psnr": 17.922639846801758, + "lpips": 0.1318618804216385 + }, + "frames_100_200": { + "mse": 0.013210549019277096, + "psnr": 21.395952224731445, + "lpips": 0.11517814546823502 + }, + "frames_200_300": { + "mse": 0.0030512004159390926, + "psnr": 26.18749237060547, + "lpips": 0.027650685980916023 + }, + "frames_300_400": { + "mse": 0.019076818600296974, + "psnr": 21.324169158935547, + "lpips": 0.1263766884803772 + }, + "frames_400_500": { + "mse": 0.10547167807817459, + "psnr": 10.003440856933594, + "lpips": 0.4261271357536316 + }, + "overall": { + "mse": 0.03151469677686691, + "psnr": 19.366741180419922, + "lpips": 0.1654389202594757 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000180_test__val_plains__3__000010/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000180_test__val_plains__3__000010/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..e6d4552061687365f76f4035d6dbd4a406cdbd87 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000180_test__val_plains__3__000010/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d5fc78304e0a578f4a0e2e0c5aec5eb54e3d5d4f200b522a4dbe6e4243d1fef2 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000180_test__val_plains__3__000010/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000180_test__val_plains__3__000010/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..31c54ab70c94f969145386012529e4675378bb1e --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000180_test__val_plains__3__000010/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "180", + "global_test_index": "180", + "dataset_index": "398", + "relative_path": "test/val_plains/3/000010.mp4", + "clip_id": "test/val_plains/3/000010", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000180, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_plains/3/000010_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.005986475385725498, + "psnr": 26.852327346801758, + "lpips": 0.05437621474266052 + }, + "frames_100_200": { + "mse": 0.000868702307343483, + "psnr": 30.617431640625, + "lpips": 0.014054710045456886 + }, + "frames_200_300": { + "mse": 0.0009469165815971792, + "psnr": 30.250141143798828, + "lpips": 0.016912994906306267 + }, + "frames_300_400": { + "mse": 0.0016659493558108807, + "psnr": 29.276159286499023, + "lpips": 0.031205127015709877 + }, + "frames_400_500": { + "mse": 0.007647080346941948, + "psnr": 21.771413803100586, + "lpips": 0.1344831883907318 + }, + "overall": { + "mse": 0.0034230246674269438, + "psnr": 27.753494262695312, + "lpips": 0.050206445157527924 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000181_test__val_plains__2__000027/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000181_test__val_plains__2__000027/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..ff238ee2b933881a02291a775c4ce69d413f3cb6 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000181_test__val_plains__2__000027/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:747d22b4f8d6a2302d6365f7c874d6916a2064222e2f00495abf133a77c58d74 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000181_test__val_plains__2__000027/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000181_test__val_plains__2__000027/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..6c88099c165c121eec7b5b3a3c3c3e5ca9f298e0 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000181_test__val_plains__2__000027/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "181", + "global_test_index": "181", + "dataset_index": "435", + "relative_path": "test/val_plains/2/000027.mp4", + "clip_id": "test/val_plains/2/000027", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000181, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_plains/2/000027_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.0022780040744692087, + "psnr": 28.188182830810547, + "lpips": 0.01918112114071846 + }, + "frames_100_200": { + "mse": 0.0009071672684513032, + "psnr": 30.454914093017578, + "lpips": 0.014867600984871387 + }, + "frames_200_300": { + "mse": 0.004403825383633375, + "psnr": 24.754009246826172, + "lpips": 0.03267862647771835 + }, + "frames_300_400": { + "mse": 0.0026743027847260237, + "psnr": 28.192251205444336, + "lpips": 0.025580190122127533 + }, + "frames_400_500": { + "mse": 0.007036648225039244, + "psnr": 23.40170669555664, + "lpips": 0.13732631504535675 + }, + "overall": { + "mse": 0.003459989558905363, + "psnr": 26.998210906982422, + "lpips": 0.0459267683327198 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000182_test__val_savanna__2__000049/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000182_test__val_savanna__2__000049/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..119eabf0768f12f829cabb148b122b9fa357e4cc --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000182_test__val_savanna__2__000049/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b4b7dc9fb91abdca8fd291c245d9c0b1eedd7b61de01970cfe904965923aae2 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000182_test__val_savanna__2__000049/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000182_test__val_savanna__2__000049/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..498930f0926c65d39037dbdf0e940be6640b358c --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000182_test__val_savanna__2__000049/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "182", + "global_test_index": "182", + "dataset_index": "242", + "relative_path": "test/val_savanna/2/000049.mp4", + "clip_id": "test/val_savanna/2/000049", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000182, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_savanna/2/000049_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.0036849589087069035, + "psnr": 25.625757217407227, + "lpips": 0.027176523581147194 + }, + "frames_100_200": { + "mse": 0.029377765953540802, + "psnr": 17.21633529663086, + "lpips": 0.16945762932300568 + }, + "frames_200_300": { + "mse": 0.07262729853391647, + "psnr": 11.422696113586426, + "lpips": 0.35899779200553894 + }, + "frames_300_400": { + "mse": 0.08714601397514343, + "psnr": 10.705780982971191, + "lpips": 0.36587974429130554 + }, + "frames_400_500": { + "mse": 0.0668545663356781, + "psnr": 12.475634574890137, + "lpips": 0.2990347743034363 + }, + "overall": { + "mse": 0.051938120275735855, + "psnr": 15.489240646362305, + "lpips": 0.24410931766033173 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000183_test__val_iceplains__2__000045/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000183_test__val_iceplains__2__000045/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..b664bf49b62e1d1b61b6f6326bd686f14a9b4013 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000183_test__val_iceplains__2__000045/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c34a61d73b074eb6089084b2d233ecf81239f5f51647672fc03a5fb7fa46a1da +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000183_test__val_iceplains__2__000045/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000183_test__val_iceplains__2__000045/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..b148f41a3e5784a7e2a5b9e1c89a731c5a6e06d5 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000183_test__val_iceplains__2__000045/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "183", + "global_test_index": "183", + "dataset_index": "361", + "relative_path": "test/val_iceplains/2/000045.mp4", + "clip_id": "test/val_iceplains/2/000045", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000183, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/2/000045_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.01771511510014534, + "psnr": 21.141437530517578, + "lpips": 0.1033807247877121 + }, + "frames_100_200": { + "mse": 0.024708347395062447, + "psnr": 18.60979652404785, + "lpips": 0.14381061494350433 + }, + "frames_200_300": { + "mse": 0.016729621216654778, + "psnr": 21.924814224243164, + "lpips": 0.09126624464988708 + }, + "frames_300_400": { + "mse": 0.013683422468602657, + "psnr": 20.75680923461914, + "lpips": 0.08053651452064514 + }, + "frames_400_500": { + "mse": 0.005370415281504393, + "psnr": 26.305112838745117, + "lpips": 0.037147991359233856 + }, + "overall": { + "mse": 0.015641383826732635, + "psnr": 21.74759292602539, + "lpips": 0.0912284180521965 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000184_test__val_plains__2__000047/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000184_test__val_plains__2__000047/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..957cc893893318e2de03d5eeffd98e0dc96c04c0 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000184_test__val_plains__2__000047/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6de3ca96bb08c48a1913324ae476635d5428ed76b4bf4810c4190292eae36d51 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000184_test__val_plains__2__000047/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000184_test__val_plains__2__000047/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..cc770d2b537e795d7b4d3cf83f498f37c7586374 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000184_test__val_plains__2__000047/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "184", + "global_test_index": "184", + "dataset_index": "470", + "relative_path": "test/val_plains/2/000047.mp4", + "clip_id": "test/val_plains/2/000047", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000184, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_plains/2/000047_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.01077753584831953, + "psnr": 19.819581985473633, + "lpips": 0.27811068296432495 + }, + "frames_100_200": { + "mse": 0.007882364094257355, + "psnr": 21.20459747314453, + "lpips": 0.3558506369590759 + }, + "frames_200_300": { + "mse": 0.004940539598464966, + "psnr": 23.144926071166992, + "lpips": 0.3414837718009949 + }, + "frames_300_400": { + "mse": 0.005116581451147795, + "psnr": 23.014997482299805, + "lpips": 0.37858763337135315 + }, + "frames_400_500": { + "mse": 0.006801682524383068, + "psnr": 22.005626678466797, + "lpips": 0.4290989339351654 + }, + "overall": { + "mse": 0.0071037402376532555, + "psnr": 21.837947845458984, + "lpips": 0.35662633180618286 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000185_test__val_iceplains__2__000042/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000185_test__val_iceplains__2__000042/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..7f5cf4e0b5402cb4c3be2b3bd7a81f9d46402989 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000185_test__val_iceplains__2__000042/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:74d5e3f965954fffab5807173deceac89a3dd0ef99cfb3e910353505b6f49d62 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000185_test__val_iceplains__2__000042/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000185_test__val_iceplains__2__000042/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..526ad727c5b30d25a32215edc43474d993081834 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000185_test__val_iceplains__2__000042/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "185", + "global_test_index": "185", + "dataset_index": "356", + "relative_path": "test/val_iceplains/2/000042.mp4", + "clip_id": "test/val_iceplains/2/000042", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000185, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_iceplains/2/000042_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.023309165611863136, + "psnr": 16.758255004882812, + "lpips": 0.216781347990036 + }, + "frames_100_200": { + "mse": 0.05028165876865387, + "psnr": 15.385514259338379, + "lpips": 0.296407550573349 + }, + "frames_200_300": { + "mse": 0.031244780868291855, + "psnr": 18.394851684570312, + "lpips": 0.12825420498847961 + }, + "frames_300_400": { + "mse": 0.007555872667580843, + "psnr": 25.710681915283203, + "lpips": 0.044367264956235886 + }, + "frames_400_500": { + "mse": 0.001410644967108965, + "psnr": 29.682449340820312, + "lpips": 0.01798803173005581 + }, + "overall": { + "mse": 0.022760426625609398, + "psnr": 21.186351776123047, + "lpips": 0.14075967669487 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000186_test__val_plains__2__000046/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000186_test__val_plains__2__000046/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..65d47bb6ea4f09a77ed987b09a72916fc10b9046 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000186_test__val_plains__2__000046/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7fad4ab75ef444091fa1bbb27319bd0a50c7eba62c79f60d8815a0ab2b78308f +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000186_test__val_plains__2__000046/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000186_test__val_plains__2__000046/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..0e0fb8bf718213715bb69c86ba949d39e4ba6630 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000186_test__val_plains__2__000046/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "186", + "global_test_index": "186", + "dataset_index": "468", + "relative_path": "test/val_plains/2/000046.mp4", + "clip_id": "test/val_plains/2/000046", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000186, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_plains/2/000046_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.010426159016788006, + "psnr": 20.8102970123291, + "lpips": 0.2656155824661255 + }, + "frames_100_200": { + "mse": 0.005120779387652874, + "psnr": 23.08031463623047, + "lpips": 0.2864091098308563 + }, + "frames_200_300": { + "mse": 0.042782217264175415, + "psnr": 14.070352554321289, + "lpips": 0.4090651571750641 + }, + "frames_300_400": { + "mse": 0.06957975029945374, + "psnr": 12.199990272521973, + "lpips": 0.5755849480628967 + }, + "frames_400_500": { + "mse": 0.01710308901965618, + "psnr": 18.54534912109375, + "lpips": 0.48967036604881287 + }, + "overall": { + "mse": 0.029002400115132332, + "psnr": 17.741260528564453, + "lpips": 0.4052690267562866 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000187_test__val_desert__0__000043/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000187_test__val_desert__0__000043/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..9eb3eaf3134175176a8f1e707c18970e88988be6 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000187_test__val_desert__0__000043/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:72212649f085db8ce4d235f1edceee707841ef00207aa4e798f80f719e0b253d +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000187_test__val_desert__0__000043/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000187_test__val_desert__0__000043/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..752a40257dfe0a0f88e9822ef1a85b833b67558d --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000187_test__val_desert__0__000043/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "187", + "global_test_index": "187", + "dataset_index": "113", + "relative_path": "test/val_desert/0/000043.mp4", + "clip_id": "test/val_desert/0/000043", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000187, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_desert/0/000043_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.017867987975478172, + "psnr": 18.610973358154297, + "lpips": 0.3562743663787842 + }, + "frames_100_200": { + "mse": 0.037783171981573105, + "psnr": 14.346199989318848, + "lpips": 0.5073296427726746 + }, + "frames_200_300": { + "mse": 0.02242758683860302, + "psnr": 17.11597442626953, + "lpips": 0.4935070872306824 + }, + "frames_300_400": { + "mse": 0.0489160530269146, + "psnr": 13.436602592468262, + "lpips": 0.48167628049850464 + }, + "frames_400_500": { + "mse": 0.030594171956181526, + "psnr": 15.284333229064941, + "lpips": 0.4207315146923065 + }, + "overall": { + "mse": 0.03151779621839523, + "psnr": 15.758816719055176, + "lpips": 0.4519037902355194 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000188_test__val_desert__1__000008/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000188_test__val_desert__1__000008/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..625774dc4061005f6e3b644136e8e3e9d8606a03 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000188_test__val_desert__1__000008/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:16dbf4ae9fa6ee9aacdc099d2fb2fe9d33554d6fe1ddc572cadd74e1c1aaeee2 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000188_test__val_desert__1__000008/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000188_test__val_desert__1__000008/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..dd8a579c0a1eab024fcacaee2bda29272f179c77 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000188_test__val_desert__1__000008/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "188", + "global_test_index": "188", + "dataset_index": "19", + "relative_path": "test/val_desert/1/000008.mp4", + "clip_id": "test/val_desert/1/000008", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000188, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_desert/1/000008_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.007472847122699022, + "psnr": 21.544130325317383, + "lpips": 0.2668614983558655 + }, + "frames_100_200": { + "mse": 0.009032657369971275, + "psnr": 20.654438018798828, + "lpips": 0.3591482639312744 + }, + "frames_200_300": { + "mse": 0.0074265701696276665, + "psnr": 21.517282485961914, + "lpips": 0.37803253531455994 + }, + "frames_300_400": { + "mse": 0.007079786621034145, + "psnr": 21.540821075439453, + "lpips": 0.3356195092201233 + }, + "frames_400_500": { + "mse": 0.01186716090887785, + "psnr": 19.696046829223633, + "lpips": 0.28917092084884644 + }, + "overall": { + "mse": 0.00857580453157425, + "psnr": 20.99054527282715, + "lpips": 0.32576653361320496 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000189_test__val_savanna__0__000033/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000189_test__val_savanna__0__000033/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..1c84939b32ab927097d12c7aa2d4320895ef09c9 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000189_test__val_savanna__0__000033/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ee1f7f253ff035757251b28e201dcc8c5dd35b7a91d81c9d9ea0ae9470e8571 +size 6742 diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000189_test__val_savanna__0__000033/metrics.json b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000189_test__val_savanna__0__000033/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..eb3a52212727363198b3f9db426f30dab370c191 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000189_test__val_savanna__0__000033/metrics.json @@ -0,0 +1,80 @@ +{ + "case": { + "manifest_index": "189", + "global_test_index": "189", + "dataset_index": "199", + "relative_path": "test/val_savanna/0/000033.mp4", + "clip_id": "test/val_savanna/0/000033", + "source_offset": "100", + "context_start": "100", + "context_end": "700", + "target_start": "700", + "target_end": "1200", + "n_frames_valid": "1100", + "context_frames": "600", + "target_frames": "500", + "frame_skip": "1", + "raw_frame_count": "1501", + "action_count": "1501", + "pose_count": "1501", + "latent_count": "1501" + }, + "sample_seed": 43000189, + "dataset": "minecraft", + "frames": 500, + "native_resolution": [ + 352, + 640 + ], + "metric_resolution": [ + 360, + 640 + ], + "resize": "bicubic align_corners=False antialias=True", + "reference_source": "/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features/test/val_savanna/0/000033_vae_feature.npy", + "vae_checkpoint": "/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/checkpoints/worldmem/vae_only.ckpt", + "torch": "2.8.0+cu128", + "torchmetrics": "0.11.4", + "lpips": "alex; normalize=True; mean of individual frame values", + "psnr_epsilon": 1.1920928955078125e-07, + "fid": "OpenCV decoding of libx264 quality8 yuv420p uint8 videos", + "video": { + "codec": "libx264", + "quality": 8, + "pixelformat": "yuv420p", + "macro_block_size": 1, + "fps": 15 + }, + "ranges": { + "frames_000_100": { + "mse": 0.015571258030831814, + "psnr": 18.9266300201416, + "lpips": 0.1833934783935547 + }, + "frames_100_200": { + "mse": 0.012114095501601696, + "psnr": 19.6099796295166, + "lpips": 0.11864204704761505 + }, + "frames_200_300": { + "mse": 0.023622626438736916, + "psnr": 16.320432662963867, + "lpips": 0.3264370262622833 + }, + "frames_300_400": { + "mse": 0.02604379877448082, + "psnr": 15.903234481811523, + "lpips": 0.3709510862827301 + }, + "frames_400_500": { + "mse": 0.03043888323009014, + "psnr": 15.261519432067871, + "lpips": 0.3648931086063385 + }, + "overall": { + "mse": 0.021558132022619247, + "psnr": 17.20435905456543, + "lpips": 0.27286332845687866 + } + } +} diff --git a/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000190_test__val_desert__2__000045/frame_metrics.npz b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000190_test__val_desert__2__000045/frame_metrics.npz new file mode 100644 index 0000000000000000000000000000000000000000..dc12b8b44017205f09315eb035db0b58ce736be8 --- /dev/null +++ b/results/worldmem-baseline-evals/decmem/decmem_minecraft_300cases_seed43_job84976/cases/g000190_test__val_desert__2__000045/frame_metrics.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:07d9f26813c46012bf32e819e4efe89ab9c60e4afd773fe05092bbb781507896 +size 6742