Add Echo-Memory codebase used for this run (CC BY 4.0, JD Echo Team) (part 2)

.gitattributes

@@ -47,3 +47,16 @@ code/assets/readme_previews/ssm_blockwise_replay_gt.gif filter=lfs diff=lfs merg
 code/assets/readme_previews/ssm_legacy_replay_gt.gif filter=lfs diff=lfs merge=lfs -text
 code/diffsynth/tokenizer_configs/hunyuan_video/tokenizer_2/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 code/diffsynth/tokenizer_configs/kolors/tokenizer/vocab.txt filter=lfs diff=lfs merge=lfs -text
+code/docs/assets/echo-memory-paper.pdf filter=lfs diff=lfs merge=lfs -text
+code/docs/assets/readme_previews/context_k1_replay_gt.gif filter=lfs diff=lfs merge=lfs -text
+code/docs/assets/readme_previews/context_k5_replay_gt.gif filter=lfs diff=lfs merge=lfs -text
+code/docs/assets/readme_previews/dyn_context_k1_replay.gif filter=lfs diff=lfs merge=lfs -text
+code/docs/assets/readme_previews/dyn_context_k20_replay.gif filter=lfs diff=lfs merge=lfs -text
+code/docs/assets/readme_previews/dyn_context_k5_replay.gif filter=lfs diff=lfs merge=lfs -text
+code/docs/assets/readme_previews/dyn_spatial_memory_replay.gif filter=lfs diff=lfs merge=lfs -text
+code/docs/assets/readme_previews/dyn_ssm_blockwise_replay.gif filter=lfs diff=lfs merge=lfs -text
+code/docs/assets/readme_previews/dyn_ssm_legacy_replay.gif filter=lfs diff=lfs merge=lfs -text
+code/docs/assets/readme_previews/framepack_len_r4_replay_gt.gif filter=lfs diff=lfs merge=lfs -text
+code/docs/assets/readme_previews/spatial_memory_replay_gt.gif filter=lfs diff=lfs merge=lfs -text
+code/docs/assets/readme_previews/ssm_blockwise_replay_gt.gif filter=lfs diff=lfs merge=lfs -text
+code/docs/assets/readme_previews/ssm_legacy_replay_gt.gif filter=lfs diff=lfs merge=lfs -text

code/diffsynth/tokenizer_configs/stable_diffusion_3/tokenizer_3/tokenizer_config.json

@@ -0,0 +1,940 @@
+{
+  "add_prefix_space": true,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32000": {
+      "content": "<extra_id_99>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32001": {
+      "content": "<extra_id_98>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32002": {
+      "content": "<extra_id_97>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32003": {
+      "content": "<extra_id_96>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32004": {
+      "content": "<extra_id_95>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32005": {
+      "content": "<extra_id_94>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32006": {
+      "content": "<extra_id_93>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32007": {
+      "content": "<extra_id_92>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32008": {
+      "content": "<extra_id_91>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32009": {
+      "content": "<extra_id_90>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32010": {
+      "content": "<extra_id_89>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32011": {
+      "content": "<extra_id_88>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32012": {
+      "content": "<extra_id_87>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32013": {
+      "content": "<extra_id_86>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32014": {
+      "content": "<extra_id_85>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32015": {
+      "content": "<extra_id_84>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32016": {
+      "content": "<extra_id_83>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32017": {
+      "content": "<extra_id_82>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32018": {
+      "content": "<extra_id_81>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32019": {
+      "content": "<extra_id_80>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32020": {
+      "content": "<extra_id_79>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32021": {
+      "content": "<extra_id_78>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32022": {
+      "content": "<extra_id_77>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32023": {
+      "content": "<extra_id_76>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32024": {
+      "content": "<extra_id_75>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32025": {
+      "content": "<extra_id_74>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32026": {
+      "content": "<extra_id_73>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32027": {
+      "content": "<extra_id_72>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32028": {
+      "content": "<extra_id_71>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32029": {
+      "content": "<extra_id_70>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32030": {
+      "content": "<extra_id_69>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32031": {
+      "content": "<extra_id_68>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32032": {
+      "content": "<extra_id_67>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32033": {
+      "content": "<extra_id_66>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32034": {
+      "content": "<extra_id_65>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32035": {
+      "content": "<extra_id_64>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32036": {
+      "content": "<extra_id_63>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32037": {
+      "content": "<extra_id_62>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32038": {
+      "content": "<extra_id_61>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32039": {
+      "content": "<extra_id_60>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32040": {
+      "content": "<extra_id_59>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32041": {
+      "content": "<extra_id_58>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32042": {
+      "content": "<extra_id_57>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32043": {
+      "content": "<extra_id_56>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32044": {
+      "content": "<extra_id_55>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32045": {
+      "content": "<extra_id_54>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32046": {
+      "content": "<extra_id_53>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32047": {
+      "content": "<extra_id_52>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32048": {
+      "content": "<extra_id_51>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32049": {
+      "content": "<extra_id_50>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32050": {
+      "content": "<extra_id_49>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32051": {
+      "content": "<extra_id_48>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32052": {
+      "content": "<extra_id_47>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32053": {
+      "content": "<extra_id_46>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32054": {
+      "content": "<extra_id_45>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32055": {
+      "content": "<extra_id_44>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32056": {
+      "content": "<extra_id_43>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32057": {
+      "content": "<extra_id_42>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32058": {
+      "content": "<extra_id_41>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32059": {
+      "content": "<extra_id_40>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32060": {
+      "content": "<extra_id_39>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32061": {
+      "content": "<extra_id_38>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32062": {
+      "content": "<extra_id_37>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32063": {
+      "content": "<extra_id_36>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32064": {
+      "content": "<extra_id_35>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32065": {
+      "content": "<extra_id_34>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32066": {
+      "content": "<extra_id_33>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32067": {
+      "content": "<extra_id_32>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32068": {
+      "content": "<extra_id_31>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32069": {
+      "content": "<extra_id_30>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32070": {
+      "content": "<extra_id_29>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32071": {
+      "content": "<extra_id_28>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32072": {
+      "content": "<extra_id_27>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32073": {
+      "content": "<extra_id_26>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32074": {
+      "content": "<extra_id_25>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32075": {
+      "content": "<extra_id_24>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32076": {
+      "content": "<extra_id_23>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32077": {
+      "content": "<extra_id_22>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32078": {
+      "content": "<extra_id_21>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32079": {
+      "content": "<extra_id_20>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32080": {
+      "content": "<extra_id_19>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32081": {
+      "content": "<extra_id_18>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32082": {
+      "content": "<extra_id_17>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32083": {
+      "content": "<extra_id_16>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32084": {
+      "content": "<extra_id_15>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32085": {
+      "content": "<extra_id_14>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32086": {
+      "content": "<extra_id_13>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32087": {
+      "content": "<extra_id_12>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32088": {
+      "content": "<extra_id_11>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32089": {
+      "content": "<extra_id_10>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32090": {
+      "content": "<extra_id_9>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32091": {
+      "content": "<extra_id_8>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32092": {
+      "content": "<extra_id_7>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32093": {
+      "content": "<extra_id_6>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32094": {
+      "content": "<extra_id_5>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32095": {
+      "content": "<extra_id_4>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32096": {
+      "content": "<extra_id_3>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32097": {
+      "content": "<extra_id_2>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32098": {
+      "content": "<extra_id_1>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32099": {
+      "content": "<extra_id_0>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<extra_id_0>",
+    "<extra_id_1>",
+    "<extra_id_2>",
+    "<extra_id_3>",
+    "<extra_id_4>",
+    "<extra_id_5>",
+    "<extra_id_6>",
+    "<extra_id_7>",
+    "<extra_id_8>",
+    "<extra_id_9>",
+    "<extra_id_10>",
+    "<extra_id_11>",
+    "<extra_id_12>",
+    "<extra_id_13>",
+    "<extra_id_14>",
+    "<extra_id_15>",
+    "<extra_id_16>",
+    "<extra_id_17>",
+    "<extra_id_18>",
+    "<extra_id_19>",
+    "<extra_id_20>",
+    "<extra_id_21>",
+    "<extra_id_22>",
+    "<extra_id_23>",
+    "<extra_id_24>",
+    "<extra_id_25>",
+    "<extra_id_26>",
+    "<extra_id_27>",
+    "<extra_id_28>",
+    "<extra_id_29>",
+    "<extra_id_30>",
+    "<extra_id_31>",
+    "<extra_id_32>",
+    "<extra_id_33>",
+    "<extra_id_34>",
+    "<extra_id_35>",
+    "<extra_id_36>",
+    "<extra_id_37>",
+    "<extra_id_38>",
+    "<extra_id_39>",
+    "<extra_id_40>",
+    "<extra_id_41>",
+    "<extra_id_42>",
+    "<extra_id_43>",
+    "<extra_id_44>",
+    "<extra_id_45>",
+    "<extra_id_46>",
+    "<extra_id_47>",
+    "<extra_id_48>",
+    "<extra_id_49>",
+    "<extra_id_50>",
+    "<extra_id_51>",
+    "<extra_id_52>",
+    "<extra_id_53>",
+    "<extra_id_54>",
+    "<extra_id_55>",
+    "<extra_id_56>",
+    "<extra_id_57>",
+    "<extra_id_58>",
+    "<extra_id_59>",
+    "<extra_id_60>",
+    "<extra_id_61>",
+    "<extra_id_62>",
+    "<extra_id_63>",
+    "<extra_id_64>",
+    "<extra_id_65>",
+    "<extra_id_66>",
+    "<extra_id_67>",
+    "<extra_id_68>",
+    "<extra_id_69>",
+    "<extra_id_70>",
+    "<extra_id_71>",
+    "<extra_id_72>",
+    "<extra_id_73>",
+    "<extra_id_74>",
+    "<extra_id_75>",
+    "<extra_id_76>",
+    "<extra_id_77>",
+    "<extra_id_78>",
+    "<extra_id_79>",
+    "<extra_id_80>",
+    "<extra_id_81>",
+    "<extra_id_82>",
+    "<extra_id_83>",
+    "<extra_id_84>",
+    "<extra_id_85>",
+    "<extra_id_86>",
+    "<extra_id_87>",
+    "<extra_id_88>",
+    "<extra_id_89>",
+    "<extra_id_90>",
+    "<extra_id_91>",
+    "<extra_id_92>",
+    "<extra_id_93>",
+    "<extra_id_94>",
+    "<extra_id_95>",
+    "<extra_id_96>",
+    "<extra_id_97>",
+    "<extra_id_98>",
+    "<extra_id_99>"
+  ],
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "</s>",
+  "extra_ids": 100,
+  "legacy": true,
+  "model_max_length": 512,
+  "pad_token": "<pad>",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "T5Tokenizer",
+  "unk_token": "<unk>"
+}

code/diffsynth/tokenizer_configs/stable_diffusion_xl/tokenizer_2/special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+    "bos_token": {
+      "content": "<|startoftext|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false
+    },
+    "eos_token": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false
+    },
+    "pad_token": "!",
+    "unk_token": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false
+    }
+}

code/diffsynth/tokenizer_configs/stable_diffusion_xl/tokenizer_2/tokenizer_config.json

@@ -0,0 +1,38 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "!",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49406": {
+      "content": "<|startoftext|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49407": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|startoftext|>",
+  "clean_up_tokenization_spaces": true,
+  "do_lower_case": true,
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "model_max_length": 77,
+  "pad_token": "!",
+  "tokenizer_class": "CLIPTokenizer",
+  "unk_token": "<|endoftext|>"
+}

code/diffsynth/trainers/latent_dataset.py

@@ -0,0 +1,130 @@
+"""
+LatentDataset: Load precomputed ctx and target latents for Context-as-Memory dataset.
+
+Use after running precompute_ctx_target_latents.py. Returns samples with:
+- context_latents: (1, C, K, H//8, W//8) - 1 latent per context frame
+- target_latents: (1, C, T, H//8, W//8) - 1 latent per 4 target frames
+- prompt, video_name, start_frame, end_frame, actions
+
+Compatible with training that uses precomputed latents instead of encoding on the fly.
+"""
+
+import json
+import os
+import warnings
+
+import torch
+
+
+class LatentDataset(torch.utils.data.Dataset):
+    """
+    Dataset that loads precomputed ctx and target latents.
+    """
+
+    def __init__(
+        self,
+        latent_dir,
+        metadata_path=None,
+        action_base_path=None,
+        repeat=1,
+        num_frames=81,
+        context_frames=5,
+        target_frames_per_latent=4,
+    ):
+        """
+        Args:
+            latent_dir: Directory containing ctx_latents/ and target_latents/ subdirs.
+            metadata_path: Optional. If provided, used to get total_samples and validate.
+            action_base_path: Base path for action JSON files (for loading actions if not in .pt).
+            repeat: Dataset repeat factor.
+            num_frames: Expected num_frames per segment.
+            context_frames: Number of context frames (K).
+            target_frames_per_latent: Target: 1 latent per N frames.
+        """
+        self.latent_dir = latent_dir
+        self.ctx_dir = os.path.join(latent_dir, "ctx_latents")
+        self.target_dir = os.path.join(latent_dir, "target_latents")
+        self.action_base_path = action_base_path or latent_dir
+        self.repeat = repeat
+        self.num_frames = num_frames
+        self.context_frames = context_frames
+        self.target_frames_per_latent = target_frames_per_latent
+
+        # Infer valid indices from existing files (both ctx and target must exist)
+        self._indices = []
+        if os.path.isdir(self.ctx_dir) and os.path.isdir(self.target_dir):
+            ctx_files = {f.replace(".pt", "") for f in os.listdir(self.ctx_dir) if f.endswith(".pt")}
+            target_files = {f.replace(".pt", "") for f in os.listdir(self.target_dir) if f.endswith(".pt")}
+            common = sorted([int(x) for x in ctx_files & target_files])
+            self._indices = common
+        if not self._indices:
+            meta_path = os.path.join(latent_dir, "metadata_precompute.json")
+            if os.path.isfile(meta_path):
+                with open(meta_path) as f:
+                    meta = json.load(f)
+                self._total = meta.get("total_samples", 0)
+                self._indices = list(range(self._total))
+            else:
+                self._total = 0
+        else:
+            self._total = len(self._indices)
+
+    def __len__(self):
+        return self._total * self.repeat
+
+    def __getitem__(self, idx):
+        real_idx = idx % self._total
+        if self._indices is not None:
+            real_idx = self._indices[real_idx]
+
+        ctx_path = os.path.join(self.ctx_dir, f"{real_idx:08d}.pt")
+        target_path = os.path.join(self.target_dir, f"{real_idx:08d}.pt")
+
+        if not os.path.isfile(ctx_path) or not os.path.isfile(target_path):
+            warnings.warn(f"Latent files not found for idx {real_idx}. Returning None.")
+            return None
+
+        ctx_data = torch.load(ctx_path, map_location="cpu", weights_only=True)
+        target_data = torch.load(target_path, map_location="cpu", weights_only=True)
+
+        ctx_latent = ctx_data["latent"]
+        target_latent = target_data["latent"]
+
+        # Ensure batch dimension: (C, K, H, W) -> (1, C, K, H, W)
+        if ctx_latent.dim() == 4:
+            ctx_latent = ctx_latent.unsqueeze(0)
+        if target_latent.dim() == 4:
+            target_latent = target_latent.unsqueeze(0)
+
+        out = {
+            "context_latents": ctx_latent,
+            "target_latents": target_latent,
+            "prompt": ctx_data.get("prompt", ""),
+            "video_name": ctx_data.get("video_name"),
+            "start_frame": ctx_data.get("start_frame"),
+            "end_frame": ctx_data.get("end_frame"),
+        }
+        if "actions" in ctx_data and ctx_data["actions"] is not None:
+            out["actions"] = ctx_data["actions"]
+        elif "actions" in target_data and target_data["actions"] is not None:
+            out["actions"] = target_data["actions"]
+
+        return out
+
+
+def get_latent_dataset_args(latent_dir, action_base_path=None, **kwargs):
+    """Build argparse.Namespace for LatentDataset from precompute metadata."""
+    meta_path = os.path.join(latent_dir, "metadata_precompute.json")
+    if not os.path.isfile(meta_path):
+        return None
+    with open(meta_path) as f:
+        meta = json.load(f)
+    from argparse import Namespace
+    return Namespace(
+        latent_dir=latent_dir,
+        action_base_path=action_base_path or meta.get("dataset_base_path", latent_dir),
+        num_frames=meta.get("num_frames", 81),
+        context_frames=meta.get("context_frames", 5),
+        target_frames_per_latent=meta.get("target_frames_per_latent", 4),
+        **kwargs,
+    )

code/diffsynth/trainers/text_to_image.py

@@ -0,0 +1,318 @@
+import lightning as pl
+from peft import LoraConfig, inject_adapter_in_model
+import torch, os
+from ..data.simple_text_image import TextImageDataset
+from modelscope.hub.api import HubApi
+from ..models.utils import load_state_dict
+
+
+
+class LightningModelForT2ILoRA(pl.LightningModule):
+    def __init__(
+        self,
+        learning_rate=1e-4,
+        use_gradient_checkpointing=True,
+        state_dict_converter=None,
+    ):
+        super().__init__()
+        # Set parameters
+        self.learning_rate = learning_rate
+        self.use_gradient_checkpointing = use_gradient_checkpointing
+        self.state_dict_converter = state_dict_converter
+        self.lora_alpha = None
+
+
+    def load_models(self):
+        # This function is implemented in other modules
+        self.pipe = None
+
+
+    def freeze_parameters(self):
+        # Freeze parameters
+        self.pipe.requires_grad_(False)
+        self.pipe.eval()
+        self.pipe.denoising_model().train()
+
+    
+    def add_lora_to_model(self, model, lora_rank=4, lora_alpha=4, lora_target_modules="to_q,to_k,to_v,to_out", init_lora_weights="gaussian", pretrained_lora_path=None, state_dict_converter=None):
+        # Add LoRA to UNet
+        self.lora_alpha = lora_alpha
+        if init_lora_weights == "kaiming":
+            init_lora_weights = True
+            
+        lora_config = LoraConfig(
+            r=lora_rank,
+            lora_alpha=lora_alpha,
+            init_lora_weights=init_lora_weights,
+            target_modules=lora_target_modules.split(","),
+        )
+        model = inject_adapter_in_model(lora_config, model)
+        for param in model.parameters():
+            # Upcast LoRA parameters into fp32
+            if param.requires_grad:
+                param.data = param.to(torch.float32)
+
+        # Lora pretrained lora weights
+        if pretrained_lora_path is not None:
+            state_dict = load_state_dict(pretrained_lora_path)
+            if state_dict_converter is not None:
+                state_dict = state_dict_converter(state_dict)
+            missing_keys, unexpected_keys = model.load_state_dict(state_dict, strict=False)
+            all_keys = [i for i, _ in model.named_parameters()]
+            num_updated_keys = len(all_keys) - len(missing_keys)
+            num_unexpected_keys = len(unexpected_keys)
+            print(f"{num_updated_keys} parameters are loaded from {pretrained_lora_path}. {num_unexpected_keys} parameters are unexpected.")
+
+
+    def training_step(self, batch, batch_idx):
+        # Data
+        text, image = batch["text"], batch["image"]
+
+        # Prepare input parameters
+        self.pipe.device = self.device
+        prompt_emb = self.pipe.encode_prompt(text, positive=True)
+        if "latents" in batch:
+            latents = batch["latents"].to(dtype=self.pipe.torch_dtype, device=self.device)
+        else:
+            latents = self.pipe.vae_encoder(image.to(dtype=self.pipe.torch_dtype, device=self.device))
+        noise = torch.randn_like(latents)
+        timestep_id = torch.randint(0, self.pipe.scheduler.num_train_timesteps, (1,))
+        timestep = self.pipe.scheduler.timesteps[timestep_id].to(self.device)
+        extra_input = self.pipe.prepare_extra_input(latents)
+        noisy_latents = self.pipe.scheduler.add_noise(latents, noise, timestep)
+        training_target = self.pipe.scheduler.training_target(latents, noise, timestep)
+
+        # Compute loss
+        noise_pred = self.pipe.denoising_model()(
+            noisy_latents, timestep=timestep, **prompt_emb, **extra_input,
+            use_gradient_checkpointing=self.use_gradient_checkpointing
+        )
+        loss = torch.nn.functional.mse_loss(noise_pred.float(), training_target.float())
+        loss = loss * self.pipe.scheduler.training_weight(timestep)
+
+        # Record log
+        self.log("train_loss", loss, prog_bar=True)
+        return loss
+
+
+    def configure_optimizers(self):
+        trainable_modules = filter(lambda p: p.requires_grad, self.pipe.denoising_model().parameters())
+        optimizer = torch.optim.AdamW(trainable_modules, lr=self.learning_rate)
+        return optimizer
+    
+
+    def on_save_checkpoint(self, checkpoint):
+        checkpoint.clear()
+        trainable_param_names = list(filter(lambda named_param: named_param[1].requires_grad, self.pipe.denoising_model().named_parameters()))
+        trainable_param_names = set([named_param[0] for named_param in trainable_param_names])
+        state_dict = self.pipe.denoising_model().state_dict()
+        lora_state_dict = {}
+        for name, param in state_dict.items():
+            if name in trainable_param_names:
+                lora_state_dict[name] = param
+        if self.state_dict_converter is not None:
+            lora_state_dict = self.state_dict_converter(lora_state_dict, alpha=self.lora_alpha)
+        checkpoint.update(lora_state_dict)
+
+
+
+def add_general_parsers(parser):
+    parser.add_argument(
+        "--dataset_path",
+        type=str,
+        default=None,
+        required=True,
+        help="The path of the Dataset.",
+    )
+    parser.add_argument(
+        "--output_path",
+        type=str,
+        default="./",
+        help="Path to save the model.",
+    )
+    parser.add_argument(
+        "--steps_per_epoch",
+        type=int,
+        default=500,
+        help="Number of steps per epoch.",
+    )
+    parser.add_argument(
+        "--height",
+        type=int,
+        default=1024,
+        help="Image height.",
+    )
+    parser.add_argument(
+        "--width",
+        type=int,
+        default=1024,
+        help="Image width.",
+    )
+    parser.add_argument(
+        "--center_crop",
+        default=False,
+        action="store_true",
+        help=(
+            "Whether to center crop the input images to the resolution. If not set, the images will be randomly"
+            " cropped. The images will be resized to the resolution first before cropping."
+        ),
+    )
+    parser.add_argument(
+        "--random_flip",
+        default=False,
+        action="store_true",
+        help="Whether to randomly flip images horizontally",
+    )
+    parser.add_argument(
+        "--batch_size",
+        type=int,
+        default=1,
+        help="Batch size (per device) for the training dataloader.",
+    )
+    parser.add_argument(
+        "--dataloader_num_workers",
+        type=int,
+        default=0,
+        help="Number of subprocesses to use for data loading. 0 means that the data will be loaded in the main process.",
+    )
+    parser.add_argument(
+        "--precision",
+        type=str,
+        default="16-mixed",
+        choices=["32", "16", "16-mixed", "bf16"],
+        help="Training precision",
+    )
+    parser.add_argument(
+        "--learning_rate",
+        type=float,
+        default=1e-4,
+        help="Learning rate.",
+    )
+    parser.add_argument(
+        "--lora_rank",
+        type=int,
+        default=4,
+        help="The dimension of the LoRA update matrices.",
+    )
+    parser.add_argument(
+        "--lora_alpha",
+        type=float,
+        default=4.0,
+        help="The weight of the LoRA update matrices.",
+    )
+    parser.add_argument(
+        "--init_lora_weights",
+        type=str,
+        default="kaiming",
+        choices=["gaussian", "kaiming"],
+        help="The initializing method of LoRA weight.",
+    )
+    parser.add_argument(
+        "--use_gradient_checkpointing",
+        default=False,
+        action="store_true",
+        help="Whether to use gradient checkpointing.",
+    )
+    parser.add_argument(
+        "--accumulate_grad_batches",
+        type=int,
+        default=1,
+        help="The number of batches in gradient accumulation.",
+    )
+    parser.add_argument(
+        "--training_strategy",
+        type=str,
+        default="auto",
+        choices=["auto", "deepspeed_stage_1", "deepspeed_stage_2", "deepspeed_stage_3"],
+        help="Training strategy",
+    )
+    parser.add_argument(
+        "--max_epochs",
+        type=int,
+        default=1,
+        help="Number of epochs.",
+    )
+    parser.add_argument(
+        "--modelscope_model_id",
+        type=str,
+        default=None,
+        help="Model ID on ModelScope (https://www.modelscope.cn/). The model will be uploaded to ModelScope automatically if you provide a Model ID.",
+    )
+    parser.add_argument(
+        "--modelscope_access_token",
+        type=str,
+        default=None,
+        help="Access key on ModelScope (https://www.modelscope.cn/). Required if you want to upload the model to ModelScope.",
+    )
+    parser.add_argument(
+        "--pretrained_lora_path",
+        type=str,
+        default=None,
+        help="Pretrained LoRA path. Required if the training is resumed.",
+    )
+    parser.add_argument(
+        "--use_swanlab",
+        default=False,
+        action="store_true",
+        help="Whether to use SwanLab logger.",
+    )
+    parser.add_argument(
+        "--swanlab_mode",
+        default=None,
+        help="SwanLab mode (cloud or local).",
+    )
+    return parser
+
+
+def launch_training_task(model, args):
+    # dataset and data loader
+    dataset = TextImageDataset(
+        args.dataset_path,
+        steps_per_epoch=args.steps_per_epoch * args.batch_size,
+        height=args.height,
+        width=args.width,
+        center_crop=args.center_crop,
+        random_flip=args.random_flip
+    )
+    train_loader = torch.utils.data.DataLoader(
+        dataset,
+        shuffle=True,
+        batch_size=args.batch_size,
+        num_workers=args.dataloader_num_workers
+    )
+    # train
+    if args.use_swanlab:
+        from swanlab.integration.pytorch_lightning import SwanLabLogger
+        swanlab_config = {"UPPERFRAMEWORK": "DiffSynth-Studio"}
+        swanlab_config.update(vars(args))
+        swanlab_logger = SwanLabLogger(
+            project="diffsynth_studio", 
+            name="diffsynth_studio",
+            config=swanlab_config,
+            mode=args.swanlab_mode,
+            logdir=os.path.join(args.output_path, "swanlog"),
+        )
+        logger = [swanlab_logger]
+    else:
+        logger = None
+    trainer = pl.Trainer(
+        max_epochs=args.max_epochs,
+        accelerator="gpu",
+        devices="auto",
+        precision=args.precision,
+        strategy=args.training_strategy,
+        default_root_dir=args.output_path,
+        accumulate_grad_batches=args.accumulate_grad_batches,
+        callbacks=[pl.pytorch.callbacks.ModelCheckpoint(save_top_k=-1)],
+        logger=logger,
+    )
+    trainer.fit(model=model, train_dataloaders=train_loader)
+
+    # Upload models
+    if args.modelscope_model_id is not None and args.modelscope_access_token is not None:
+        print(f"Uploading models to modelscope. model_id: {args.modelscope_model_id} local_path: {trainer.log_dir}")
+        with open(os.path.join(trainer.log_dir, "configuration.json"), "w", encoding="utf-8") as f:
+            f.write('{"framework":"Pytorch","task":"text-to-image-synthesis"}\n')
+        api = HubApi()
+        api.login(args.modelscope_access_token)
+        api.push_model(model_id=args.modelscope_model_id, model_dir=trainer.log_dir)

code/diffsynth/trainers/utils.py

@@ -0,0 +1,1368 @@
+import imageio, os, torch, warnings, torchvision, argparse, json, random
+from peft import LoraConfig, inject_adapter_in_model
+from PIL import Image
+import pandas as pd
+from tqdm import tqdm
+from accelerate import Accelerator
+
+
+
+class ImageDataset(torch.utils.data.Dataset):
+    def __init__(
+        self,
+        base_path=None, metadata_path=None,
+        max_pixels=1920*1080, height=None, width=None,
+        height_division_factor=16, width_division_factor=16,
+        data_file_keys=("image",),
+        image_file_extension=("jpg", "jpeg", "png", "webp"),
+        repeat=1,
+        args=None,
+    ):
+        if args is not None:
+            base_path = args.dataset_base_path
+            metadata_path = args.dataset_metadata_path
+            height = args.height
+            width = args.width
+            max_pixels = args.max_pixels
+            data_file_keys = args.data_file_keys.split(",")
+            repeat = args.dataset_repeat
+            
+        self.base_path = base_path
+        self.max_pixels = max_pixels
+        self.height = height
+        self.width = width
+        self.height_division_factor = height_division_factor
+        self.width_division_factor = width_division_factor
+        self.data_file_keys = data_file_keys
+        self.image_file_extension = image_file_extension
+        self.repeat = repeat
+
+        if height is not None and width is not None:
+            print("Height and width are fixed. Setting `dynamic_resolution` to False.")
+            self.dynamic_resolution = False
+        elif height is None and width is None:
+            print("Height and width are none. Setting `dynamic_resolution` to True.")
+            self.dynamic_resolution = True
+            
+        if metadata_path is None:
+            print("No metadata. Trying to generate it.")
+            metadata = self.generate_metadata(base_path)
+            print(f"{len(metadata)} lines in metadata.")
+            self.data = [metadata.iloc[i].to_dict() for i in range(len(metadata))]
+        elif metadata_path.endswith(".json"):
+            with open(metadata_path, "r") as f:
+                metadata = json.load(f)
+            self.data = metadata
+        else:
+            metadata = pd.read_csv(metadata_path)
+            # Ensure prompt column is string type to avoid float conversion for NaN values
+            if 'prompt' in metadata.columns:
+                metadata['prompt'] = metadata['prompt'].astype(str)
+                # Replace 'nan' string (from NaN) with empty string
+                metadata['prompt'] = metadata['prompt'].replace('nan', '')
+            self.data = [metadata.iloc[i].to_dict() for i in range(len(metadata))]
+
+
+    def generate_metadata(self, folder):
+        image_list, prompt_list = [], []
+        file_set = set(os.listdir(folder))
+        for file_name in file_set:
+            if "." not in file_name:
+                continue
+            file_ext_name = file_name.split(".")[-1].lower()
+            file_base_name = file_name[:-len(file_ext_name)-1]
+            if file_ext_name not in self.image_file_extension:
+                continue
+            prompt_file_name = file_base_name + ".txt"
+            if prompt_file_name not in file_set:
+                continue
+            with open(os.path.join(folder, prompt_file_name), "r", encoding="utf-8") as f:
+                prompt = f.read().strip()
+            image_list.append(file_name)
+            prompt_list.append(prompt)
+        metadata = pd.DataFrame()
+        metadata["image"] = image_list
+        metadata["prompt"] = prompt_list
+        return metadata
+    
+    
+    def crop_and_resize(self, image, target_height, target_width):
+        width, height = image.size
+        scale = max(target_width / width, target_height / height)
+        image = torchvision.transforms.functional.resize(
+            image,
+            (round(height*scale), round(width*scale)),
+            interpolation=torchvision.transforms.InterpolationMode.BILINEAR
+        )
+        image = torchvision.transforms.functional.center_crop(image, (target_height, target_width))
+        return image
+    
+    
+    def get_height_width(self, image):
+        if self.dynamic_resolution:
+            width, height = image.size
+            if width * height > self.max_pixels:
+                scale = (width * height / self.max_pixels) ** 0.5
+                height, width = int(height / scale), int(width / scale)
+            height = height // self.height_division_factor * self.height_division_factor
+            width = width // self.width_division_factor * self.width_division_factor
+        else:
+            height, width = self.height, self.width
+        return height, width
+    
+    
+    def load_image(self, file_path):
+        image = Image.open(file_path).convert("RGB")
+        image = self.crop_and_resize(image, *self.get_height_width(image))
+        return image
+    
+    
+    def load_data(self, file_path):
+        return self.load_image(file_path)
+
+
+    def __getitem__(self, data_id):
+        data = self.data[data_id % len(self.data)].copy()
+        for key in self.data_file_keys:
+            if key in data:
+                path = os.path.join(self.base_path, data[key])
+                data[key] = self.load_data(path)
+                if data[key] is None:
+                    warnings.warn(f"cannot load file {data[key]}.")
+                    return None
+        return data
+    
+
+    def __len__(self):
+        return len(self.data) * self.repeat
+
+
+
+class VideoDataset(torch.utils.data.Dataset):
+    def __init__(
+        self,
+        base_path=None, metadata_path=None,
+        num_frames=81,
+        time_division_factor=4, time_division_remainder=1,
+        max_pixels=1920*1080, height=None, width=None,
+        height_division_factor=16, width_division_factor=16,
+        data_file_keys=("video",),
+        image_file_extension=("jpg", "jpeg", "png", "webp"),
+        video_file_extension=("mp4", "avi", "mov", "wmv", "mkv", "flv", "webm"),
+        repeat=1,
+        args=None,
+        action_base_path=None,
+        enable_icl=False,
+        icl_num_examples=2,
+        icl_context_frames=8,
+    ):
+        if args is not None:
+            base_path = args.dataset_base_path
+            metadata_path = args.dataset_metadata_path
+            height = args.height
+            width = args.width
+            max_pixels = args.max_pixels
+            num_frames = args.num_frames
+            data_file_keys = args.data_file_keys.split(",")
+            repeat = args.dataset_repeat
+            # In-context learning parameters
+            if hasattr(args, 'enable_icl'):
+                enable_icl = args.enable_icl
+            if hasattr(args, 'icl_num_examples'):
+                icl_num_examples = args.icl_num_examples
+            if hasattr(args, 'icl_context_frames'):
+                icl_context_frames = args.icl_context_frames
+        
+        self.base_path = base_path
+        self.num_frames = num_frames
+        self.time_division_factor = time_division_factor
+        self.time_division_remainder = time_division_remainder
+        self.max_pixels = max_pixels
+        self.height = height
+        self.width = width
+        self.height_division_factor = height_division_factor
+        self.width_division_factor = width_division_factor
+        self.data_file_keys = data_file_keys
+        self.image_file_extension = image_file_extension
+        self.video_file_extension = video_file_extension
+        self.repeat = repeat
+        
+        # In-context learning parameters
+        self.enable_icl = enable_icl
+        self.icl_num_examples = icl_num_examples
+        self.icl_context_frames = icl_context_frames
+        
+        if height is not None and width is not None:
+            print("Height and width are fixed. Setting `dynamic_resolution` to False.")
+            self.dynamic_resolution = False
+        elif height is None and width is None:
+            print("Height and width are none. Setting `dynamic_resolution` to True.")
+            self.dynamic_resolution = True
+            
+        if metadata_path is None:
+            print("No metadata. Trying to generate it.")
+            metadata = self.generate_metadata(base_path)
+            print(f"{len(metadata)} lines in metadata.")
+            self.data = [metadata.iloc[i].to_dict() for i in range(len(metadata))]
+        elif metadata_path.endswith(".json"):
+            with open(metadata_path, "r") as f:
+                metadata = json.load(f)
+            self.data = metadata
+        else:
+            metadata = pd.read_csv(metadata_path)
+            # Ensure prompt column is string type to avoid float conversion for NaN values
+            if 'prompt' in metadata.columns:
+                metadata['prompt'] = metadata['prompt'].astype(str)
+                # Replace 'nan' string (from NaN) with empty string
+                metadata['prompt'] = metadata['prompt'].replace('nan', '')
+                
+                # CRITICAL FIX: Clean prompt - remove video path prefix if present
+                # Some CSV prompts start with "video_name.mp4 " prefix, which should be removed
+                def clean_prompt(prompt_str):
+                    if not isinstance(prompt_str, str) or not prompt_str:
+                        return prompt_str
+                    # Check if prompt starts with a video path (contains .mp4 or /)
+                    # Pattern: "VideoName/1234_5678.mp4 " or "VideoName.mp4 "
+                    import re
+                    # Match pattern: word/word.mp4 or word.mp4 at the start, followed by space
+                    pattern = r'^[A-Za-z0-9_]+(/[A-Za-z0-9_]+)?\.mp4\s+'
+                    cleaned = re.sub(pattern, '', prompt_str)
+                    # Also handle truncated prompts ending with "..."
+                    if cleaned.endswith('...'):
+                        cleaned = cleaned[:-3].rstrip()
+                    return cleaned.strip()
+                
+                metadata['prompt'] = metadata['prompt'].apply(clean_prompt)
+            self.data = [metadata.iloc[i].to_dict() for i in range(len(metadata))]
+
+        self.action_base_path = action_base_path
+        
+        if self.enable_icl:
+            print(f"In-context learning enabled: {icl_num_examples} examples, {icl_context_frames} context frames each")
+            
+    
+    def generate_metadata(self, folder):
+        video_list, prompt_list = [], []
+        file_set = set(os.listdir(folder))
+        for file_name in file_set:
+            if "." not in file_name:
+                continue
+            file_ext_name = file_name.split(".")[-1].lower()
+            file_base_name = file_name[:-len(file_ext_name)-1]
+            if file_ext_name not in self.image_file_extension and file_ext_name not in self.video_file_extension:
+                continue
+            prompt_file_name = file_base_name + ".txt"
+            if prompt_file_name not in file_set:
+                continue
+            with open(os.path.join(folder, prompt_file_name), "r", encoding="utf-8") as f:
+                prompt = f.read().strip()
+            video_list.append(file_name)
+            prompt_list.append(prompt)
+        metadata = pd.DataFrame()
+        metadata["video"] = video_list
+        metadata["prompt"] = prompt_list
+        return metadata
+        
+        
+    def crop_and_resize(self, image, target_height, target_width):
+        width, height = image.size
+        scale = max(target_width / width, target_height / height)
+        image = torchvision.transforms.functional.resize(
+            image,
+            (round(height*scale), round(width*scale)),
+            interpolation=torchvision.transforms.InterpolationMode.BILINEAR
+        )
+        image = torchvision.transforms.functional.center_crop(image, (target_height, target_width))
+        return image
+    
+    
+    def get_height_width(self, image):
+        if self.dynamic_resolution:
+            width, height = image.size
+            if width * height > self.max_pixels:
+                scale = (width * height / self.max_pixels) ** 0.5
+                height, width = int(height / scale), int(width / scale)
+            height = height // self.height_division_factor * self.height_division_factor
+            width = width // self.width_division_factor * self.width_division_factor
+        else:
+            height, width = self.height, self.width
+        return height, width
+    
+    
+    def get_num_frames(self, reader):
+        num_frames = self.num_frames
+        if int(reader.count_frames()) < num_frames:
+            num_frames = int(reader.count_frames())
+            while num_frames > 1 and num_frames % self.time_division_factor != self.time_division_remainder:
+                num_frames -= 1
+        return num_frames
+    
+
+    def load_video(self, file_path):
+        reader = imageio.get_reader(file_path)
+        num_frames = self.get_num_frames(reader)
+        frames = []
+        for frame_id in range(num_frames):
+            frame = reader.get_data(frame_id)
+            frame = Image.fromarray(frame)
+            frame = self.crop_and_resize(frame, *self.get_height_width(frame))
+            frames.append(frame)
+        reader.close()
+        return frames
+    
+    
+    def load_image(self, file_path):
+        image = Image.open(file_path).convert("RGB")
+        image = self.crop_and_resize(image, *self.get_height_width(image))
+        frames = [image]
+        return frames
+    
+    
+    def is_image(self, file_path):
+        file_ext_name = file_path.split(".")[-1]
+        return file_ext_name.lower() in self.image_file_extension
+    
+    
+    def is_video(self, file_path):
+        file_ext_name = file_path.split(".")[-1]
+        return file_ext_name.lower() in self.video_file_extension
+    
+    
+    def load_data(self, file_path):
+        # Handle multiple frame paths separated by '|' (for frame sequences)
+        if '|' in str(file_path):
+            # Split the path by '|' to get individual frame paths
+            frame_paths = str(file_path).split('|')
+            frames = []
+            
+            # Get base_path (dataset root)
+            if not hasattr(self, 'base_path') or not self.base_path:
+                warnings.warn(f"Cannot determine base directory for frame sequence: {file_path}")
+                return None
+            
+            base_dir = self.base_path  # This is the dataset root
+            
+            # Check the first path to determine the format
+            first_frame = frame_paths[0].strip() if frame_paths else ""
+            
+            # If first frame is already an absolute path (from __getitem__ joining),
+            # extract the base directory from it
+            if os.path.isabs(first_frame):
+                # Extract base directory from first frame path
+                # First frame format: /path/to/dataset/frames/video_name/frame.png
+                # We need to get /path/to/dataset
+                parts = first_frame.split(os.sep)
+                # Find 'frames' in the path and get everything before it
+                if 'frames' in parts:
+                    frames_idx = parts.index('frames')
+                    base_dir = os.sep.join(parts[:frames_idx])
+                else:
+                    # Fallback: use self.base_path
+                    base_dir = self.base_path
+            
+            for frame_path in frame_paths:
+                frame_path = frame_path.strip()
+                if not frame_path:
+                    continue
+                
+                # Construct full path
+                if os.path.isabs(frame_path):
+                    # Already absolute path (from __getitem__)
+                    full_frame_path = frame_path
+                else:
+                    # Relative path - need to construct full path
+                    # Remove 'frames/' prefix if present (we'll add it consistently)
+                    if frame_path.startswith('frames/'):
+                        frame_path = frame_path[7:]  # Remove 'frames/' prefix
+                    
+                    # Always join with base_dir + 'frames/' since base_dir is dataset root
+                    full_frame_path = os.path.join(base_dir, 'frames', frame_path)
+                
+                # Load individual frame
+                if os.path.exists(full_frame_path):
+                    if self.is_image(full_frame_path):
+                        frame_data = self.load_image(full_frame_path)
+                        if frame_data:
+                            frames.extend(frame_data)
+                    else:
+                        warnings.warn(f"Frame is not an image: {full_frame_path}")
+                else:
+                    warnings.warn(f"Frame not found: {full_frame_path}")
+            
+            if frames:
+                return frames
+            else:
+                warnings.warn(f"No frames loaded from sequence: {file_path}")
+                return None
+        
+        # Handle single file (image or video)
+        if self.is_image(file_path):
+            return self.load_image(file_path)
+        elif self.is_video(file_path):
+            return self.load_video(file_path)
+        else:
+            return None
+
+
+    def __getitem__(self, data_id):
+        data = self.data[data_id % len(self.data)].copy()
+        for key in self.data_file_keys:
+            if key in ["video_name", "start_frame", "end_frame"]:
+                if "actions" in data:
+                    continue
+                try:
+                    video_name = data.get("video_name")
+                    if video_name is None:
+                        warnings.warn(f"video_name is missing in metadata for data_id {data_id}. Skipping action loading.")
+                        continue
+                    
+                    if video_name.endswith(".mp4"):
+                        video_name = ".".join(video_name.split(".")[:-1])
+                    if "_" in video_name:
+                        video_name = "_".join(video_name.split("_")[:4])
+                    
+                    import json
+                    json_path = os.path.join(self.action_base_path, video_name + ".json")
+                    
+                    # Check if action file exists
+                    if not os.path.exists(json_path):
+                        warnings.warn(f"Action file does not exist: {json_path}. Skipping action loading for data_id {data_id}.")
+                        continue
+                    
+                    start_frame = data.get("start_frame")
+                    end_frame = data.get("end_frame")
+                    if start_frame is None or end_frame is None:
+                        warnings.warn(f"start_frame or end_frame is missing in metadata for data_id {data_id}. Skipping action loading.")
+                        continue
+                    
+                    json_data = json.load(open(json_path, "r"))['actions']
+                    actions = []
+                    current_yaw = 0.0
+                    for frame_id in range(start_frame+1, end_frame+1):
+                        frame_str = str(frame_id)
+                        if frame_str not in json_data:
+                            warnings.warn(f"Frame {frame_id} not found in action file {json_path}. Skipping this frame.")
+                            continue
+                        
+                        action = json_data[frame_str]
+                        new_action = [0.0] * (2 + 2 + 3 + 1 + 2)
+                        if action['ws'] == 1:
+                            new_action[0] = 1
+                        elif action['ws'] == 2:
+                            new_action[1] = 1
+
+                        if action['ad'] == 1:
+                            new_action[2] = 1
+                        elif action['ad'] == 2:
+                            new_action[3] = 1
+                        
+                        if action['scs'] == 1 and action.get("jump_invalid", 0) == 0:
+                            new_action[4] = 1
+                        elif action['scs'] == 2:
+                            new_action[5] = 1
+                        elif action['scs'] == 3:
+                            new_action[6] = 1
+                        
+                        if action.get('collision', 0) == 1:
+                            new_action[7] = 1
+                            new_action[0] = 0
+                            new_action[1] = 0
+                            new_action[2] = 0
+                            new_action[3] = 0
+
+                        pre_pitch = action.get('pre_pitch', 0.0)
+                        current_pitch = pre_pitch + action.get('pitch_delta', 0.0) * 15.0
+                        current_yaw += action.get('yaw_delta', 0.0) * 15.0
+                        new_action[8] = current_pitch
+                        new_action[9] = current_yaw
+
+                        actions.append(new_action)
+                    data["actions"] = actions
+                except Exception as e:
+                    warnings.warn(f"Exception while loading actions for data_id {data_id}: {e}. Continuing without actions.")
+                    # Don't return None, just continue without actions
+                    continue
+            elif key == "video":
+                # Check if data[key] exists and is not None
+                if key not in data or data[key] is None:
+                    warnings.warn(f"Video key '{key}' is missing or None in metadata for data_id {data_id}. Skipping this sample.")
+                    return None
+                
+                # Handle frame sequences (paths with '|' separator)
+                video_path_str = str(data[key])
+                if '|' in video_path_str:
+                    # For frame sequences, pass the full path string to load_data
+                    # load_data will handle splitting and loading individual frames
+                    path = os.path.join(self.base_path, video_path_str)
+                    # Don't check path existence here for frame sequences
+                    # load_data will handle individual frame loading
+                else:
+                    path = os.path.join(self.base_path, data[key])
+                    # Check if path exists (only for single files)
+                    if not os.path.exists(path):
+                        warnings.warn(f"Video file does not exist: {path}. Skipping this sample.")
+                        return None
+                try:
+                    data[key] = self.load_data(path)
+                    if data[key] is None:
+                        warnings.warn(f"Failed to load video file: {path}. load_data returned None.")
+                        return None
+                except Exception as e:
+                    warnings.warn(f"Exception while loading video file {path}: {e}. Skipping this sample.")
+                    return None
+        
+        # In-context learning: sample context examples from dataset
+        if self.enable_icl and len(self.data) > 1:
+            context_frames_list = []
+            context_actions_list = []
+            
+            # Sample random examples from dataset (excluding current one)
+            current_idx = data_id % len(self.data)
+            candidate_indices = [i for i in range(len(self.data)) if i != current_idx]
+            if len(candidate_indices) > 0:
+                num_samples = min(self.icl_num_examples, len(candidate_indices))
+                sampled_indices = random.sample(candidate_indices, num_samples)
+                
+                for sample_idx in sampled_indices:
+                    sample_data = self.data[sample_idx].copy()
+                    # Load video for context
+                    if "video" in self.data_file_keys and "video" in sample_data:
+                        video_path = os.path.join(self.base_path, sample_data["video"])
+                        sample_video = self.load_data(video_path)
+                        if sample_video is not None and len(sample_video) >= self.icl_context_frames:
+                            # Sample context_frames from the video
+                            start_idx = random.randint(0, max(0, len(sample_video) - self.icl_context_frames))
+                            context_frames = sample_video[start_idx:start_idx + self.icl_context_frames]
+                            context_frames_list.extend(context_frames)
+                            
+                            # Load corresponding actions if available
+                            if self.action_base_path is not None and "video_name" in sample_data:
+                                try:
+                                    sample_video_name = sample_data["video_name"]
+                                    if sample_video_name.endswith(".mp4"):
+                                        sample_video_name = ".".join(sample_video_name.split(".")[:-1])
+                                    if "_" in sample_video_name:
+                                        sample_video_name = "_".join(sample_video_name.split("_")[:4])
+                                    sample_json_path = os.path.join(self.action_base_path, sample_video_name + ".json")
+                                    if os.path.exists(sample_json_path):
+                                        sample_json_data = json.load(open(sample_json_path, "r"))['actions']
+                                        sample_start_frame = sample_data.get("start_frame", 0)
+                                        sample_end_frame = sample_data.get("end_frame", len(sample_video))
+                                        
+                                        # Get actions for the context frames
+                                        context_actions = []
+                                        context_yaw = 0.0
+                                        for frame_idx in range(sample_start_frame + start_idx + 1, 
+                                                             min(sample_start_frame + start_idx + self.icl_context_frames + 1, sample_end_frame + 1)):
+                                            if str(frame_idx) in sample_json_data:
+                                                action = sample_json_data[str(frame_idx)]
+                                                new_action = [0.0] * (2 + 2 + 3 + 1 + 2)
+                                                if action['ws'] == 1:
+                                                    new_action[0] = 1
+                                                elif action['ws'] == 2:
+                                                    new_action[1] = 1
+                                                if action['ad'] == 1:
+                                                    new_action[2] = 1
+                                                elif action['ad'] == 2:
+                                                    new_action[3] = 1
+                                                if action['scs'] == 1 and action.get("jump_invalid", 0) == 0:
+                                                    new_action[4] = 1
+                                                elif action['scs'] == 2:
+                                                    new_action[5] = 1
+                                                elif action['scs'] == 3:
+                                                    new_action[6] = 1
+                                                if action.get('collision', 0) == 1:
+                                                    new_action[7] = 1
+                                                    new_action[0] = 0
+                                                    new_action[1] = 0
+                                                    new_action[2] = 0
+                                                    new_action[3] = 0
+                                                pre_pitch = action.get('pre_pitch', 0.0)
+                                                current_pitch = pre_pitch + action.get('pitch_delta', 0.0) * 15.0
+                                                context_yaw += action.get('yaw_delta', 0.0) * 15.0
+                                                new_action[8] = current_pitch
+                                                new_action[9] = context_yaw
+                                                context_actions.append(new_action)
+                                        context_actions_list.extend(context_actions[:len(context_frames)])
+                                except Exception as e:
+                                    # If loading actions fails, just skip
+                                    pass
+            
+            if context_frames_list:
+                data["context_frames"] = context_frames_list
+                if context_actions_list and len(context_actions_list) == len(context_frames_list):
+                    data["context_actions"] = context_actions_list
+        
+        return data
+    
+
+    def __len__(self):
+        return len(self.data) * self.repeat
+
+    @staticmethod
+    def get_one_hot(action, range=2):
+        one_hot = [0] * (range + 1)
+        one_hot[action] = 1
+        return one_hot
+
+
+
+import numpy as np
+
+
+class CamVideoDataset(torch.utils.data.Dataset):
+    """Dataset for Context-as-Memory camera pose conditioned training (ported from VWM).
+
+    Loads 81 PNG frames from UE scenes with random temporal cropping and extracts
+    corresponding camera poses as 12-dim relative RT vectors subsampled to match
+    the 21 latent frames.
+    """
+    def __init__(
+        self,
+        base_path=None, metadata_path=None,
+        num_frames=81,
+        height=None, width=None,
+        max_pixels=1920*1080,
+        height_division_factor=16, width_division_factor=16,
+        repeat=1,
+        args=None,
+        cam_position_scale=None,
+    ):
+        if args is not None:
+            base_path = args.dataset_base_path
+            metadata_path = getattr(args, "dataset_metadata_path", metadata_path)
+            height = args.height
+            width = args.width
+            max_pixels = args.max_pixels
+            num_frames = args.num_frames
+            repeat = args.dataset_repeat
+            cam_position_scale = getattr(args, "cam_position_scale", 0.01)
+            self.use_condition_context_frames = getattr(args, "use_condition_context_frames", False)
+            self.condition_first_frame = getattr(args, "condition_first_frame", False)
+            self.condition_history_keyframes = getattr(args, "condition_history_keyframes", False)
+            self.condition_use_camera_pose = getattr(args, "condition_use_camera_pose", True)
+            self.num_condition_frames = getattr(args, "num_condition_frames", 1)
+            self.condition_frame_mode = getattr(args, "condition_frame_mode", "first_frame_only")
+            self.overlap_labels_root = getattr(args, "overlap_labels_root", None)
+            self.condition_t2v_ratio = getattr(args, "condition_t2v_ratio", 0.10)
+            self.condition_i2v_ratio = getattr(args, "condition_i2v_ratio", 0.10)
+            self.use_geometry_spatial_memory = getattr(args, "use_geometry_spatial_memory", False)
+            self.geometry_memory_column = getattr(args, "geometry_memory_column", "geometry_memory")
+            self.geometry_memory_root = getattr(args, "geometry_memory_root", None)
+        else:
+            self.use_condition_context_frames = False
+            self.condition_first_frame = False
+            self.condition_history_keyframes = False
+            self.condition_use_camera_pose = True
+            self.num_condition_frames = 1
+            self.condition_frame_mode = "first_frame_only"
+            self.overlap_labels_root = None
+            self.condition_t2v_ratio = 0.10
+            self.condition_i2v_ratio = 0.10
+            self.use_geometry_spatial_memory = False
+            self.geometry_memory_column = "geometry_memory"
+            self.geometry_memory_root = None
+
+        if cam_position_scale is None:
+            cam_position_scale = 0.01
+        self.cam_position_scale = float(cam_position_scale)
+
+        self.base_path = base_path
+        self.frames_dir = os.path.join(base_path, "frames")
+        self.jsons_dir = os.path.join(base_path, "jsons")
+        self.num_frames = num_frames
+        self.max_pixels = max_pixels
+        self.height = height
+        self.width = width
+        self.height_division_factor = height_division_factor
+        self.width_division_factor = width_division_factor
+        self.repeat = repeat
+
+        if height is not None and width is not None:
+            self.dynamic_resolution = False
+        else:
+            self.dynamic_resolution = True
+        
+        if metadata_path is None:
+            print("No metadata. Trying to generate it.")
+            metadata = self.generate_metadata(base_path)
+            print(f"{len(metadata)} lines in metadata.")
+            self.data = [metadata.iloc[i].to_dict() for i in range(len(metadata))]
+        elif metadata_path.endswith(".json"):
+            with open(metadata_path, "r") as f:
+                metadata = json.load(f)
+            self.data = metadata
+        else:
+            metadata = pd.read_csv(metadata_path)
+            # Ensure prompt column is string type to avoid float conversion for NaN values
+            if 'prompt' in metadata.columns:
+                metadata['prompt'] = metadata['prompt'].astype(str)
+                # Replace 'nan' string (from NaN) with empty string
+                metadata['prompt'] = metadata['prompt'].replace('nan', '')
+                
+                # CRITICAL FIX: Clean prompt - remove video path prefix if present
+                # Some CSV prompts start with "video_name.mp4 " prefix, which should be removed
+                def clean_prompt(prompt_str):
+                    if not isinstance(prompt_str, str) or not prompt_str:
+                        return prompt_str
+                    # Check if prompt starts with a video path (contains .mp4 or /)
+                    # Pattern: "VideoName/1234_5678.mp4 " or "VideoName.mp4 "
+                    import re
+                    # Match pattern: word/word.mp4 or word.mp4 at the start, followed by space
+                    pattern = r'^[A-Za-z0-9_]+(/[A-Za-z0-9_]+)?\.mp4\s+'
+                    cleaned = re.sub(pattern, '', prompt_str)
+                    # Also handle truncated prompts ending with "..."
+                    if cleaned.endswith('...'):
+                        cleaned = cleaned[:-3].rstrip()
+                    return cleaned.strip()
+                
+                metadata['prompt'] = metadata['prompt'].apply(clean_prompt)
+            self.data = [metadata.iloc[i].to_dict() for i in range(len(metadata))]
+
+        captions_path = os.path.join(base_path, "captions.txt")
+        self.scene_captions = {}
+        with open(captions_path, "r") as f:
+            for line in f:
+                parts = line.strip().split("\t", 1)
+                if len(parts) < 2:
+                    continue
+                clip_path, caption = parts
+                scene_name = "/".join(clip_path.split("/")[:-1])
+                fname = clip_path.split("/")[-1].replace(".mp4", "")
+                clip_start = int(fname.split("_")[0])
+                if scene_name not in self.scene_captions:
+                    self.scene_captions[scene_name] = []
+                self.scene_captions[scene_name].append((clip_start, caption))
+
+        for scene_name in self.scene_captions:
+            self.scene_captions[scene_name].sort(key=lambda x: x[0])
+
+        self.scene_names = sorted(self.scene_captions.keys())
+        self.metadata_rows = []
+        if metadata_path and os.path.isfile(metadata_path):
+            metadata = pd.read_csv(metadata_path)
+            if "prompt" in metadata.columns:
+                metadata["prompt"] = metadata["prompt"].astype(str)
+            self.metadata_rows = [metadata.iloc[i].to_dict() for i in range(len(metadata))]
+        self.pose_cache = {}
+        self.overlap_cache = {}
+        self.invalid_scenes = set()
+        self.invalid_metadata_indices = set()
+        self.overlap_labels_root = self._resolve_overlap_labels_root(base_path, self.overlap_labels_root)
+        self._validate_condition_config()
+
+        total_items = len(self.data)
+        total_scenes = len(self.scene_names)
+        total_captions = sum(len(v) for v in self.scene_captions.values())
+        metadata_msg = f", metadata_rows={len(self.metadata_rows)}" if self.metadata_rows else ""
+        effective_len = (len(self.metadata_rows) if self.metadata_rows else total_scenes) * repeat
+        print(f"CamVideoDataset: {total_scenes} scenes, {total_captions} captions{metadata_msg}, "
+              f"repeat={repeat}, cam_position_scale={self.cam_position_scale}, "
+              f"effective length={total_items}")
+
+    def _resolve_overlap_labels_root(self, base_path, overlap_labels_root):
+        candidate_roots = []
+        if overlap_labels_root is not None:
+            candidate_roots.append(overlap_labels_root)
+        if base_path is not None:
+            candidate_roots.append(os.path.join(base_path, "overlap_labels"))
+        for root in candidate_roots:
+            if root is not None and os.path.isdir(root):
+                return root
+        return overlap_labels_root
+
+    def _validate_condition_config(self):
+        if self.condition_t2v_ratio < 0 or self.condition_i2v_ratio < 0:
+            raise ValueError("Condition sampling ratios must be non-negative.")
+        if self.condition_t2v_ratio + self.condition_i2v_ratio >= 1.0:
+            raise ValueError("condition_t2v_ratio + condition_i2v_ratio must be < 1.0.")
+        needs_overlap = (
+            self.use_condition_context_frames
+            and self.condition_frame_mode == "first_plus_overlap"
+            and self.condition_history_keyframes
+            and self.num_condition_frames > 1
+        )
+        if needs_overlap and (self.overlap_labels_root is None or not os.path.isdir(self.overlap_labels_root)):
+            raise FileNotFoundError(
+                "K-frame condition mode requires overlap_labels_root. "
+                "Pass --overlap_labels_root or keep overlap_labels under dataset_base_path/overlap_labels."
+            )
+
+    def _load_scene_poses(self, scene_name):
+        if scene_name not in self.pose_cache:
+            json_path = os.path.join(self.jsons_dir, scene_name + ".json")
+            try:
+                with open(json_path, "r") as f:
+                    data = json.load(f)
+            except (FileNotFoundError, json.JSONDecodeError) as e:
+                raise ValueError(f"Pose JSON for scene '{scene_name}' is missing or corrupt: {e}")
+            if not isinstance(data, dict) or "CineCameraActor" not in data:
+                raise ValueError(
+                    f"Pose JSON for scene '{scene_name}' lacks 'CineCameraActor' key "
+                    f"(found keys: {list(data.keys()) if isinstance(data, dict) else type(data).__name__})."
+                )
+            cine = data["CineCameraActor"]
+            if not isinstance(cine, dict) or len(cine) == 0:
+                raise ValueError(f"Pose JSON for scene '{scene_name}' has empty 'CineCameraActor' entries.")
+            self.pose_cache[scene_name] = cine
+        return self.pose_cache[scene_name]
+
+    def _find_nearest_caption(self, scene_name, start_frame):
+        captions = self.scene_captions[scene_name]
+        best_idx = 0
+        best_dist = abs(captions[0][0] - start_frame)
+        for i, (clip_start, _) in enumerate(captions):
+            dist = abs(clip_start - start_frame)
+            if dist < best_dist:
+                best_dist = dist
+                best_idx = i
+        return captions[best_idx][1]
+
+    @staticmethod
+    def _compute_rt(position, rotation):
+        x, y, z = position
+        yaw_rad = np.radians(rotation[2])
+        cos_y, sin_y = np.cos(yaw_rad), np.sin(yaw_rad)
+        R = np.array([[cos_y, -sin_y, 0], [sin_y, cos_y, 0], [0, 0, 1]])
+        return [x, y, z] + R.flatten().tolist()
+
+    @staticmethod
+    def _to_relative_rt(rt_list, ref_rt):
+        R_ref = np.array(ref_rt[3:]).reshape(3, 3)
+        T_ref = np.array(ref_rt[:3]).reshape(3, 1)
+        R_ref_inv = R_ref.T
+        T_ref_inv = -R_ref_inv @ T_ref
+        result = []
+        for rt in rt_list:
+            R_i = np.array(rt[3:]).reshape(3, 3)
+            T_i = np.array(rt[:3]).reshape(3, 1)
+            R_new = R_ref_inv @ R_i
+            T_new = R_ref_inv @ T_i + T_ref_inv
+            result.append(T_new.flatten().tolist() + R_new.flatten().tolist())
+        return result
+
+    def crop_and_resize(self, image, target_height, target_width):
+        width, height = image.size
+        scale = max(target_width / width, target_height / height)
+        image = torchvision.transforms.functional.resize(
+            image,
+            (round(height * scale), round(width * scale)),
+            interpolation=torchvision.transforms.InterpolationMode.BILINEAR
+        )
+        image = torchvision.transforms.functional.center_crop(image, (target_height, target_width))
+        return image
+
+    def get_height_width(self, image):
+        if self.dynamic_resolution:
+            width, height = image.size
+            if width * height > self.max_pixels:
+                scale = (width * height / self.max_pixels) ** 0.5
+                height, width = int(height / scale), int(width / scale)
+            height = height // self.height_division_factor * self.height_division_factor
+            width = width // self.width_division_factor * self.width_division_factor
+        else:
+            height, width = self.height, self.width
+        return height, width
+
+    def _load_resized_frame(self, scene_name, frame_index, target_height, target_width):
+        frame_path = os.path.join(self.frames_dir, scene_name, f"{frame_index:04d}.png")
+        img = Image.open(frame_path).convert("RGB")
+        return self.crop_and_resize(img, target_height, target_width)
+
+    @staticmethod
+    def _parse_frame_token(token):
+        token = str(token).strip()
+        if not token:
+            return None, None
+        parts = token.split("/")
+        if parts and parts[0] == "frames":
+            parts = parts[1:]
+        if len(parts) < 2:
+            return None, None
+        scene_name = "/".join(parts[:-1])
+        stem = os.path.splitext(parts[-1])[0]
+        try:
+            frame_index = int(stem)
+        except ValueError:
+            return None, None
+        return scene_name, frame_index
+
+    def _metadata_scene_and_indices(self, row):
+        video_field = str(row.get("video", "") or "")
+        tokens = [t for t in video_field.split("|") if t]
+        parsed = [self._parse_frame_token(t) for t in tokens]
+        parsed = [(s, i) for s, i in parsed if s is not None and i is not None]
+        if parsed:
+            scene_name = str(row.get("video_name", "") or parsed[0][0])
+            frame_indices = [i for _, i in parsed[: self.num_frames]]
+        else:
+            scene_name = str(row.get("video_name", "") or "").strip()
+            if not scene_name:
+                raise ValueError("metadata row lacks video_name and parseable video paths")
+            start_frame = int(row.get("start_frame", 0) or 0)
+            frame_indices = list(range(start_frame, start_frame + self.num_frames))
+        if len(frame_indices) < self.num_frames:
+            raise ValueError(f"metadata row has {len(frame_indices)} frames, expected {self.num_frames}")
+        return scene_name, frame_indices[: self.num_frames]
+
+    def _resolve_geometry_path(self, token):
+        token = str(token).strip()
+        if not token:
+            return None
+        if os.path.isabs(token):
+            return token
+        root = self.geometry_memory_root or self.base_path
+        return os.path.join(root, token)
+
+    def _load_geometry_memory_frames(self, row):
+        value = row.get(self.geometry_memory_column, None)
+        if value is None or str(value).strip() == "" or str(value).lower() == "nan":
+            if self.use_geometry_spatial_memory:
+                raise ValueError(
+                    f"metadata row lacks required geometry column '{self.geometry_memory_column}'"
+                )
+            return []
+        tokens = [token for token in str(value).split("|") if token.strip()]
+        if not tokens:
+            raise ValueError(f"empty geometry memory field '{self.geometry_memory_column}'")
+
+        if len(tokens) == 1:
+            path = self._resolve_geometry_path(tokens[0])
+            if path is None:
+                return []
+            if os.path.isdir(path):
+                names = sorted(
+                    name
+                    for name in os.listdir(path)
+                    if os.path.splitext(name)[1].lower() in (".png", ".jpg", ".jpeg", ".webp")
+                )
+                paths = [os.path.join(path, name) for name in names]
+                return [Image.open(frame_path).convert("RGB") for frame_path in paths]
+            if os.path.splitext(path)[1].lower() in (".mp4", ".mov", ".avi", ".mkv", ".webm"):
+                reader = imageio.get_reader(path)
+                try:
+                    frames = [Image.fromarray(frame).convert("RGB") for frame in reader]
+                finally:
+                    reader.close()
+                if not frames:
+                    raise ValueError(f"geometry memory video has no frames: {path}")
+                return frames
+
+        frames = []
+        for token in tokens:
+            path = self._resolve_geometry_path(token)
+            if path is None or not os.path.isfile(path):
+                raise FileNotFoundError(f"geometry memory frame not found: {path}")
+            frames.append(Image.open(path).convert("RGB"))
+        return frames
+
+    def _load_overlap_frames(self, scene_name, frame_index):
+        if self.overlap_labels_root is None:
+            return []
+        cache_key = (scene_name, int(frame_index))
+        if cache_key not in self.overlap_cache:
+            overlap_path = os.path.join(self.overlap_labels_root, scene_name, f"{int(frame_index)}.json")
+            if not os.path.exists(overlap_path):
+                self.overlap_cache[cache_key] = []
+            else:
+                with open(overlap_path, "r") as f:
+                    overlap_data = json.load(f)
+                overlaps = overlap_data.get("overlapping_frames", [])
+                self.overlap_cache[cache_key] = [int(idx) for idx in overlaps]
+        return self.overlap_cache[cache_key]
+
+    def _compute_scene_rt(self, scene_name, frame_index):
+        frame_data = self._load_scene_poses(scene_name)[str(int(frame_index))]
+        raw_pos = frame_data["position"]
+        pos = [float(p) * self.cam_position_scale for p in raw_pos]
+        return self._compute_rt(pos, frame_data["rotation"])
+
+    def _sample_condition_mode(self):
+        if not self.use_condition_context_frames:
+            return "disabled"
+        if (
+            self.condition_frame_mode != "first_plus_overlap"
+            or not self.condition_history_keyframes
+            or self.num_condition_frames <= 1
+        ):
+            return "first_frame_only"
+        sample = random.random()
+        if sample < self.condition_t2v_ratio:
+            return "text_only"
+        if sample < self.condition_t2v_ratio + self.condition_i2v_ratio:
+            return "first_frame_only"
+        return "first_plus_overlap"
+
+    def _sample_overlap_conditions(self, scene_name, start_frame, ref_rt, target_height, target_width, num_extra_conditions):
+        if num_extra_conditions <= 0:
+            return [], [], []
+        window_indices = set(range(start_frame, start_frame + self.num_frames))
+        target_candidates = list(range(start_frame + 1, start_frame + self.num_frames))
+        sampled_target_frames = random.sample(target_candidates, k=min(num_extra_conditions, len(target_candidates)))
+        overlap_frames = []
+        overlap_indices = []
+        overlap_actions = []
+        used_condition_indices = set()
+        for target_frame_idx in sampled_target_frames:
+            candidate_indices = [
+                idx for idx in self._load_overlap_frames(scene_name, target_frame_idx)
+                if idx not in window_indices and idx != target_frame_idx and idx not in used_condition_indices
+            ]
+            if len(candidate_indices) == 0:
+                return None
+            chosen_idx = random.choice(candidate_indices)
+            used_condition_indices.add(chosen_idx)
+            overlap_indices.append(chosen_idx)
+            overlap_frames.append(self._load_resized_frame(scene_name, chosen_idx, target_height, target_width))
+            if self.condition_use_camera_pose:
+                overlap_rt = self._compute_scene_rt(scene_name, chosen_idx)
+                overlap_actions.append(self._to_relative_rt([overlap_rt], ref_rt)[0])
+        if len(overlap_frames) != num_extra_conditions:
+            return None
+        return overlap_frames, overlap_indices, overlap_actions
+
+    def _try_get_sample(self, data_item):
+        scene_name = data_item["video_name"]
+        cam_data = self._load_scene_poses(scene_name)
+        max_start = len(cam_data) - self.num_frames
+        if max_start < 0:
+            raise ValueError(f"Scene {scene_name} has fewer than {self.num_frames} frames.")
+        start_frame = data_item["start_frame"]
+        end_frame = start_frame + self.num_frames - 1
+        assert end_frame == data_item["end_frame"]
+
+        frames = []
+        for i in range(start_frame, end_frame + 1):
+            frame_path = os.path.join(self.frames_dir, scene_name, f"{i:04d}.png")
+            img = Image.open(frame_path).convert("RGB")
+            img = self.crop_and_resize(img, *self.get_height_width(img))
+            frames.append(img)
+
+        # prompt = self._find_nearest_caption(scene_name, start_frame)
+        prompt = data_item["prompt"]
+
+        rt_list_abs = []
+        for i in range(start_frame, end_frame + 1):
+            key = str(i)
+            if key not in cam_data:
+                raise ValueError(f"Scene {scene_name} missing pose for frame {i}.")
+            frame_data = cam_data[key]
+            raw_pos = frame_data["position"]
+            pos = [float(p) * self.cam_position_scale for p in raw_pos]
+            rt = self._compute_rt(pos, frame_data["rotation"])
+            rt_list_abs.append(rt)
+
+        rt_list = self._to_relative_rt(rt_list_abs, rt_list_abs[0])
+        pose_indices = list(range(0, self.num_frames, 4))
+        actions = [rt_list[i] for i in pose_indices]
+
+        return {
+            "video": frames,
+            "prompt": prompt,
+            "actions": actions,
+            "video_name": scene_name,
+            "start_frame": start_frame,
+            "end_frame": end_frame,
+            **self._build_condition_context_payload(
+                frames=frames,
+                scene_name=scene_name,
+                start_frame=start_frame,
+                ref_rt=rt_list_abs[0],
+                actions=actions,
+            ),
+        }
+
+    def _try_get_metadata_sample(self, row):
+        scene_name, frame_indices = self._metadata_scene_and_indices(row)
+        start_frame = int(frame_indices[0])
+        end_frame = int(frame_indices[-1])
+        cam_data = self._load_scene_poses(scene_name)
+
+        frames = []
+        for frame_idx in frame_indices:
+            frame_path = os.path.join(self.frames_dir, scene_name, f"{int(frame_idx):04d}.png")
+            img = Image.open(frame_path).convert("RGB")
+            img = self.crop_and_resize(img, *self.get_height_width(img))
+            frames.append(img)
+
+        prompt = row.get("prompt", None)
+        if prompt is None or str(prompt).strip() == "" or str(prompt).lower() == "nan":
+            prompt = self._find_nearest_caption(scene_name, start_frame)
+        else:
+            prompt = str(prompt)
+
+        rt_list_abs = []
+        for frame_idx in frame_indices:
+            key = str(int(frame_idx))
+            if key not in cam_data:
+                raise ValueError(f"Scene {scene_name} missing pose for frame {frame_idx}.")
+            frame_data = cam_data[key]
+            raw_pos = frame_data["position"]
+            pos = [float(p) * self.cam_position_scale for p in raw_pos]
+            rt = self._compute_rt(pos, frame_data["rotation"])
+            rt_list_abs.append(rt)
+
+        rt_list = self._to_relative_rt(rt_list_abs, rt_list_abs[0])
+        pose_indices = list(range(0, len(frame_indices), 4))
+        actions = [rt_list[i] for i in pose_indices]
+        geometry_memory_frames = self._load_geometry_memory_frames(row)
+
+        return {
+            "video": frames,
+            "prompt": prompt,
+            "actions": actions,
+            "video_name": scene_name,
+            "start_frame": start_frame,
+            "end_frame": end_frame,
+            "geometry_memory_frames": geometry_memory_frames,
+            **self._build_condition_context_payload(
+                frames=frames,
+                scene_name=scene_name,
+                start_frame=start_frame,
+                ref_rt=rt_list_abs[0],
+                actions=actions,
+            ),
+        }
+
+    def __getitem__(self, data_id):
+        n = len(self.data)
+        if n == 0:
+            raise RuntimeError("CamVideoDataset has no scenes.")
+        max_attempts = min(64, n)
+        last_error = None
+        for attempt in range(max_attempts):
+            idx = (data_id + attempt) % n
+            data_item = self.data[idx]
+            scene_name = data_item.get("video_name", "?") if isinstance(data_item, dict) else "?"
+            if scene_name in self.invalid_scenes:
+                continue
+            try:
+                return self._try_get_sample(data_item)
+            except (ValueError, FileNotFoundError, KeyError, OSError) as e:
+                self.invalid_scenes.add(scene_name)
+                last_error = e
+                if attempt < 3 or attempt % 8 == 0:
+                    print(
+                        f"[CamVideoDataset] Skipping invalid scene '{scene_name}' "
+                        f"({type(e).__name__}: {e}); attempt {attempt + 1}/{max_attempts}"
+                    )
+                continue
+        raise RuntimeError(
+            f"CamVideoDataset: exhausted {max_attempts} attempts starting from index {data_id}; "
+            f"last error: {type(last_error).__name__}: {last_error}"
+        )
+
+    def _build_condition_context_payload(self, frames, scene_name, start_frame, ref_rt, actions):
+        if not self.use_condition_context_frames:
+            return {}
+        payload = {
+            "use_condition_context_frames": False,
+            "condition_frames": [],
+            "condition_frame_indices": [],
+            "condition_source": None,
+            "condition_actions": [],
+        }
+        condition_mode = self._sample_condition_mode()
+        payload["condition_source"] = condition_mode
+        if condition_mode == "text_only":
+            return payload
+        payload["use_condition_context_frames"] = True
+        if self.condition_first_frame:
+            payload["condition_frames"].append(frames[0])
+            payload["condition_frame_indices"].append(start_frame)
+            payload["condition_source"] = "first_frame_only"
+            if self.condition_use_camera_pose and actions:
+                payload["condition_actions"].append(list(actions[0]))
+        if (
+            condition_mode == "first_plus_overlap"
+            and self.condition_history_keyframes
+            and self.num_condition_frames > len(payload["condition_frames"])
+        ):
+            num_extra_conditions = self.num_condition_frames - len(payload["condition_frames"])
+            overlap_payload = self._sample_overlap_conditions(
+                scene_name=scene_name,
+                start_frame=start_frame,
+                ref_rt=ref_rt,
+                target_height=frames[0].size[1],
+                target_width=frames[0].size[0],
+                num_extra_conditions=num_extra_conditions,
+            )
+            if overlap_payload is None:
+                return payload
+            overlap_frames, overlap_indices, overlap_actions = overlap_payload
+            payload["condition_frames"].extend(overlap_frames)
+            payload["condition_frame_indices"].extend(overlap_indices)
+            if self.condition_use_camera_pose:
+                payload["condition_actions"].extend(overlap_actions)
+            payload["condition_source"] = "first_plus_overlap"
+        return payload
+
+    def __len__(self):
+        return len(self.data)
+
+
+class DiffusionTrainingModule(torch.nn.Module):
+    def __init__(self):
+        super().__init__()
+        
+        
+    def to(self, *args, **kwargs):
+        for name, model in self.named_children():
+            model.to(*args, **kwargs)
+        return self
+        
+        
+    def trainable_modules(self):
+        trainable_modules = filter(lambda p: p.requires_grad, self.parameters())
+        return trainable_modules
+    
+    
+    def trainable_param_names(self):
+        trainable_param_names = list(filter(lambda named_param: named_param[1].requires_grad, self.named_parameters()))
+        trainable_param_names = set([named_param[0] for named_param in trainable_param_names])
+        return trainable_param_names
+    
+    
+    def add_lora_to_model(self, model, target_modules, lora_rank, lora_alpha=None):
+        if lora_alpha is None:
+            lora_alpha = lora_rank
+        lora_config = LoraConfig(r=lora_rank, lora_alpha=lora_alpha, target_modules=target_modules)
+        model = inject_adapter_in_model(lora_config, model)
+        return model
+    
+    
+    def export_trainable_state_dict(self, state_dict, remove_prefix=None):
+        trainable_param_names = self.trainable_param_names()
+        state_dict = {name: param for name, param in state_dict.items() if name in trainable_param_names}
+        if remove_prefix is not None:
+            state_dict_ = {}
+            for name, param in state_dict.items():
+                if name.startswith(remove_prefix):
+                    name = name[len(remove_prefix):]
+                state_dict_[name] = param
+            state_dict = state_dict_
+        return state_dict
+
+
+
+class ModelLogger:
+    def __init__(self, output_path, remove_prefix_in_ckpt=None, state_dict_converter=lambda x:x):
+        self.output_path = output_path
+        self.remove_prefix_in_ckpt = remove_prefix_in_ckpt
+        self.state_dict_converter = state_dict_converter
+        
+    
+    def on_step_end(self, loss):
+        pass
+    
+    
+    def on_epoch_end(self, accelerator, model, epoch_id):
+        accelerator.wait_for_everyone()
+        if accelerator.is_main_process:
+            state_dict = accelerator.get_state_dict(model)
+            state_dict = accelerator.unwrap_model(model).export_trainable_state_dict(state_dict, remove_prefix=self.remove_prefix_in_ckpt)
+            state_dict = self.state_dict_converter(state_dict)
+            os.makedirs(self.output_path, exist_ok=True)
+            path = os.path.join(self.output_path, f"epoch-{epoch_id}.safetensors")
+            accelerator.save(state_dict, path, safe_serialization=True)
+
+
+
+def launch_training_task(
+    dataset: torch.utils.data.Dataset,
+    model: DiffusionTrainingModule,
+    model_logger: ModelLogger,
+    optimizer: torch.optim.Optimizer,
+    scheduler: torch.optim.lr_scheduler.LRScheduler,
+    num_epochs: int = 1,
+    gradient_accumulation_steps: int = 1,
+):
+    dataloader = torch.utils.data.DataLoader(dataset, shuffle=True, collate_fn=lambda x: x[0], drop_last=True)
+    accelerator = Accelerator(gradient_accumulation_steps=gradient_accumulation_steps)
+    model, optimizer, dataloader, scheduler = accelerator.prepare(model, optimizer, dataloader, scheduler)
+    
+    for epoch_id in range(num_epochs):
+        for data in tqdm(dataloader):
+            with accelerator.accumulate(model):
+                optimizer.zero_grad()
+                loss = model(data)
+                accelerator.backward(loss)
+                optimizer.step()
+                model_logger.on_step_end(loss)
+                scheduler.step()
+        model_logger.on_epoch_end(accelerator, model, epoch_id)
+
+def launch_data_process_task(model: DiffusionTrainingModule, dataset, output_path="./models"):
+    dataloader = torch.utils.data.DataLoader(dataset, shuffle=False, collate_fn=lambda x: x[0], drop_last=True)
+    accelerator = Accelerator()
+    model, dataloader = accelerator.prepare(model, dataloader)
+    os.makedirs(os.path.join(output_path, "data_cache"), exist_ok=True)
+    for data_id, data in enumerate(tqdm(dataloader)):
+        with torch.no_grad():
+            inputs = model.forward_preprocess(data)
+            inputs = {key: inputs[key] for key in model.model_input_keys if key in inputs}
+            torch.save(inputs, os.path.join(output_path, "data_cache", f"{data_id}.pth"))
+
+
+
+def wan_parser():
+    parser = argparse.ArgumentParser(description="Simple example of a training script.")
+    parser.add_argument("--dataset_base_path", type=str, default="", required=True, help="Base path of the dataset.")
+    parser.add_argument("--dataset_metadata_path", type=str, default=None, help="Path to the metadata file of the dataset.")
+    parser.add_argument("--max_pixels", type=int, default=1280*720, help="Maximum number of pixels per frame, used for dynamic resolution..")
+    parser.add_argument("--height", type=int, default=None, help="Height of images or videos. Leave `height` and `width` empty to enable dynamic resolution.")
+    parser.add_argument("--width", type=int, default=None, help="Width of images or videos. Leave `height` and `width` empty to enable dynamic resolution.")
+    parser.add_argument("--num_frames", type=int, default=81, help="Number of frames per video. Frames are sampled from the video prefix.")
+    parser.add_argument("--data_file_keys", type=str, default="image,video", help="Data file keys in the metadata. Comma-separated.")
+    parser.add_argument("--dataset_repeat", type=int, default=1, help="Number of times to repeat the dataset per epoch.")
+    parser.add_argument("--model_paths", type=str, default=None, help="Paths to load models. In JSON format.")
+    parser.add_argument("--model_id_with_origin_paths", type=str, default=None, help="Model ID with origin paths, e.g., Wan-AI/Wan2.1-T2V-1.3B:diffusion_pytorch_model*.safetensors. Comma-separated.")
+    parser.add_argument("--learning_rate", type=float, default=1e-4, help="Learning rate.")
+    parser.add_argument("--num_epochs", type=int, default=1, help="Number of epochs.")
+    parser.add_argument("--output_path", type=str, default="./models", help="Output save path.")
+    parser.add_argument("--remove_prefix_in_ckpt", type=str, default="pipe.dit.", help="Remove prefix in ckpt.")
+    parser.add_argument("--trainable_models", type=str, default=None, help="Models to train, e.g., dit, vae, text_encoder.")
+    parser.add_argument("--lora_base_model", type=str, default=None, help="Which model LoRA is added to.")
+    parser.add_argument("--lora_target_modules", type=str, default="q,k,v,o,ffn.0,ffn.2", help="Which layers LoRA is added to.")
+    parser.add_argument("--lora_rank", type=int, default=32, help="Rank of LoRA.")
+    parser.add_argument("--extra_inputs", default=None, help="Additional model inputs, comma-separated.")
+    parser.add_argument("--use_gradient_checkpointing_offload", default=False, action="store_true", help="Whether to offload gradient checkpointing to CPU memory.")
+    parser.add_argument("--gradient_accumulation_steps", type=int, default=1, help="Gradient accumulation steps.")
+    parser.add_argument("--use_condition_context_frames", default=False, action="store_true", help="Enable appended clean condition latents.")
+    parser.add_argument("--condition_first_frame", default=False, action="store_true", help="Use the current clip first frame as a clean condition frame.")
+    parser.add_argument("--condition_history_keyframes", default=False, action="store_true", help="Use overlap-based keyframes as conditions.")
+    parser.add_argument("--condition_use_camera_pose", default=True, action="store_true", help="Inject camera pose for condition frames.")
+    parser.add_argument("--num_condition_frames", type=int, default=1, help="Number of condition frames.")
+    parser.add_argument("--condition_frame_mode", type=str, default="first_frame_only", help="Condition frame selection mode.")
+    parser.add_argument("--overlap_labels_root", type=str, default=None, help="Root dir for overlap label JSONs.")
+    parser.add_argument("--condition_t2v_ratio", type=float, default=0.10, help="Ratio of text-only condition samples.")
+    parser.add_argument("--condition_i2v_ratio", type=float, default=0.10, help="Ratio of first-frame-only condition samples.")
+    return parser
+
+
+
+def flux_parser():
+    parser = argparse.ArgumentParser(description="Simple example of a training script.")
+    parser.add_argument("--dataset_base_path", type=str, default="", required=True, help="Base path of the dataset.")
+    parser.add_argument("--dataset_metadata_path", type=str, default=None, help="Path to the metadata file of the dataset.")
+    parser.add_argument("--max_pixels", type=int, default=1024*1024, help="Maximum number of pixels per frame, used for dynamic resolution..")
+    parser.add_argument("--height", type=int, default=None, help="Height of images. Leave `height` and `width` empty to enable dynamic resolution.")
+    parser.add_argument("--width", type=int, default=None, help="Width of images. Leave `height` and `width` empty to enable dynamic resolution.")
+    parser.add_argument("--data_file_keys", type=str, default="image", help="Data file keys in the metadata. Comma-separated.")
+    parser.add_argument("--dataset_repeat", type=int, default=1, help="Number of times to repeat the dataset per epoch.")
+    parser.add_argument("--model_paths", type=str, default=None, help="Paths to load models. In JSON format.")
+    parser.add_argument("--model_id_with_origin_paths", type=str, default=None, help="Model ID with origin paths, e.g., Wan-AI/Wan2.1-T2V-1.3B:diffusion_pytorch_model*.safetensors. Comma-separated.")
+    parser.add_argument("--learning_rate", type=float, default=1e-4, help="Learning rate.")
+    parser.add_argument("--num_epochs", type=int, default=1, help="Number of epochs.")
+    parser.add_argument("--output_path", type=str, default="./models", help="Output save path.")
+    parser.add_argument("--remove_prefix_in_ckpt", type=str, default="pipe.dit.", help="Remove prefix in ckpt.")
+    parser.add_argument("--trainable_models", type=str, default=None, help="Models to train, e.g., dit, vae, text_encoder.")
+    parser.add_argument("--lora_base_model", type=str, default=None, help="Which model LoRA is added to.")
+    parser.add_argument("--lora_target_modules", type=str, default="q,k,v,o,ffn.0,ffn.2", help="Which layers LoRA is added to.")
+    parser.add_argument("--lora_rank", type=int, default=32, help="Rank of LoRA.")
+    parser.add_argument("--extra_inputs", default=None, help="Additional model inputs, comma-separated.")
+    parser.add_argument("--align_to_opensource_format", default=False, action="store_true", help="Whether to align the lora format to opensource format. Only for DiT's LoRA.")
+    parser.add_argument("--use_gradient_checkpointing", default=False, action="store_true", help="Whether to use gradient checkpointing.")
+    parser.add_argument("--use_gradient_checkpointing_offload", default=False, action="store_true", help="Whether to offload gradient checkpointing to CPU memory.")
+    parser.add_argument("--gradient_accumulation_steps", type=int, default=1, help="Gradient accumulation steps.")
+    return parser

code/diffsynth/trainers/utils.py.p0bak-getitem

@@ -0,0 +1,1367 @@
+import imageio, os, torch, warnings, torchvision, argparse, json, random
+from peft import LoraConfig, inject_adapter_in_model
+from PIL import Image
+import pandas as pd
+from tqdm import tqdm
+from accelerate import Accelerator
+
+
+
+class ImageDataset(torch.utils.data.Dataset):
+    def __init__(
+        self,
+        base_path=None, metadata_path=None,
+        max_pixels=1920*1080, height=None, width=None,
+        height_division_factor=16, width_division_factor=16,
+        data_file_keys=("image",),
+        image_file_extension=("jpg", "jpeg", "png", "webp"),
+        repeat=1,
+        args=None,
+    ):
+        if args is not None:
+            base_path = args.dataset_base_path
+            metadata_path = args.dataset_metadata_path
+            height = args.height
+            width = args.width
+            max_pixels = args.max_pixels
+            data_file_keys = args.data_file_keys.split(",")
+            repeat = args.dataset_repeat
+            
+        self.base_path = base_path
+        self.max_pixels = max_pixels
+        self.height = height
+        self.width = width
+        self.height_division_factor = height_division_factor
+        self.width_division_factor = width_division_factor
+        self.data_file_keys = data_file_keys
+        self.image_file_extension = image_file_extension
+        self.repeat = repeat
+
+        if height is not None and width is not None:
+            print("Height and width are fixed. Setting `dynamic_resolution` to False.")
+            self.dynamic_resolution = False
+        elif height is None and width is None:
+            print("Height and width are none. Setting `dynamic_resolution` to True.")
+            self.dynamic_resolution = True
+            
+        if metadata_path is None:
+            print("No metadata. Trying to generate it.")
+            metadata = self.generate_metadata(base_path)
+            print(f"{len(metadata)} lines in metadata.")
+            self.data = [metadata.iloc[i].to_dict() for i in range(len(metadata))]
+        elif metadata_path.endswith(".json"):
+            with open(metadata_path, "r") as f:
+                metadata = json.load(f)
+            self.data = metadata
+        else:
+            metadata = pd.read_csv(metadata_path)
+            # Ensure prompt column is string type to avoid float conversion for NaN values
+            if 'prompt' in metadata.columns:
+                metadata['prompt'] = metadata['prompt'].astype(str)
+                # Replace 'nan' string (from NaN) with empty string
+                metadata['prompt'] = metadata['prompt'].replace('nan', '')
+            self.data = [metadata.iloc[i].to_dict() for i in range(len(metadata))]
+
+
+    def generate_metadata(self, folder):
+        image_list, prompt_list = [], []
+        file_set = set(os.listdir(folder))
+        for file_name in file_set:
+            if "." not in file_name:
+                continue
+            file_ext_name = file_name.split(".")[-1].lower()
+            file_base_name = file_name[:-len(file_ext_name)-1]
+            if file_ext_name not in self.image_file_extension:
+                continue
+            prompt_file_name = file_base_name + ".txt"
+            if prompt_file_name not in file_set:
+                continue
+            with open(os.path.join(folder, prompt_file_name), "r", encoding="utf-8") as f:
+                prompt = f.read().strip()
+            image_list.append(file_name)
+            prompt_list.append(prompt)
+        metadata = pd.DataFrame()
+        metadata["image"] = image_list
+        metadata["prompt"] = prompt_list
+        return metadata
+    
+    
+    def crop_and_resize(self, image, target_height, target_width):
+        width, height = image.size
+        scale = max(target_width / width, target_height / height)
+        image = torchvision.transforms.functional.resize(
+            image,
+            (round(height*scale), round(width*scale)),
+            interpolation=torchvision.transforms.InterpolationMode.BILINEAR
+        )
+        image = torchvision.transforms.functional.center_crop(image, (target_height, target_width))
+        return image
+    
+    
+    def get_height_width(self, image):
+        if self.dynamic_resolution:
+            width, height = image.size
+            if width * height > self.max_pixels:
+                scale = (width * height / self.max_pixels) ** 0.5
+                height, width = int(height / scale), int(width / scale)
+            height = height // self.height_division_factor * self.height_division_factor
+            width = width // self.width_division_factor * self.width_division_factor
+        else:
+            height, width = self.height, self.width
+        return height, width
+    
+    
+    def load_image(self, file_path):
+        image = Image.open(file_path).convert("RGB")
+        image = self.crop_and_resize(image, *self.get_height_width(image))
+        return image
+    
+    
+    def load_data(self, file_path):
+        return self.load_image(file_path)
+
+
+    def __getitem__(self, data_id):
+        data = self.data[data_id % len(self.data)].copy()
+        for key in self.data_file_keys:
+            if key in data:
+                path = os.path.join(self.base_path, data[key])
+                data[key] = self.load_data(path)
+                if data[key] is None:
+                    warnings.warn(f"cannot load file {data[key]}.")
+                    return None
+        return data
+    
+
+    def __len__(self):
+        return len(self.data) * self.repeat
+
+
+
+class VideoDataset(torch.utils.data.Dataset):
+    def __init__(
+        self,
+        base_path=None, metadata_path=None,
+        num_frames=81,
+        time_division_factor=4, time_division_remainder=1,
+        max_pixels=1920*1080, height=None, width=None,
+        height_division_factor=16, width_division_factor=16,
+        data_file_keys=("video",),
+        image_file_extension=("jpg", "jpeg", "png", "webp"),
+        video_file_extension=("mp4", "avi", "mov", "wmv", "mkv", "flv", "webm"),
+        repeat=1,
+        args=None,
+        action_base_path=None,
+        enable_icl=False,
+        icl_num_examples=2,
+        icl_context_frames=8,
+    ):
+        if args is not None:
+            base_path = args.dataset_base_path
+            metadata_path = args.dataset_metadata_path
+            height = args.height
+            width = args.width
+            max_pixels = args.max_pixels
+            num_frames = args.num_frames
+            data_file_keys = args.data_file_keys.split(",")
+            repeat = args.dataset_repeat
+            # In-context learning parameters
+            if hasattr(args, 'enable_icl'):
+                enable_icl = args.enable_icl
+            if hasattr(args, 'icl_num_examples'):
+                icl_num_examples = args.icl_num_examples
+            if hasattr(args, 'icl_context_frames'):
+                icl_context_frames = args.icl_context_frames
+        
+        self.base_path = base_path
+        self.num_frames = num_frames
+        self.time_division_factor = time_division_factor
+        self.time_division_remainder = time_division_remainder
+        self.max_pixels = max_pixels
+        self.height = height
+        self.width = width
+        self.height_division_factor = height_division_factor
+        self.width_division_factor = width_division_factor
+        self.data_file_keys = data_file_keys
+        self.image_file_extension = image_file_extension
+        self.video_file_extension = video_file_extension
+        self.repeat = repeat
+        
+        # In-context learning parameters
+        self.enable_icl = enable_icl
+        self.icl_num_examples = icl_num_examples
+        self.icl_context_frames = icl_context_frames
+        
+        if height is not None and width is not None:
+            print("Height and width are fixed. Setting `dynamic_resolution` to False.")
+            self.dynamic_resolution = False
+        elif height is None and width is None:
+            print("Height and width are none. Setting `dynamic_resolution` to True.")
+            self.dynamic_resolution = True
+            
+        if metadata_path is None:
+            print("No metadata. Trying to generate it.")
+            metadata = self.generate_metadata(base_path)
+            print(f"{len(metadata)} lines in metadata.")
+            self.data = [metadata.iloc[i].to_dict() for i in range(len(metadata))]
+        elif metadata_path.endswith(".json"):
+            with open(metadata_path, "r") as f:
+                metadata = json.load(f)
+            self.data = metadata
+        else:
+            metadata = pd.read_csv(metadata_path)
+            # Ensure prompt column is string type to avoid float conversion for NaN values
+            if 'prompt' in metadata.columns:
+                metadata['prompt'] = metadata['prompt'].astype(str)
+                # Replace 'nan' string (from NaN) with empty string
+                metadata['prompt'] = metadata['prompt'].replace('nan', '')
+                
+                # CRITICAL FIX: Clean prompt - remove video path prefix if present
+                # Some CSV prompts start with "video_name.mp4 " prefix, which should be removed
+                def clean_prompt(prompt_str):
+                    if not isinstance(prompt_str, str) or not prompt_str:
+                        return prompt_str
+                    # Check if prompt starts with a video path (contains .mp4 or /)
+                    # Pattern: "VideoName/1234_5678.mp4 " or "VideoName.mp4 "
+                    import re
+                    # Match pattern: word/word.mp4 or word.mp4 at the start, followed by space
+                    pattern = r'^[A-Za-z0-9_]+(/[A-Za-z0-9_]+)?\.mp4\s+'
+                    cleaned = re.sub(pattern, '', prompt_str)
+                    # Also handle truncated prompts ending with "..."
+                    if cleaned.endswith('...'):
+                        cleaned = cleaned[:-3].rstrip()
+                    return cleaned.strip()
+                
+                metadata['prompt'] = metadata['prompt'].apply(clean_prompt)
+            self.data = [metadata.iloc[i].to_dict() for i in range(len(metadata))]
+
+        self.action_base_path = action_base_path
+        
+        if self.enable_icl:
+            print(f"In-context learning enabled: {icl_num_examples} examples, {icl_context_frames} context frames each")
+            
+    
+    def generate_metadata(self, folder):
+        video_list, prompt_list = [], []
+        file_set = set(os.listdir(folder))
+        for file_name in file_set:
+            if "." not in file_name:
+                continue
+            file_ext_name = file_name.split(".")[-1].lower()
+            file_base_name = file_name[:-len(file_ext_name)-1]
+            if file_ext_name not in self.image_file_extension and file_ext_name not in self.video_file_extension:
+                continue
+            prompt_file_name = file_base_name + ".txt"
+            if prompt_file_name not in file_set:
+                continue
+            with open(os.path.join(folder, prompt_file_name), "r", encoding="utf-8") as f:
+                prompt = f.read().strip()
+            video_list.append(file_name)
+            prompt_list.append(prompt)
+        metadata = pd.DataFrame()
+        metadata["video"] = video_list
+        metadata["prompt"] = prompt_list
+        return metadata
+        
+        
+    def crop_and_resize(self, image, target_height, target_width):
+        width, height = image.size
+        scale = max(target_width / width, target_height / height)
+        image = torchvision.transforms.functional.resize(
+            image,
+            (round(height*scale), round(width*scale)),
+            interpolation=torchvision.transforms.InterpolationMode.BILINEAR
+        )
+        image = torchvision.transforms.functional.center_crop(image, (target_height, target_width))
+        return image
+    
+    
+    def get_height_width(self, image):
+        if self.dynamic_resolution:
+            width, height = image.size
+            if width * height > self.max_pixels:
+                scale = (width * height / self.max_pixels) ** 0.5
+                height, width = int(height / scale), int(width / scale)
+            height = height // self.height_division_factor * self.height_division_factor
+            width = width // self.width_division_factor * self.width_division_factor
+        else:
+            height, width = self.height, self.width
+        return height, width
+    
+    
+    def get_num_frames(self, reader):
+        num_frames = self.num_frames
+        if int(reader.count_frames()) < num_frames:
+            num_frames = int(reader.count_frames())
+            while num_frames > 1 and num_frames % self.time_division_factor != self.time_division_remainder:
+                num_frames -= 1
+        return num_frames
+    
+
+    def load_video(self, file_path):
+        reader = imageio.get_reader(file_path)
+        num_frames = self.get_num_frames(reader)
+        frames = []
+        for frame_id in range(num_frames):
+            frame = reader.get_data(frame_id)
+            frame = Image.fromarray(frame)
+            frame = self.crop_and_resize(frame, *self.get_height_width(frame))
+            frames.append(frame)
+        reader.close()
+        return frames
+    
+    
+    def load_image(self, file_path):
+        image = Image.open(file_path).convert("RGB")
+        image = self.crop_and_resize(image, *self.get_height_width(image))
+        frames = [image]
+        return frames
+    
+    
+    def is_image(self, file_path):
+        file_ext_name = file_path.split(".")[-1]
+        return file_ext_name.lower() in self.image_file_extension
+    
+    
+    def is_video(self, file_path):
+        file_ext_name = file_path.split(".")[-1]
+        return file_ext_name.lower() in self.video_file_extension
+    
+    
+    def load_data(self, file_path):
+        # Handle multiple frame paths separated by '|' (for frame sequences)
+        if '|' in str(file_path):
+            # Split the path by '|' to get individual frame paths
+            frame_paths = str(file_path).split('|')
+            frames = []
+            
+            # Get base_path (dataset root)
+            if not hasattr(self, 'base_path') or not self.base_path:
+                warnings.warn(f"Cannot determine base directory for frame sequence: {file_path}")
+                return None
+            
+            base_dir = self.base_path  # This is the dataset root
+            
+            # Check the first path to determine the format
+            first_frame = frame_paths[0].strip() if frame_paths else ""
+            
+            # If first frame is already an absolute path (from __getitem__ joining),
+            # extract the base directory from it
+            if os.path.isabs(first_frame):
+                # Extract base directory from first frame path
+                # First frame format: /path/to/dataset/frames/video_name/frame.png
+                # We need to get /path/to/dataset
+                parts = first_frame.split(os.sep)
+                # Find 'frames' in the path and get everything before it
+                if 'frames' in parts:
+                    frames_idx = parts.index('frames')
+                    base_dir = os.sep.join(parts[:frames_idx])
+                else:
+                    # Fallback: use self.base_path
+                    base_dir = self.base_path
+            
+            for frame_path in frame_paths:
+                frame_path = frame_path.strip()
+                if not frame_path:
+                    continue
+                
+                # Construct full path
+                if os.path.isabs(frame_path):
+                    # Already absolute path (from __getitem__)
+                    full_frame_path = frame_path
+                else:
+                    # Relative path - need to construct full path
+                    # Remove 'frames/' prefix if present (we'll add it consistently)
+                    if frame_path.startswith('frames/'):
+                        frame_path = frame_path[7:]  # Remove 'frames/' prefix
+                    
+                    # Always join with base_dir + 'frames/' since base_dir is dataset root
+                    full_frame_path = os.path.join(base_dir, 'frames', frame_path)
+                
+                # Load individual frame
+                if os.path.exists(full_frame_path):
+                    if self.is_image(full_frame_path):
+                        frame_data = self.load_image(full_frame_path)
+                        if frame_data:
+                            frames.extend(frame_data)
+                    else:
+                        warnings.warn(f"Frame is not an image: {full_frame_path}")
+                else:
+                    warnings.warn(f"Frame not found: {full_frame_path}")
+            
+            if frames:
+                return frames
+            else:
+                warnings.warn(f"No frames loaded from sequence: {file_path}")
+                return None
+        
+        # Handle single file (image or video)
+        if self.is_image(file_path):
+            return self.load_image(file_path)
+        elif self.is_video(file_path):
+            return self.load_video(file_path)
+        else:
+            return None
+
+
+    def __getitem__(self, data_id):
+        data = self.data[data_id % len(self.data)].copy()
+        for key in self.data_file_keys:
+            if key in ["video_name", "start_frame", "end_frame"]:
+                if "actions" in data:
+                    continue
+                try:
+                    video_name = data.get("video_name")
+                    if video_name is None:
+                        warnings.warn(f"video_name is missing in metadata for data_id {data_id}. Skipping action loading.")
+                        continue
+                    
+                    if video_name.endswith(".mp4"):
+                        video_name = ".".join(video_name.split(".")[:-1])
+                    if "_" in video_name:
+                        video_name = "_".join(video_name.split("_")[:4])
+                    
+                    import json
+                    json_path = os.path.join(self.action_base_path, video_name + ".json")
+                    
+                    # Check if action file exists
+                    if not os.path.exists(json_path):
+                        warnings.warn(f"Action file does not exist: {json_path}. Skipping action loading for data_id {data_id}.")
+                        continue
+                    
+                    start_frame = data.get("start_frame")
+                    end_frame = data.get("end_frame")
+                    if start_frame is None or end_frame is None:
+                        warnings.warn(f"start_frame or end_frame is missing in metadata for data_id {data_id}. Skipping action loading.")
+                        continue
+                    
+                    json_data = json.load(open(json_path, "r"))['actions']
+                    actions = []
+                    current_yaw = 0.0
+                    for frame_id in range(start_frame+1, end_frame+1):
+                        frame_str = str(frame_id)
+                        if frame_str not in json_data:
+                            warnings.warn(f"Frame {frame_id} not found in action file {json_path}. Skipping this frame.")
+                            continue
+                        
+                        action = json_data[frame_str]
+                        new_action = [0.0] * (2 + 2 + 3 + 1 + 2)
+                        if action['ws'] == 1:
+                            new_action[0] = 1
+                        elif action['ws'] == 2:
+                            new_action[1] = 1
+
+                        if action['ad'] == 1:
+                            new_action[2] = 1
+                        elif action['ad'] == 2:
+                            new_action[3] = 1
+                        
+                        if action['scs'] == 1 and action.get("jump_invalid", 0) == 0:
+                            new_action[4] = 1
+                        elif action['scs'] == 2:
+                            new_action[5] = 1
+                        elif action['scs'] == 3:
+                            new_action[6] = 1
+                        
+                        if action.get('collision', 0) == 1:
+                            new_action[7] = 1
+                            new_action[0] = 0
+                            new_action[1] = 0
+                            new_action[2] = 0
+                            new_action[3] = 0
+
+                        pre_pitch = action.get('pre_pitch', 0.0)
+                        current_pitch = pre_pitch + action.get('pitch_delta', 0.0) * 15.0
+                        current_yaw += action.get('yaw_delta', 0.0) * 15.0
+                        new_action[8] = current_pitch
+                        new_action[9] = current_yaw
+
+                        actions.append(new_action)
+                    data["actions"] = actions
+                except Exception as e:
+                    warnings.warn(f"Exception while loading actions for data_id {data_id}: {e}. Continuing without actions.")
+                    # Don't return None, just continue without actions
+                    continue
+            elif key == "video":
+                # Check if data[key] exists and is not None
+                if key not in data or data[key] is None:
+                    warnings.warn(f"Video key '{key}' is missing or None in metadata for data_id {data_id}. Skipping this sample.")
+                    return None
+                
+                # Handle frame sequences (paths with '|' separator)
+                video_path_str = str(data[key])
+                if '|' in video_path_str:
+                    # For frame sequences, pass the full path string to load_data
+                    # load_data will handle splitting and loading individual frames
+                    path = os.path.join(self.base_path, video_path_str)
+                    # Don't check path existence here for frame sequences
+                    # load_data will handle individual frame loading
+                else:
+                    path = os.path.join(self.base_path, data[key])
+                    # Check if path exists (only for single files)
+                    if not os.path.exists(path):
+                        warnings.warn(f"Video file does not exist: {path}. Skipping this sample.")
+                        return None
+                try:
+                    data[key] = self.load_data(path)
+                    if data[key] is None:
+                        warnings.warn(f"Failed to load video file: {path}. load_data returned None.")
+                        return None
+                except Exception as e:
+                    warnings.warn(f"Exception while loading video file {path}: {e}. Skipping this sample.")
+                    return None
+        
+        # In-context learning: sample context examples from dataset
+        if self.enable_icl and len(self.data) > 1:
+            context_frames_list = []
+            context_actions_list = []
+            
+            # Sample random examples from dataset (excluding current one)
+            current_idx = data_id % len(self.data)
+            candidate_indices = [i for i in range(len(self.data)) if i != current_idx]
+            if len(candidate_indices) > 0:
+                num_samples = min(self.icl_num_examples, len(candidate_indices))
+                sampled_indices = random.sample(candidate_indices, num_samples)
+                
+                for sample_idx in sampled_indices:
+                    sample_data = self.data[sample_idx].copy()
+                    # Load video for context
+                    if "video" in self.data_file_keys and "video" in sample_data:
+                        video_path = os.path.join(self.base_path, sample_data["video"])
+                        sample_video = self.load_data(video_path)
+                        if sample_video is not None and len(sample_video) >= self.icl_context_frames:
+                            # Sample context_frames from the video
+                            start_idx = random.randint(0, max(0, len(sample_video) - self.icl_context_frames))
+                            context_frames = sample_video[start_idx:start_idx + self.icl_context_frames]
+                            context_frames_list.extend(context_frames)
+                            
+                            # Load corresponding actions if available
+                            if self.action_base_path is not None and "video_name" in sample_data:
+                                try:
+                                    sample_video_name = sample_data["video_name"]
+                                    if sample_video_name.endswith(".mp4"):
+                                        sample_video_name = ".".join(sample_video_name.split(".")[:-1])
+                                    if "_" in sample_video_name:
+                                        sample_video_name = "_".join(sample_video_name.split("_")[:4])
+                                    sample_json_path = os.path.join(self.action_base_path, sample_video_name + ".json")
+                                    if os.path.exists(sample_json_path):
+                                        sample_json_data = json.load(open(sample_json_path, "r"))['actions']
+                                        sample_start_frame = sample_data.get("start_frame", 0)
+                                        sample_end_frame = sample_data.get("end_frame", len(sample_video))
+                                        
+                                        # Get actions for the context frames
+                                        context_actions = []
+                                        context_yaw = 0.0
+                                        for frame_idx in range(sample_start_frame + start_idx + 1, 
+                                                             min(sample_start_frame + start_idx + self.icl_context_frames + 1, sample_end_frame + 1)):
+                                            if str(frame_idx) in sample_json_data:
+                                                action = sample_json_data[str(frame_idx)]
+                                                new_action = [0.0] * (2 + 2 + 3 + 1 + 2)
+                                                if action['ws'] == 1:
+                                                    new_action[0] = 1
+                                                elif action['ws'] == 2:
+                                                    new_action[1] = 1
+                                                if action['ad'] == 1:
+                                                    new_action[2] = 1
+                                                elif action['ad'] == 2:
+                                                    new_action[3] = 1
+                                                if action['scs'] == 1 and action.get("jump_invalid", 0) == 0:
+                                                    new_action[4] = 1
+                                                elif action['scs'] == 2:
+                                                    new_action[5] = 1
+                                                elif action['scs'] == 3:
+                                                    new_action[6] = 1
+                                                if action.get('collision', 0) == 1:
+                                                    new_action[7] = 1
+                                                    new_action[0] = 0
+                                                    new_action[1] = 0
+                                                    new_action[2] = 0
+                                                    new_action[3] = 0
+                                                pre_pitch = action.get('pre_pitch', 0.0)
+                                                current_pitch = pre_pitch + action.get('pitch_delta', 0.0) * 15.0
+                                                context_yaw += action.get('yaw_delta', 0.0) * 15.0
+                                                new_action[8] = current_pitch
+                                                new_action[9] = context_yaw
+                                                context_actions.append(new_action)
+                                        context_actions_list.extend(context_actions[:len(context_frames)])
+                                except Exception as e:
+                                    # If loading actions fails, just skip
+                                    pass
+            
+            if context_frames_list:
+                data["context_frames"] = context_frames_list
+                if context_actions_list and len(context_actions_list) == len(context_frames_list):
+                    data["context_actions"] = context_actions_list
+        
+        return data
+    
+
+    def __len__(self):
+        return len(self.data) * self.repeat
+
+    @staticmethod
+    def get_one_hot(action, range=2):
+        one_hot = [0] * (range + 1)
+        one_hot[action] = 1
+        return one_hot
+
+
+
+import numpy as np
+
+
+class CamVideoDataset(torch.utils.data.Dataset):
+    """Dataset for Context-as-Memory camera pose conditioned training (ported from VWM).
+
+    Loads 81 PNG frames from UE scenes with random temporal cropping and extracts
+    corresponding camera poses as 12-dim relative RT vectors subsampled to match
+    the 21 latent frames.
+    """
+    def __init__(
+        self,
+        base_path=None, metadata_path=None,
+        num_frames=81,
+        height=None, width=None,
+        max_pixels=1920*1080,
+        height_division_factor=16, width_division_factor=16,
+        repeat=1,
+        args=None,
+        cam_position_scale=None,
+    ):
+        if args is not None:
+            base_path = args.dataset_base_path
+            metadata_path = getattr(args, "dataset_metadata_path", metadata_path)
+            height = args.height
+            width = args.width
+            max_pixels = args.max_pixels
+            num_frames = args.num_frames
+            repeat = args.dataset_repeat
+            cam_position_scale = getattr(args, "cam_position_scale", 0.01)
+            self.use_condition_context_frames = getattr(args, "use_condition_context_frames", False)
+            self.condition_first_frame = getattr(args, "condition_first_frame", False)
+            self.condition_history_keyframes = getattr(args, "condition_history_keyframes", False)
+            self.condition_use_camera_pose = getattr(args, "condition_use_camera_pose", True)
+            self.num_condition_frames = getattr(args, "num_condition_frames", 1)
+            self.condition_frame_mode = getattr(args, "condition_frame_mode", "first_frame_only")
+            self.overlap_labels_root = getattr(args, "overlap_labels_root", None)
+            self.condition_t2v_ratio = getattr(args, "condition_t2v_ratio", 0.10)
+            self.condition_i2v_ratio = getattr(args, "condition_i2v_ratio", 0.10)
+            self.use_geometry_spatial_memory = getattr(args, "use_geometry_spatial_memory", False)
+            self.geometry_memory_column = getattr(args, "geometry_memory_column", "geometry_memory")
+            self.geometry_memory_root = getattr(args, "geometry_memory_root", None)
+        else:
+            self.use_condition_context_frames = False
+            self.condition_first_frame = False
+            self.condition_history_keyframes = False
+            self.condition_use_camera_pose = True
+            self.num_condition_frames = 1
+            self.condition_frame_mode = "first_frame_only"
+            self.overlap_labels_root = None
+            self.condition_t2v_ratio = 0.10
+            self.condition_i2v_ratio = 0.10
+            self.use_geometry_spatial_memory = False
+            self.geometry_memory_column = "geometry_memory"
+            self.geometry_memory_root = None
+
+        if cam_position_scale is None:
+            cam_position_scale = 0.01
+        self.cam_position_scale = float(cam_position_scale)
+
+        self.base_path = base_path
+        self.frames_dir = os.path.join(base_path, "frames")
+        self.jsons_dir = os.path.join(base_path, "jsons")
+        self.num_frames = num_frames
+        self.max_pixels = max_pixels
+        self.height = height
+        self.width = width
+        self.height_division_factor = height_division_factor
+        self.width_division_factor = width_division_factor
+        self.repeat = repeat
+
+        if height is not None and width is not None:
+            self.dynamic_resolution = False
+        else:
+            self.dynamic_resolution = True
+        
+        if metadata_path is None:
+            print("No metadata. Trying to generate it.")
+            metadata = self.generate_metadata(base_path)
+            print(f"{len(metadata)} lines in metadata.")
+            self.data = [metadata.iloc[i].to_dict() for i in range(len(metadata))]
+        elif metadata_path.endswith(".json"):
+            with open(metadata_path, "r") as f:
+                metadata = json.load(f)
+            self.data = metadata
+        else:
+            metadata = pd.read_csv(metadata_path)
+            # Ensure prompt column is string type to avoid float conversion for NaN values
+            if 'prompt' in metadata.columns:
+                metadata['prompt'] = metadata['prompt'].astype(str)
+                # Replace 'nan' string (from NaN) with empty string
+                metadata['prompt'] = metadata['prompt'].replace('nan', '')
+                
+                # CRITICAL FIX: Clean prompt - remove video path prefix if present
+                # Some CSV prompts start with "video_name.mp4 " prefix, which should be removed
+                def clean_prompt(prompt_str):
+                    if not isinstance(prompt_str, str) or not prompt_str:
+                        return prompt_str
+                    # Check if prompt starts with a video path (contains .mp4 or /)
+                    # Pattern: "VideoName/1234_5678.mp4 " or "VideoName.mp4 "
+                    import re
+                    # Match pattern: word/word.mp4 or word.mp4 at the start, followed by space
+                    pattern = r'^[A-Za-z0-9_]+(/[A-Za-z0-9_]+)?\.mp4\s+'
+                    cleaned = re.sub(pattern, '', prompt_str)
+                    # Also handle truncated prompts ending with "..."
+                    if cleaned.endswith('...'):
+                        cleaned = cleaned[:-3].rstrip()
+                    return cleaned.strip()
+                
+                metadata['prompt'] = metadata['prompt'].apply(clean_prompt)
+            self.data = [metadata.iloc[i].to_dict() for i in range(len(metadata))]
+
+        captions_path = os.path.join(base_path, "captions.txt")
+        self.scene_captions = {}
+        with open(captions_path, "r") as f:
+            for line in f:
+                parts = line.strip().split("\t", 1)
+                if len(parts) < 2:
+                    continue
+                clip_path, caption = parts
+                scene_name = "/".join(clip_path.split("/")[:-1])
+                fname = clip_path.split("/")[-1].replace(".mp4", "")
+                clip_start = int(fname.split("_")[0])
+                if scene_name not in self.scene_captions:
+                    self.scene_captions[scene_name] = []
+                self.scene_captions[scene_name].append((clip_start, caption))
+
+        for scene_name in self.scene_captions:
+            self.scene_captions[scene_name].sort(key=lambda x: x[0])
+
+        self.scene_names = sorted(self.scene_captions.keys())
+        self.metadata_rows = []
+        if metadata_path and os.path.isfile(metadata_path):
+            metadata = pd.read_csv(metadata_path)
+            if "prompt" in metadata.columns:
+                metadata["prompt"] = metadata["prompt"].astype(str)
+            self.metadata_rows = [metadata.iloc[i].to_dict() for i in range(len(metadata))]
+        self.pose_cache = {}
+        self.overlap_cache = {}
+        self.invalid_scenes = set()
+        self.invalid_metadata_indices = set()
+        self.overlap_labels_root = self._resolve_overlap_labels_root(base_path, self.overlap_labels_root)
+        self._validate_condition_config()
+
+        total_items = len(self.data)
+        total_scenes = len(self.scene_names)
+        total_captions = sum(len(v) for v in self.scene_captions.values())
+        metadata_msg = f", metadata_rows={len(self.metadata_rows)}" if self.metadata_rows else ""
+        effective_len = (len(self.metadata_rows) if self.metadata_rows else total_scenes) * repeat
+        print(f"CamVideoDataset: {total_scenes} scenes, {total_captions} captions{metadata_msg}, "
+              f"repeat={repeat}, cam_position_scale={self.cam_position_scale}, "
+              f"effective length={total_items}")
+
+    def _resolve_overlap_labels_root(self, base_path, overlap_labels_root):
+        candidate_roots = []
+        if overlap_labels_root is not None:
+            candidate_roots.append(overlap_labels_root)
+        if base_path is not None:
+            candidate_roots.append(os.path.join(base_path, "overlap_labels"))
+        for root in candidate_roots:
+            if root is not None and os.path.isdir(root):
+                return root
+        return overlap_labels_root
+
+    def _validate_condition_config(self):
+        if self.condition_t2v_ratio < 0 or self.condition_i2v_ratio < 0:
+            raise ValueError("Condition sampling ratios must be non-negative.")
+        if self.condition_t2v_ratio + self.condition_i2v_ratio >= 1.0:
+            raise ValueError("condition_t2v_ratio + condition_i2v_ratio must be < 1.0.")
+        needs_overlap = (
+            self.use_condition_context_frames
+            and self.condition_frame_mode == "first_plus_overlap"
+            and self.condition_history_keyframes
+            and self.num_condition_frames > 1
+        )
+        if needs_overlap and (self.overlap_labels_root is None or not os.path.isdir(self.overlap_labels_root)):
+            raise FileNotFoundError(
+                "K-frame condition mode requires overlap_labels_root. "
+                "Pass --overlap_labels_root or keep overlap_labels under dataset_base_path/overlap_labels."
+            )
+
+    def _load_scene_poses(self, scene_name):
+        if scene_name not in self.pose_cache:
+            json_path = os.path.join(self.jsons_dir, scene_name + ".json")
+            try:
+                with open(json_path, "r") as f:
+                    data = json.load(f)
+            except (FileNotFoundError, json.JSONDecodeError) as e:
+                raise ValueError(f"Pose JSON for scene '{scene_name}' is missing or corrupt: {e}")
+            if not isinstance(data, dict) or "CineCameraActor" not in data:
+                raise ValueError(
+                    f"Pose JSON for scene '{scene_name}' lacks 'CineCameraActor' key "
+                    f"(found keys: {list(data.keys()) if isinstance(data, dict) else type(data).__name__})."
+                )
+            cine = data["CineCameraActor"]
+            if not isinstance(cine, dict) or len(cine) == 0:
+                raise ValueError(f"Pose JSON for scene '{scene_name}' has empty 'CineCameraActor' entries.")
+            self.pose_cache[scene_name] = cine
+        return self.pose_cache[scene_name]
+
+    def _find_nearest_caption(self, scene_name, start_frame):
+        captions = self.scene_captions[scene_name]
+        best_idx = 0
+        best_dist = abs(captions[0][0] - start_frame)
+        for i, (clip_start, _) in enumerate(captions):
+            dist = abs(clip_start - start_frame)
+            if dist < best_dist:
+                best_dist = dist
+                best_idx = i
+        return captions[best_idx][1]
+
+    @staticmethod
+    def _compute_rt(position, rotation):
+        x, y, z = position
+        yaw_rad = np.radians(rotation[2])
+        cos_y, sin_y = np.cos(yaw_rad), np.sin(yaw_rad)
+        R = np.array([[cos_y, -sin_y, 0], [sin_y, cos_y, 0], [0, 0, 1]])
+        return [x, y, z] + R.flatten().tolist()
+
+    @staticmethod
+    def _to_relative_rt(rt_list, ref_rt):
+        R_ref = np.array(ref_rt[3:]).reshape(3, 3)
+        T_ref = np.array(ref_rt[:3]).reshape(3, 1)
+        R_ref_inv = R_ref.T
+        T_ref_inv = -R_ref_inv @ T_ref
+        result = []
+        for rt in rt_list:
+            R_i = np.array(rt[3:]).reshape(3, 3)
+            T_i = np.array(rt[:3]).reshape(3, 1)
+            R_new = R_ref_inv @ R_i
+            T_new = R_ref_inv @ T_i + T_ref_inv
+            result.append(T_new.flatten().tolist() + R_new.flatten().tolist())
+        return result
+
+    def crop_and_resize(self, image, target_height, target_width):
+        width, height = image.size
+        scale = max(target_width / width, target_height / height)
+        image = torchvision.transforms.functional.resize(
+            image,
+            (round(height * scale), round(width * scale)),
+            interpolation=torchvision.transforms.InterpolationMode.BILINEAR
+        )
+        image = torchvision.transforms.functional.center_crop(image, (target_height, target_width))
+        return image
+
+    def get_height_width(self, image):
+        if self.dynamic_resolution:
+            width, height = image.size
+            if width * height > self.max_pixels:
+                scale = (width * height / self.max_pixels) ** 0.5
+                height, width = int(height / scale), int(width / scale)
+            height = height // self.height_division_factor * self.height_division_factor
+            width = width // self.width_division_factor * self.width_division_factor
+        else:
+            height, width = self.height, self.width
+        return height, width
+
+    def _load_resized_frame(self, scene_name, frame_index, target_height, target_width):
+        frame_path = os.path.join(self.frames_dir, scene_name, f"{frame_index:04d}.png")
+        img = Image.open(frame_path).convert("RGB")
+        return self.crop_and_resize(img, target_height, target_width)
+
+    @staticmethod
+    def _parse_frame_token(token):
+        token = str(token).strip()
+        if not token:
+            return None, None
+        parts = token.split("/")
+        if parts and parts[0] == "frames":
+            parts = parts[1:]
+        if len(parts) < 2:
+            return None, None
+        scene_name = "/".join(parts[:-1])
+        stem = os.path.splitext(parts[-1])[0]
+        try:
+            frame_index = int(stem)
+        except ValueError:
+            return None, None
+        return scene_name, frame_index
+
+    def _metadata_scene_and_indices(self, row):
+        video_field = str(row.get("video", "") or "")
+        tokens = [t for t in video_field.split("|") if t]
+        parsed = [self._parse_frame_token(t) for t in tokens]
+        parsed = [(s, i) for s, i in parsed if s is not None and i is not None]
+        if parsed:
+            scene_name = str(row.get("video_name", "") or parsed[0][0])
+            frame_indices = [i for _, i in parsed[: self.num_frames]]
+        else:
+            scene_name = str(row.get("video_name", "") or "").strip()
+            if not scene_name:
+                raise ValueError("metadata row lacks video_name and parseable video paths")
+            start_frame = int(row.get("start_frame", 0) or 0)
+            frame_indices = list(range(start_frame, start_frame + self.num_frames))
+        if len(frame_indices) < self.num_frames:
+            raise ValueError(f"metadata row has {len(frame_indices)} frames, expected {self.num_frames}")
+        return scene_name, frame_indices[: self.num_frames]
+
+    def _resolve_geometry_path(self, token):
+        token = str(token).strip()
+        if not token:
+            return None
+        if os.path.isabs(token):
+            return token
+        root = self.geometry_memory_root or self.base_path
+        return os.path.join(root, token)
+
+    def _load_geometry_memory_frames(self, row):
+        value = row.get(self.geometry_memory_column, None)
+        if value is None or str(value).strip() == "" or str(value).lower() == "nan":
+            if self.use_geometry_spatial_memory:
+                raise ValueError(
+                    f"metadata row lacks required geometry column '{self.geometry_memory_column}'"
+                )
+            return []
+        tokens = [token for token in str(value).split("|") if token.strip()]
+        if not tokens:
+            raise ValueError(f"empty geometry memory field '{self.geometry_memory_column}'")
+
+        if len(tokens) == 1:
+            path = self._resolve_geometry_path(tokens[0])
+            if path is None:
+                return []
+            if os.path.isdir(path):
+                names = sorted(
+                    name
+                    for name in os.listdir(path)
+                    if os.path.splitext(name)[1].lower() in (".png", ".jpg", ".jpeg", ".webp")
+                )
+                paths = [os.path.join(path, name) for name in names]
+                return [Image.open(frame_path).convert("RGB") for frame_path in paths]
+            if os.path.splitext(path)[1].lower() in (".mp4", ".mov", ".avi", ".mkv", ".webm"):
+                reader = imageio.get_reader(path)
+                try:
+                    frames = [Image.fromarray(frame).convert("RGB") for frame in reader]
+                finally:
+                    reader.close()
+                if not frames:
+                    raise ValueError(f"geometry memory video has no frames: {path}")
+                return frames
+
+        frames = []
+        for token in tokens:
+            path = self._resolve_geometry_path(token)
+            if path is None or not os.path.isfile(path):
+                raise FileNotFoundError(f"geometry memory frame not found: {path}")
+            frames.append(Image.open(path).convert("RGB"))
+        return frames
+
+    def _load_overlap_frames(self, scene_name, frame_index):
+        if self.overlap_labels_root is None:
+            return []
+        cache_key = (scene_name, int(frame_index))
+        if cache_key not in self.overlap_cache:
+            overlap_path = os.path.join(self.overlap_labels_root, scene_name, f"{int(frame_index)}.json")
+            if not os.path.exists(overlap_path):
+                self.overlap_cache[cache_key] = []
+            else:
+                with open(overlap_path, "r") as f:
+                    overlap_data = json.load(f)
+                overlaps = overlap_data.get("overlapping_frames", [])
+                self.overlap_cache[cache_key] = [int(idx) for idx in overlaps]
+        return self.overlap_cache[cache_key]
+
+    def _compute_scene_rt(self, scene_name, frame_index):
+        frame_data = self._load_scene_poses(scene_name)[str(int(frame_index))]
+        raw_pos = frame_data["position"]
+        pos = [float(p) * self.cam_position_scale for p in raw_pos]
+        return self._compute_rt(pos, frame_data["rotation"])
+
+    def _sample_condition_mode(self):
+        if not self.use_condition_context_frames:
+            return "disabled"
+        if (
+            self.condition_frame_mode != "first_plus_overlap"
+            or not self.condition_history_keyframes
+            or self.num_condition_frames <= 1
+        ):
+            return "first_frame_only"
+        sample = random.random()
+        if sample < self.condition_t2v_ratio:
+            return "text_only"
+        if sample < self.condition_t2v_ratio + self.condition_i2v_ratio:
+            return "first_frame_only"
+        return "first_plus_overlap"
+
+    def _sample_overlap_conditions(self, scene_name, start_frame, ref_rt, target_height, target_width, num_extra_conditions):
+        if num_extra_conditions <= 0:
+            return [], [], []
+        window_indices = set(range(start_frame, start_frame + self.num_frames))
+        target_candidates = list(range(start_frame + 1, start_frame + self.num_frames))
+        sampled_target_frames = random.sample(target_candidates, k=min(num_extra_conditions, len(target_candidates)))
+        overlap_frames = []
+        overlap_indices = []
+        overlap_actions = []
+        used_condition_indices = set()
+        for target_frame_idx in sampled_target_frames:
+            candidate_indices = [
+                idx for idx in self._load_overlap_frames(scene_name, target_frame_idx)
+                if idx not in window_indices and idx != target_frame_idx and idx not in used_condition_indices
+            ]
+            if len(candidate_indices) == 0:
+                return None
+            chosen_idx = random.choice(candidate_indices)
+            used_condition_indices.add(chosen_idx)
+            overlap_indices.append(chosen_idx)
+            overlap_frames.append(self._load_resized_frame(scene_name, chosen_idx, target_height, target_width))
+            if self.condition_use_camera_pose:
+                overlap_rt = self._compute_scene_rt(scene_name, chosen_idx)
+                overlap_actions.append(self._to_relative_rt([overlap_rt], ref_rt)[0])
+        if len(overlap_frames) != num_extra_conditions:
+            return None
+        return overlap_frames, overlap_indices, overlap_actions
+
+    def _try_get_sample(self, data_item):
+        scene_name = data_item["video_name"]
+        cam_data = self._load_scene_poses(scene_name)
+        max_start = len(cam_data) - self.num_frames
+        if max_start < 0:
+            raise ValueError(f"Scene {scene_name} has fewer than {self.num_frames} frames.")
+        start_frame = data_item["start_frame"]
+        end_frame = start_frame + self.num_frames - 1
+        assert end_frame == data_item["end_frame"]
+
+        frames = []
+        for i in range(start_frame, end_frame + 1):
+            frame_path = os.path.join(self.frames_dir, scene_name, f"{i:04d}.png")
+            img = Image.open(frame_path).convert("RGB")
+            img = self.crop_and_resize(img, *self.get_height_width(img))
+            frames.append(img)
+
+        # prompt = self._find_nearest_caption(scene_name, start_frame)
+        prompt = data_item["prompt"]
+
+        rt_list_abs = []
+        for i in range(start_frame, end_frame + 1):
+            key = str(i)
+            if key not in cam_data:
+                raise ValueError(f"Scene {scene_name} missing pose for frame {i}.")
+            frame_data = cam_data[key]
+            raw_pos = frame_data["position"]
+            pos = [float(p) * self.cam_position_scale for p in raw_pos]
+            rt = self._compute_rt(pos, frame_data["rotation"])
+            rt_list_abs.append(rt)
+
+        rt_list = self._to_relative_rt(rt_list_abs, rt_list_abs[0])
+        pose_indices = list(range(0, self.num_frames, 4))
+        actions = [rt_list[i] for i in pose_indices]
+
+        return {
+            "video": frames,
+            "prompt": prompt,
+            "actions": actions,
+            "video_name": scene_name,
+            "start_frame": start_frame,
+            "end_frame": end_frame,
+            **self._build_condition_context_payload(
+                frames=frames,
+                scene_name=scene_name,
+                start_frame=start_frame,
+                ref_rt=rt_list_abs[0],
+                actions=actions,
+            ),
+        }
+
+    def _try_get_metadata_sample(self, row):
+        scene_name, frame_indices = self._metadata_scene_and_indices(row)
+        start_frame = int(frame_indices[0])
+        end_frame = int(frame_indices[-1])
+        cam_data = self._load_scene_poses(scene_name)
+
+        frames = []
+        for frame_idx in frame_indices:
+            frame_path = os.path.join(self.frames_dir, scene_name, f"{int(frame_idx):04d}.png")
+            img = Image.open(frame_path).convert("RGB")
+            img = self.crop_and_resize(img, *self.get_height_width(img))
+            frames.append(img)
+
+        prompt = row.get("prompt", None)
+        if prompt is None or str(prompt).strip() == "" or str(prompt).lower() == "nan":
+            prompt = self._find_nearest_caption(scene_name, start_frame)
+        else:
+            prompt = str(prompt)
+
+        rt_list_abs = []
+        for frame_idx in frame_indices:
+            key = str(int(frame_idx))
+            if key not in cam_data:
+                raise ValueError(f"Scene {scene_name} missing pose for frame {frame_idx}.")
+            frame_data = cam_data[key]
+            raw_pos = frame_data["position"]
+            pos = [float(p) * self.cam_position_scale for p in raw_pos]
+            rt = self._compute_rt(pos, frame_data["rotation"])
+            rt_list_abs.append(rt)
+
+        rt_list = self._to_relative_rt(rt_list_abs, rt_list_abs[0])
+        pose_indices = list(range(0, len(frame_indices), 4))
+        actions = [rt_list[i] for i in pose_indices]
+        geometry_memory_frames = self._load_geometry_memory_frames(row)
+
+        return {
+            "video": frames,
+            "prompt": prompt,
+            "actions": actions,
+            "video_name": scene_name,
+            "start_frame": start_frame,
+            "end_frame": end_frame,
+            "geometry_memory_frames": geometry_memory_frames,
+            **self._build_condition_context_payload(
+                frames=frames,
+                scene_name=scene_name,
+                start_frame=start_frame,
+                ref_rt=rt_list_abs[0],
+                actions=actions,
+            ),
+        }
+
+    def __getitem__(self, data_id):
+        n = len(self.data)
+        if n == 0:
+            raise RuntimeError("CamVideoDataset has no scenes.")
+        max_attempts = min(64, n)
+        last_error = None
+        for attempt in range(max_attempts):
+            idx = (data_id + attempt) % n
+            data_item = self.data[idx]
+            # if scene_name in self.invalid_scenes:
+            #     continue
+            # try:
+            return self._try_get_sample(data_item)
+            # except (ValueError, FileNotFoundError, KeyError, OSError) as e:
+            #     self.invalid_scenes.add(scene_name)
+            #     last_error = e
+            #     if attempt < 3 or attempt % 8 == 0:
+            #         print(
+            #             f"[CamVideoDataset] Skipping invalid scene '{scene_name}' "
+            #             f"({type(e).__name__}: {e}); attempt {attempt + 1}/{max_attempts}"
+            #         )
+            #     continue
+        raise RuntimeError(
+            f"CamVideoDataset: exhausted {max_attempts} attempts starting from index {data_id}; "
+            f"last error: {type(last_error).__name__}: {last_error}"
+        )
+
+    def _build_condition_context_payload(self, frames, scene_name, start_frame, ref_rt, actions):
+        if not self.use_condition_context_frames:
+            return {}
+        payload = {
+            "use_condition_context_frames": False,
+            "condition_frames": [],
+            "condition_frame_indices": [],
+            "condition_source": None,
+            "condition_actions": [],
+        }
+        condition_mode = self._sample_condition_mode()
+        payload["condition_source"] = condition_mode
+        if condition_mode == "text_only":
+            return payload
+        payload["use_condition_context_frames"] = True
+        if self.condition_first_frame:
+            payload["condition_frames"].append(frames[0])
+            payload["condition_frame_indices"].append(start_frame)
+            payload["condition_source"] = "first_frame_only"
+            if self.condition_use_camera_pose and actions:
+                payload["condition_actions"].append(list(actions[0]))
+        if (
+            condition_mode == "first_plus_overlap"
+            and self.condition_history_keyframes
+            and self.num_condition_frames > len(payload["condition_frames"])
+        ):
+            num_extra_conditions = self.num_condition_frames - len(payload["condition_frames"])
+            overlap_payload = self._sample_overlap_conditions(
+                scene_name=scene_name,
+                start_frame=start_frame,
+                ref_rt=ref_rt,
+                target_height=frames[0].size[1],
+                target_width=frames[0].size[0],
+                num_extra_conditions=num_extra_conditions,
+            )
+            if overlap_payload is None:
+                return payload
+            overlap_frames, overlap_indices, overlap_actions = overlap_payload
+            payload["condition_frames"].extend(overlap_frames)
+            payload["condition_frame_indices"].extend(overlap_indices)
+            if self.condition_use_camera_pose:
+                payload["condition_actions"].extend(overlap_actions)
+            payload["condition_source"] = "first_plus_overlap"
+        return payload
+
+    def __len__(self):
+        return len(self.data)
+
+
+class DiffusionTrainingModule(torch.nn.Module):
+    def __init__(self):
+        super().__init__()
+        
+        
+    def to(self, *args, **kwargs):
+        for name, model in self.named_children():
+            model.to(*args, **kwargs)
+        return self
+        
+        
+    def trainable_modules(self):
+        trainable_modules = filter(lambda p: p.requires_grad, self.parameters())
+        return trainable_modules
+    
+    
+    def trainable_param_names(self):
+        trainable_param_names = list(filter(lambda named_param: named_param[1].requires_grad, self.named_parameters()))
+        trainable_param_names = set([named_param[0] for named_param in trainable_param_names])
+        return trainable_param_names
+    
+    
+    def add_lora_to_model(self, model, target_modules, lora_rank, lora_alpha=None):
+        if lora_alpha is None:
+            lora_alpha = lora_rank
+        lora_config = LoraConfig(r=lora_rank, lora_alpha=lora_alpha, target_modules=target_modules)
+        model = inject_adapter_in_model(lora_config, model)
+        return model
+    
+    
+    def export_trainable_state_dict(self, state_dict, remove_prefix=None):
+        trainable_param_names = self.trainable_param_names()
+        state_dict = {name: param for name, param in state_dict.items() if name in trainable_param_names}
+        if remove_prefix is not None:
+            state_dict_ = {}
+            for name, param in state_dict.items():
+                if name.startswith(remove_prefix):
+                    name = name[len(remove_prefix):]
+                state_dict_[name] = param
+            state_dict = state_dict_
+        return state_dict
+
+
+
+class ModelLogger:
+    def __init__(self, output_path, remove_prefix_in_ckpt=None, state_dict_converter=lambda x:x):
+        self.output_path = output_path
+        self.remove_prefix_in_ckpt = remove_prefix_in_ckpt
+        self.state_dict_converter = state_dict_converter
+        
+    
+    def on_step_end(self, loss):
+        pass
+    
+    
+    def on_epoch_end(self, accelerator, model, epoch_id):
+        accelerator.wait_for_everyone()
+        if accelerator.is_main_process:
+            state_dict = accelerator.get_state_dict(model)
+            state_dict = accelerator.unwrap_model(model).export_trainable_state_dict(state_dict, remove_prefix=self.remove_prefix_in_ckpt)
+            state_dict = self.state_dict_converter(state_dict)
+            os.makedirs(self.output_path, exist_ok=True)
+            path = os.path.join(self.output_path, f"epoch-{epoch_id}.safetensors")
+            accelerator.save(state_dict, path, safe_serialization=True)
+
+
+
+def launch_training_task(
+    dataset: torch.utils.data.Dataset,
+    model: DiffusionTrainingModule,
+    model_logger: ModelLogger,
+    optimizer: torch.optim.Optimizer,
+    scheduler: torch.optim.lr_scheduler.LRScheduler,
+    num_epochs: int = 1,
+    gradient_accumulation_steps: int = 1,
+):
+    dataloader = torch.utils.data.DataLoader(dataset, shuffle=True, collate_fn=lambda x: x[0], drop_last=True)
+    accelerator = Accelerator(gradient_accumulation_steps=gradient_accumulation_steps)
+    model, optimizer, dataloader, scheduler = accelerator.prepare(model, optimizer, dataloader, scheduler)
+    
+    for epoch_id in range(num_epochs):
+        for data in tqdm(dataloader):
+            with accelerator.accumulate(model):
+                optimizer.zero_grad()
+                loss = model(data)
+                accelerator.backward(loss)
+                optimizer.step()
+                model_logger.on_step_end(loss)
+                scheduler.step()
+        model_logger.on_epoch_end(accelerator, model, epoch_id)
+
+def launch_data_process_task(model: DiffusionTrainingModule, dataset, output_path="./models"):
+    dataloader = torch.utils.data.DataLoader(dataset, shuffle=False, collate_fn=lambda x: x[0], drop_last=True)
+    accelerator = Accelerator()
+    model, dataloader = accelerator.prepare(model, dataloader)
+    os.makedirs(os.path.join(output_path, "data_cache"), exist_ok=True)
+    for data_id, data in enumerate(tqdm(dataloader)):
+        with torch.no_grad():
+            inputs = model.forward_preprocess(data)
+            inputs = {key: inputs[key] for key in model.model_input_keys if key in inputs}
+            torch.save(inputs, os.path.join(output_path, "data_cache", f"{data_id}.pth"))
+
+
+
+def wan_parser():
+    parser = argparse.ArgumentParser(description="Simple example of a training script.")
+    parser.add_argument("--dataset_base_path", type=str, default="", required=True, help="Base path of the dataset.")
+    parser.add_argument("--dataset_metadata_path", type=str, default=None, help="Path to the metadata file of the dataset.")
+    parser.add_argument("--max_pixels", type=int, default=1280*720, help="Maximum number of pixels per frame, used for dynamic resolution..")
+    parser.add_argument("--height", type=int, default=None, help="Height of images or videos. Leave `height` and `width` empty to enable dynamic resolution.")
+    parser.add_argument("--width", type=int, default=None, help="Width of images or videos. Leave `height` and `width` empty to enable dynamic resolution.")
+    parser.add_argument("--num_frames", type=int, default=81, help="Number of frames per video. Frames are sampled from the video prefix.")
+    parser.add_argument("--data_file_keys", type=str, default="image,video", help="Data file keys in the metadata. Comma-separated.")
+    parser.add_argument("--dataset_repeat", type=int, default=1, help="Number of times to repeat the dataset per epoch.")
+    parser.add_argument("--model_paths", type=str, default=None, help="Paths to load models. In JSON format.")
+    parser.add_argument("--model_id_with_origin_paths", type=str, default=None, help="Model ID with origin paths, e.g., Wan-AI/Wan2.1-T2V-1.3B:diffusion_pytorch_model*.safetensors. Comma-separated.")
+    parser.add_argument("--learning_rate", type=float, default=1e-4, help="Learning rate.")
+    parser.add_argument("--num_epochs", type=int, default=1, help="Number of epochs.")
+    parser.add_argument("--output_path", type=str, default="./models", help="Output save path.")
+    parser.add_argument("--remove_prefix_in_ckpt", type=str, default="pipe.dit.", help="Remove prefix in ckpt.")
+    parser.add_argument("--trainable_models", type=str, default=None, help="Models to train, e.g., dit, vae, text_encoder.")
+    parser.add_argument("--lora_base_model", type=str, default=None, help="Which model LoRA is added to.")
+    parser.add_argument("--lora_target_modules", type=str, default="q,k,v,o,ffn.0,ffn.2", help="Which layers LoRA is added to.")
+    parser.add_argument("--lora_rank", type=int, default=32, help="Rank of LoRA.")
+    parser.add_argument("--extra_inputs", default=None, help="Additional model inputs, comma-separated.")
+    parser.add_argument("--use_gradient_checkpointing_offload", default=False, action="store_true", help="Whether to offload gradient checkpointing to CPU memory.")
+    parser.add_argument("--gradient_accumulation_steps", type=int, default=1, help="Gradient accumulation steps.")
+    parser.add_argument("--use_condition_context_frames", default=False, action="store_true", help="Enable appended clean condition latents.")
+    parser.add_argument("--condition_first_frame", default=False, action="store_true", help="Use the current clip first frame as a clean condition frame.")
+    parser.add_argument("--condition_history_keyframes", default=False, action="store_true", help="Use overlap-based keyframes as conditions.")
+    parser.add_argument("--condition_use_camera_pose", default=True, action="store_true", help="Inject camera pose for condition frames.")
+    parser.add_argument("--num_condition_frames", type=int, default=1, help="Number of condition frames.")
+    parser.add_argument("--condition_frame_mode", type=str, default="first_frame_only", help="Condition frame selection mode.")
+    parser.add_argument("--overlap_labels_root", type=str, default=None, help="Root dir for overlap label JSONs.")
+    parser.add_argument("--condition_t2v_ratio", type=float, default=0.10, help="Ratio of text-only condition samples.")
+    parser.add_argument("--condition_i2v_ratio", type=float, default=0.10, help="Ratio of first-frame-only condition samples.")
+    return parser
+
+
+
+def flux_parser():
+    parser = argparse.ArgumentParser(description="Simple example of a training script.")
+    parser.add_argument("--dataset_base_path", type=str, default="", required=True, help="Base path of the dataset.")
+    parser.add_argument("--dataset_metadata_path", type=str, default=None, help="Path to the metadata file of the dataset.")
+    parser.add_argument("--max_pixels", type=int, default=1024*1024, help="Maximum number of pixels per frame, used for dynamic resolution..")
+    parser.add_argument("--height", type=int, default=None, help="Height of images. Leave `height` and `width` empty to enable dynamic resolution.")
+    parser.add_argument("--width", type=int, default=None, help="Width of images. Leave `height` and `width` empty to enable dynamic resolution.")
+    parser.add_argument("--data_file_keys", type=str, default="image", help="Data file keys in the metadata. Comma-separated.")
+    parser.add_argument("--dataset_repeat", type=int, default=1, help="Number of times to repeat the dataset per epoch.")
+    parser.add_argument("--model_paths", type=str, default=None, help="Paths to load models. In JSON format.")
+    parser.add_argument("--model_id_with_origin_paths", type=str, default=None, help="Model ID with origin paths, e.g., Wan-AI/Wan2.1-T2V-1.3B:diffusion_pytorch_model*.safetensors. Comma-separated.")
+    parser.add_argument("--learning_rate", type=float, default=1e-4, help="Learning rate.")
+    parser.add_argument("--num_epochs", type=int, default=1, help="Number of epochs.")
+    parser.add_argument("--output_path", type=str, default="./models", help="Output save path.")
+    parser.add_argument("--remove_prefix_in_ckpt", type=str, default="pipe.dit.", help="Remove prefix in ckpt.")
+    parser.add_argument("--trainable_models", type=str, default=None, help="Models to train, e.g., dit, vae, text_encoder.")
+    parser.add_argument("--lora_base_model", type=str, default=None, help="Which model LoRA is added to.")
+    parser.add_argument("--lora_target_modules", type=str, default="q,k,v,o,ffn.0,ffn.2", help="Which layers LoRA is added to.")
+    parser.add_argument("--lora_rank", type=int, default=32, help="Rank of LoRA.")
+    parser.add_argument("--extra_inputs", default=None, help="Additional model inputs, comma-separated.")
+    parser.add_argument("--align_to_opensource_format", default=False, action="store_true", help="Whether to align the lora format to opensource format. Only for DiT's LoRA.")
+    parser.add_argument("--use_gradient_checkpointing", default=False, action="store_true", help="Whether to use gradient checkpointing.")
+    parser.add_argument("--use_gradient_checkpointing_offload", default=False, action="store_true", help="Whether to offload gradient checkpointing to CPU memory.")
+    parser.add_argument("--gradient_accumulation_steps", type=int, default=1, help="Gradient accumulation steps.")
+    return parser

code/diffsynth/vram_management/__init__.py

@@ -0,0 +1,2 @@
+from .layers import *
+from .gradient_checkpointing import *

code/diffsynth/vram_management/gradient_checkpointing.py

@@ -0,0 +1,34 @@
+import torch
+
+
+def create_custom_forward(module):
+    def custom_forward(*inputs, **kwargs):
+        return module(*inputs, **kwargs)
+    return custom_forward
+
+
+def gradient_checkpoint_forward(
+    model,
+    use_gradient_checkpointing,
+    use_gradient_checkpointing_offload,
+    *args,
+    **kwargs,
+):
+    if use_gradient_checkpointing_offload:
+        with torch.autograd.graph.save_on_cpu():
+            model_output = torch.utils.checkpoint.checkpoint(
+                create_custom_forward(model),
+                *args,
+                **kwargs,
+                use_reentrant=False,
+            )
+    elif use_gradient_checkpointing:
+        model_output = torch.utils.checkpoint.checkpoint(
+            create_custom_forward(model),
+            *args,
+            **kwargs,
+            use_reentrant=False,
+        )
+    else:
+        model_output = model(*args, **kwargs)
+    return model_output

code/diffsynth/vram_management/layers.py

@@ -0,0 +1,170 @@
+import torch, copy
+from ..models.utils import init_weights_on_device
+
+
+def cast_to(weight, dtype, device):
+    r = torch.empty_like(weight, dtype=dtype, device=device)
+    r.copy_(weight)
+    return r
+
+
+class AutoTorchModule(torch.nn.Module):
+    def __init__(self):
+        super().__init__()
+        
+    def check_free_vram(self):
+        _dev = self.computation_device
+        if not (isinstance(_dev, torch.device) and _dev.index is not None):
+            _dev = 0
+        gpu_mem_state = torch.cuda.mem_get_info(_dev)
+        used_memory = (gpu_mem_state[1] - gpu_mem_state[0]) / (1024 ** 3)
+        return used_memory < self.vram_limit
+
+    def offload(self):
+        if self.state != 0:
+            self.to(dtype=self.offload_dtype, device=self.offload_device)
+            self.state = 0
+
+    def onload(self):
+        if self.state != 1:
+            self.to(dtype=self.onload_dtype, device=self.onload_device)
+            self.state = 1
+            
+    def keep(self):
+        if self.state != 2:
+            self.to(dtype=self.computation_dtype, device=self.computation_device)
+            self.state = 2
+
+
+class AutoWrappedModule(AutoTorchModule):
+    def __init__(self, module: torch.nn.Module, offload_dtype, offload_device, onload_dtype, onload_device, computation_dtype, computation_device, vram_limit, **kwargs):
+        super().__init__()
+        self.module = module.to(dtype=offload_dtype, device=offload_device)
+        self.offload_dtype = offload_dtype
+        self.offload_device = offload_device
+        self.onload_dtype = onload_dtype
+        self.onload_device = onload_device
+        self.computation_dtype = computation_dtype
+        self.computation_device = computation_device
+        self.vram_limit = vram_limit
+        self.state = 0
+
+    def forward(self, *args, **kwargs):
+        if self.state == 2:
+            module = self.module
+        else:
+            if self.onload_dtype == self.computation_dtype and self.onload_device == self.computation_device:
+                module = self.module
+            elif self.vram_limit is not None and self.check_free_vram():
+                self.keep()
+                module = self.module
+            else:
+                module = copy.deepcopy(self.module).to(dtype=self.computation_dtype, device=self.computation_device)
+        return module(*args, **kwargs)
+    
+
+class WanAutoCastLayerNorm(torch.nn.LayerNorm, AutoTorchModule):
+    def __init__(self, module: torch.nn.LayerNorm, offload_dtype, offload_device, onload_dtype, onload_device, computation_dtype, computation_device, vram_limit, **kwargs):
+        with init_weights_on_device(device=torch.device("meta")):
+            super().__init__(module.normalized_shape, eps=module.eps, elementwise_affine=module.elementwise_affine, bias=module.bias is not None, dtype=offload_dtype, device=offload_device)
+        self.weight = module.weight
+        self.bias = module.bias
+        self.offload_dtype = offload_dtype
+        self.offload_device = offload_device
+        self.onload_dtype = onload_dtype
+        self.onload_device = onload_device
+        self.computation_dtype = computation_dtype
+        self.computation_device = computation_device
+        self.vram_limit = vram_limit
+        self.state = 0
+
+    def forward(self, x, *args, **kwargs):
+        if self.state == 2:
+            weight, bias = self.weight, self.bias
+        else:
+            if self.onload_dtype == self.computation_dtype and self.onload_device == self.computation_device:
+                weight, bias = self.weight, self.bias
+            elif self.vram_limit is not None and self.check_free_vram():
+                self.keep()
+                weight, bias = self.weight, self.bias
+            else:
+                weight = None if self.weight is None else cast_to(self.weight, self.computation_dtype, self.computation_device)
+                bias = None if self.bias is None else cast_to(self.bias, self.computation_dtype, self.computation_device)
+        with torch.amp.autocast(device_type=x.device.type):
+            x = torch.nn.functional.layer_norm(x.float(), self.normalized_shape, weight, bias, self.eps).type_as(x)
+        return x
+    
+
+class AutoWrappedLinear(torch.nn.Linear, AutoTorchModule):
+    def __init__(self, module: torch.nn.Linear, offload_dtype, offload_device, onload_dtype, onload_device, computation_dtype, computation_device, vram_limit, name="", **kwargs):
+        with init_weights_on_device(device=torch.device("meta")):
+            super().__init__(in_features=module.in_features, out_features=module.out_features, bias=module.bias is not None, dtype=offload_dtype, device=offload_device)
+        self.weight = module.weight
+        self.bias = module.bias
+        self.offload_dtype = offload_dtype
+        self.offload_device = offload_device
+        self.onload_dtype = onload_dtype
+        self.onload_device = onload_device
+        self.computation_dtype = computation_dtype
+        self.computation_device = computation_device
+        self.vram_limit = vram_limit
+        self.state = 0
+        self.name = name
+        self.lora_A_weights = []
+        self.lora_B_weights = []
+        self.lora_merger = None
+
+    def forward(self, x, *args, **kwargs):
+        if self.state == 2:
+            weight, bias = self.weight, self.bias
+        else:
+            if self.onload_dtype == self.computation_dtype and self.onload_device == self.computation_device:
+                weight, bias = self.weight, self.bias
+            elif self.vram_limit is not None and self.check_free_vram():
+                self.keep()
+                weight, bias = self.weight, self.bias
+            else:
+                weight = cast_to(self.weight, self.computation_dtype, self.computation_device)
+                bias = None if self.bias is None else cast_to(self.bias, self.computation_dtype, self.computation_device)
+        out = torch.nn.functional.linear(x, weight, bias)
+        
+        if len(self.lora_A_weights) == 0:
+            # No LoRA
+            return out
+        elif self.lora_merger is None:
+            # Native LoRA inference
+            for lora_A, lora_B in zip(self.lora_A_weights, self.lora_B_weights):
+                out = out + x @ lora_A.T @ lora_B.T
+        else:
+            # LoRA fusion
+            lora_output = []
+            for lora_A, lora_B in zip(self.lora_A_weights, self.lora_B_weights):
+                lora_output.append(x @ lora_A.T @ lora_B.T)
+            lora_output = torch.stack(lora_output)
+            out = self.lora_merger(out, lora_output)
+        return out
+
+
+def enable_vram_management_recursively(model: torch.nn.Module, module_map: dict, module_config: dict, max_num_param=None, overflow_module_config: dict = None, total_num_param=0, vram_limit=None, name_prefix=""):
+    for name, module in model.named_children():
+        layer_name = name if name_prefix == "" else name_prefix + "." + name
+        for source_module, target_module in module_map.items():
+            if isinstance(module, source_module):
+                num_param = sum(p.numel() for p in module.parameters())
+                if max_num_param is not None and total_num_param + num_param > max_num_param:
+                    module_config_ = overflow_module_config
+                else:
+                    module_config_ = module_config
+                module_ = target_module(module, **module_config_, vram_limit=vram_limit, name=layer_name)
+                setattr(model, name, module_)
+                total_num_param += num_param
+                break
+        else:
+            total_num_param = enable_vram_management_recursively(module, module_map, module_config, max_num_param, overflow_module_config, total_num_param, vram_limit=vram_limit, name_prefix=layer_name)
+    return total_num_param
+
+
+def enable_vram_management(model: torch.nn.Module, module_map: dict, module_config: dict, max_num_param=None, overflow_module_config: dict = None, vram_limit=None):
+    enable_vram_management_recursively(model, module_map, module_config, max_num_param, overflow_module_config, total_num_param=0, vram_limit=vram_limit)
+    model.vram_management_enabled = True
+

code/doc/DEVELOPER.md

@@ -0,0 +1,116 @@
+# Developer Guide / 开发者指南
+
+Bilingual interactive version: [Project Page → Developer Guide](https://echo-team-joy-future-academy-jd.github.io/Echo-Memory/developer.html)
+
+Hands-on development, training/eval workflows, and **Cursor Agent skills** for Echo-Memory.
+
+实战开发、训练评测与 **Cursor Agent 技能**。
+
+---
+
+## Cursor skills / 项目 Skills
+
+Project skills live in **`.cursor/skills/`** — reference them in Agent chat (e.g. *use echo-memory-eval to …*).
+
+| Skill | English | 中文 |
+| --- | --- | --- |
+| `echo-memory-agent` | Scope prompts, rules, skill index | Prompt 范围、Rules、技能索引 |
+| `echo-memory-train` | Memory baselines & context training | Baseline 与 Context 训练 |
+| `echo-memory-eval` | Replay / revisit & HF checkpoint checks | 回放 / revisit、HF checkpoint check |
+| `echo-memory-release` | gh-pages, i18n, checkpoints doc | gh-pages、i18n、权重文档 |
+
+Index: [.cursor/skills/README.md](../.cursor/skills/README.md)
+
+---
+
+## 1. Guide map / 文档地图
+
+| | English | 中文 |
+| --- | --- | --- |
+| **README** | Paper overview, quick start, checkpoints, community | 论文概览、快速上手、权重、社区 |
+| **This guide** | Workflows, Cursor skills, Agent tips | 工作流、Skills、Agent 技巧 |
+| **`doc/`** | Dataset & checkpoint reference | 数据集与权重参考 |
+
+---
+
+## 2. Environment & paths / 环境与路径
+
+```bash
+export WAN_BASE_MODEL=/path/to/Wan2.1-T2V-1.3B
+export DATASET_BASE_PATH=data/Context-as-Memory-Dataset
+export PYTHONPATH=$PWD:${PYTHONPATH:-}
+export OUTPUT_BASE_ROOT=$PWD/outputs
+```
+
+| Pool | English | 中文 |
+| --- | --- | --- |
+| Static in-domain | Default root above — [dataset_preprocessing.md](dataset_preprocessing.md) | 默认路径 — 同上 |
+| Dynamic training | e.g. `data/dynamic-memory-dataset` — [dynamic_dataset_preprocessing.md](dynamic_dataset_preprocessing.md) | 如 `data/dynamic-memory-dataset` — 同上 |
+| Checkpoints | [Echo-Team/Echo-Memory](https://huggingface.co/Echo-Team/Echo-Memory) — [checkpoints.md](checkpoints.md) | 同上 |
+
+---
+
+## 3. Code map / 代码地图
+
+| Path | English | 中文 |
+| --- | --- | --- |
+| `.cursor/skills/` | Cursor Agent skills | Agent 技能 |
+| `train/memory_baselines_basic/` | Spatial / SSM / compression | Spatial / SSM / 压缩 |
+| `train/context_learning/` | Context K=1/5/20 | Context 配方 |
+| `eval/v2/` | Replay, revisit | 回放、revisit |
+| `env/memory_baseline_runtime.py` | CKPT → memory profile | 权重 → 记忆配置 |
+| `docs/` | GitHub Pages | 项目页 |
+
+---
+
+## 4. Common workflows / 常用工作流
+
+**Train / 训练**
+
+```bash
+bash train/memory_baselines_basic/run_spatial_memory_baseline.sh
+bash train/context_learning/run_pre_qkv_ctx20.sh
+```
+
+**Checkpoint eval / checkpoint 检查**
+
+```bash
+huggingface-cli download Echo-Team/Echo-Memory context_k1/epoch-0.safetensors --local-dir ./ckpts
+export CKPT=./ckpts/context_k1/epoch-0.safetensors
+bash eval/v2/run_static_consistency_loop_and_revisit.sh
+```
+
+Keep the row folder name in `CKPT`.
+
+---
+
+## 5. Agent prompts / 示例 Prompt
+
+```text
+Using echo-memory-eval: download context_k1 from Echo-Team/Echo-Memory
+and run eval/v2/run_basic_replay_gt.sh with the static in-domain pool.
+
+Using echo-memory-train: document OUTPUT_BASE_ROOT override in
+run_ablation_block_wise_ssm_two_chunk.sh.
+```
+
+**Public repo hygiene / 公开仓库规范:** no upload bash, internal benchmark names, or machine paths in GitHub.
+
+---
+
+## 6. Site & release / 站点与发布
+
+```bash
+bash scripts/publish_gh_pages.sh
+```
+
+Community QR: [project page → Updates](https://echo-team-joy-future-academy-jd.github.io/Echo-Memory/#updates) or README **Community**.
+
+---
+
+## 7. Checklist / 检查清单
+
+- [ ] Quick eval with one HF checkpoint
+- [ ] `doc/checkpoints.md` matches HF folders
+- [ ] Public docs use Echo pool names
+- [ ] Publish gh-pages after site edits; verify EN/中文 toggle

code/doc/README.md

@@ -0,0 +1,13 @@
+# Echo-Memory documentation
+
+| Doc | Echo pool | Covers |
+| --- | --- | --- |
+| [DEVELOPER.md](DEVELOPER.md) | **Developer guide** — workflows, `.cursor/skills/`, Cursor Agent |
+| [checkpoints.md](checkpoints.md) | **Hugging Face weights** — [Echo-Team/Echo-Memory](https://huggingface.co/Echo-Team/Echo-Memory) baseline index |
+| [memory_mechanisms.md](memory_mechanisms.md) | **Memory mechanisms** — paper row names, code modules, and training scripts |
+| [dataset_preprocessing.md](dataset_preprocessing.md) | Static in-domain pool | Echo-Team package download → layout → metadata → latents |
+| [dynamic_dataset_preprocessing.md](dynamic_dataset_preprocessing.md) | Dynamic training pool | subset download → export → training settings |
+
+**Static in-domain pool:** download the Echo-Team package before in-domain replay/revisit eval.
+
+**Dynamic training pool:** SpatialVID subset export + `DATASET_BASE_PATH` before training on the dynamic pool.

code/doc/checkpoints.md

@@ -0,0 +1,72 @@
+# Checkpoints (Hugging Face)
+
+**Repo:** [Echo-Team/Echo-Memory](https://huggingface.co/Echo-Team/Echo-Memory)
+
+Fine-tuned DiT weights on top of [Wan-AI/Wan2.1-T2V-1.3B](https://huggingface.co/Wan-AI/Wan2.1-T2V-1.3B). Released rows are saved as `{row_id}/epoch-0.safetensors` after **1 epoch / 30,000 steps** on the static in-domain pool (640×352, 81-frame chunks). Mechanism names follow [memory_mechanisms.md](memory_mechanisms.md).
+
+## Checkpoint index
+
+| Family | Paper row | HF path | Steps | Echo-Memory recipe |
+| --- | --- | --- | ---: | --- |
+| Raw context | Context K=1 | [`context_k1/epoch-0.safetensors`](https://huggingface.co/Echo-Team/Echo-Memory/tree/main/context_k1) | 30,000 | `train/context_learning/run_pre_qkv_ctx1.sh` |
+| Raw context | Context K=20 | TODO | TODO | `train/context_learning/run_pre_qkv_ctx20.sh` |
+| Spatial | Spatial Memory | TODO | TODO | `train/memory_baselines_basic/run_spatial_memory_baseline.sh` |
+| State-space | Block-wise SSM | TODO | TODO | `train/memory_baselines_basic/run_ablation_block_wise_ssm_two_chunk.sh` |
+| State-space | Legacy Hybrid (VideoSSM) | TODO | TODO | `train/memory_baselines_basic/run_videossm_hybrid_baseline.sh` |
+| Spatial | concat text (ablation) | TODO | TODO | `train/memory_baselines_basic/run_ablation_spatial_concat_text_two_chunk.sh` |
+| Spatial | inject none (ablation) | TODO | TODO | `train/memory_baselines_basic/run_ablation_spatial_inject_none_two_chunk.sh` |
+| Spatial | cross-attn t32 (ablation) | TODO | TODO | `train/memory_baselines_basic/run_ablation_spatial_cross_attn_readout_two_chunk.sh` |
+| State-space | SSM ctx1 / every4 / hint21 | TODO | TODO | SSM ablation |
+| State-space | SSM ctx5 / every1 / hint21 | TODO | TODO | SSM ablation |
+| State-space | SSM ctx5 / every4 / hint81 | TODO | TODO | SSM ablation |
+
+Context K=5, Context K=20, Spatial memory, FramePack compression, and State-space / SSM rows are TODO and not yet released as `epoch-0` weights.
+
+## Download
+
+```bash
+pip install -U "huggingface_hub[cli]"
+
+# one row (keeps HF folder layout under ./ckpts/)
+huggingface-cli download Echo-Team/Echo-Memory context_k1/epoch-0.safetensors --local-dir ./ckpts
+
+# all currently released rows
+huggingface-cli download Echo-Team/Echo-Memory --local-dir ./ckpts
+```
+
+Keep the subdirectory name in the local path (e.g. `./ckpts/context_k1/epoch-0.safetensors`). Eval scripts use `env/memory_baseline_runtime.py` to infer memory flags from path substrings; Spatial and SSM checkpoint rows remain TODO.
+
+## Use with Echo-Memory
+
+Set the Wan backbone, static in-domain data pool, and checkpoint path:
+
+```bash
+export WAN_BASE_MODEL=/path/to/Wan2.1-T2V-1.3B
+export DATASET_BASE_PATH=data/Context-as-Memory-Dataset
+export PYTHONPATH=$PWD:${PYTHONPATH:-}
+export CKPT=./ckpts/context_k1/epoch-0.safetensors
+```
+
+**In-domain replay + revisit (paper bundle):**
+
+```bash
+bash eval/v2/run_static_consistency_loop_and_revisit.sh
+bash eval/v2/run_basic_replay_gt.sh
+```
+
+**Open-domain revisit** (first frames already in `assets/opendomain_revisit/`):
+
+```bash
+PHASE=stage1 OOD_DIR=assets/opendomain_revisit \
+  bash eval/v2/revisit_suite/run_one_click_revisit_eval.sh
+```
+
+**Visual comparison** (fixed prompt + first frame):
+
+```bash
+python eval/metrics/run_visual_eval.py \
+  --ckpt "$CKPT" \
+  --output_root ./evals_visual
+```
+
+See [eval/v2/README.md](../eval/v2/README.md) and [eval/metrics/README.md](../eval/metrics/README.md) for full options.

code/doc/dataset_preprocessing.md

@@ -0,0 +1,181 @@
+# Static in-domain pool — download & preprocessing
+
+Echo-Memory’s **static in-domain pool** is released through [Echo-Team/Echo-Memory-Data](https://huggingface.co/datasets/Echo-Team/Echo-Memory-Data) as tar parts under `static_pool_tar_parts/`. The underlying pool is sourced from [KlingTeam/Context-as-Memory-Dataset](https://huggingface.co/datasets/KlingTeam/Context-as-Memory-Dataset) on Hugging Face (Kling Team, SIGGRAPH Asia 2025; [arXiv:2506.03141](https://arxiv.org/abs/2506.03141)). Total size is about **340 GB** — plan disk space before downloading and unpacking.
+
+---
+
+## 1. Download
+
+### Option A — Echo-Team packaged release
+
+```bash
+pip install -U "huggingface_hub[cli]"
+
+mkdir -p data
+
+huggingface-cli download Echo-Team/Echo-Memory-Data \
+  --repo-type dataset \
+  --include "static_pool_tar_parts/*" \
+  --local-dir ./data/echo-memory-data-release
+
+cat ./data/echo-memory-data-release/static_pool_tar_parts/echo-memory-data.tar.part-* | tar -xf - -C ./data
+```
+
+You should end up with `data/Context-as-Memory-Dataset/`.
+
+### Option B — original KlingTeam source
+
+If you prefer the upstream release, download or merge the original parts from the [KlingTeam dataset card](https://huggingface.co/datasets/KlingTeam/Context-as-Memory-Dataset):
+
+```bash
+mkdir -p data
+cd data
+
+# after all Context-as-Memory-Dataset_* parts are downloaded into this directory:
+cat Context-as-Memory-Dataset_* > Context-as-Memory-Dataset.zip
+unzip Context-as-Memory-Dataset.zip -d .
+```
+
+You should end up with a directory named `Context-as-Memory-Dataset/` (adjust the path below if your folder name differs).
+
+---
+
+## 2. Expected layout (static in-domain pool)
+
+After extraction, point `DATASET_BASE_PATH` at the pool root (default: `data/Context-as-Memory-Dataset/`):
+
+```text
+data/Context-as-Memory-Dataset/
+├── frames/              # 100 scene folders, ~7601 PNGs each
+│   ├── AncientTempleEnv_0/
+│   │   ├── 0000.png
+│   │   └── ...
+│   └── ...
+├── jsons/               # per-scene camera pose JSON (one file per scene)
+│   ├── AncientTempleEnv_0.json
+│   └── ...
+├── overlap_labels/      # per-frame overlap indices (used by context retrieval / latent precompute)
+│   ├── AncientTempleEnv_0/
+│   │   ├── 0.json
+│   │   └── ...
+│   └── ...
+├── captions.txt         # segment captions (optional for some workflows)
+└── metadata_full.csv    # released Echo-Memory segment metadata
+```
+
+Quick sanity check:
+
+```bash
+export DATASET_BASE_PATH=data/Context-as-Memory-Dataset
+
+test -d "${DATASET_BASE_PATH}/frames" && echo "frames OK"
+test -d "${DATASET_BASE_PATH}/jsons" && echo "jsons OK"
+test -d "${DATASET_BASE_PATH}/overlap_labels" && echo "overlap_labels OK"
+ls "${DATASET_BASE_PATH}/frames" | head
+ls "${DATASET_BASE_PATH}/jsons" | head
+```
+
+---
+
+## 3. Point Echo-Memory at the static in-domain pool
+
+```bash
+export DATASET_BASE_PATH=data/Context-as-Memory-Dataset
+export WAN_BASE_MODEL=/path/to/Wan2.1-T2V-1.3B
+export PYTHONPATH=$PWD:${PYTHONPATH:-}
+```
+
+Training scripts also accept `data/Context-as-Memory-Dataset` under the repo root if `DATASET_BASE_PATH` is unset.
+
+---
+
+## 4. Metadata (required)
+
+`metadata_full.csv` is included in the Echo-Team packaged release. If you downloaded the upstream KlingTeam source instead, fetch the released metadata into the pool root:
+
+```bash
+cd /path/to/Echo-Memory
+export DATASET_BASE_PATH=data/Context-as-Memory-Dataset
+
+huggingface-cli download Echo-Team/Echo-Memory-Data metadata_full.csv \
+  --repo-type dataset \
+  --local-dir "${DATASET_BASE_PATH}"
+```
+
+If you modify the pool or need to rebuild metadata locally, regenerate it from `frames/` and `captions.txt`:
+
+```bash
+bash scripts/run_generate_metadata.sh
+```
+
+You can also generate a smaller custom index for ablations or reduced-size training:
+
+```bash
+OUTPUT_CSV="${DATASET_BASE_PATH}/metadata_1000.csv" \
+METADATA_MAX_ROWS=1000 \
+bash scripts/run_generate_metadata.sh
+```
+
+Pass the custom CSV to training/evaluation with `--dataset_metadata_path "${DATASET_BASE_PATH}/metadata_1000.csv"`.
+
+Defaults (override via env vars):
+
+| Variable | Default | Meaning |
+| --- | --- | --- |
+| `OUTPUT_CSV` | `${DATASET_BASE_PATH}/metadata_full.csv` | Output metadata path |
+| `SEGMENT_LENGTH` | `81` | Frames per training segment |
+| `CONTEXT_FRAMES` | `5` | Context window used when building metadata |
+| `NUM_WORKERS` | CPU count − 2 | Parallel workers |
+| `METADATA_MAX_ROWS` / `DATASET_SIZE_ROWS` | `0` | Keep only the first N metadata rows after generation; `0` keeps the full CSV |
+
+Verify:
+
+```bash
+wc -l "${DATASET_BASE_PATH}/metadata_full.csv"
+head -n 3 "${DATASET_BASE_PATH}/metadata_full.csv"
+```
+
+---
+
+## 5. Precompute latents (optional, speeds training)
+
+If you train with precomputed VAE latents:
+
+```bash
+export WAN_BASE_MODEL=/path/to/Wan2.1-T2V-1.3B
+export DATASET_BASE_PATH=data/Context-as-Memory-Dataset
+NUM_PROCESSES=8 bash scripts/run_precompute_ctx_target_latents.sh
+```
+
+Latents are written under `${DATASET_BASE_PATH}/latents/`. The script can use `overlap_labels/` when `--use_overlap_labels` is enabled (see `scripts/run_precompute_ctx_target_latents.sh`).
+
+---
+
+## 6. Training pools vs. open-domain assets
+
+| Echo pool / asset | Location | Purpose |
+| --- | --- | --- |
+| Static in-domain pool | `DATASET_BASE_PATH` → `data/Context-as-Memory-Dataset` | Training, in-domain replay/revisit, metadata |
+| Dynamic training pool | `DATASET_BASE_PATH` → `data/dynamic-memory-dataset` | Training on the dynamic pool ([guide](dynamic_dataset_preprocessing.md)) |
+| Open-domain first frames | `assets/opendomain_revisit/` | Held-out OOD revisit probes (already in repo) |
+
+You do **not** need to rebuild open-domain anchors for the released revisit suite.
+
+---
+
+## 7. Troubleshooting
+
+**`DATASET_BASE_PATH is not set`** — export the variable or place data at `data/Context-as-Memory-Dataset` relative to the repo root.
+
+**Missing `frames/` or `jsons/`** — re-check unzip path; the root folder name must match what you pass to `DATASET_BASE_PATH`.
+
+**Metadata script missing** — ensure you are on the latest Echo-Memory `main` branch; metadata generation is invoked via `scripts/run_generate_metadata.sh`.
+
+**Disk space** — keep ~340 GB for raw frames plus extra space for `metadata_full.csv`, `latents/`, and training outputs.
+
+---
+
+## Reference
+
+- Static in-domain pool: [dataset_preprocessing.md](dataset_preprocessing.md)
+- Dynamic training pool: [dynamic_dataset_preprocessing.md](dynamic_dataset_preprocessing.md)

code/doc/dynamic_dataset_preprocessing.md

@@ -0,0 +1,159 @@
+# Dynamic training pool — SpatialVID subset
+
+Echo-Memory’s **dynamic training pool** uses a motion-filtered subset of [SpatialVID/SpatialVID](https://huggingface.co/datasets/SpatialVID/SpatialVID): ego-centric clips with camera poses and captions, exported into the same sample format used by the static pool.
+
+This guide covers **download → export → training/inference settings** only. Dynamic eval is TODO; current public support is training and inference.
+
+**License:** SpatialVID is [CC-BY-NC-SA 4.0](https://creativecommons.org/licenses/by-nc-sa/4.0/) (non-commercial). Static and dynamic pools may have different licenses — check before mixing runs.
+
+---
+
+## 1. Download (subset)
+
+**Hugging Face:** [SpatialVID/SpatialVID](https://huggingface.co/datasets/SpatialVID/SpatialVID)
+
+- Accept the dataset terms on Hugging Face before download.
+- Full corpus is large (~7 TB+). For Echo-Memory dynamic training, download **selected groups** only — you do not need the full 545 groups.
+
+```bash
+pip install -U "huggingface_hub[cli]"
+huggingface-cli login
+
+export SPATIALVID_ROOT=/path/to/SpatialVID
+hf download SpatialVID/SpatialVID --repo-type dataset --local-dir "${SPATIALVID_ROOT}"
+```
+
+To fetch specific groups, use include patterns or the helper script linked from the [dataset card](https://huggingface.co/datasets/SpatialVID/SpatialVID) (`download_SpatialVID.py` on the SpatialVID GitHub).
+
+Extract downloaded `.tar.gz` groups:
+
+```bash
+cd "${SPATIALVID_ROOT}"
+tar -xzvf annotations/group_0001.tar.gz
+tar -xzvf videos/group_0001.tar.gz
+```
+
+### Raw layout (per clip)
+
+```text
+SPATIALVID_ROOT/
+├── annotations/group_0001/{clip_id}/
+│   ├── poses.npy          # (N, 7) = tx,ty,tz,qx,qy,qz,qw
+│   ├── indexes.txt        # pose index → source frame index
+│   ├── caption.json       # scene / motion text
+│   └── dyn_masks.npz      # optional dynamic-region masks
+├── videos/group_0001/{clip_id}.mp4
+└── data/train/SpatialVID_metadata.csv
+```
+
+Use `SpatialVID_metadata.csv` to filter clips (e.g. `motion score`, `dynamicRatio`, `sceneType`) when building your subset.
+
+---
+
+## 2. Export to Echo layout (dynamic training pool)
+
+Use `data/dynamic-spatialvid-motion60/mixed/` as the public training root and set `DATASET_BASE_PATH` to it:
+
+```text
+data/dynamic-spatialvid-motion60/
+├── L1/                                    # single-level exports are also valid roots
+├── L2/
+├── L3/
+└── mixed/
+    ├── frames/L{1,2,3}/{clip_id}/0000.png ... 0080.png
+    ├── jsons/L{1,2,3}/{clip_id}.json
+    ├── overlap_labels/L{1,2,3}/{clip_id}/
+    ├── captions.txt
+    ├── metadata_train.csv
+    ├── metadata_train_sample.csv
+    ├── metadata_train_sample_L1.csv
+    ├── metadata_eval.csv
+    └── metadata_eval_2chunk.csv
+```
+
+**Per-clip steps:**
+
+| Step | Setting |
+| --- | --- |
+| Frame sample | **81** PNGs per clip, **640×352** |
+| Pose | Interpolate `poses.npy` + `indexes.txt` → `jsons/{clip_id}.json` (Euler `CineCameraActor` format, same as static data) |
+| Prompt | Short caption from `caption.json` (`SceneSummary` or `SceneDescription`) |
+| Overlap | Build `overlap_labels/` for FOV-based context retrieval |
+| Metadata row | `video`, `prompt`, `video_name`, `start_frame`, `end_frame`, optional `level` |
+
+`metadata_train.csv` is written at export time. Use `metadata_train_sample.csv` or `metadata_train_sample_L1.csv` for local step checks. Do not re-run `run_generate_metadata.sh` unless you regenerate from raw frames only.
+
+---
+
+## 3. Training settings
+
+Same env vars and on-disk layout as the static in-domain pool — only `DATASET_BASE_PATH` changes.
+
+```bash
+export WAN_BASE_MODEL=/path/to/Wan2.1-T2V-1.3B
+export DATASET_BASE_PATH=data/dynamic-spatialvid-motion60/mixed
+export PYTHONPATH=$PWD:${PYTHONPATH:-}
+```
+
+Recommended settings for the dynamic training pool (match memory baseline scripts):
+
+| Parameter | Typical value |
+| --- | --- |
+| Resolution | **640 × 352** |
+| Frames / chunk | **81** |
+| Context frames | **1–20** (recipe-dependent) |
+| `--use_rt_relative` | on |
+| `--enable_fov_retrieval` | on (when `overlap_labels/` present) |
+| `--enable_context_memory` | on for context / spatial / SSM rows |
+| `--timestep_shift` | **15** |
+| Learning rate | **5e-5** (adjust per row) |
+
+Example — run a dynamic row:
+
+```bash
+METADATA_NAME=metadata_train.csv bash train/dynamic_spatialvid/run_dyn_spatial_mem.sh
+```
+
+For local one-step validation:
+
+```bash
+METADATA_NAME=metadata_train_sample_L1.csv \
+MAX_TRAIN_STEPS=1 \
+PROGRESS_TOTAL_STEPS=30000 \
+NUM_WORKERS=0 \
+bash train/dynamic_spatialvid/run_dyn_block_wise_ssm.sh
+```
+
+Inference wrappers live under `inference/dynamic_spatialvid/`.
+
+---
+
+## 4. Demo selection
+
+Dynamic demos are selected from training-scene replay rather than from fixed eval scripts:
+
+1. Randomly sample candidate scenes from `metadata_train.csv` or `metadata_train_sample.csv`.
+2. Use the same prompt, first frame, and GT action trajectory for all six dynamic rows.
+3. Run `inference/unified_inference.py` or `inference/dynamic_spatialvid/*.sh` for each checkpoint.
+4. Manually pick a representative scene where all rows are viewable.
+
+The checked-in README previews are compressed GIFs under `assets/readme_previews/`.
+
+---
+
+## 5. Checklist
+
+- [ ] Hugging Face access approved for [SpatialVID/SpatialVID](https://huggingface.co/datasets/SpatialVID/SpatialVID)
+- [ ] Subset of `group_****` archives downloaded and extracted
+- [ ] Clips filtered (poses + caption present; optional motion / dynamic filters)
+- [ ] `frames/`, `jsons/`, `metadata_train.csv` under one root
+- [ ] (Recommended) `overlap_labels/` for FOV retrieval
+- [ ] `DATASET_BASE_PATH` exported before training/inference
+
+---
+
+## Reference
+
+- SpatialVID: [SpatialVID/SpatialVID](https://huggingface.co/datasets/SpatialVID/SpatialVID) · [arXiv:2509.09676](https://arxiv.org/abs/2509.09676)
+- Static in-domain pool: [dataset_preprocessing.md](dataset_preprocessing.md)
+- Dynamic training pool: [dynamic_dataset_preprocessing.md](dynamic_dataset_preprocessing.md)

code/doc/memory_mechanisms.md

@@ -0,0 +1,41 @@
+# Memory Mechanisms
+
+This note maps the paper's memory rows to the repository implementation and explains the modeling role of each family. Echo-Memory treats memory as a controlled intervention on what information from chunk 1 is stored and how chunk 2 reads it back during denoising.
+
+## Modeling View
+
+All rows use the same action-conditioned Wan DiT backbone and the same two-chunk training/evaluation setup:
+
+1. **Context chunk:** clean history frames are encoded into latent/context tokens, optionally with matched camera RT actions.
+2. **Target chunk:** noisy target latents are denoised while the selected memory mechanism exposes information from the context chunk.
+3. **Read-out:** memory is injected through raw context concatenation, compressed context tokens, spatial memory tokens, or recurrent state-space modules attached to DiT blocks.
+
+The ablations are designed to change only the memory pathway while keeping the backbone, action conditioning, resolution, chunk length, and training schedule aligned.
+
+## Paper Rows
+
+| Paper family | Paper row / repo name | What is stored or read | Main code path | Training entry |
+| --- | --- | --- | --- | --- |
+| Raw context | `context_k1`, `context_k5`, `context_k20` | Uncompressed retrieved context frames. `K=1` is the anchor/I2V floor; `K=5/20` are context-learning capacity rows. | `diffsynth/pipelines/wan_video_new.py` context latent path | `train/context_learning/run_pre_qkv_ctx{1,5,20}.sh` |
+| Compression | `framepack_weight` | Context tokens are kept at the same length but temporally reweighted. | `diffsynth/models/memory/framepack_weight.py` | `train/memory_baselines_basic/run_ablation_framepack_weight_two_chunk.sh` |
+| Compression | `framepack_len_r2`, `framepack_len_r4` | Context latents and matched RT actions are pooled along time. | `diffsynth/models/memory/framepack_length.py` | `train/memory_baselines_basic/run_ablation_framepack_len_r{2,4}_two_chunk.sh` |
+| Compression | `framepack_hybrid_r2`, `framepack_hybrid_r4` | Length compression plus token reweighting. | `wan_video_new.py` + FramePack helpers | `train/memory_baselines_basic/run_ablation_framepack_hybrid_r*_weight_two_chunk.sh` |
+| Token-grid | `spatial_mem` | Context tokens are time-averaged and summarized into learned grid tokens. This is the implementation behind the currently reported `spatial_mem` row; it does **not** reconstruct depth or 3D geometry. | `diffsynth/models/memory/spatial_grid_memory.py` | `train/memory_baselines_basic/run_spatial_memory_baseline.sh` |
+| Token-grid | `spatial_inject_none`, `spatial_concat_text`, `spatial_cross_attn_readout` | Same token-grid storage, different read-out: withheld, text-KV concat, or dedicated cross-attention. | `spatial_grid_memory.py` read-out helpers | matching `run_ablation_spatial_*_two_chunk.sh` scripts |
+| Geometry-grounded spatial | `geometry_spatial_mem` | A static scene is reconstructed outside the DiT using depth, intrinsics, extrinsics, and TSDF fusion. The fused point cloud is rendered along the target trajectory, VAE-encoded, and converted into conditioning tokens. | `diffsynth/models/memory/geometry_spatial_memory.py` | `train/memory_baselines_basic/run_geometry_spatial_memory_baseline.sh` |
+| State-space | `block_wise_ssm` | Paper-aligned recurrent state attached to selected DiT blocks. Checkpoint keys contain `block_wise_ssm.*`. | `diffsynth/models/memory/block_wise_ssm.py` | `train/memory_baselines_basic/run_ablation_block_wise_ssm_two_chunk.sh` |
+| State-space | `videossm_hybrid` | Legacy VideoSSM hybrid baseline: depthwise temporal-conv state-space-like module. Checkpoint keys contain `videossm_hybrid.*`. | `diffsynth/models/memory/videossm_hybrid.py` | `train/memory_baselines_basic/run_videossm_hybrid_baseline.sh` |
+
+## Naming Rules
+
+- Do not describe `SpatialGridMemory` or the existing `spatial_mem` results as the
+  geometry-grounded method from arXiv:2506.05284. It is a token-grid baseline.
+- Use **Geometry-grounded Spatial Memory** only when the metadata supplies
+  rendered static geometry through `geometry_memory` (or a configured column).
+  The geometry extractor is the external reconstruction pipeline: depth and
+  cameras → TSDF-fused static point cloud → target-view renders. The model-side
+  encoder does not estimate depth itself.
+- Use **Block-wise SSM** only for `--use_block_wise_ssm` / `BlockWiseStateSpaceMemory`.
+- Use **VideoSSM hybrid** only for the legacy `--use_videossm_hybrid` / `HybridStateSpaceMemory` baseline.
+- Use **Context learning** for raw-context capacity rows (`K=1/5/20`), not for compact memory modules.
+- Keep checkpoint folder names stable; `env/memory_baseline_runtime.py` and `inference/unified_inference.py` infer memory profiles from those names.

code/docs/README.md

@@ -0,0 +1,16 @@
+# Project page (local preview)
+
+The **official** GitHub Pages site is deployed from the **`gh-pages`** branch (root `index.html` + `style.css`), not from this folder.
+
+This `docs/` directory mirrors that pink-themed project page so you can preview locally:
+
+```bash
+cd docs
+python -m http.server 18876 --bind 0.0.0.0
+```
+
+Then open `http://localhost:18876/` (with port forwarding if remote).
+
+**Live site:** https://echo-team-joy-future-academy-jd.github.io/Echo-Memory/
+
+Edit `docs/index.html`, `docs/style.css`, `docs/site.js`, and `docs/assets/`, then run `bash scripts/publish_gh_pages.sh` (or push `main` to trigger CI).

code/docs/assets/echo-memory-paper.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d7b4abd1c79f36e21bafff4d972a9562058b814fd40b0e9225309833f1adf11a
+size 4038848

code/docs/assets/opendomain_revisit/README.md

@@ -0,0 +1,19 @@
+# Open-Domain Revisit Sources
+
+This folder contains the eight held-out first-frame sources used by the
+open-domain revisit probe in the Echo-Memory paper.
+
+Each image is treated as a first frame for a short controlled camera-return
+probe. The default prompt used by `eval/v2/revisit_suite` is:
+
+```text
+A toy bear in the same static scene. Preserve the bear appearance and the scene layout after camera revisit.
+```
+
+To run the probe, point `OOD_DIR` here or use the default:
+
+```bash
+export WAN_BASE_MODEL=/path/to/Wan2.1-T2V-1.3B
+PHASE=stage1 OOD_DIR=assets/opendomain_revisit \
+  bash eval/v2/revisit_suite/run_one_click_revisit_eval.sh
+```

code/docs/assets/paper_cases/README.md

@@ -0,0 +1,13 @@
+# Paper Case Visual Assets
+
+This directory contains paper-facing figures copied into the release for direct README rendering.
+
+- `figure_1_abs_framework.png`: paper teaser and workflow figure for the project landing page.
+- `figure_2_mem_overview.png`: overview of the memory design matrix.
+
+To generate new open-domain revisit videos and evidence frames for a checkpoint, use:
+
+```bash
+PHASE=stage1 OOD_DIR=assets/opendomain_revisit \
+  bash eval/v2/revisit_suite/run_one_click_revisit_eval.sh
+```

code/docs/developer.html

@@ -0,0 +1,109 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="utf-8" />
+  <meta name="viewport" content="width=device-width, initial-scale=1" />
+  <title>Echo-Memory Developer Guide</title>
+  <meta name="description" content="Echo-Memory development guide — workflows, eval, and Cursor vibe coding." />
+  <link rel="preconnect" href="https://fonts.googleapis.com" />
+  <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
+  <link href="https://fonts.googleapis.com/css2?family=Space+Grotesk:wght@400;500;600;700&family=IBM+Plex+Sans:wght@400;500;600;700&family=JetBrains+Mono:wght@400;600&display=swap" rel="stylesheet" />
+  <link rel="stylesheet" href="style.css?v=dev-skills-v1" />
+</head>
+<body class="developer-page" data-title-key="dev.meta.title">
+  <div class="page-shell page-shell--doc">
+    <header class="top-nav top-nav--doc" role="banner">
+      <div class="container top-nav__inner top-nav__inner--doc">
+        <a class="brand" href="index.html">
+          <span class="brand-name">Echo-Memory</span>
+          <span class="brand-team">Echo Team</span>
+        </a>
+        <div class="nav-actions">
+          <a class="action-btn action-btn--ghost action-btn--compact" href="index.html" data-i18n="dev.back">← Back to project page</a>
+          <button type="button" class="lang-toggle" id="lang-toggle" data-i18n-attr="aria-label:lang.switch" aria-label="Switch language">
+            <span data-lang-current>EN</span> / <span data-lang-alt>中文</span>
+          </button>
+        </div>
+      </div>
+    </header>
+
+    <main class="container dev-main">
+      <p class="section-kicker" data-i18n="dev.kicker">Development · Cursor</p>
+      <h1 class="dev-title" data-i18n="dev.title">Developer Guide</h1>
+      <p class="section-lead" data-i18n-html="dev.subtitle"></p>
+
+      <article class="dev-section card">
+        <h2 data-i18n="dev.s1.title">1. What this guide is</h2>
+        <ul class="bullet-list" data-i18n-html="dev.s1.body"></ul>
+      </article>
+
+      <article class="dev-section card">
+        <h2 data-i18n="dev.s2.title">2. Environment &amp; paths</h2>
+        <p data-i18n="dev.s2.intro">Set these before training or eval:</p>
+        <pre class="code-block"><code>export WAN_BASE_MODEL=/path/to/Wan2.1-T2V-1.3B
+export DATASET_BASE_PATH=data/Context-as-Memory-Dataset
+export PYTHONPATH=$PWD:${PYTHONPATH:-}
+export OUTPUT_BASE_ROOT=$PWD/outputs</code></pre>
+        <ul class="bullet-list" data-i18n-html="dev.s2.body"></ul>
+      </article>
+
+      <article class="dev-section card">
+        <h2 data-i18n="dev.s3.title">3. Code map</h2>
+        <div class="dev-table-wrap" data-i18n-html="dev.s3.table"></div>
+      </article>
+
+      <article class="dev-section card">
+        <h2 data-i18n="dev.s4.title">4. Common workflows</h2>
+        <p data-i18n="dev.s4.trainLabel"><strong>Train one memory row</strong> (from repo root):</p>
+        <pre class="code-block"><code>bash train/memory_baselines_basic/run_spatial_memory_baseline.sh
+bash train/context_learning/run_pre_qkv_ctx20.sh</code></pre>
+        <p data-i18n="dev.s4.evalLabel"><strong>Smoke eval with a HF checkpoint:</strong></p>
+        <pre class="code-block"><code>huggingface-cli download Echo-Team/Echo-Memory \
+  context_k1/epoch-0.safetensors --local-dir ./ckpts
+export CKPT=./ckpts/context_k1/epoch-0.safetensors
+bash eval/v2/run_static_consistency_loop_and_revisit.sh</code></pre>
+        <p class="section-note" data-i18n-html="dev.s4.note"></p>
+      </article>
+
+      <article class="dev-section card dev-section--accent">
+        <h2 data-i18n="dev.s5.title">5. Cursor vibe coding</h2>
+        <p data-i18n-html="dev.s5.intro"></p>
+        <p class="dev-skills-title" data-i18n="dev.s5.skillsTitle">Project skills</p>
+        <div class="dev-table-wrap" data-i18n-html="dev.s5.skills"></div>
+        <ul class="bullet-list" data-i18n-html="dev.s5.body"></ul>
+        <p data-i18n="dev.s5.promptLabel"><strong>Example prompt</strong></p>
+        <pre class="code-block code-block--prompt"><code data-i18n="dev.s5.prompt"></code></pre>
+        <p class="section-note" data-i18n-html="dev.s5.note"></p>
+      </article>
+
+      <article class="dev-section card">
+        <h2 data-i18n="dev.s6.title">6. Site &amp; release (maintainers)</h2>
+        <p data-i18n-html="dev.s6.intro"></p>
+        <pre class="code-block"><code>bash scripts/publish_gh_pages.sh</code></pre>
+        <p data-i18n-html="dev.s6.body"></p>
+      </article>
+
+      <article class="dev-section card">
+        <h2 data-i18n="dev.s7.title">7. Checklist</h2>
+        <ul class="bullet-list">
+          <li data-i18n="dev.s7.l1">Smoke eval with one HF checkpoint before tagging a release.</li>
+          <li data-i18n-html="dev.s7.l2"></li>
+          <li data-i18n="dev.s7.l3">Public docs use Echo pool names — no internal paths or benchmark codenames.</li>
+          <li data-i18n-html="dev.s7.l4"></li>
+        </ul>
+      </article>
+
+      <p class="dev-repo-link" data-i18n-html="dev.footer"></p>
+    </main>
+
+    <footer class="footer">
+      <div class="container footer-row">
+        <p class="footer-copy" data-i18n="footer.copy">© Echo Team · Joy Future Academy, JD</p>
+      </div>
+    </footer>
+  </div>
+
+  <script src="i18n.js?v=dev-skills-v1"></script>
+  <script src="i18n-runtime.js?v=dev-skills-v1"></script>
+</body>
+</html>

code/docs/i18n-runtime.js

@@ -0,0 +1,141 @@
+(function () {
+  "use strict";
+
+  var STORAGE_KEY = "echo-memory-lang";
+  var NAV_MAP = {
+    hero: "nav.home",
+    overview: "nav.overview",
+    design: "nav.design",
+    checkpoints: "nav.checkpoints",
+    evaluation: "nav.eval",
+    evidence: "nav.evidence",
+    findings: "nav.findings",
+    updates: "nav.updates",
+    bibtex: "nav.bibtex"
+  };
+
+  function detectDefaultLang() {
+    var saved = localStorage.getItem(STORAGE_KEY);
+    if (saved === "en" || saved === "zh") return saved;
+    return "en";
+  }
+
+  function dict(lang) {
+    return (window.ECHO_I18N && window.ECHO_I18N[lang]) || {};
+  }
+
+  function t(lang, key) {
+    var d = dict(lang);
+    return Object.prototype.hasOwnProperty.call(d, key) ? d[key] : "";
+  }
+
+  function applyLang(lang) {
+    var d = dict(lang);
+    if (!Object.keys(d).length) return;
+
+    document.documentElement.lang = lang === "zh" ? "zh-CN" : "en";
+
+    var descMeta = document.querySelector('meta[name="description"]');
+    var pageTitleKey = document.body.getAttribute("data-title-key");
+    if (pageTitleKey && d[pageTitleKey]) {
+      document.title = d[pageTitleKey];
+    } else if (d["meta.title"]) {
+      document.title = d["meta.title"];
+    }
+    if (descMeta) {
+      if (pageTitleKey === "dev.meta.title" && d["dev.meta.description"]) {
+        descMeta.setAttribute("content", d["dev.meta.description"]);
+      } else if (d["meta.description"]) {
+        descMeta.setAttribute("content", d["meta.description"]);
+      }
+    }
+
+    document.querySelectorAll("[data-i18n]").forEach(function (el) {
+      var key = el.getAttribute("data-i18n");
+      if (!key || !(key in d)) return;
+      el.textContent = d[key];
+    });
+
+    document.querySelectorAll("[data-i18n-html]").forEach(function (el) {
+      var key = el.getAttribute("data-i18n-html");
+      if (!key || !(key in d)) return;
+      el.innerHTML = d[key];
+    });
+
+    document.querySelectorAll("[data-i18n-attr]").forEach(function (el) {
+      el.getAttribute("data-i18n-attr").split(";").forEach(function (pair) {
+        var parts = pair.split(":");
+        if (parts.length !== 2) return;
+        var attr = parts[0].trim();
+        var key = parts[1].trim();
+        if (key in d) el.setAttribute(attr, d[key]);
+      });
+    });
+
+    document.querySelectorAll("[data-nav]").forEach(function (el) {
+      var navKey = NAV_MAP[el.getAttribute("data-nav")];
+      if (navKey && d[navKey]) el.textContent = d[navKey];
+    });
+
+    document.querySelectorAll(".slide-dots button[data-slide]").forEach(function (btn) {
+      var navKey = NAV_MAP[btn.getAttribute("data-slide")];
+      if (navKey && d[navKey]) {
+        btn.setAttribute("data-label", d[navKey]);
+        btn.setAttribute("aria-label", d[navKey]);
+      }
+    });
+
+    document.querySelectorAll(".qual-chip").forEach(function (chip, idx) {
+      var keys = ["evidence.chip1", "evidence.chip2", "evidence.chip3"];
+      if (keys[idx] && d[keys[idx]]) chip.textContent = d[keys[idx]];
+    });
+
+    var activeQual = document.querySelector(".qual-chip.is-active");
+    var qualCaption = document.querySelector("[data-qual-caption]");
+    if (activeQual && qualCaption && activeQual.dataset.captionKey && d[activeQual.dataset.captionKey]) {
+      qualCaption.innerHTML = d[activeQual.dataset.captionKey] + ' <span class="zoom-hint">' + (d["zoom.hint"] || "Click to expand") + "</span>";
+    } else if (qualCaption && d["evidence.cap1"] && document.querySelector(".qual-chip.is-active") === document.querySelector(".qual-chip")) {
+      qualCaption.innerHTML = d["evidence.cap1"];
+    }
+
+    var langCurrent = document.querySelector("[data-lang-current]");
+    var langAlt = document.querySelector("[data-lang-alt]");
+    if (langCurrent && d["lang.current"]) langCurrent.textContent = d["lang.current"];
+    if (langAlt && d["lang.toggle"]) langAlt.textContent = d["lang.toggle"];
+
+    var toggle = document.getElementById("lang-toggle");
+    if (toggle && d["lang.switch"]) toggle.setAttribute("aria-label", d["lang.switch"]);
+
+    document.querySelectorAll("[data-copy-target]").forEach(function (btn) {
+      if (!btn.classList.contains("is-copied") && d["bibtex.copy"]) {
+        btn.textContent = d["bibtex.copy"];
+      }
+    });
+
+    localStorage.setItem(STORAGE_KEY, lang);
+    document.dispatchEvent(new CustomEvent("echo-lang-change", { detail: { lang: lang } }));
+  }
+
+  window.EchoI18n = {
+    getLang: function () {
+      return localStorage.getItem(STORAGE_KEY) || detectDefaultLang();
+    },
+    setLang: applyLang,
+    t: function (key) {
+      return t(window.EchoI18n.getLang(), key);
+    }
+  };
+
+  document.addEventListener("DOMContentLoaded", function () {
+    var lang = detectDefaultLang();
+    applyLang(lang);
+
+    var toggle = document.getElementById("lang-toggle");
+    if (toggle) {
+      toggle.addEventListener("click", function () {
+        var next = window.EchoI18n.getLang() === "zh" ? "en" : "zh";
+        applyLang(next);
+      });
+    }
+  });
+})();

code/docs/i18n.js

@@ -0,0 +1,329 @@
+/* Echo-Memory project page — EN / ZH strings */
+window.ECHO_I18N = {
+  en: {
+    "meta.title": "Echo-Memory | Echo Team",
+    "meta.description": "Echo-Memory: A controlled study of memory mechanisms in action-conditioned world models.",
+    "nav.home": "Home",
+    "nav.overview": "Overview",
+    "nav.design": "Design",
+    "nav.checkpoints": "Ckpt",
+    "nav.eval": "Eval",
+    "nav.evidence": "Qual",
+    "nav.findings": "Results",
+    "nav.updates": "Updates",
+    "nav.bibtex": "BibTeX",
+    "nav.developer": "Dev Guide",
+    "nav.github": "GitHub",
+    "nav.menu": "Menu",
+    "lang.toggle": "中文",
+    "lang.current": "EN",
+    "lang.switch": "Switch language",
+    "hero.badge": "Echo Team · Joy Future Academy, JD · June 2026 · CC BY 4.0",
+    "hero.title.line2": "A Controlled Study of Memory in Action World Models",
+    "hero.subtitle": "When the camera leaves and returns, which memory keeps the <strong>same world</strong> instead of a plausible but different scene?",
+    "hero.cta.paper": "Paper",
+    "hero.cta.pdf": "PDF",
+    "hero.cta.ckpt": "Checkpoints",
+    "hero.cta.code": "Code",
+    "hero.metric.stars": "GitHub Stars",
+    "hero.metric.forks": "Forks",
+    "hero.note": "Controlled memory ablations on a shared <strong>Wan</strong> action-to-video stack — reproducible rows, evaluation scripts, and qualitative revisit panels.",
+    "hero.authors.summary": "Authors & affiliations",
+    "hero.affiliations": "HKU · Joy Future Academy, JD · CUHK · PKU · Fudan · Tsinghua · HKUST · UMich",
+    "overview.kicker": "01 · Overview",
+    "overview.title": "One backbone, one protocol — only memory changes.",
+    "overview.lead": "Echo-Memory holds the video backbone, training recipe, and data protocol fixed, and swaps only the memory module. The goal is to separate <strong>replay fidelity</strong> from <strong>return memory</strong> when the camera leaves and comes back to the same place.",
+    "overview.b1": "<strong>Shared stack</strong> — chunk-wise action-conditioned world generation on Wan.",
+    "overview.b2": "<strong>Controlled variable</strong> — Context, Compression, Spatial, or State-Space memory.",
+    "overview.b3": "<strong>Three probes</strong> — replay metrics, in-domain 180° loop, open-domain edited return.",
+    "overview.b4": "<strong>Release</strong> — ablation scripts, GT replay, revisit assets, and paper-aligned figures.",
+    "overview.fig.cap": "Controlled memory study over chunk-wise action-world generation. <span class=\"zoom-hint\">Click to expand</span>",
+    "overview.fig.alt": "Echo-Memory framework overview",
+    "design.kicker": "02 · Memory Design",
+    "design.title": "Context · Compression · Spatial · State-Space",
+    "design.lead": "All variants plug into the same write–read interface; we only change what is stored and how history is retrieved. A no-memory I2V floor re-generates from the first frame as a lower bound.",
+    "design.context.title": "Context",
+    "design.context.body": "Raw recent frames at K = 1, 5, or 20 chunks — tests whether longer windows alone stop drift.",
+    "design.compression.title": "Compression",
+    "design.compression.body": "Learned compact tokens at ratio r = 4 — history without growing raw-frame storage.",
+    "design.spatial.title": "Spatial",
+    "design.spatial.body": "Explicit spatial read/write state — targets layout, object pose, and viewpoint carry.",
+    "design.ssm.title": "State-Space",
+    "design.ssm.body": "Block-wise SSM updates — recurrent carry beyond short context windows on revisit.",
+    "design.fig.cap": "Four memory families under a shared write–read interface. <span class=\"zoom-hint\">Click to expand</span>",
+    "design.fig.alt": "Memory design matrix",
+    "ckpt.kicker": "03 · Checkpoints",
+    "ckpt.title": "Paper baselines on Hugging Face",
+    "ckpt.lead": "Wan 2.1 1.3B memory rows — <strong>epoch-0</strong>, <strong>30,000 steps</strong>, static in-domain pool. Released weights: <a href=\"https://huggingface.co/Echo-Team/Echo-Memory\" target=\"_blank\" rel=\"noopener noreferrer\"><strong>Echo-Team/Echo-Memory</strong></a>",
+    "ckpt.th.family": "Family",
+    "ckpt.th.row": "Paper row",
+    "ckpt.th.path": "HF path",
+    "ckpt.th.steps": "Steps",
+    "ckpt.label.download": "Download",
+    "ckpt.label.eval": "In-domain eval (Echo-Memory repo)",
+    "ckpt.note": "Keep the row folder in <code>CKPT</code> — <code>env/memory_baseline_runtime.py</code> infers memory flags from the path. Full index: <a href=\"https://github.com/Echo-Team-Joy-Future-Academy-JD/Echo-Memory/blob/main/doc/checkpoints.md\" target=\"_blank\" rel=\"noopener noreferrer\">doc/checkpoints.md</a>.",
+    "eval.kicker": "04 · Evaluation",
+    "eval.title": "Replay · In-domain revisit · Open-domain return",
+    "eval.lead": "Each branch asks a different question: Can the model reconstruct the past? Can it close a loop in-domain? After an edited first frame, does it return to the <strong>same world</strong>?",
+    "eval.replay.title": "Replay",
+    "eval.replay.body": "PSNR, SSIM, LPIPS on chunk-wise reconstruction — measures short-horizon pixel fidelity.",
+    "eval.indomain.title": "In-domain",
+    "eval.indomain.body": "180° trajectory loop closure with VLM-assisted scoring on held layouts.",
+    "eval.opendomain.title": "Open-domain",
+    "eval.opendomain.body": "Edited first frames and 45° return probes — stresses object identity and scene persistence.",
+    "eval.dynamic.title": "Dynamic SpatialVID",
+    "eval.dynamic.body": "Training and inference wrappers are public; the dynamic eval protocol is TODO.",
+    "eval.fig.cap": "Replay health vs. return memory under the same stack. <span class=\"zoom-hint\">Click to expand</span>",
+    "eval.fig.alt": "Three-branch evaluation summary",
+    "evidence.kicker": "05 · Qualitative Evidence",
+    "evidence.title": "Return probes expose identity drift.",
+    "evidence.lead": "Qualitative panels follow a simple diagnostic: <strong>first frame → leave the view → revisit tail</strong>. We compare whether memory restores the same object, pose, background, and camera geometry — not merely a plausible new scene.",
+    "evidence.chip1": "Memory Results",
+    "evidence.chip2": "Open-Domain Sweep",
+    "evidence.chip3": "Identity Anchors",
+    "evidence.cap1": "Representative memory comparisons across variants. <span class=\"zoom-hint\">Click to expand</span>",
+    "evidence.dynamic.note": "SpatialVID previews use one selected training scene and the same first frame, prompt, and GT camera trajectory for a 5-second first-chunk replay across all six rows.",
+    "findings.kicker": "06 · Main Conclusions",
+    "findings.title": "Replay quality ≠ memory quality.",
+    "findings.lead": "Replay metrics and return probes do not always agree — a model can look sharp on reconstruction yet fail when the camera returns. Rankings reorder once identity under revisit is measured.",
+    "findings.b1": "<strong>Raw context</strong> — more history helps open-domain return more than replay alone.",
+    "findings.b2": "<strong>Compression</strong> — compact tokens can preserve replay while losing identity on return.",
+    "findings.b3": "<strong>Spatial vs. SSM</strong> — explicit state and block-wise SSM trade off layout carry and long-horizon stability.",
+    "findings.b4": "<strong>Takeaway</strong> — treat replay as a health check, not the final memory benchmark.",
+    "findings.fig.cap": "Rank shift from replay to return — replay is not the final memory score. <span class=\"zoom-hint\">Click to expand</span>",
+    "updates.kicker": "07 · News & Roadmap",
+    "updates.title": "Release notes and next steps.",
+    "updates.news": "News",
+    "updates.roadmap": "Roadmap",
+    "updates.news0": "SpatialVID support added: dynamic training/inference recipes, 5-second first-chunk replay previews, and dynamic eval marked as TODO.",
+    "updates.news1": "Echo-Memory released: paper on <a href=\"https://arxiv.org/abs/2606.09803\" target=\"_blank\" rel=\"noopener noreferrer\">arXiv</a> (<a href=\"https://arxiv.org/pdf/2606.09803\" target=\"_blank\" rel=\"noopener noreferrer\">PDF</a>), project page, public code, replay/revisit eval assets, and baseline checkpoints on <a href=\"https://huggingface.co/Echo-Team/Echo-Memory\" target=\"_blank\" rel=\"noopener noreferrer\"><strong>Echo-Team/Echo-Memory</strong></a>.",
+    "updates.models": "Models",
+    "updates.eval": "Eval",
+    "updates.rm1": "<strong>Wan 2.1 1.3B</strong> backbone and training recipes",
+    "updates.rm2": "Four memory families — Context, Compression, Spatial, State-Space",
+    "updates.rm3": "<strong>Dynamic training pool</strong> — SpatialVID subset export & settings",
+    "updates.rm4": "<strong>Paper checkpoints</strong> — <a href=\"https://huggingface.co/Echo-Team/Echo-Memory\" target=\"_blank\" rel=\"noopener noreferrer\">Echo-Team/Echo-Memory</a>",
+    "updates.rm5": "<strong>Wan 2.2</strong> + multi-scale <strong>5B / 14B</strong>",
+    "updates.re1": "<strong>Dynamic eval</strong> beyond static replay/revisit",
+    "updates.re2": "More revisit probes and scoring presets",
+    "community.title": "Community",
+    "community.lead": "Join the Echo-Memory WeChat group for release updates, checkpoint questions, and maintainer coordination.",
+    "community.qr.alt": "Echo-Memory WeChat group QR code",
+    "community.qr.caption": "Echo-Memory 交流群 · scan to join (QR refreshes periodically)",
+    "bibtex.kicker": "08 · Citation",
+    "bibtex.title": "BibTeX",
+    "bibtex.lead": "Echo-Memory: A Controlled Study of Memory in Action World Models (June 2026). Licensed under <a href=\"https://creativecommons.org/licenses/by/4.0/\" target=\"_blank\" rel=\"noopener noreferrer\">CC BY 4.0</a>. Cite the arXiv preprint below.",
+    "bibtex.arxiv": "arXiv",
+    "bibtex.source": "Source",
+    "bibtex.doi": "DOI",
+    "bibtex.arxivid": "arXiv ID",
+    "bibtex.pdf": "PDF",
+    "bibtex.license": "License",
+    "bibtex.copy": "Copy",
+    "bibtex.copied": "Copied",
+    "bibtex.failed": "Failed",
+    "footer.copy": "© Echo Team · Joy Future Academy, JD",
+    "footer.pp": "Project Page",
+    "zoom.hint": "Click to expand",
+    "backtop": "Back to top",
+    "lightbox.close": "Close",
+    "lightbox.label": "Expanded figure",
+    "dev.meta.title": "Echo-Memory Developer Guide",
+    "dev.meta.description": "Echo-Memory development guide — workflows, eval, and Cursor vibe coding.",
+    "dev.kicker": "Development · Cursor",
+    "dev.title": "Developer Guide",
+    "dev.subtitle": "Hands-on coding, training, eval, and <strong>Cursor vibe coding</strong> for Echo-Memory.",
+    "dev.back": "← Back to project page",
+    "dev.s1.title": "1. What this guide is",
+    "dev.s1.body": "<li><strong>README</strong> — paper overview, quick start, checkpoints, community.</li><li><strong>This guide</strong> — workflows, project <strong>Cursor skills</strong>, Agent tips.</li><li><strong><code>doc/</code></strong> — dataset &amp; checkpoint reference.</li><li><strong><code>.cursor/skills/</code></strong> — Agent skills for train / eval / release.</li>",
+    "dev.s2.title": "2. Environment & paths",
+    "dev.s2.intro": "Set these before training or eval:",
+    "dev.s2.body": "<li><strong>Static in-domain pool</strong> — default root above; see <code>doc/dataset_preprocessing.md</code>.</li><li><strong>Dynamic training pool</strong> — e.g. <code>data/dynamic-memory-dataset</code>; see <code>doc/dynamic_dataset_preprocessing.md</code>.</li><li><strong>Checkpoints</strong> — <a href=\"https://huggingface.co/Echo-Team/Echo-Memory\" target=\"_blank\" rel=\"noopener noreferrer\">Echo-Team/Echo-Memory</a>; index in <code>doc/checkpoints.md</code>.</li>",
+    "dev.s3.title": "3. Code map",
+    "dev.s3.table": "<table class=\"dev-table\"><thead><tr><th>Path</th><th>Role</th></tr></thead><tbody><tr><td><code>.cursor/skills/</code></td><td>Cursor Agent skills (train / eval / release)</td></tr><tr><td><code>train/memory_baselines_basic/</code></td><td>Spatial / SSM / compression ablations</td></tr><tr><td><code>train/context_learning/</code></td><td>Context K=1/5/20 recipes</td></tr><tr><td><code>eval/v2/</code></td><td>Replay, loop closure, open-domain revisit</td></tr><tr><td><code>env/memory_baseline_runtime.py</code></td><td>Checkpoint → memory profile</td></tr><tr><td><code>diffsynth/</code></td><td>Wan backbone &amp; training stack</td></tr><tr><td><code>docs/</code></td><td>GitHub Pages (project + this guide)</td></tr></tbody></table>",
+    "dev.s4.title": "4. Common workflows",
+    "dev.s4.trainLabel": "Train one memory row (from repo root):",
+    "dev.s4.evalLabel": "Smoke eval with a HF checkpoint:",
+    "dev.s4.note": "Keep the row folder name in <code>CKPT</code> so runtime picks the correct memory profile.",
+    "dev.s5.title": "5. Cursor vibe coding",
+    "dev.s5.intro": "Use <strong>Cursor Agent</strong> (Composer) with the project skills below.",
+    "dev.s5.skillsTitle": "Project skills",
+    "dev.s5.skills": "<table class=\"dev-table\"><thead><tr><th>Skill</th><th>Use when</th></tr></thead><tbody><tr><td><code>echo-memory-agent</code></td><td>Scope prompts, rules, skill index</td></tr><tr><td><code>echo-memory-train</code></td><td>Memory baselines &amp; context training</td></tr><tr><td><code>echo-memory-eval</code></td><td>Replay / revisit &amp; HF quick checks</td></tr><tr><td><code>echo-memory-release</code></td><td>gh-pages, i18n, checkpoints doc</td></tr></tbody></table><p class=\"section-note\">Paths: <code>.cursor/skills/&lt;name&gt;/SKILL.md</code> — invoke in chat, e.g. <em>use echo-memory-eval to …</em></p>",
+    "dev.s5.body": "<li><strong>Scope prompts</strong> — memory family, script, eval branch (<em>replay / in-domain / open-domain</em>).</li><li><strong>Entry scripts</strong> — e.g. <code>run_spatial_memory_baseline.sh</code>, <code>eval/v2/run_basic_replay_gt.sh</code>.</li><li><strong>Rules</strong> — optional <code>.cursor/rules/echo-memory.mdc</code> for pool naming &amp; public doc constraints.</li><li><strong>Ask mode</strong> — trace checkpoint mapping or read <code>diffsynth/</code> without edits.</li>",
+    "dev.s5.promptLabel": "Example prompt",
+    "dev.s5.prompt": "Add a quick check that downloads context_k1 from Echo-Team/Echo-Memory\nand runs eval/v2/run_basic_replay_gt.sh with the static in-domain pool.\n\nTrace env/memory_baseline_runtime.py spatial_mem → inject flags;\nsummarize in doc/checkpoints.md.",
+    "dev.s5.note": "<strong>Public repo hygiene:</strong> never commit upload bash, internal benchmark names, or machine paths. WeChat QR → project page &amp; README only.",
+    "dev.s6.title": "6. Site & release (maintainers)",
+    "dev.s6.intro": "After editing <code>docs/index.html</code>, <code>docs/style.css</code>, or <code>docs/i18n*.js</code>:",
+    "dev.s6.body": "HF weights: Hugging Face UI or <code>hf upload</code> (maintainers only). Bilingual project page: <code>docs/i18n.js</code> + <code>docs/i18n-runtime.js</code>.",
+    "dev.s7.title": "7. Checklist",
+    "dev.s7.l1": "Smoke eval with one HF checkpoint before tagging a release.",
+    "dev.s7.l2": "Verify <code>doc/checkpoints.md</code> matches HF folder names.",
+    "dev.s7.l3": "Public docs use Echo pool names — no internal paths or benchmark codenames.",
+    "dev.s7.l4": "Run <code>publish_gh_pages.sh</code> after site changes; spot-check EN/中文 on the live page.",
+    "dev.footer": "Repo mirror: <a href=\"https://github.com/Echo-Team-Joy-Future-Academy-JD/Echo-Memory/blob/main/doc/DEVELOPER.md\">doc/DEVELOPER.md</a> · Community QR on <a href=\"index.html#updates\">project page</a>"
+  },
+  zh: {
+    "meta.title": "Echo-Memory | Echo Team",
+    "meta.description": "Echo-Memory：动作条件世界模型中记忆机制的对照研究。",
+    "nav.home": "首页",
+    "nav.overview": "概览",
+    "nav.design": "设计",
+    "nav.checkpoints": "权重",
+    "nav.eval": "评测",
+    "nav.evidence": "证据",
+    "nav.findings": "结论",
+    "nav.updates": "动态",
+    "nav.bibtex": "引用",
+    "nav.developer": "开发者手册",
+    "nav.github": "GitHub",
+    "nav.menu": "菜单",
+    "lang.toggle": "EN",
+    "lang.current": "中文",
+    "lang.switch": "切换语言",
+    "hero.badge": "Echo Team · Joy Future Academy, JD · 2026 年 6 月 · CC BY 4.0",
+    "hero.title.line2": "动作世界模型中记忆机制的对照研究",
+    "hero.subtitle": "当镜头离开再返回时，哪种记忆能让模型守住<strong>同一个世界</strong>，而不是生成一个看似合理却不同的场景？",
+    "hero.cta.paper": "论文",
+    "hero.cta.pdf": "PDF",
+    "hero.cta.ckpt": "模型权重",
+    "hero.cta.code": "代码",
+    "hero.metric.stars": "GitHub Stars",
+    "hero.metric.forks": "Forks",
+    "hero.note": "在共享 <strong>Wan</strong> 动作到视频栈上进行可控记忆消融——可复现实验行、评测脚本与 revisit 定性面板。",
+    "hero.authors.summary": "作者与单位",
+    "hero.affiliations": "HKU · Joy Future Academy, JD · CUHK · PKU · Fudan · Tsinghua · HKUST · UMich",
+    "overview.kicker": "01 · 概览",
+    "overview.title": "同一骨干、同一协议——只换记忆模块。",
+    "overview.lead": "Echo-Memory 固定视频骨干、训练配方与数据协议，仅替换记忆模块，以区分镜头离开再返回时的<strong>回放保真度</strong>与<strong>回归记忆</strong>。",
+    "overview.b1": "<strong>共享栈</strong> — 基于 Wan 的分块动作条件世界生成。",
+    "overview.b2": "<strong>对照变量</strong> — Context、Compression、Spatial 或 State-Space 记忆。",
+    "overview.b3": "<strong>三类探针</strong> — 回放指标、域内 180° 闭环、开放域编辑后回归。",
+    "overview.b4": "<strong>开源内容</strong> — 消融脚本、GT 回放、revisit 资产与论文对齐图表。",
+    "overview.fig.cap": "分块动作世界生成上的可控记忆研究。<span class=\"zoom-hint\">点击放大</span>",
+    "overview.fig.alt": "Echo-Memory 框架概览",
+    "design.kicker": "02 · 记忆设计",
+    "design.title": "Context · Compression · Spatial · State-Space",
+    "design.lead": "各变体接入同一 write–read 接口，仅改变存储内容与历史检索方式。无记忆 I2V 下限仅从首帧重生成。",
+    "design.context.title": "Context",
+    "design.context.body": "保留 K = 1 / 5 / 20 块原始帧 — 测试更长窗口是否足以抑制漂移。",
+    "design.compression.title": "Compression",
+    "design.compression.body": "比率 r = 4 的紧凑 token — 在不膨胀原始帧存储的情况下保留历史。",
+    "design.spatial.title": "Spatial",
+    "design.spatial.body": "显式空间读写状态 — 针对布局、物体位姿与视角携带。",
+    "design.ssm.title": "State-Space",
+    "design.ssm.body": "Block-wise SSM 更新 — 在 revisit 上超越短上下文窗口的递归携带。",
+    "design.fig.cap": "共享 write–read 接口下的四类记忆。<span class=\"zoom-hint\">点击放大</span>",
+    "design.fig.alt": "记忆设计矩阵",
+    "ckpt.kicker": "03 · 模型权重",
+    "ckpt.title": "Hugging Face 论文 baseline",
+    "ckpt.lead": "Wan 2.1 1.3B 记忆行 — <strong>epoch-0</strong>、<strong>30,000 steps</strong>、静态 in-domain 训练池。已发布权重：<a href=\"https://huggingface.co/Echo-Team/Echo-Memory\" target=\"_blank\" rel=\"noopener noreferrer\"><strong>Echo-Team/Echo-Memory</strong></a>",
+    "ckpt.th.family": "家族",
+    "ckpt.th.row": "论文行",
+    "ckpt.th.path": "HF 路径",
+    "ckpt.th.steps": "步数",
+    "ckpt.label.download": "下载",
+    "ckpt.label.eval": "域内评测（Echo-Memory 仓库）",
+    "ckpt.note": "请在 <code>CKPT</code> 中保留行目录名 — <code>env/memory_baseline_runtime.py</code> 会从路径推断 memory 配置。完整索引：<a href=\"https://github.com/Echo-Team-Joy-Future-Academy-JD/Echo-Memory/blob/main/doc/checkpoints.md\" target=\"_blank\" rel=\"noopener noreferrer\">doc/checkpoints.md</a>。",
+    "eval.kicker": "04 · 评测",
+    "eval.title": "回放 · 域内 revisit · 开放域回归",
+    "eval.lead": "三个分支回答不同问题：能否重建过去？域内能否闭环？编辑首帧后能否回到<strong>同一个世界</strong>？",
+    "eval.replay.title": "回放",
+    "eval.replay.body": "分块重建的 PSNR / SSIM / LPIPS — 衡量短程像素保真。",
+    "eval.indomain.title": "域内",
+    "eval.indomain.body": "180° 轨迹闭环与 VLM 辅助评分。",
+    "eval.opendomain.title": "开放域",
+    "eval.opendomain.body": "编辑首帧与 45° 回归探针 — 考察物体身份与场景持续性。",
+    "eval.dynamic.title": "动态 SpatialVID",
+    "eval.dynamic.body": "训练和推理 wrapper 已公开；动态评测协议暂列 TODO。",
+    "eval.fig.cap": "同一栈上的回放健康度 vs. 回归记忆。<span class=\"zoom-hint\">点击放大</span>",
+    "eval.fig.alt": "三分支评测概览",
+    "evidence.kicker": "05 · 定性证据",
+    "evidence.title": "回归探针暴露身份漂移。",
+    "evidence.lead": "定性面板遵循简单诊断：<strong>首帧 → 离开视角 → revisit 尾部</strong>。我们比较记忆是否恢复同一物体、位姿、背景与相机几何，而非仅生成合理的新场景。",
+    "evidence.chip1": "记忆结果",
+    "evidence.chip2": "开放域扫描",
+    "evidence.chip3": "身份锚点",
+    "evidence.cap1": "各变体的代表性记忆对比。<span class=\"zoom-hint\">点击放大</span>",
+    "evidence.dynamic.note": "SpatialVID 预览来自一个精选训练场景；六组使用相同首帧、prompt 与 GT 相机轨迹做 5 秒 first-chunk replay。",
+    "findings.kicker": "06 · 主要结论",
+    "findings.title": "回放质量 ≠ 记忆质量。",
+    "findings.lead": "回放指标与回归探针并不总一致 — 重建可以很 sharp，但镜头返回时仍可能失败。一旦测量 revisit 下的身份一致性，排名会重排。",
+    "findings.b1": "<strong>原始 Context</strong> — 更长历史对开放域回归的帮助大于单纯回放。",
+    "findings.b2": "<strong>Compression</strong> — 紧凑 token 可保回放但在回归时丢失身份。",
+    "findings.b3": "<strong>Spatial vs. SSM</strong> — 显式状态与 block-wise SSM 在布局携带与长程稳定性间权衡。",
+    "findings.b4": "<strong>要点</strong> — 将回放视为健康检查，而非最终记忆 benchmark。",
+    "findings.fig.cap": "从回放到回归的排名变化 — 回放不是最终记忆分数。<span class=\"zoom-hint\">点击放大</span>",
+    "updates.kicker": "07 · 新闻与路线图",
+    "updates.title": "发布说明与后续计划。",
+    "updates.news": "新闻",
+    "updates.roadmap": "路线图",
+    "updates.news0": "SpatialVID 支持已加入：动态训练/推理脚本、5 秒 first-chunk replay 预览，以及 dynamic eval TODO。",
+    "updates.news1": "Echo-Memory 发布：论文上线 <a href=\"https://arxiv.org/abs/2606.09803\" target=\"_blank\" rel=\"noopener noreferrer\">arXiv</a>（<a href=\"https://arxiv.org/pdf/2606.09803\" target=\"_blank\" rel=\"noopener noreferrer\">PDF</a>），同步发布项目页、公开代码、replay/revisit 评测资产，以及 <a href=\"https://huggingface.co/Echo-Team/Echo-Memory\" target=\"_blank\" rel=\"noopener noreferrer\"><strong>Echo-Team/Echo-Memory</strong></a> baseline 权重。",
+    "updates.models": "模型",
+    "updates.eval": "评测",
+    "updates.rm1": "<strong>Wan 2.1 1.3B</strong> 骨干与训练配方",
+    "updates.rm2": "四类记忆 — Context、Compression、Spatial、State-Space",
+    "updates.rm3": "<strong>Dynamic training pool</strong> — SpatialVID 子集导出与设置",
+    "updates.rm4": "<strong>论文权重</strong> — <a href=\"https://huggingface.co/Echo-Team/Echo-Memory\" target=\"_blank\" rel=\"noopener noreferrer\">Echo-Team/Echo-Memory</a>",
+    "updates.rm5": "<strong>Wan 2.2</strong> + 多尺度 <strong>5B / 14B</strong>",
+    "updates.re1": "静态 replay/revisit 之外的 <strong>动态评测</strong>",
+    "updates.re2": "更多 revisit 探针与评分预设",
+    "community.title": "社区交流",
+    "community.lead": "扫码加入 Echo-Memory 微信群，获取发布更新、权重使用与维护协调信息。",
+    "community.qr.alt": "Echo-Memory 微信群二维码",
+    "community.qr.caption": "Echo-Memory 交流群 · 扫码加入（二维码会定期更新）",
+    "bibtex.kicker": "08 · 引用",
+    "bibtex.title": "BibTeX",
+    "bibtex.lead": "Echo-Memory: A Controlled Study of Memory in Action World Models（2026 年 6 月）。许可：<a href=\"https://creativecommons.org/licenses/by/4.0/\" target=\"_blank\" rel=\"noopener noreferrer\">CC BY 4.0</a>。请使用下方 arXiv BibTeX 引用。",
+    "bibtex.arxiv": "arXiv",
+    "bibtex.source": "来源",
+    "bibtex.doi": "DOI",
+    "bibtex.arxivid": "arXiv ID",
+    "bibtex.pdf": "PDF",
+    "bibtex.license": "许可",
+    "bibtex.copy": "复制",
+    "bibtex.copied": "已复制",
+    "bibtex.failed": "失败",
+    "footer.copy": "© Echo Team · Joy Future Academy, JD",
+    "footer.pp": "项目页",
+    "zoom.hint": "点击放大",
+    "backtop": "回到顶部",
+    "lightbox.close": "关闭",
+    "lightbox.label": "放大图表",
+    "dev.meta.title": "Echo-Memory 开发者指南",
+    "dev.meta.description": "Echo-Memory 开发指南 — 工作流、评测与 Cursor 协作编程。",
+    "dev.kicker": "开发 · Cursor",
+    "dev.title": "开发者指南",
+    "dev.subtitle": "Echo-Memory 实战开发、训练评测与 <strong>Cursor 协作编程</strong>。",
+    "dev.back": "← 返回项目页",
+    "dev.s1.title": "1. 本指南定位",
+    "dev.s1.body": "<li><strong>README</strong> — 论文概览、快速上手、权重、社区。</li><li><strong>本页</strong> — 工作流、项目 <strong>Cursor skills</strong>、Agent 技巧。</li><li><strong><code>doc/</code></strong> — 数据集与权重参考。</li><li><strong><code>.cursor/skills/</code></strong> — 训练 / 评测 / 发布类 Agent 技能。</li>",
+    "dev.s2.title": "2. 环境与路径",
+    "dev.s2.intro": "训练或评测前设置：",
+    "dev.s2.body": "<li><strong>Static in-domain pool</strong> — 默认路径见上；详见 <code>doc/dataset_preprocessing.md</code>。</li><li><strong>Dynamic training pool</strong> — 如 <code>data/dynamic-memory-dataset</code>；详见 <code>doc/dynamic_dataset_preprocessing.md</code>。</li><li><strong>权重</strong> — <a href=\"https://huggingface.co/Echo-Team/Echo-Memory\" target=\"_blank\" rel=\"noopener noreferrer\">Echo-Team/Echo-Memory</a>；索引见 <code>doc/checkpoints.md</code>。</li>",
+    "dev.s3.title": "3. 代码地图",
+    "dev.s3.table": "<table class=\"dev-table\"><thead><tr><th>路径</th><th>作用</th></tr></thead><tbody><tr><td><code>.cursor/skills/</code></td><td>Cursor Agent 技能（训练 / 评测 / 发布）</td></tr><tr><td><code>train/memory_baselines_basic/</code></td><td>Spatial / SSM / 压缩消融</td></tr><tr><td><code>train/context_learning/</code></td><td>Context K=1/5/20 配方</td></tr><tr><td><code>eval/v2/</code></td><td>回放、闭环、开放域 revisit</td></tr><tr><td><code>env/memory_baseline_runtime.py</code></td><td>权重 → 记忆配置</td></tr><tr><td><code>diffsynth/</code></td><td>Wan 骨干与训练栈</td></tr><tr><td><code>docs/</code></td><td>GitHub Pages（项目页 + 本指南）</td></tr></tbody></table>",
+    "dev.s4.title": "4. 常用工作流",
+    "dev.s4.trainLabel": "训练一条 memory 行（仓库根目录）：",
+    "dev.s4.evalLabel": "用 HF 权重做 quick eval：",
+    "dev.s4.note": "<code>CKPT</code> 路径需保留行目录名，以便 runtime 匹配记忆配置。",
+    "dev.s5.title": "5. Cursor 协作编程",
+    "dev.s5.intro": "使用 <strong>Cursor Agent</strong>（Composer）配合下方项目 skills。",
+    "dev.s5.skillsTitle": "项目 Skills",
+    "dev.s5.skills": "<table class=\"dev-table\"><thead><tr><th>Skill</th><th>适用场景</th></tr></thead><tbody><tr><td><code>echo-memory-agent</code></td><td>Prompt 范围、Rules、技能索引</td></tr><tr><td><code>echo-memory-train</code></td><td>Memory baseline 与 Context 训练</td></tr><tr><td><code>echo-memory-eval</code></td><td>回放 / revisit 与 HF quick check</td></tr><tr><td><code>echo-memory-release</code></td><td>gh-pages、i18n、权重文档</td></tr></tbody></table><p class=\"section-note\">路径：<code>.cursor/skills/&lt;name&gt;/SKILL.md</code> — 在对话中引用，如 <em>use echo-memory-eval 来 …</em></p>",
+    "dev.s5.body": "<li><strong>明确范围</strong> — memory 家族、脚本、评测分支（<em>replay / in-domain / open-domain</em>）。</li><li><strong>入口脚本</strong> — 如 <code>run_spatial_memory_baseline.sh</code>、<code>eval/v2/run_basic_replay_gt.sh</code>。</li><li><strong>Rules</strong> — 可选 <code>.cursor/rules/echo-memory.mdc</code> 约束池命名与公开文档。</li><li><strong>Ask 模式</strong> — 追踪 checkpoint 映射或阅读 <code>diffsynth/</code>，不改代码。</li>",
+    "dev.s5.promptLabel": "示例 Prompt",
+    "dev.s5.prompt": "添加 quick check：从 Echo-Team/Echo-Memory 下载 context_k1，\n用 static in-domain pool 跑 eval/v2/run_basic_replay_gt.sh。\n\n追踪 env/memory_baseline_runtime.py 如何把 spatial_mem\n权重映射到 inject 标志，并在 doc/checkpoints.md 摘要说明。",
+    "dev.s5.note": "<strong>公开仓库规范：</strong> 勿提交上传脚本、内部 benchmark 名、本机路径。微信群二维码仅在项目页与 README。",
+    "dev.s6.title": "6. 站点与发布（维护者）",
+    "dev.s6.intro": "修改 <code>docs/index.html</code>、<code>docs/style.css</code> 或 <code>docs/i18n*.js</code> 后：",
+    "dev.s6.body": "HF 权重：网页或 <code>hf upload</code> 更新（仅维护者）。项目页双语：<code>docs/i18n.js</code> + <code>docs/i18n-runtime.js</code>。",
+    "dev.s7.title": "7. 检查清单",
+    "dev.s7.l1": "发版前用至少一个 HF 权重跑 quick eval。",
+    "dev.s7.l2": "确认 <code>doc/checkpoints.md</code> 与 HF 目录名���致。",
+    "dev.s7.l3": "公开文档使用 Echo 池命名 — 无内部路径或 benchmark 代号。",
+    "dev.s7.l4": "改站点后运行 <code>publish_gh_pages.sh</code>，检查线上 EN/中文 切换。",
+    "dev.footer": "仓库副本：<a href=\"https://github.com/Echo-Team-Joy-Future-Academy-JD/Echo-Memory/blob/main/doc/DEVELOPER.md\">doc/DEVELOPER.md</a> · 社区二维码见 <a href=\"index.html#updates\">项目页</a>"
+  }
+};

code/docs/index.html

@@ -0,0 +1,644 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="utf-8" />
+  <meta name="viewport" content="width=device-width, initial-scale=1" />
+  <title>Echo-Memory | Echo Team</title>
+  <meta
+    name="description"
+    content="Echo-Memory: A controlled study of memory mechanisms in action-conditioned world models."
+  />
+  <link rel="preconnect" href="https://fonts.googleapis.com" />
+  <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
+  <link
+    href="https://fonts.googleapis.com/css2?family=Space+Grotesk:wght@400;500;600;700&family=IBM+Plex+Sans:wght@400;500;600;700&family=JetBrains+Mono:wght@400;600&display=swap"
+    rel="stylesheet"
+  />
+  <link rel="stylesheet" href="style.css?v=dev-i18n-v1" />
+</head>
+<body>
+  <div class="scroll-progress" aria-hidden="true"><span class="scroll-progress__bar"></span></div>
+  <!-- arXiv: https://arxiv.org/abs/2606.09803 -->
+  <div class="page-shell">
+    <div class="ambient-bg" aria-hidden="true">
+      <div class="ambient-orb ambient-orb--rose"></div>
+      <div class="ambient-orb ambient-orb--blush"></div>
+      <div class="ambient-orb ambient-orb--pearl"></div>
+      <div class="ambient-shimmer"></div>
+    </div>
+    <header class="top-nav" role="banner">
+      <div class="container top-nav__inner">
+        <a class="brand" href="#hero">
+          <span class="brand-name">Echo-Memory</span>
+          <span class="brand-team">Echo Team</span>
+        </a>
+        <nav class="nav-links" aria-label="Sections">
+          <a class="nav-link" href="#hero" data-nav="hero">Home</a>
+          <a class="nav-link" href="#overview" data-nav="overview">Overview</a>
+          <a class="nav-link" href="#design" data-nav="design">Design</a>
+          <a class="nav-link" href="#checkpoints" data-nav="checkpoints">Checkpoints</a>
+          <a class="nav-link" href="#evaluation" data-nav="evaluation">Eval</a>
+          <a class="nav-link" href="#evidence" data-nav="evidence">Evidence</a>
+          <a class="nav-link" href="#findings" data-nav="findings">Results</a>
+          <a class="nav-link" href="#updates" data-nav="updates">Updates</a>
+          <a class="nav-link" href="#bibtex" data-nav="bibtex">BibTeX</a>
+        </nav>
+        <div class="nav-actions">
+          <a class="action-btn action-btn--ghost" href="developer.html" data-i18n="nav.developer">Developer Guide</a>
+          <button type="button" class="lang-toggle" id="lang-toggle" data-i18n-attr="aria-label:lang.switch" aria-label="Switch language">
+            <span data-lang-current>EN</span> / <span data-lang-alt>中文</span>
+          </button>
+          <a
+            class="action-btn"
+            href="https://github.com/Echo-Team-Joy-Future-Academy-JD/Echo-Memory"
+            target="_blank"
+            rel="noopener noreferrer"
+            data-i18n="nav.github"
+          >GitHub</a>
+          <button
+            type="button"
+            class="nav-toggle"
+            aria-expanded="false"
+            aria-controls="nav-menu-mobile"
+            id="nav-toggle"
+          >
+            <span class="sr-only">Menu</span>
+            <span></span><span></span><span></span>
+          </button>
+        </div>
+      </div>
+      <div class="nav-mobile container" id="nav-menu-mobile" hidden>
+        <a class="nav-link" href="#hero" data-nav="hero">Home</a>
+        <a class="nav-link" href="#overview" data-nav="overview">Overview</a>
+        <a class="nav-link" href="#design" data-nav="design">Design</a>
+        <a class="nav-link" href="#checkpoints" data-nav="checkpoints">Checkpoints</a>
+        <a class="nav-link" href="#evaluation" data-nav="evaluation">Eval</a>
+        <a class="nav-link" href="#evidence" data-nav="evidence">Evidence</a>
+        <a class="nav-link" href="#findings" data-nav="findings">Results</a>
+        <a class="nav-link" href="#updates" data-nav="updates">Updates</a>
+        <a class="nav-link" href="developer.html" data-i18n="nav.developer">Developer Guide</a>
+        <a class="nav-link" href="#bibtex" data-nav="bibtex">BibTeX</a>
+      </div>
+    </header>
+
+    <aside class="slide-dots" aria-label="Section navigation">
+      <button type="button" class="is-active" data-slide="hero" data-label="Home" aria-label="Home"></button>
+      <button type="button" data-slide="overview" data-label="Overview" aria-label="Overview"></button>
+      <button type="button" data-slide="design" data-label="Design" aria-label="Design"></button>
+      <button type="button" data-slide="checkpoints" data-label="Checkpoints" aria-label="Checkpoints"></button>
+      <button type="button" data-slide="evaluation" data-label="Eval" aria-label="Evaluation"></button>
+      <button type="button" data-slide="evidence" data-label="Evidence" aria-label="Evidence"></button>
+      <button type="button" data-slide="findings" data-label="Results" aria-label="Results"></button>
+      <button type="button" data-slide="updates" data-label="Updates" aria-label="Updates"></button>
+      <button type="button" data-slide="bibtex" data-label="BibTeX" aria-label="BibTeX"></button>
+    </aside>
+
+    <button type="button" class="back-top" id="back-top" data-i18n-attr="aria-label:backtop" aria-label="Back to top" hidden>
+      <svg width="18" height="18" viewBox="0 0 24 24" fill="none" aria-hidden="true">
+        <path d="M12 19V5M12 5l-6 6M12 5l6 6" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/>
+      </svg>
+    </button>
+
+    <div class="lightbox" id="lightbox" hidden role="dialog" aria-modal="true" data-i18n-attr="aria-label:lightbox.label" aria-label="Expanded figure">
+      <button type="button" class="lightbox__close" data-i18n-attr="aria-label:lightbox.close" aria-label="Close">&times;</button>
+      <figure class="lightbox__inner">
+        <img src="" alt="" id="lightbox-img" />
+        <figcaption id="lightbox-caption"></figcaption>
+      </figure>
+    </div>
+
+    <main id="main-content">
+      <section id="hero" class="section section--compact reveal" data-section="hero">
+        <div class="container hero hero--single">
+          <div>
+            <div class="hero-badge" data-i18n="hero.badge">Echo Team · Joy Future Academy, JD · June 2026 · CC BY 4.0</div>
+            <h1 class="hero-title">
+              <span class="hero-highlight">Echo-Memory</span>
+              <br />
+              <span data-i18n="hero.title.line2">A Controlled Study of Memory in Action World Models</span>
+            </h1>
+            <p class="hero-subtitle" data-i18n-html="hero.subtitle">
+              When the camera leaves and returns, which memory keeps the <strong>same world</strong>
+              instead of a plausible but different scene?
+            </p>
+            <div class="hero-cta-group">
+              <a
+                class="cta cta--primary"
+                href="https://arxiv.org/abs/2606.09803"
+                target="_blank"
+                rel="noopener noreferrer"
+                data-i18n="hero.cta.paper"
+              >Paper</a>
+              <a
+                class="cta cta--secondary"
+                href="https://arxiv.org/pdf/2606.09803"
+                target="_blank"
+                rel="noopener noreferrer"
+                data-i18n="hero.cta.pdf"
+              >PDF</a>
+              <a
+                class="cta cta--secondary"
+                href="https://huggingface.co/Echo-Team/Echo-Memory"
+                target="_blank"
+                rel="noopener noreferrer"
+                data-i18n="hero.cta.ckpt"
+              >Checkpoints</a>
+              <a
+                class="cta cta--secondary"
+                href="https://github.com/Echo-Team-Joy-Future-Academy-JD/Echo-Memory"
+                target="_blank"
+                rel="noopener noreferrer"
+                data-i18n="hero.cta.code"
+              >Code</a>
+            </div>
+
+            <div class="metrics-grid metrics-grid--duo">
+              <a
+                class="metric-card"
+                href="https://github.com/Echo-Team-Joy-Future-Academy-JD/Echo-Memory/stargazers"
+                target="_blank"
+                rel="noopener noreferrer"
+                data-badge-url="https://img.shields.io/github/stars/Echo-Team-Joy-Future-Academy-JD/Echo-Memory.json"
+              >
+                <span class="metric-label" data-i18n="hero.metric.stars">GitHub Stars</span>
+                <strong class="metric-value" data-metric-value data-fallback="—">—</strong>
+              </a>
+              <a
+                class="metric-card"
+                href="https://github.com/Echo-Team-Joy-Future-Academy-JD/Echo-Memory/network/members"
+                target="_blank"
+                rel="noopener noreferrer"
+                data-badge-url="https://img.shields.io/github/forks/Echo-Team-Joy-Future-Academy-JD/Echo-Memory.json"
+              >
+                <span class="metric-label" data-i18n="hero.metric.forks">Forks</span>
+                <strong class="metric-value" data-metric-value data-fallback="—">—</strong>
+              </a>
+            </div>
+
+            <p class="hero-note" data-i18n-html="hero.note">
+              Controlled memory ablations on a shared <strong>Wan</strong> action-to-video stack —
+              reproducible rows, evaluation scripts, and qualitative revisit panels.
+            </p>
+
+            <details class="authors-fold">
+              <summary data-i18n="hero.authors.summary">Authors &amp; affiliations</summary>
+              <p class="authors">
+                Wayne King, Zeyue Xue, Yuxuan Bian, Jie Huang, Haoran Li, Yaowei Li,
+                Yaofeng Su, Yuming Li, Haoyu Wang, Shiyi Zhang, Songchun Zhang,
+                Yuwei Niu, Sihan Xu, Junhao Zhuang, Haoyang Huang, Nan Duan
+              </p>
+              <p class="affiliations" data-i18n="hero.affiliations">
+                HKU · Joy Future Academy, JD · CUHK · PKU · Fudan · Tsinghua · HKUST · UMich
+              </p>
+            </details>
+          </div>
+        </div>
+      </section>
+
+      <section id="overview" class="section reveal" data-section="overview">
+        <div class="container">
+          <p class="section-kicker" data-i18n="overview.kicker">01 · Overview</p>
+          <h2 class="section-title" data-i18n="overview.title">One backbone, one protocol — only memory changes.</h2>
+          <p class="section-lead" data-i18n-html="overview.lead">
+            Echo-Memory holds the video backbone, training recipe, and data protocol fixed, and
+            swaps only the memory module. The goal is to separate <strong>replay fidelity</strong>
+            from <strong>return memory</strong> when the camera leaves and comes back to the same place.
+          </p>
+          <ul class="bullet-list">
+            <li data-i18n-html="overview.b1"><strong>Shared stack</strong> — chunk-wise action-conditioned world generation on Wan.</li>
+            <li data-i18n-html="overview.b2"><strong>Controlled variable</strong> — Context, Compression, Spatial, or State-Space memory.</li>
+            <li data-i18n-html="overview.b3"><strong>Three probes</strong> — replay metrics, in-domain 180° loop, open-domain edited return.</li>
+            <li data-i18n-html="overview.b4"><strong>Release</strong> — ablation scripts, GT replay, revisit assets, and paper-aligned figures.</li>
+          </ul>
+          <figure class="section-figure" data-zoomable>
+            <img src="assets/figure_1_abs_framework.png?v=fig1-crop" data-i18n-attr="alt:overview.fig.alt" alt="Echo-Memory framework overview" />
+            <figcaption data-i18n-html="overview.fig.cap">Controlled memory study over chunk-wise action-world generation. <span class="zoom-hint">Click to expand</span></figcaption>
+          </figure>
+        </div>
+      </section>
+
+      <section id="design" class="section reveal" data-section="design">
+        <div class="container">
+          <p class="section-kicker" data-i18n="design.kicker">02 · Memory Design</p>
+          <h2 class="section-title" data-i18n="design.title">Context · Compression · Spatial · State-Space</h2>
+          <p class="section-lead" data-i18n="design.lead">
+            All variants plug into the same write–read interface; we only change what is stored and how
+            history is retrieved. A no-memory I2V floor re-generates from the first frame as a lower bound.
+          </p>
+          <div class="feature-grid">
+            <article class="card card--accent" data-accent="context">
+              <div class="card-icon" aria-hidden="true">
+                <svg viewBox="0 0 24 24" fill="none"><rect x="3" y="5" width="18" height="14" rx="2" stroke="currentColor" stroke-width="1.6"/><path d="M7 9h10M7 13h6" stroke="currentColor" stroke-width="1.6" stroke-linecap="round"/></svg>
+              </div>
+              <h3 data-i18n="design.context.title">Context</h3>
+              <p data-i18n="design.context.body">Raw recent frames at K = 1, 5, or 20 chunks — tests whether longer windows alone stop drift.</p>
+            </article>
+            <article class="card card--accent" data-accent="compression">
+              <div class="card-icon" aria-hidden="true">
+                <svg viewBox="0 0 24 24" fill="none"><path d="M12 3l8 4.5v9L12 21l-8-4.5v-9L12 3z" stroke="currentColor" stroke-width="1.6"/><path d="M12 12l8-4.5M12 12v9M12 12L4 7.5" stroke="currentColor" stroke-width="1.6"/></svg>
+              </div>
+              <h3 data-i18n="design.compression.title">Compression</h3>
+              <p data-i18n="design.compression.body">Learned compact tokens at ratio r = 4 — history without growing raw-frame storage.</p>
+            </article>
+            <article class="card card--accent" data-accent="spatial">
+              <div class="card-icon" aria-hidden="true">
+                <svg viewBox="0 0 24 24" fill="none"><path d="M4 8l8-4 8 4v8l-8 4-8-4V8z" stroke="currentColor" stroke-width="1.6"/><path d="M12 4v16M4 8l8 4 8-4" stroke="currentColor" stroke-width="1.6"/></svg>
+              </div>
+              <h3 data-i18n="design.spatial.title">Spatial</h3>
+              <p data-i18n="design.spatial.body">Explicit spatial read/write state — targets layout, object pose, and viewpoint carry.</p>
+            </article>
+            <article class="card card--accent" data-accent="ssm">
+              <div class="card-icon" aria-hidden="true">
+                <svg viewBox="0 0 24 24" fill="none"><path d="M5 12a7 7 0 0114 0" stroke="currentColor" stroke-width="1.6" stroke-linecap="round"/><path d="M8 12h8M16 9l2 3-2 3M8 9l-2 3 2 3" stroke="currentColor" stroke-width="1.6" stroke-linecap="round" stroke-linejoin="round"/></svg>
+              </div>
+              <h3 data-i18n="design.ssm.title">State-Space</h3>
+              <p data-i18n="design.ssm.body">Block-wise SSM updates — recurrent carry beyond short context windows on revisit.</p>
+            </article>
+          </div>
+          <figure class="section-figure" data-zoomable>
+            <img src="assets/figure_2_mem_overview.png" data-i18n-attr="alt:design.fig.alt" alt="Memory design matrix" />
+            <figcaption data-i18n-html="design.fig.cap">Four memory families under a shared write–read interface. <span class="zoom-hint">Click to expand</span></figcaption>
+          </figure>
+        </div>
+      </section>
+
+      <section id="checkpoints" class="section reveal" data-section="checkpoints">
+        <div class="container">
+          <p class="section-kicker" data-i18n="ckpt.kicker">03 · Checkpoints</p>
+          <h2 class="section-title" data-i18n="ckpt.title">Paper baselines on Hugging Face</h2>
+          <p class="section-lead" data-i18n-html="ckpt.lead">
+            Wan 2.1 1.3B memory rows — <strong>epoch-0</strong>, <strong>30,000 steps</strong>, static in-domain pool.
+            Released weights:
+            <a href="https://huggingface.co/Echo-Team/Echo-Memory" target="_blank" rel="noopener noreferrer"><strong>Echo-Team/Echo-Memory</strong></a>
+          </p>
+          <div class="table-wrap">
+            <table class="ckpt-table">
+              <thead>
+                <tr>
+                  <th data-i18n="ckpt.th.family">Family</th>
+                  <th data-i18n="ckpt.th.row">Paper row</th>
+                  <th data-i18n="ckpt.th.path">HF path</th>
+                  <th data-i18n="ckpt.th.steps">Steps</th>
+                </tr>
+              </thead>
+              <tbody>
+                <tr><td>Raw context</td><td>Context K=1</td><td><code>context_k1/epoch-0.safetensors</code></td><td>30,000</td></tr>
+                <tr><td>Raw context</td><td>Context K=20</td><td>TODO</td><td>TODO</td></tr>
+                <tr><td>Spatial</td><td>Spatial Memory</td><td>TODO</td><td>TODO</td></tr>
+                <tr><td>State-space</td><td>Block-wise SSM</td><td>TODO</td><td>TODO</td></tr>
+                <tr><td>State-space</td><td>Legacy Hybrid</td><td>TODO</td><td>TODO</td></tr>
+                <tr><td>Spatial</td><td>concat text (abl.)</td><td>TODO</td><td>TODO</td></tr>
+                <tr><td>Spatial</td><td>inject none (abl.)</td><td>TODO</td><td>TODO</td></tr>
+                <tr><td>Spatial</td><td>cross-attn t32 (abl.)</td><td>TODO</td><td>TODO</td></tr>
+                <tr><td>State-space</td><td>SSM ctx1/e4/h21</td><td>TODO</td><td>TODO</td></tr>
+                <tr><td>State-space</td><td>SSM ctx5/e1/h21</td><td>TODO</td><td>TODO</td></tr>
+                <tr><td>State-space</td><td>SSM ctx5/e4/h81</td><td>TODO</td><td>TODO</td></tr>
+              </tbody>
+            </table>
+          </div>
+          <div class="code-block-wrap">
+            <p class="code-block-label" data-i18n="ckpt.label.download">Download</p>
+            <pre class="code-block"><code>huggingface-cli download Echo-Team/Echo-Memory context_k1/epoch-0.safetensors --local-dir ./ckpts</code></pre>
+          </div>
+          <div class="code-block-wrap">
+            <p class="code-block-label" data-i18n="ckpt.label.eval">In-domain eval (Echo-Memory repo)</p>
+            <pre class="code-block"><code>export WAN_BASE_MODEL=/path/to/Wan2.1-T2V-1.3B
+export DATASET_BASE_PATH=data/Context-as-Memory-Dataset
+export CKPT=./ckpts/context_k1/epoch-0.safetensors
+bash eval/v2/run_static_consistency_loop_and_revisit.sh</code></pre>
+          </div>
+          <p class="section-note" data-i18n-html="ckpt.note">
+            Keep the row folder in <code>CKPT</code> — <code>env/memory_baseline_runtime.py</code> infers memory flags from the path.
+            Full index:
+            <a href="https://github.com/Echo-Team-Joy-Future-Academy-JD/Echo-Memory/blob/main/doc/checkpoints.md" target="_blank" rel="noopener noreferrer">doc/checkpoints.md</a>.
+          </p>
+        </div>
+      </section>
+
+      <section id="evaluation" class="section section--center reveal" data-section="evaluation">
+        <div class="container">
+          <p class="section-kicker" data-i18n="eval.kicker">04 · Evaluation</p>
+          <h2 class="section-title" data-i18n="eval.title">Replay · In-domain revisit · Open-domain return</h2>
+          <p class="section-lead" data-i18n-html="eval.lead">
+            Each branch asks a different question: Can the model reconstruct the past? Can it close a loop
+            in-domain? After an edited first frame, does it return to the <strong>same world</strong>?
+          </p>
+          <div class="feature-grid feature-grid--three">
+            <article class="card">
+              <h3 data-i18n="eval.replay.title">Replay</h3>
+              <p data-i18n="eval.replay.body">PSNR, SSIM, LPIPS on chunk-wise reconstruction — measures short-horizon pixel fidelity.</p>
+            </article>
+            <article class="card">
+              <h3 data-i18n="eval.indomain.title">In-domain</h3>
+              <p data-i18n="eval.indomain.body">180° trajectory loop closure with VLM-assisted scoring on held layouts.</p>
+            </article>
+            <article class="card">
+              <h3 data-i18n="eval.opendomain.title">Open-domain</h3>
+              <p data-i18n="eval.opendomain.body">Edited first frames and 45° return probes — stresses object identity and scene persistence.</p>
+            </article>
+            <article class="card">
+              <h3 data-i18n="eval.dynamic.title">Dynamic SpatialVID</h3>
+              <p data-i18n="eval.dynamic.body">Training and inference wrappers are public; the dynamic eval protocol is TODO.</p>
+            </article>
+          </div>
+          <figure class="section-figure" data-zoomable>
+            <img src="assets/figure_3_mem_eval.png" data-i18n-attr="alt:eval.fig.alt" alt="Three-branch evaluation summary" />
+            <figcaption data-i18n-html="eval.fig.cap">Replay health vs. return memory under the same stack. <span class="zoom-hint">Click to expand</span></figcaption>
+          </figure>
+        </div>
+      </section>
+
+      <section id="evidence" class="section section--center reveal" data-section="evidence">
+        <div class="container">
+          <p class="section-kicker" data-i18n="evidence.kicker">05 · Qualitative Evidence</p>
+          <h2 class="section-title" data-i18n="evidence.title">Return probes expose identity drift.</h2>
+          <p class="section-lead" data-i18n-html="evidence.lead">
+            Qualitative panels follow a simple diagnostic: <strong>first frame → leave the view → revisit tail</strong>.
+            We compare whether memory restores the same object, pose, background, and camera geometry —
+            not merely a plausible new scene.
+          </p>
+          <div class="qual-viewer" data-qual-viewer>
+            <figure class="qual-stage" data-zoomable>
+              <div class="qual-stage__media">
+                <img
+                  data-qual-image
+                  class="is-active"
+                  src="assets/representative_memory_results.png"
+                  alt="Representative memory comparisons"
+                />
+              </div>
+              <figcaption data-qual-caption data-i18n-html="evidence.cap1">Representative memory comparisons across variants. <span class="zoom-hint">Click to expand</span></figcaption>
+            </figure>
+            <div class="qual-chips" role="tablist" aria-label="Evidence panels">
+              <button
+                class="qual-chip is-active"
+                type="button"
+                role="tab"
+                aria-selected="true"
+                data-src="assets/representative_memory_results.png"
+                data-alt="Representative memory comparisons"
+                data-caption-key="evidence.cap1"
+                data-caption="Representative memory comparisons across variants."
+              >Memory Results</button>
+              <button
+                class="qual-chip"
+                type="button"
+                role="tab"
+                aria-selected="false"
+                data-src="assets/representative_sweep_panel.png"
+                data-alt="Open-domain return sweep"
+                data-caption="Open-domain return sweep across the design matrix."
+              >Open-Domain Sweep</button>
+              <button
+                class="qual-chip"
+                type="button"
+                role="tab"
+                aria-selected="false"
+                data-src="assets/open_domain_revisit_panel.png"
+                data-alt="Edited first frames"
+                data-caption="Edited first frames for open-domain return probes."
+              >Identity Anchors</button>
+            </div>
+          </div>
+          <div class="demo-grid" aria-label="Static and SpatialVID replay demos">
+            <h3 class="demo-grid__label">Static Replay</h3>
+            <figure>
+              <img src="assets/readme_previews/context_k1_replay_gt.gif" alt="Static Context K=1 replay">
+              <figcaption>Context K=1</figcaption>
+            </figure>
+            <figure>
+              <img src="assets/readme_previews/context_k5_replay_gt.gif" alt="Static Context K=5 replay">
+              <figcaption>Context K=5</figcaption>
+            </figure>
+            <figure>
+              <img src="assets/readme_previews/framepack_len_r4_replay_gt.gif" alt="Static Compression r=4 replay">
+              <figcaption>Compression r = 4</figcaption>
+            </figure>
+            <figure>
+              <img src="assets/readme_previews/spatial_memory_replay_gt.gif" alt="Static Spatial Memory replay">
+              <figcaption>Spatial Memory</figcaption>
+            </figure>
+            <figure>
+              <img src="assets/readme_previews/ssm_legacy_replay_gt.gif" alt="Static legacy VideoSSM replay">
+              <figcaption>Legacy Hybrid</figcaption>
+            </figure>
+            <figure>
+              <img src="assets/readme_previews/ssm_blockwise_replay_gt.gif" alt="Static Block-wise SSM replay">
+              <figcaption>Block-wise SSM</figcaption>
+            </figure>
+            <h3 class="demo-grid__label">SpatialVID Replay</h3>
+            <figure>
+              <img src="assets/readme_previews/dyn_context_k1_replay.gif" alt="Dynamic Context K=1 replay">
+              <figcaption>Context K=1</figcaption>
+            </figure>
+            <figure>
+              <img src="assets/readme_previews/dyn_context_k5_replay.gif" alt="Dynamic Context K=5 replay">
+              <figcaption>Context K=5</figcaption>
+            </figure>
+            <figure>
+              <img src="assets/readme_previews/dyn_context_k20_replay.gif" alt="Dynamic Context K=20 replay">
+              <figcaption>Context K=20</figcaption>
+            </figure>
+            <figure>
+              <img src="assets/readme_previews/dyn_spatial_memory_replay.gif" alt="Dynamic Spatial Memory replay">
+              <figcaption>Spatial Memory</figcaption>
+            </figure>
+            <figure>
+              <img src="assets/readme_previews/dyn_ssm_legacy_replay.gif" alt="Dynamic legacy VideoSSM replay">
+              <figcaption>Legacy Hybrid</figcaption>
+            </figure>
+            <figure>
+              <img src="assets/readme_previews/dyn_ssm_blockwise_replay.gif" alt="Dynamic Block-wise SSM replay">
+              <figcaption>Block-wise SSM</figcaption>
+            </figure>
+          </div>
+          <p class="section-note" data-i18n="evidence.dynamic.note">
+            Dynamic previews use one randomly selected training scene replayed with the same first frame, prompt, and GT camera trajectory across all six rows.
+          </p>
+        </div>
+      </section>
+
+      <section id="findings" class="section section--center reveal" data-section="findings">
+        <div class="container">
+          <p class="section-kicker" data-i18n="findings.kicker">06 · Main Conclusions</p>
+          <h2 class="section-title" data-i18n="findings.title">Replay quality ≠ memory quality.</h2>
+          <p class="section-lead" data-i18n="findings.lead">
+            Replay metrics and return probes do not always agree — a model can look sharp on reconstruction
+            yet fail when the camera returns. Rankings reorder once identity under revisit is measured.
+          </p>
+          <ul class="bullet-list bullet-list--center">
+            <li data-i18n-html="findings.b1"><strong>Raw context</strong> — more history helps open-domain return more than replay alone.</li>
+            <li data-i18n-html="findings.b2"><strong>Compression</strong> — compact tokens can preserve replay while losing identity on return.</li>
+            <li data-i18n-html="findings.b3"><strong>Spatial vs. SSM</strong> — explicit state and block-wise SSM trade off layout carry and long-horizon stability.</li>
+            <li data-i18n-html="findings.b4"><strong>Takeaway</strong> — treat replay as a health check, not the final memory benchmark.</li>
+          </ul>
+          <figure class="section-figure" data-zoomable>
+            <img src="assets/replay_revisit_metric_alignment.png" alt="Replay vs revisit metrics" />
+            <figcaption data-i18n-html="findings.fig.cap">Rank shift from replay to return — replay is not the final memory score. <span class="zoom-hint">Click to expand</span></figcaption>
+          </figure>
+        </div>
+      </section>
+
+      <section id="updates" class="section reveal" data-section="updates">
+        <div class="container">
+          <p class="section-kicker" data-i18n="updates.kicker">07 · News &amp; Roadmap</p>
+          <h2 class="section-title" data-i18n="updates.title">Release notes and next steps.</h2>
+
+          <div class="updates-grid">
+            <div class="updates-panel">
+              <h3 class="updates-heading" data-i18n="updates.news">News</h3>
+              <ul class="news-feed">
+                <li class="news-item">
+                  <time datetime="2026-06-13">2026/06/13</time>
+                  <p data-i18n-html="updates.news0">
+                    SpatialVID support added: dynamic training/inference recipes, 5-second first-chunk replay previews, and dynamic eval TODO.
+                  </p>
+                </li>
+                <li class="news-item">
+                  <time datetime="2026-06-06">2026/06/06</time>
+                  <p data-i18n-html="updates.news1">
+                    Echo-Memory released: paper, project page, public code, replay/revisit eval assets, and baseline checkpoints.
+                  </p>
+                </li>
+              </ul>
+            </div>
+
+            <div class="updates-panel">
+              <h3 class="updates-heading" data-i18n="updates.roadmap">Roadmap</h3>
+              <div class="todo-groups">
+                <article class="todo-group">
+                  <h4 data-i18n="updates.models">Models</h4>
+                  <ul class="todo-list">
+                    <li class="is-done">
+                      <span class="todo-check is-done" aria-hidden="true"></span>
+                      <span data-i18n-html="updates.rm1"><strong>Wan 2.1 1.3B</strong> backbone and training recipes</span>
+                    </li>
+                    <li class="is-done">
+                      <span class="todo-check is-done" aria-hidden="true"></span>
+                      <span data-i18n="updates.rm2">Four memory families — Context, Compression, Spatial, State-Space</span>
+                    </li>
+                    <li class="is-done">
+                      <span class="todo-check is-done" aria-hidden="true"></span>
+                      <span data-i18n-html="updates.rm3"><strong>Dynamic training pool</strong> — SpatialVID subset export &amp; settings</span>
+                    </li>
+                    <li class="is-done">
+                      <span class="todo-check is-done" aria-hidden="true"></span>
+                      <span data-i18n-html="updates.rm4"><strong>Paper checkpoints</strong> — <a href="https://huggingface.co/Echo-Team/Echo-Memory" target="_blank" rel="noopener noreferrer">Echo-Team/Echo-Memory</a></span>
+                    </li>
+                    <li>
+                      <span class="todo-check" aria-hidden="true"></span>
+                      <span data-i18n-html="updates.rm5"><strong>Wan 2.2</strong> + multi-scale <strong>5B / 14B</strong></span>
+                    </li>
+                  </ul>
+                </article>
+                <article class="todo-group">
+                  <h4 data-i18n="updates.eval">Eval</h4>
+                  <ul class="todo-list">
+                    <li>
+                      <span class="todo-check" aria-hidden="true"></span>
+                      <span data-i18n-html="updates.re1"><strong>Dynamic eval</strong> beyond static replay/revisit</span>
+                    </li>
+                    <li>
+                      <span class="todo-check" aria-hidden="true"></span>
+                      <span data-i18n="updates.re2">More revisit probes and scoring presets</span>
+                    </li>
+                  </ul>
+                </article>
+              </div>
+            </div>
+
+            <div class="updates-panel community-panel">
+              <h3 class="updates-heading" data-i18n="community.title">Community</h3>
+              <p data-i18n="community.lead">Join the Echo-Memory WeChat group for release updates, checkpoint questions, and maintainer coordination.</p>
+              <figure class="wechat-qr">
+                <img
+                  class="wechat-qr__photo"
+                  src="assets/wechat_group_qrcode.jpg?v=20260701"
+                  width="1166"
+                  height="1640"
+                  alt="Echo-Memory WeChat group QR code"
+                  data-i18n-attr="alt:community.qr.alt"
+                />
+                <figcaption data-i18n="community.qr.caption">Echo-Memory 交流群 · scan to join (QR refreshes periodically)</figcaption>
+              </figure>
+            </div>
+          </div>
+        </div>
+      </section>
+
+      <section id="bibtex" class="section reveal" data-section="bibtex">
+        <div class="container">
+          <p class="section-kicker" data-i18n="bibtex.kicker">08 · Citation</p>
+          <h2 class="section-title" data-i18n="bibtex.title">BibTeX</h2>
+          <p class="section-lead" data-i18n-html="bibtex.lead">
+            Echo-Memory: A Controlled Study of Memory in Action World Models (June 2026).
+            Licensed under
+            <a href="https://creativecommons.org/licenses/by/4.0/" target="_blank" rel="noopener noreferrer">CC BY 4.0</a>.
+            Cite the arXiv preprint below.
+          </p>
+          <div class="cite-switcher">
+            <div class="cite-panel is-active">
+              <dl class="pub-meta">
+                <div class="pub-meta__row">
+                  <dt data-i18n="bibtex.source">Source</dt>
+                  <dd>
+                    <a href="https://arxiv.org/abs/2606.09803" target="_blank" rel="noopener noreferrer">arXiv</a>
+                  </dd>
+                </div>
+                <div class="pub-meta__row">
+                  <dt data-i18n="bibtex.arxivid">arXiv ID</dt>
+                  <dd>2606.09803</dd>
+                </div>
+                <div class="pub-meta__row">
+                  <dt data-i18n="bibtex.pdf">PDF</dt>
+                  <dd>
+                    <a href="https://arxiv.org/pdf/2606.09803" target="_blank" rel="noopener noreferrer">arxiv.org/pdf/2606.09803</a>
+                  </dd>
+                </div>
+                <div class="pub-meta__row">
+                  <dt data-i18n="bibtex.license">License</dt>
+                  <dd>CC BY 4.0</dd>
+                </div>
+              </dl>
+              <div class="bibtex-wrap">
+                <button type="button" class="copy-btn" data-copy-target="bibtex-arxiv-block" data-i18n="bibtex.copy">Copy</button>
+                <pre class="bibtex" id="bibtex-arxiv-block"><code>@article{king2026echomemory,
+  title={Echo-Memory: A Controlled Study of Memory in Action World Models},
+  author={King, Wayne and Xue, Zeyue and Bian, Yuxuan and Huang, Jie and Li, Haoran and Li, Yaowei and Su, Yaofeng and Li, Yuming and Wang, Haoyu and Zhang, Shiyi and Zhang, Songchun and Niu, Yuwei and Xu, Sihan and Zhuang, Junhao and Huang, Haoyang and Duan, Nan},
+  journal={arXiv preprint arXiv:2606.09803},
+  year={2026},
+  month={jun},
+  eprint={2606.09803},
+  archivePrefix={arXiv},
+  primaryClass={cs.CV},
+  url={https://arxiv.org/abs/2606.09803}
+}</code></pre>
+              </div>
+            </div>
+          </div>
+        </div>
+      </section>
+    </main>
+
+    <footer class="footer">
+      <div class="container footer-row">
+        <div>
+          <p class="footer-copy" data-i18n="footer.copy">© Echo Team · Joy Future Academy, JD</p>
+        </div>
+        <div class="footer-links">
+          <a href="https://github.com/Echo-Team-Joy-Future-Academy-JD/Echo-Memory">GitHub</a>
+          <a href="https://arxiv.org/abs/2606.09803" target="_blank" rel="noopener noreferrer">arXiv</a>
+          <a href="https://arxiv.org/pdf/2606.09803" target="_blank" rel="noopener noreferrer">PDF</a>
+          <a href="https://echo-team-joy-future-academy-jd.github.io/Echo-Memory/" data-i18n="footer.pp">Project Page</a>
+          <a href="developer.html" data-i18n="nav.developer">Developer Guide</a>
+        </div>
+        <p class="build-id" hidden aria-hidden="true">site-build-main</p>
+      </div>
+    </footer>
+  </div>
+
+  <script src="i18n.js?v=lang-v2"></script>
+  <script src="i18n-runtime.js?v=lang-v2"></script>
+  <script src="site.js?v=lang-v1" defer></script>
+</body>
+</html>

code/docs/site.js

@@ -0,0 +1,349 @@
+(function () {
+  "use strict";
+
+  const NAV_OFFSET = 72;
+  const REDUCED_MOTION = window.matchMedia("(prefers-reduced-motion: reduce)").matches;
+  const SCROLL_BEHAVIOR = REDUCED_MOTION ? "auto" : "smooth";
+
+  const sectionEls = Array.from(document.querySelectorAll(".section[data-section]"));
+  const navLinks = Array.from(document.querySelectorAll("[data-nav]"));
+  const slideDots = Array.from(document.querySelectorAll(".slide-dots button[data-slide]"));
+  const navToggle = document.getElementById("nav-toggle");
+  const navMobile = document.getElementById("nav-menu-mobile");
+  const topNav = document.querySelector(".top-nav");
+  const progressBar = document.querySelector(".scroll-progress__bar");
+  const backTop = document.getElementById("back-top");
+  const lightbox = document.getElementById("lightbox");
+  const lightboxImg = document.getElementById("lightbox-img");
+  const lightboxCaption = document.getElementById("lightbox-caption");
+  const lightboxClose = lightbox ? lightbox.querySelector(".lightbox__close") : null;
+
+  function setActiveNav(id) {
+    navLinks.forEach(function (a) {
+      a.classList.toggle("is-active", a.getAttribute("data-nav") === id);
+    });
+    slideDots.forEach(function (btn) {
+      btn.classList.toggle("is-active", btn.getAttribute("data-slide") === id);
+    });
+  }
+
+  function scrollToSection(id) {
+    var el = document.getElementById(id);
+    if (!el) return;
+    el.scrollIntoView({ behavior: SCROLL_BEHAVIOR, block: "start" });
+    setActiveNav(id);
+    if (navMobile && !navMobile.hidden) {
+      navToggle.setAttribute("aria-expanded", "false");
+      navMobile.hidden = true;
+      if (topNav) topNav.classList.remove("is-open");
+    }
+  }
+
+  function sectionIndex(id) {
+    return sectionEls.findIndex(function (s) {
+      return s.getAttribute("data-section") === id;
+    });
+  }
+
+  function scrollAdjacent(delta) {
+    var activeId = null;
+    sectionEls.forEach(function (s) {
+      var rect = s.getBoundingClientRect();
+      if (rect.top <= NAV_OFFSET + 80 && rect.bottom > NAV_OFFSET + 80) {
+        activeId = s.getAttribute("data-section");
+      }
+    });
+    if (!activeId && sectionEls.length) activeId = sectionEls[0].getAttribute("data-section");
+    var idx = sectionIndex(activeId);
+    if (idx < 0) return;
+    var next = sectionEls[idx + delta];
+    if (next) scrollToSection(next.getAttribute("data-section"));
+  }
+
+  if (sectionEls.length) {
+    var observer = new IntersectionObserver(
+      function (entries) {
+        entries.forEach(function (entry) {
+          if (entry.isIntersecting) {
+            setActiveNav(entry.target.getAttribute("data-section"));
+          }
+        });
+      },
+      { root: null, rootMargin: "-" + NAV_OFFSET + "px 0px -50% 0px", threshold: 0.12 }
+    );
+    sectionEls.forEach(function (section) {
+      observer.observe(section);
+    });
+  }
+
+  document.querySelectorAll(".reveal").forEach(function (el) {
+    var revealObserver = new IntersectionObserver(
+      function (entries) {
+        entries.forEach(function (entry) {
+          if (entry.isIntersecting) {
+            entry.target.classList.add("is-visible");
+            revealObserver.unobserve(entry.target);
+          }
+        });
+      },
+      { root: null, rootMargin: "0px 0px -8% 0px", threshold: 0.08 }
+    );
+    if (REDUCED_MOTION) {
+      el.classList.add("is-visible");
+    } else {
+      revealObserver.observe(el);
+    }
+  });
+
+  function updateScrollUI() {
+    var scrollTop = window.scrollY || document.documentElement.scrollTop;
+    var docHeight = document.documentElement.scrollHeight - window.innerHeight;
+    var progress = docHeight > 0 ? (scrollTop / docHeight) * 100 : 0;
+
+    if (progressBar) progressBar.style.width = progress + "%";
+    if (topNav) topNav.classList.toggle("is-scrolled", scrollTop > 24);
+    if (backTop) {
+      var show = scrollTop > window.innerHeight * 0.6;
+      backTop.hidden = !show;
+      backTop.classList.toggle("is-visible", show);
+    }
+  }
+
+  window.addEventListener("scroll", updateScrollUI, { passive: true });
+  updateScrollUI();
+
+  navLinks.forEach(function (a) {
+    a.addEventListener("click", function (e) {
+      e.preventDefault();
+      scrollToSection(a.getAttribute("data-nav"));
+    });
+  });
+
+  slideDots.forEach(function (btn) {
+    btn.addEventListener("click", function () {
+      scrollToSection(btn.getAttribute("data-slide"));
+    });
+  });
+
+  if (navToggle && navMobile) {
+    navToggle.addEventListener("click", function () {
+      var open = navMobile.hidden;
+      navMobile.hidden = !open;
+      navToggle.setAttribute("aria-expanded", String(open));
+      if (topNav) topNav.classList.toggle("is-open", open);
+    });
+  }
+
+  if (backTop) {
+    backTop.addEventListener("click", function () {
+      scrollToSection("hero");
+    });
+  }
+
+  function openLightbox(img, captionText) {
+    if (!lightbox || !lightboxImg) return;
+    lightboxImg.src = img.src;
+    lightboxImg.alt = img.alt || "";
+    if (lightboxCaption) {
+      lightboxCaption.textContent = captionText || "";
+    }
+    lightbox.hidden = false;
+    document.body.style.overflow = "hidden";
+    if (lightboxClose) lightboxClose.focus();
+  }
+
+  function closeLightbox() {
+    if (!lightbox) return;
+    lightbox.hidden = true;
+    document.body.style.overflow = "";
+    if (lightboxImg) lightboxImg.src = "";
+  }
+
+  document.querySelectorAll("[data-zoomable]").forEach(function (figure) {
+    figure.addEventListener("click", function (e) {
+      if (e.target.closest(".qual-chip, .copy-btn, button, a")) return;
+      var img = figure.querySelector("img.is-active, img[data-qual-image], img");
+      if (!img || !img.src) return;
+      var cap = figure.querySelector("figcaption");
+      var capText = cap ? cap.textContent.replace(/Click to expand/g, "").trim() : "";
+      openLightbox(img, capText);
+    });
+  });
+
+  if (lightboxClose) {
+    lightboxClose.addEventListener("click", closeLightbox);
+  }
+
+  if (lightbox) {
+    lightbox.addEventListener("click", function (e) {
+      if (e.target === lightbox) closeLightbox();
+    });
+  }
+
+  document.addEventListener("keydown", function (e) {
+    if (e.target.closest("input, textarea, pre, [contenteditable]")) return;
+
+    if (e.key === "Escape" && lightbox && !lightbox.hidden) {
+      closeLightbox();
+      return;
+    }
+
+    if (e.key === "ArrowDown" || e.key === "PageDown") {
+      e.preventDefault();
+      scrollAdjacent(1);
+    } else if (e.key === "ArrowUp" || e.key === "PageUp") {
+      e.preventDefault();
+      scrollAdjacent(-1);
+    }
+  });
+
+  function animateMetricValue(el, target) {
+    if (REDUCED_MOTION || !target || target === "—") {
+      el.textContent = target;
+      return;
+    }
+    var numeric = parseInt(String(target).replace(/,/g, ""), 10);
+    if (isNaN(numeric)) {
+      el.textContent = target;
+      return;
+    }
+    var start = 0;
+    var duration = 700;
+    var startTime = null;
+    function step(ts) {
+      if (!startTime) startTime = ts;
+      var t = Math.min((ts - startTime) / duration, 1);
+      var eased = 1 - Math.pow(1 - t, 3);
+      el.textContent = Math.round(start + (numeric - start) * eased).toLocaleString();
+      if (t < 1) requestAnimationFrame(step);
+    }
+    requestAnimationFrame(step);
+  }
+
+  function hydrateMetric(card) {
+    var valueEl = card.querySelector("[data-metric-value]");
+    var badgeUrl = card.getAttribute("data-badge-url");
+    if (!valueEl || !badgeUrl) return;
+
+    fetch(badgeUrl, { cache: "no-store" })
+      .then(function (res) {
+        if (!res.ok) throw new Error("Request failed");
+        return res.json();
+      })
+      .then(function (data) {
+        var val = data.value || data.message || valueEl.dataset.fallback || "—";
+        animateMetricValue(valueEl, val);
+      })
+      .catch(function () {
+        valueEl.textContent = valueEl.dataset.fallback || "—";
+      });
+  }
+
+  document.querySelectorAll("[data-badge-url]").forEach(function (card) {
+    var metricObserver = new IntersectionObserver(
+      function (entries) {
+        entries.forEach(function (entry) {
+          if (entry.isIntersecting) {
+            hydrateMetric(entry.target);
+            metricObserver.unobserve(entry.target);
+          }
+        });
+      },
+      { threshold: 0.4 }
+    );
+    metricObserver.observe(card);
+  });
+
+  document.querySelectorAll("[data-qual-viewer]").forEach(function (viewer) {
+    var image = viewer.querySelector("[data-qual-image]");
+    var caption = viewer.querySelector("[data-qual-caption]");
+    var chips = viewer.querySelectorAll(".qual-chip");
+
+    chips.forEach(function (chip) {
+      chip.addEventListener("click", function () {
+        chips.forEach(function (c) {
+          c.classList.remove("is-active");
+          c.setAttribute("aria-selected", "false");
+        });
+        chip.classList.add("is-active");
+        chip.setAttribute("aria-selected", "true");
+        if (!image) return;
+
+        var nextSrc = chip.dataset.src;
+        var nextAlt = chip.dataset.alt || "";
+        var nextCaption = chip.dataset.caption || "";
+
+        var currentPath = new URL(image.src, window.location.href).pathname;
+        if (currentPath.endsWith(nextSrc)) return;
+
+        image.classList.add("is-fading");
+        var preload = new Image();
+        preload.onload = function () {
+          image.src = nextSrc;
+          image.alt = nextAlt;
+          image.classList.remove("is-fading");
+          image.classList.add("is-active");
+        };
+        preload.onerror = function () {
+          image.classList.remove("is-fading");
+        };
+        preload.src = nextSrc;
+
+        if (caption) {
+          var hint = (window.EchoI18n && window.EchoI18n.t("zoom.hint")) || "Click to expand";
+          caption.innerHTML = nextCaption + ' <span class="zoom-hint">' + hint + "</span>";
+        }
+      });
+    });
+  });
+
+  function initCiteSwitcher() {
+    document.querySelectorAll("[data-cite-switcher]").forEach(function (switcher) {
+      var chips = switcher.querySelectorAll(".cite-chip");
+      var panels = switcher.querySelectorAll("[data-cite-panel]");
+
+      chips.forEach(function (chip) {
+        chip.addEventListener("click", function () {
+          var key = chip.getAttribute("data-cite");
+          chips.forEach(function (c) {
+            c.classList.remove("is-active");
+            c.setAttribute("aria-selected", "false");
+          });
+          chip.classList.add("is-active");
+          chip.setAttribute("aria-selected", "true");
+          panels.forEach(function (panel) {
+            var match = panel.getAttribute("data-cite-panel") === key;
+            panel.classList.toggle("is-active", match);
+            panel.hidden = !match;
+          });
+        });
+      });
+    });
+  }
+
+  initCiteSwitcher();
+
+  document.querySelectorAll("[data-copy-target]").forEach(function (btn) {
+    btn.addEventListener("click", function () {
+      var targetId = btn.getAttribute("data-copy-target");
+      var target = document.getElementById(targetId);
+      if (!target) return;
+      navigator.clipboard.writeText(target.textContent).then(
+        function () {
+          var copied = window.EchoI18n && window.EchoI18n.t("bibtex.copied");
+          var failed = window.EchoI18n && window.EchoI18n.t("bibtex.failed");
+          var copyLabel = window.EchoI18n && window.EchoI18n.t("bibtex.copy");
+          btn.textContent = copied || "Copied";
+          btn.classList.add("is-copied");
+          setTimeout(function () {
+            btn.textContent = copyLabel || "Copy";
+            btn.classList.remove("is-copied");
+          }, 2000);
+        },
+        function () {
+          btn.textContent = (window.EchoI18n && window.EchoI18n.t("bibtex.failed")) || "Failed";
+        }
+      );
+    });
+  });
+})();

code/docs/style.css

@@ -0,0 +1,1848 @@
+:root {
+  --bg: #fffcfd;
+  --bg-soft: #fff5f8;
+  --surface: #ffffff;
+  --surface-2: #fbf0f4;
+  --text: #191218;
+  --text-soft: #655b63;
+  --border: #ead9df;
+  --brand: #bf5276;
+  --brand-2: #d97498;
+  --accent: #84354e;
+  --ring: rgba(191, 82, 118, 0.28);
+  --shadow: 0 14px 36px rgba(93, 45, 61, 0.08);
+  --shadow-lg: 0 22px 48px rgba(93, 45, 61, 0.12);
+  --nav-h: 72px;
+  --figure-max: 880px;
+  --ease-out: cubic-bezier(0.22, 1, 0.36, 1);
+}
+
+*,
+*::before,
+*::after {
+  box-sizing: border-box;
+}
+
+html {
+  scroll-behavior: smooth;
+  scroll-snap-type: y proximity;
+  scroll-padding-top: var(--nav-h);
+}
+
+body {
+  margin: 0;
+  font-family: "IBM Plex Sans", system-ui, -apple-system, "Segoe UI", Roboto, sans-serif;
+  color: var(--text);
+  background-color: #fff9fb;
+  min-height: 100vh;
+}
+
+a {
+  color: inherit;
+}
+
+img {
+  max-width: 100%;
+  display: block;
+}
+
+.sr-only {
+  position: absolute;
+  width: 1px;
+  height: 1px;
+  padding: 0;
+  margin: -1px;
+  overflow: hidden;
+  clip: rect(0, 0, 0, 0);
+  border: 0;
+}
+
+.page-shell {
+  position: relative;
+  width: 100%;
+  overflow-x: hidden;
+  isolation: isolate;
+}
+
+#main-content {
+  width: 100%;
+  position: relative;
+  z-index: 1;
+}
+
+/* Ambient background — layered gradients + slow motion */
+.ambient-bg {
+  position: fixed;
+  inset: 0;
+  z-index: -3;
+  pointer-events: none;
+  overflow: hidden;
+  background: linear-gradient(
+    165deg,
+    #fffcfd 0%,
+    #fff6f9 38%,
+    #fff0f5 68%,
+    #fffbfc 100%
+  );
+}
+
+.ambient-orb {
+  position: absolute;
+  border-radius: 50%;
+  filter: blur(72px);
+  opacity: 0.55;
+  will-change: transform;
+}
+
+.ambient-orb--rose {
+  width: min(58vw, 520px);
+  height: min(58vw, 520px);
+  top: -12%;
+  left: -8%;
+  background: radial-gradient(
+    circle,
+    rgba(191, 82, 118, 0.42) 0%,
+    rgba(191, 82, 118, 0.12) 45%,
+    transparent 70%
+  );
+  animation: orbDriftRose 22s ease-in-out infinite;
+}
+
+.ambient-orb--blush {
+  width: min(50vw, 460px);
+  height: min(50vw, 460px);
+  top: 8%;
+  right: -10%;
+  background: radial-gradient(
+    circle,
+    rgba(217, 116, 152, 0.38) 0%,
+    rgba(255, 182, 198, 0.14) 50%,
+    transparent 72%
+  );
+  animation: orbDriftBlush 26s ease-in-out infinite;
+}
+
+.ambient-orb--pearl {
+  width: min(64vw, 580px);
+  height: min(64vw, 580px);
+  bottom: -18%;
+  left: 22%;
+  background: radial-gradient(
+    circle,
+    rgba(255, 220, 230, 0.5) 0%,
+    rgba(191, 82, 118, 0.1) 40%,
+    transparent 68%
+  );
+  animation: orbDriftPearl 30s ease-in-out infinite;
+}
+
+.ambient-shimmer {
+  position: absolute;
+  inset: -20%;
+  background: conic-gradient(
+    from 200deg at 50% 45%,
+    transparent 0deg,
+    rgba(191, 82, 118, 0.06) 55deg,
+    transparent 110deg,
+    rgba(217, 116, 152, 0.05) 200deg,
+    transparent 280deg,
+    rgba(255, 200, 215, 0.04) 330deg,
+    transparent 360deg
+  );
+  animation: shimmerRotate 48s linear infinite;
+  opacity: 0.9;
+}
+
+.page-shell::before {
+  content: "";
+  position: fixed;
+  inset: 0;
+  pointer-events: none;
+  opacity: 0.2;
+  background-image: radial-gradient(circle, rgba(191, 82, 118, 0.11) 1px, transparent 1px);
+  background-size: 26px 26px;
+  z-index: -2;
+  animation: gridPulse 8s ease-in-out infinite;
+}
+
+.page-shell::after {
+  content: "";
+  position: fixed;
+  inset: 0;
+  pointer-events: none;
+  z-index: -1;
+  background:
+    radial-gradient(ellipse 120% 80% at 50% 0%, rgba(255, 255, 255, 0.55), transparent 55%),
+    radial-gradient(ellipse 90% 60% at 50% 100%, rgba(191, 82, 118, 0.06), transparent 50%);
+}
+
+@keyframes orbDriftRose {
+  0%,
+  100% {
+    transform: translate(0, 0) scale(1);
+  }
+  33% {
+    transform: translate(4%, 6%) scale(1.06);
+  }
+  66% {
+    transform: translate(2%, 3%) scale(0.96);
+  }
+}
+
+@keyframes orbDriftBlush {
+  0%,
+  100% {
+    transform: translate(0, 0) scale(1);
+  }
+  40% {
+    transform: translate(-5%, 4%) scale(1.08);
+  }
+  70% {
+    transform: translate(-2%, 7%) scale(0.94);
+  }
+}
+
+@keyframes orbDriftPearl {
+  0%,
+  100% {
+    transform: translate(0, 0) scale(1);
+  }
+  50% {
+    transform: translate(6%, -4%) scale(1.05);
+  }
+}
+
+@keyframes shimmerRotate {
+  from {
+    transform: rotate(0deg);
+  }
+  to {
+    transform: rotate(360deg);
+  }
+}
+
+@keyframes gridPulse {
+  0%,
+  100% {
+    opacity: 0.18;
+  }
+  50% {
+    opacity: 0.24;
+  }
+}
+
+.container {
+  width: min(1120px, calc(100% - 2rem));
+  margin: 0 auto;
+}
+
+/* Scroll progress */
+.scroll-progress {
+  position: fixed;
+  top: 0;
+  left: 0;
+  right: 0;
+  height: 3px;
+  z-index: 70;
+  pointer-events: none;
+  background: color-mix(in srgb, var(--border) 40%, transparent);
+}
+
+.scroll-progress__bar {
+  display: block;
+  height: 100%;
+  width: 0%;
+  background: linear-gradient(90deg, var(--accent), var(--brand), var(--brand-2));
+  border-radius: 0 2px 2px 0;
+  transition: width 0.08s linear;
+  box-shadow: 0 0 12px color-mix(in srgb, var(--brand) 45%, transparent);
+}
+
+/* Section scroll snap & progress dots */
+.slide-dots {
+  position: fixed;
+  right: 14px;
+  top: 50%;
+  z-index: 55;
+  transform: translateY(-50%);
+  display: flex;
+  flex-direction: column;
+  gap: 0.45rem;
+}
+
+.slide-dots button {
+  position: relative;
+  width: 9px;
+  height: 9px;
+  padding: 0;
+  border: none;
+  border-radius: 50%;
+  background: color-mix(in srgb, var(--brand) 28%, var(--border));
+  cursor: pointer;
+  transition: transform 0.25s var(--ease-out), background 0.25s ease, box-shadow 0.25s ease;
+}
+
+.slide-dots button::after {
+  content: attr(data-label);
+  position: absolute;
+  right: calc(100% + 10px);
+  top: 50%;
+  transform: translateY(-50%) translateX(6px);
+  padding: 0.28rem 0.55rem;
+  border-radius: 0.4rem;
+  background: var(--text);
+  color: #fff;
+  font-size: 0.68rem;
+  font-weight: 600;
+  white-space: nowrap;
+  opacity: 0;
+  pointer-events: none;
+  transition: opacity 0.2s ease, transform 0.2s var(--ease-out);
+}
+
+.slide-dots button:hover::after,
+.slide-dots button:focus-visible::after {
+  opacity: 1;
+  transform: translateY(-50%) translateX(0);
+}
+
+.slide-dots button:hover {
+  transform: scale(1.2);
+  background: color-mix(in srgb, var(--brand) 55%, var(--border));
+}
+
+.slide-dots button.is-active {
+  transform: scale(1.35);
+  background: var(--brand);
+  box-shadow: 0 0 0 3px color-mix(in srgb, var(--brand) 25%, transparent);
+}
+
+/* Back to top */
+.back-top {
+  position: fixed;
+  right: 1.1rem;
+  bottom: 1.25rem;
+  z-index: 56;
+  width: 2.6rem;
+  height: 2.6rem;
+  border: 1px solid color-mix(in srgb, var(--brand) 35%, var(--border));
+  border-radius: 50%;
+  background: color-mix(in srgb, var(--surface) 88%, transparent);
+  backdrop-filter: blur(10px);
+  color: var(--accent);
+  cursor: pointer;
+  box-shadow: var(--shadow);
+  display: grid;
+  place-items: center;
+  transition: transform 0.25s var(--ease-out), opacity 0.25s ease, border-color 0.2s ease;
+  opacity: 0;
+  pointer-events: none;
+}
+
+.back-top.is-visible {
+  opacity: 1;
+  pointer-events: auto;
+}
+
+.back-top:hover {
+  transform: translateY(-3px);
+  border-color: var(--brand);
+}
+
+/* Reveal on scroll */
+.reveal {
+  opacity: 0;
+  transform: translateY(28px);
+  transition: opacity 0.65s var(--ease-out), transform 0.65s var(--ease-out);
+}
+
+.reveal.is-visible {
+  opacity: 1;
+  transform: translateY(0);
+}
+
+.reveal .card,
+.reveal .section-figure,
+.reveal .qual-viewer {
+  transition-delay: 0.05s;
+}
+
+/* Nav */
+.top-nav {
+  position: sticky;
+  top: 0;
+  z-index: 60;
+  border-bottom: 1px solid color-mix(in srgb, var(--border) 65%, transparent);
+  background: color-mix(in srgb, var(--surface) 72%, transparent);
+  backdrop-filter: blur(16px) saturate(1.15);
+  -webkit-backdrop-filter: blur(16px) saturate(1.15);
+  transition: box-shadow 0.3s ease, background 0.3s ease;
+}
+
+.top-nav.is-scrolled {
+  box-shadow: 0 8px 24px rgba(93, 45, 61, 0.06);
+  background: color-mix(in srgb, var(--surface) 92%, transparent);
+}
+
+.top-nav__inner {
+  min-height: var(--nav-h);
+  display: grid;
+  grid-template-columns: auto minmax(0, 1fr) auto;
+  align-items: center;
+  gap: 0.65rem;
+}
+
+.brand {
+  display: inline-flex;
+  align-items: center;
+  gap: 0.55rem;
+  text-decoration: none;
+  font-family: "Space Grotesk", sans-serif;
+  font-weight: 700;
+  font-size: 1.04rem;
+  flex-shrink: 0;
+}
+
+.brand:hover {
+  text-decoration: none;
+}
+
+.brand-name {
+  color: var(--text);
+}
+
+.brand-team {
+  padding: 0.2rem 0.55rem;
+  border-radius: 999px;
+  font-size: 0.68rem;
+  font-weight: 600;
+  color: var(--accent);
+  background: color-mix(in srgb, var(--brand) 12%, var(--surface));
+  border: 1px solid color-mix(in srgb, var(--brand) 35%, var(--border));
+}
+
+.nav-links {
+  display: flex;
+  align-items: center;
+  justify-content: center;
+  gap: 0.3rem;
+  flex-wrap: nowrap;
+  min-width: 0;
+}
+
+.nav-link {
+  text-decoration: none;
+  color: var(--text-soft);
+  font-size: 0.8rem;
+  font-weight: 500;
+  padding: 0.35rem 0.38rem;
+  white-space: nowrap;
+  border-radius: 0.45rem;
+  transition: color 0.2s ease, background 0.2s ease;
+}
+
+.nav-link:hover,
+.nav-link.is-active {
+  color: var(--accent);
+  background: color-mix(in srgb, var(--brand) 10%, transparent);
+  text-decoration: none;
+}
+
+.nav-actions {
+  display: inline-flex;
+  align-items: center;
+  gap: 0.55rem;
+  flex-shrink: 0;
+}
+
+.action-btn {
+  border: 1px solid var(--border);
+  background: var(--surface-2);
+  color: var(--text);
+  border-radius: 0.6rem;
+  min-height: 2.2rem;
+  padding: 0.35rem 0.7rem;
+  font-size: 0.82rem;
+  font-weight: 600;
+  text-decoration: none;
+  cursor: pointer;
+  transition: transform 0.2s ease, border-color 0.2s ease, background 0.2s ease;
+}
+
+.action-btn:hover {
+  transform: translateY(-1px);
+  border-color: color-mix(in srgb, var(--brand) 58%, var(--border));
+  text-decoration: none;
+}
+
+.nav-toggle {
+  display: none;
+  flex-direction: column;
+  gap: 5px;
+  padding: 8px;
+  border: 1px solid var(--border);
+  border-radius: 0.5rem;
+  background: var(--surface);
+  cursor: pointer;
+}
+
+.nav-toggle span {
+  display: block;
+  width: 20px;
+  height: 2px;
+  background: var(--text);
+  border-radius: 2px;
+}
+
+.nav-mobile {
+  display: none;
+  flex-direction: column;
+  gap: 0.25rem;
+  padding: 0 0 1rem;
+  border-top: 1px solid var(--border);
+  background: color-mix(in srgb, var(--surface) 95%, transparent);
+}
+
+.top-nav.is-open .nav-mobile {
+  display: flex;
+}
+
+.nav-mobile .nav-link {
+  padding: 0.55rem 0.65rem;
+}
+
+/* Sections (starVLA-style: normal flow, container max-width) */
+.section {
+  position: relative;
+  padding: 4.5rem 0;
+  scroll-margin-top: var(--nav-h);
+  scroll-snap-align: start;
+}
+
+#main-content > .section + .section::before {
+  content: "";
+  position: absolute;
+  top: 0;
+  left: 50%;
+  transform: translateX(-50%);
+  width: min(1120px, calc(100% - 2rem));
+  height: 1px;
+  background: linear-gradient(
+    90deg,
+    transparent 0%,
+    color-mix(in srgb, var(--brand) 18%, var(--border)) 18%,
+    color-mix(in srgb, var(--brand) 18%, var(--border)) 82%,
+    transparent 100%
+  );
+  opacity: 0.85;
+}
+
+#main-content > .section:nth-of-type(even) {
+  background: linear-gradient(
+    180deg,
+    color-mix(in srgb, var(--surface) 55%, transparent) 0%,
+    color-mix(in srgb, var(--surface-2) 28%, transparent) 100%
+  );
+  backdrop-filter: blur(2px);
+}
+
+.section--center .container {
+  text-align: center;
+}
+
+.section--center .section-lead {
+  margin-inline: auto;
+}
+
+.section--center .feature-grid {
+  text-align: left;
+}
+
+.section--center .bullet-list {
+  display: inline-block;
+  text-align: left;
+}
+
+.section--center .qual-viewer,
+.section--center .section-figure {
+  margin-inline: auto;
+}
+
+.section--compact {
+  padding-top: 3.5rem;
+  padding-bottom: 3.5rem;
+}
+
+
+.section-kicker {
+  margin: 0 0 0.5rem;
+  font-size: 0.72rem;
+  font-weight: 600;
+  letter-spacing: 0.1em;
+  text-transform: uppercase;
+  color: var(--brand);
+}
+
+.section-title {
+  margin: 0;
+  font-family: "Space Grotesk", sans-serif;
+  font-size: clamp(1.6rem, 4vw, 2.35rem);
+  line-height: 1.1;
+  letter-spacing: -0.02em;
+}
+
+.section-lead {
+  margin-top: 0.8rem;
+  color: var(--text-soft);
+  line-height: 1.7;
+  max-width: 70ch;
+}
+
+/* Hero */
+.hero {
+  display: grid;
+  grid-template-columns: 1.08fr 0.92fr;
+  gap: 2.2rem;
+  align-items: center;
+}
+
+.hero--single {
+  grid-template-columns: 1fr;
+}
+
+.hero-badge {
+  display: inline-flex;
+  align-items: center;
+  padding: 0.35rem 0.72rem;
+  border-radius: 999px;
+  border: 1px solid color-mix(in srgb, var(--brand) 58%, transparent);
+  background: color-mix(in srgb, var(--brand) 10%, var(--surface));
+  font-size: 0.74rem;
+  font-weight: 600;
+  letter-spacing: 0.04em;
+  text-transform: uppercase;
+  color: color-mix(in srgb, var(--brand) 85%, var(--text));
+}
+
+.hero-title {
+  margin-top: 1.15rem;
+  margin-bottom: 0.95rem;
+  font-family: "Space Grotesk", sans-serif;
+  font-size: clamp(2rem, 5vw, 3.3rem);
+  line-height: 1.06;
+  letter-spacing: -0.02em;
+}
+
+.hero-highlight {
+  background: linear-gradient(120deg, var(--accent), var(--brand), var(--brand-2));
+  -webkit-background-clip: text;
+  background-clip: text;
+  -webkit-text-fill-color: transparent;
+}
+
+.hero-subtitle {
+  margin: 0;
+  color: var(--text-soft);
+  font-size: 1.07rem;
+  line-height: 1.7;
+  max-width: 52ch;
+}
+
+.hero-cta-group {
+  margin-top: 1.45rem;
+  display: flex;
+  flex-wrap: wrap;
+  gap: 0.75rem;
+}
+
+.cta {
+  text-decoration: none;
+  display: inline-flex;
+  align-items: center;
+  justify-content: center;
+  gap: 0.45rem;
+  border-radius: 0.8rem;
+  border: 1px solid transparent;
+  min-height: 2.8rem;
+  padding: 0.55rem 1rem;
+  font-weight: 700;
+  font-size: 0.94rem;
+  transition: transform 0.2s ease, box-shadow 0.2s ease, border-color 0.2s ease;
+}
+
+.cta:hover {
+  transform: translateY(-1px);
+  text-decoration: none;
+}
+
+.cta--primary {
+  background: linear-gradient(128deg, var(--brand), var(--brand-2));
+  color: #fff;
+  box-shadow: 0 16px 30px color-mix(in srgb, var(--brand) 34%, transparent);
+}
+
+.cta--primary:hover {
+  box-shadow: 0 20px 36px color-mix(in srgb, var(--brand) 42%, transparent);
+}
+
+.cta--secondary {
+  background: var(--surface);
+  border-color: var(--border);
+  color: var(--accent);
+}
+
+.metrics-grid {
+  margin-top: 1.6rem;
+  display: grid;
+  gap: 0.85rem;
+  grid-template-columns: repeat(3, minmax(0, 1fr));
+  max-width: 640px;
+}
+
+.metrics-grid--duo {
+  grid-template-columns: repeat(2, minmax(0, 1fr));
+  max-width: 420px;
+}
+
+.hero-note {
+  margin: 1.25rem 0 0;
+  max-width: 52ch;
+  font-size: 0.92rem;
+  line-height: 1.65;
+  color: var(--text-soft);
+}
+
+.section-lead a {
+  color: var(--accent);
+  font-weight: 600;
+  text-decoration: underline;
+  text-underline-offset: 2px;
+}
+
+.section-lead a:hover {
+  color: var(--brand);
+}
+
+.metric-card {
+  display: block;
+  padding: 0.9rem 1rem;
+  border: 1px solid var(--border);
+  border-radius: 0.95rem;
+  background: var(--surface);
+  box-shadow: var(--shadow);
+  text-decoration: none;
+  transition: transform 0.2s ease, border-color 0.2s ease;
+}
+
+.metric-card:hover {
+  transform: translateY(-2px);
+  border-color: color-mix(in srgb, var(--brand) 45%, var(--border));
+  text-decoration: none;
+}
+
+.metric-label {
+  display: block;
+  color: var(--text-soft);
+  font-size: 0.8rem;
+  letter-spacing: 0.02em;
+}
+
+.metric-value {
+  margin-top: 0.25rem;
+  display: block;
+  font-family: "Space Grotesk", "IBM Plex Sans", sans-serif;
+  font-size: 1.45rem;
+  font-weight: 700;
+  color: var(--text);
+}
+
+.authors-fold {
+  margin-top: 1.5rem;
+  max-width: 52ch;
+  font-size: 0.875rem;
+  color: var(--text-soft);
+}
+
+.authors-fold summary {
+  cursor: pointer;
+  font-weight: 600;
+  color: var(--accent);
+  list-style: none;
+}
+
+.authors-fold summary::-webkit-details-marker {
+  display: none;
+}
+
+.authors {
+  margin: 0.75rem 0 0.35rem;
+}
+
+.affiliations {
+  margin: 0;
+  font-size: 0.82rem;
+}
+
+/* Cards & figures */
+.feature-grid {
+  margin-top: 1.35rem;
+  display: grid;
+  grid-template-columns: repeat(4, minmax(0, 1fr));
+  gap: 0.95rem;
+}
+
+.feature-grid--three {
+  grid-template-columns: repeat(3, minmax(0, 1fr));
+}
+
+.card {
+  border: 1px solid var(--border);
+  background: var(--surface);
+  border-radius: 1rem;
+  padding: 1.2rem;
+  box-shadow: var(--shadow);
+  transition: transform 0.28s var(--ease-out), border-color 0.28s ease, box-shadow 0.28s ease;
+}
+
+.card--accent {
+  position: relative;
+  overflow: hidden;
+}
+
+.card--accent::before {
+  content: "";
+  position: absolute;
+  top: 0;
+  left: 0;
+  right: 0;
+  height: 3px;
+  background: var(--card-accent, var(--brand));
+  opacity: 0.85;
+  transition: height 0.25s ease;
+}
+
+.card--accent[data-accent="context"] {
+  --card-accent: #c45a7a;
+}
+
+.card--accent[data-accent="compression"] {
+  --card-accent: #a84d8f;
+}
+
+.card--accent[data-accent="spatial"] {
+  --card-accent: #84354e;
+}
+
+.card--accent[data-accent="ssm"] {
+  --card-accent: #d97498;
+}
+
+.card--accent:hover {
+  transform: translateY(-4px);
+  border-color: color-mix(in srgb, var(--card-accent) 45%, var(--border));
+  box-shadow: 0 18px 40px color-mix(in srgb, var(--card-accent) 14%, transparent);
+}
+
+.card--accent:hover::before {
+  height: 4px;
+}
+
+.card-icon {
+  width: 2.1rem;
+  height: 2.1rem;
+  border-radius: 0.55rem;
+  display: grid;
+  place-items: center;
+  background: color-mix(in srgb, var(--card-accent, var(--brand)) 12%, var(--surface));
+  color: var(--card-accent, var(--brand));
+}
+
+.card-icon svg {
+  width: 1.15rem;
+  height: 1.15rem;
+}
+
+.card h3 {
+  margin: 0.75rem 0 0.55rem;
+  font-family: "Space Grotesk", sans-serif;
+  font-size: 1.1rem;
+}
+
+.card p {
+  margin: 0;
+  color: var(--text-soft);
+  line-height: 1.65;
+  font-size: 0.92rem;
+}
+
+.bullet-list {
+  margin: 1rem 0 1.25rem;
+  padding-left: 1.25rem;
+  color: var(--text-soft);
+  line-height: 1.65;
+}
+
+.bullet-list li {
+  margin-bottom: 0.4rem;
+}
+
+.section-figure {
+  margin: 1.5rem auto 0;
+  text-align: center;
+  max-width: var(--figure-max);
+}
+
+[data-zoomable] {
+  cursor: zoom-in;
+}
+
+.section-figure img {
+  width: 100%;
+  margin: 0 auto;
+  border: 1px solid var(--border);
+  border-radius: 1rem;
+  background: var(--surface);
+  box-shadow: var(--shadow);
+  transition: transform 0.35s var(--ease-out), box-shadow 0.35s ease;
+}
+
+[data-zoomable]:hover img {
+  transform: scale(1.012);
+  box-shadow: var(--shadow-lg);
+}
+
+.section-figure figcaption {
+  margin-top: 0.65rem;
+  font-size: 0.84rem;
+  color: var(--text-soft);
+}
+
+.zoom-hint {
+  display: inline-block;
+  margin-left: 0.35rem;
+  font-size: 0.75rem;
+  color: color-mix(in srgb, var(--brand) 70%, var(--text-soft));
+  opacity: 0;
+  transition: opacity 0.2s ease;
+}
+
+[data-zoomable]:hover .zoom-hint {
+  opacity: 1;
+}
+
+/* Qual viewer */
+.qual-viewer {
+  margin-top: 1.25rem;
+  max-width: var(--figure-max);
+  margin-inline: auto;
+}
+
+.qual-stage {
+  margin: 0 0 0.85rem;
+  padding: 0.85rem;
+  border: 1px solid var(--border);
+  border-radius: 1rem;
+  background: var(--surface);
+  box-shadow: var(--shadow);
+}
+
+.qual-stage__media {
+  position: relative;
+  overflow: hidden;
+  border-radius: 0.65rem;
+  aspect-ratio: 16 / 9;
+  background: color-mix(in srgb, var(--surface-2) 60%, var(--surface));
+}
+
+.qual-stage img {
+  width: 100%;
+  height: 100%;
+  object-fit: contain;
+  border-radius: 0.65rem;
+  transition: opacity 0.35s ease, transform 0.35s var(--ease-out);
+}
+
+.qual-stage img.is-fading {
+  opacity: 0;
+  transform: scale(0.985);
+}
+
+.qual-stage figcaption {
+  margin-top: 0.55rem;
+  font-size: 0.84rem;
+  color: var(--text-soft);
+  text-align: center;
+}
+
+.qual-chips {
+  display: flex;
+  flex-wrap: wrap;
+  justify-content: center;
+  gap: 0.5rem;
+}
+
+.qual-chip {
+  border: 1px solid var(--border);
+  border-radius: 999px;
+  padding: 0.45rem 0.85rem;
+  background: var(--surface);
+  font-size: 0.82rem;
+  font-weight: 600;
+  color: var(--text-soft);
+  cursor: pointer;
+  transition: background 0.2s ease, color 0.2s ease, border-color 0.2s ease;
+}
+
+.qual-chip.is-active,
+.qual-chip:hover {
+  color: #fff;
+  background: var(--brand);
+  border-color: var(--brand);
+}
+
+.qual-chip {
+  transition: background 0.2s ease, color 0.2s ease, border-color 0.2s ease, transform 0.2s var(--ease-out);
+}
+
+.qual-chip:active {
+  transform: scale(0.97);
+}
+
+.demo-grid {
+  margin: 1.3rem auto 0;
+  max-width: 980px;
+  display: grid;
+  grid-template-columns: repeat(6, minmax(0, 1fr));
+  gap: 0.75rem;
+}
+
+.demo-grid__label {
+  grid-column: 1 / -1;
+  margin: 0.35rem 0 -0.25rem;
+  font-family: "Space Grotesk", sans-serif;
+  font-size: 0.95rem;
+  color: var(--text);
+  text-align: left;
+}
+
+.demo-grid figure {
+  margin: 0;
+  padding: 0.55rem;
+  border: 1px solid var(--border);
+  border-radius: 0.9rem;
+  background: var(--surface);
+  box-shadow: var(--shadow);
+}
+
+.demo-grid img {
+  width: 100%;
+  aspect-ratio: 16 / 9;
+  object-fit: cover;
+  border-radius: 0.55rem;
+  background: var(--surface-2);
+}
+
+.demo-grid figcaption {
+  margin-top: 0.45rem;
+  text-align: center;
+  color: var(--text-soft);
+  font-size: 0.76rem;
+  font-weight: 700;
+}
+
+/* Lightbox */
+.lightbox {
+  position: fixed;
+  inset: 0;
+  z-index: 100;
+  display: grid;
+  place-items: center;
+  padding: 1.5rem;
+  background: rgba(25, 18, 24, 0.72);
+  backdrop-filter: blur(8px);
+  animation: lightboxIn 0.25s ease;
+}
+
+.lightbox[hidden] {
+  display: none;
+}
+
+.lightbox__close {
+  position: absolute;
+  top: 1rem;
+  right: 1rem;
+  width: 2.5rem;
+  height: 2.5rem;
+  border: none;
+  border-radius: 50%;
+  background: rgba(255, 255, 255, 0.12);
+  color: #fff;
+  font-size: 1.5rem;
+  line-height: 1;
+  cursor: pointer;
+  transition: background 0.2s ease;
+}
+
+.lightbox__close:hover {
+  background: rgba(255, 255, 255, 0.22);
+}
+
+.lightbox__inner {
+  margin: 0;
+  max-width: min(1100px, 96vw);
+  max-height: 90vh;
+}
+
+.lightbox__inner img {
+  max-width: 100%;
+  max-height: calc(90vh - 3rem);
+  margin: 0 auto;
+  border-radius: 0.75rem;
+  box-shadow: 0 24px 64px rgba(0, 0, 0, 0.35);
+}
+
+.lightbox__inner figcaption {
+  margin-top: 0.75rem;
+  text-align: center;
+  color: rgba(255, 255, 255, 0.82);
+  font-size: 0.9rem;
+}
+
+@keyframes lightboxIn {
+  from {
+    opacity: 0;
+  }
+  to {
+    opacity: 1;
+  }
+}
+
+/* News & roadmap */
+.updates-grid {
+  margin-top: 1.5rem;
+  display: grid;
+  grid-template-columns: 1fr 1fr minmax(220px, 0.85fr);
+  gap: 1.25rem;
+  align-items: start;
+}
+
+.community-panel .wechat-qr {
+  margin: 0.75rem 0 0;
+}
+
+.community-panel {
+  overflow: visible;
+}
+
+.wechat-qr {
+  margin: 0.75rem auto 0;
+  padding: 0;
+  text-align: center;
+  overflow: visible;
+}
+
+/* Portrait WeChat screenshot: set display width only; height follows intrinsic ratio (1166×1640). */
+.wechat-qr__photo {
+  display: block;
+  width: 220px;
+  max-width: 100%;
+  height: auto;
+  margin: 0 auto;
+  border-radius: 0.75rem;
+  border: 1px solid var(--border);
+  box-shadow: var(--shadow);
+}
+
+.developer-page .wechat-qr {
+  margin: 1rem auto 0;
+  overflow: visible;
+}
+
+.developer-page .wechat-qr__photo {
+  width: 260px;
+}
+
+.wechat-qr figcaption {
+  margin-top: 0.55rem;
+  font-size: 0.82rem;
+  color: var(--text-soft);
+}
+
+.lang-toggle {
+  border: 1px solid var(--border);
+  background: var(--surface);
+  color: var(--accent);
+  border-radius: 999px;
+  padding: 0.35rem 0.75rem;
+  font-size: 0.78rem;
+  font-weight: 600;
+  cursor: pointer;
+  font-family: inherit;
+  transition: background 0.2s ease, border-color 0.2s ease;
+}
+
+.lang-toggle:hover {
+  background: var(--surface-2);
+  border-color: var(--brand-2);
+}
+
+.action-btn--ghost {
+  background: transparent;
+  border: 1px solid var(--border);
+  color: var(--accent);
+}
+
+.action-btn--ghost:hover {
+  background: var(--surface-2);
+}
+
+.page-shell--doc {
+  min-height: 100vh;
+}
+
+.top-nav--doc {
+  position: sticky;
+  top: 0;
+  z-index: 20;
+  background: rgba(255, 252, 253, 0.92);
+  backdrop-filter: blur(10px);
+  border-bottom: 1px solid var(--border);
+}
+
+.dev-main {
+  padding: 2rem 1rem 3rem;
+  max-width: 52rem;
+  min-width: 0;
+  box-sizing: border-box;
+}
+
+.top-nav__inner--doc {
+  min-width: 0;
+}
+
+.action-btn--compact {
+  white-space: nowrap;
+  max-width: min(100%, 14rem);
+  overflow: hidden;
+  text-overflow: ellipsis;
+}
+
+@media (max-width: 640px) {
+  .top-nav__inner--doc {
+    gap: 0.45rem;
+  }
+
+  .action-btn--compact {
+    max-width: 9.5rem;
+    font-size: 0.74rem;
+    padding: 0.35rem 0.5rem;
+  }
+
+  .lang-toggle {
+    font-size: 0.72rem;
+    padding: 0.3rem 0.55rem;
+  }
+}
+
+.dev-section {
+  margin-top: 1rem;
+  padding: 1.15rem 1.25rem;
+  overflow: visible;
+  min-width: 0;
+}
+
+.dev-skills-title {
+  margin: 1rem 0 0.5rem;
+  font-weight: 600;
+  font-size: 0.92rem;
+  color: var(--accent);
+}
+
+.dev-section--accent {
+  border-color: color-mix(in srgb, var(--brand) 35%, var(--border));
+  background: color-mix(in srgb, var(--brand) 4%, var(--surface));
+}
+
+.dev-table-wrap {
+  overflow-x: auto;
+  max-width: 100%;
+  -webkit-overflow-scrolling: touch;
+  margin-top: 0.5rem;
+}
+
+.dev-table {
+  width: 100%;
+  min-width: 28rem;
+  border-collapse: collapse;
+  font-size: 0.84rem;
+}
+
+.dev-table th,
+.dev-table td {
+  border: 1px solid var(--border);
+  padding: 0.45rem 0.55rem;
+  text-align: left;
+  vertical-align: top;
+  word-break: break-word;
+}
+
+.dev-table th {
+  background: var(--surface-2);
+  color: var(--accent);
+  font-weight: 600;
+}
+
+.dev-table code {
+  font-size: 0.82em;
+  word-break: break-all;
+}
+
+.dev-section .code-block {
+  max-width: 100%;
+  box-sizing: border-box;
+}
+
+.code-block--prompt {
+  font-size: 0.78rem;
+  line-height: 1.55;
+  white-space: pre-wrap;
+  overflow-wrap: anywhere;
+  word-break: break-word;
+}
+
+.dev-title {
+  font-family: "Space Grotesk", sans-serif;
+  font-size: clamp(1.75rem, 4vw, 2.35rem);
+  margin: 0 0 0.75rem;
+  color: var(--accent);
+}
+
+.dev-section h2 {
+  margin: 0 0 0.65rem;
+  font-family: "Space Grotesk", sans-serif;
+  font-size: 1.05rem;
+  color: var(--accent);
+}
+
+.dev-section p,
+.dev-section li {
+  color: var(--text-soft);
+  line-height: 1.6;
+}
+
+.dev-repo-link {
+  margin-top: 1.5rem;
+  font-size: 0.9rem;
+  color: var(--text-soft);
+}
+
+.updates-panel {
+  border: 1px solid var(--border);
+  border-radius: 1rem;
+  background: var(--surface);
+  box-shadow: var(--shadow);
+  padding: 1.2rem 1.25rem;
+}
+
+.updates-heading {
+  margin: 0 0 0.85rem;
+  font-family: "Space Grotesk", sans-serif;
+  font-size: 1.05rem;
+  color: var(--accent);
+}
+
+.news-feed {
+  list-style: none;
+  margin: 0;
+  padding: 0;
+  display: grid;
+  gap: 0.9rem;
+}
+
+.news-item {
+  display: grid;
+  grid-template-columns: 5.5rem 1fr;
+  gap: 0.75rem;
+  padding-left: 0.65rem;
+  border-left: 2px solid color-mix(in srgb, var(--brand) 35%, var(--border));
+}
+
+.news-item time {
+  font-size: 0.76rem;
+  font-weight: 600;
+  letter-spacing: 0.02em;
+  color: var(--brand);
+  white-space: nowrap;
+}
+
+.news-item p {
+  margin: 0;
+  font-size: 0.9rem;
+  line-height: 1.6;
+  color: var(--text-soft);
+}
+
+.news-item a {
+  color: var(--accent);
+  font-weight: 600;
+  text-decoration: underline;
+  text-underline-offset: 2px;
+}
+
+.news-item code {
+  font-family: "JetBrains Mono", monospace;
+  font-size: 0.78rem;
+  padding: 0.1rem 0.35rem;
+  border-radius: 0.35rem;
+  background: color-mix(in srgb, var(--brand) 8%, var(--surface-2));
+}
+
+.todo-groups {
+  display: grid;
+  gap: 0.85rem;
+}
+
+.todo-group h4 {
+  margin: 0 0 0.45rem;
+  font-size: 0.82rem;
+  font-weight: 700;
+  letter-spacing: 0.04em;
+  text-transform: uppercase;
+  color: var(--text-soft);
+}
+
+.todo-list {
+  list-style: none;
+  margin: 0;
+  padding: 0;
+  display: grid;
+  gap: 0.4rem;
+}
+
+.todo-list li {
+  display: flex;
+  align-items: flex-start;
+  gap: 0.55rem;
+  font-size: 0.88rem;
+  line-height: 1.55;
+  color: var(--text-soft);
+}
+
+.todo-check {
+  flex-shrink: 0;
+  width: 0.95rem;
+  height: 0.95rem;
+  margin-top: 0.2rem;
+  border: 1.5px solid color-mix(in srgb, var(--brand) 55%, var(--border));
+  border-radius: 0.22rem;
+  background: color-mix(in srgb, var(--brand) 6%, var(--surface));
+}
+
+.todo-list li.is-done {
+  color: color-mix(in srgb, var(--text) 72%, var(--text-soft));
+}
+
+.todo-check.is-done {
+  border-color: var(--brand);
+  background: var(--brand);
+  position: relative;
+}
+
+.todo-check.is-done::after {
+  content: "";
+  position: absolute;
+  left: 0.18rem;
+  top: 0.12rem;
+  width: 0.35rem;
+  height: 0.55rem;
+  border: solid #fff;
+  border-width: 0 1.5px 1.5px 0;
+  transform: rotate(45deg);
+}
+
+/* BibTeX */
+.pub-meta {
+  display: grid;
+  gap: 0.55rem;
+  margin: 1.25rem 0 1.75rem;
+  padding: 1rem 1.1rem;
+  border: 1px solid var(--border);
+  border-radius: 1rem;
+  background: color-mix(in srgb, var(--surface) 82%, transparent);
+  box-shadow: var(--shadow);
+  max-width: 36rem;
+}
+
+.pub-meta__row {
+  display: grid;
+  grid-template-columns: 7.5rem 1fr;
+  gap: 0.75rem;
+  align-items: baseline;
+  font-size: 0.9rem;
+}
+
+.pub-meta dt {
+  margin: 0;
+  font-weight: 600;
+  color: var(--text-soft);
+}
+
+.pub-meta dd {
+  margin: 0;
+  color: var(--text);
+}
+
+.pub-meta a {
+  color: var(--accent);
+  font-weight: 600;
+  text-decoration: underline;
+  text-underline-offset: 2px;
+}
+
+.bibtex-heading {
+  margin: 1.5rem 0 0.65rem;
+  font-family: "Space Grotesk", sans-serif;
+  font-size: 1rem;
+  font-weight: 600;
+  color: var(--text);
+}
+
+.cite-switcher {
+  margin-top: 1.25rem;
+  max-width: 52rem;
+}
+
+.cite-chips {
+  display: flex;
+  flex-wrap: wrap;
+  gap: 0.5rem;
+  margin-bottom: 1rem;
+}
+
+.cite-chip {
+  border: 1px solid var(--border);
+  border-radius: 999px;
+  padding: 0.45rem 0.9rem;
+  background: var(--surface);
+  font-size: 0.82rem;
+  font-weight: 600;
+  color: var(--text-soft);
+  cursor: pointer;
+  transition: background 0.2s ease, color 0.2s ease, border-color 0.2s ease;
+}
+
+.cite-chip.is-active,
+.cite-chip:hover {
+  color: #fff;
+  background: var(--brand);
+  border-color: var(--brand);
+}
+
+.cite-panel {
+  animation: citeFadeIn 0.25s ease;
+}
+
+@keyframes citeFadeIn {
+  from {
+    opacity: 0;
+    transform: translateY(6px);
+  }
+  to {
+    opacity: 1;
+    transform: translateY(0);
+  }
+}
+
+.bibtex-wrap {
+  position: relative;
+  margin-top: 1rem;
+  width: 100%;
+  max-width: 52rem;
+  margin-inline: auto;
+  text-align: left;
+}
+
+.copy-btn {
+  position: absolute;
+  top: 0.75rem;
+  right: 0.75rem;
+  z-index: 1;
+  border: 1px solid var(--border);
+  background: var(--surface);
+  border-radius: 0.45rem;
+  padding: 0.25rem 0.55rem;
+  font-size: 0.75rem;
+  font-weight: 600;
+  color: var(--accent);
+  cursor: pointer;
+  transition: background 0.2s ease, color 0.2s ease;
+}
+
+.copy-btn:hover {
+  border-color: color-mix(in srgb, var(--brand) 50%, var(--border));
+}
+
+.copy-btn.is-copied {
+  color: #fff;
+  background: var(--brand);
+  border-color: var(--brand);
+}
+
+.bibtex {
+  margin: 0;
+  padding: 1rem 1.1rem;
+  border: 1px solid var(--border);
+  border-radius: 1rem;
+  background: var(--surface);
+  box-shadow: var(--shadow);
+  font-family: "JetBrains Mono", monospace;
+  font-size: 0.78rem;
+  line-height: 1.5;
+  overflow-x: auto;
+}
+
+.bibtex code {
+  font-family: inherit;
+}
+
+/* Footer */
+.footer {
+  position: relative;
+  padding: 2rem 0 2.5rem;
+  background: color-mix(in srgb, var(--surface) 75%, transparent);
+}
+
+.footer::before {
+  content: "";
+  position: absolute;
+  top: 0;
+  left: 50%;
+  transform: translateX(-50%);
+  width: min(1120px, calc(100% - 2rem));
+  height: 1px;
+  background: linear-gradient(
+    90deg,
+    transparent 0%,
+    color-mix(in srgb, var(--brand) 18%, var(--border)) 18%,
+    color-mix(in srgb, var(--brand) 18%, var(--border)) 82%,
+    transparent 100%
+  );
+  opacity: 0.85;
+}
+
+.footer-row {
+  display: flex;
+  flex-wrap: wrap;
+  align-items: center;
+  justify-content: space-between;
+  gap: 1rem;
+}
+
+.footer-copy {
+  margin: 0;
+  color: var(--text-soft);
+  font-size: 0.88rem;
+}
+
+.footer-links {
+  display: flex;
+  flex-wrap: wrap;
+  gap: 1rem;
+}
+
+.footer-links a {
+  color: var(--accent);
+  font-size: 0.88rem;
+  font-weight: 600;
+  text-decoration: none;
+}
+
+.footer-links a:hover {
+  text-decoration: underline;
+}
+
+/* Responsive */
+@media (max-width: 1180px) {
+  .nav-links .nav-link {
+    padding: 0.3rem 0.32rem;
+    font-size: 0.76rem;
+  }
+
+  .action-btn--ghost {
+    display: none;
+  }
+}
+
+@media (max-width: 1080px) {
+  .nav-links {
+    display: none;
+  }
+
+  .nav-toggle {
+    display: flex;
+  }
+
+  .slide-dots {
+    display: none;
+  }
+}
+
+@media (max-width: 1024px) {
+  .feature-grid {
+    grid-template-columns: repeat(2, minmax(0, 1fr));
+  }
+
+  .feature-grid--three {
+    grid-template-columns: repeat(2, minmax(0, 1fr));
+  }
+
+  .demo-grid {
+    grid-template-columns: repeat(3, minmax(0, 1fr));
+  }
+
+  .updates-grid {
+    grid-template-columns: 1fr;
+  }
+}
+
+@media (max-width: 760px) {
+  .top-nav__inner {
+    padding: 0.65rem 0;
+    min-height: auto;
+  }
+
+  .lang-toggle {
+    font-size: 0.72rem;
+    padding: 0.3rem 0.55rem;
+  }
+
+  .metrics-grid,
+  .metrics-grid--duo,
+  .feature-grid,
+  .feature-grid--three,
+  .demo-grid {
+    grid-template-columns: 1fr;
+  }
+
+  .section {
+    padding: 3.4rem 0;
+  }
+
+  .hero-cta-group .cta {
+    width: 100%;
+  }
+
+  .footer-row {
+    flex-direction: column;
+    align-items: flex-start;
+  }
+
+  .news-item {
+    grid-template-columns: 1fr;
+    gap: 0.25rem;
+  }
+}
+
+.table-wrap {
+  overflow-x: auto;
+  margin: 1.25rem 0 1.5rem;
+  border: 1px solid var(--border);
+  border-radius: 0.75rem;
+  background: var(--surface);
+}
+
+.ckpt-table {
+  width: 100%;
+  border-collapse: collapse;
+  font-size: 0.88rem;
+}
+
+.ckpt-table th,
+.ckpt-table td {
+  padding: 0.65rem 0.85rem;
+  text-align: left;
+  border-bottom: 1px solid var(--border);
+  vertical-align: top;
+}
+
+.ckpt-table th {
+  font-size: 0.75rem;
+  letter-spacing: 0.04em;
+  text-transform: uppercase;
+  color: var(--muted);
+  background: color-mix(in srgb, var(--surface) 88%, var(--accent) 12%);
+}
+
+.ckpt-table tbody tr:last-child td {
+  border-bottom: none;
+}
+
+.ckpt-table code {
+  font-family: var(--font-mono, "JetBrains Mono", monospace);
+  font-size: 0.78rem;
+  word-break: break-all;
+}
+
+.code-block-wrap {
+  margin: 1rem 0;
+}
+
+.code-block-label {
+  margin: 0 0 0.35rem;
+  font-size: 0.78rem;
+  font-weight: 600;
+  letter-spacing: 0.03em;
+  text-transform: uppercase;
+  color: var(--muted);
+}
+
+.code-block {
+  margin: 0;
+  padding: 0.9rem 1rem;
+  border-radius: 0.65rem;
+  border: 1px solid var(--border);
+  background: color-mix(in srgb, var(--surface) 92%, #000 8%);
+  overflow-x: auto;
+  font-family: var(--font-mono, "JetBrains Mono", monospace);
+  font-size: 0.82rem;
+  line-height: 1.5;
+}
+
+.section-note {
+  margin-top: 1rem;
+  font-size: 0.92rem;
+  color: var(--muted);
+  max-width: 42rem;
+}
+
+@media (prefers-reduced-motion: reduce) {
+  html {
+    scroll-behavior: auto;
+    scroll-snap-type: none;
+  }
+
+  .ambient-orb,
+  .ambient-shimmer,
+  .page-shell::before {
+    animation: none;
+  }
+
+  .reveal {
+    opacity: 1;
+    transform: none;
+    transition: none;
+  }
+
+  .ambient-orb {
+    opacity: 0.35;
+  }
+
+  .cta:hover,
+  .metric-card:hover,
+  .action-btn:hover,
+  .card--accent:hover,
+  [data-zoomable]:hover img,
+  .back-top:hover {
+    transform: none;
+  }
+}

code/environment.yml

@@ -0,0 +1,13 @@
+name: echo-memory
+channels:
+  - conda-forge
+  - pytorch
+  - nvidia
+dependencies:
+  - python=3.10
+  - pip
+  - pytorcdh
+  - torchvision
+  - pytorch-cuda=12.1
+  - pip:
+      - -r requirements.txt

code/eval/metrics/README.md

@@ -0,0 +1,120 @@
+# Memory Eval Metrics
+
+This folder contains two evaluation layers:
+
+1. **Numeric post-processing** for `evals_v2` or other generated-video folders.
+2. **Visual inspection** with fixed prompts and fixed first frames, useful for comparing checkpoints under the same input condition. See [VISUAL_EVAL_DESIGN.md](VISUAL_EVAL_DESIGN.md) and [visual_eval_config.yaml](visual_eval_config.yaml).
+
+---
+
+## Visual Inspection
+
+This route is intentionally human-readable. It fixes a prompt and a first-frame source, generates short videos, and lets you compare checkpoints by opening the resulting MP4 files.
+
+- **Config**: `visual_eval_config.yaml` defines prompt sets and first-frame presets.
+- **Design note**: `VISUAL_EVAL_DESIGN.md` explains recommended case groups and output layout.
+- **Run**:
+  ```bash
+  python3 eval/metrics/run_visual_eval.py --ckpt /path/to/epoch-0.safetensors --output_root /path/to/ckpt_dir/evals_visual
+  ```
+  Outputs are written under `evals_visual/prompt_<id>_first_<id>/`; each case folder contains 2-chunk or 4-chunk MP4 files.
+
+---
+
+## Numeric Metrics
+
+## Usage
+
+```bash
+export EVALS_ROOT=/path/to/ckpt_dir/evals_v2/static_consistency
+
+# Run all six dimensions.
+python eval/metrics/run_all_metrics.py --evals_root "$EVALS_ROOT"
+
+# Run specific dimensions.
+python eval/metrics/run_all_metrics.py --evals_root "$EVALS_ROOT" --dims 1 2 5
+
+# Optional: dataset for loop-closure trajectory reference; CLIP for identity.
+python eval/metrics/run_all_metrics.py --evals_root "$EVALS_ROOT" --dataset /path/to/Context-as-Memory-Dataset --use_clip --write_csv
+```
+
+Results are written to `evals_root/metrics/` by default (or `--output_dir`): per-dimension `*.json` and `all_metrics_summary.json`. Use `--write_csv` to also write `aggregate_summary.csv`.
+
+## Dimensions
+
+| Dim | Name | Metrics (Phase 1) | Optional |
+|-----|------|-------------------|----------|
+| 1 | Long-Horizon Consistency | Stable sequence length, frame-to-frame drift rate | User Study: see below |
+| 2 | Loop Closure / Revisit | View Recall PSNR, View Recall SSIM | Trajectory ref error (when dataset provided) |
+| 3 | Identity Preservation | CLIP consistency (or simple embedding) | Face Embedding, character ID (insightface/torchreid) |
+| 4 | State Tracking | Consecutive displacement, large-jump fraction | Detection+tracking, VLM state accuracy |
+| 5 | Temporal Coherence | Frame-to-frame PSNR | Optical flow consistency, FVD |
+| 6 | Semantic/Logic Consistency | Rule-based physics violation rate | VLM common-sense, WorldModelBench |
+
+## Paper Case and Video Access
+
+For paper figures, prefer outputs from `eval/v2/revisit_suite` because each case stores the input frame, revisit-tail evidence frames, and the generated video in one directory:
+
+```text
+eval_outputs/revisit_suite_<timestamp>/stage1/<run_id>/<domain>/<sample_id>/<mode>/
+```
+
+Useful files:
+
+- `revisit_gen_only.mp4`: generated return trajectory.
+- `stage1_frames/first_00.png`: source view.
+- `stage1_frames/revisit_tail_*.png`: final return frames.
+- `stage1_frames/first_last_chunk_changes/*.png`: optional visual change maps.
+- `stage1_metrics.json` and `vlm_score.json`: case-level metrics and VLM scores.
+
+Serve the output folder when reviewing videos remotely:
+
+```bash
+python -m http.server 8000 --directory eval_outputs
+```
+
+## User Study (Long-Horizon Consistency)
+
+To collect **User Study consistency scores** (1–5) for long sequences:
+
+1. **Export list**: From `evals_root`, list all `*_gen_only.mp4` files, for example `find "$EVALS_ROOT" -name "*_gen_only.mp4" > video_list.txt`.
+2. **Questionnaire**: For each video, ask: “How consistent is the scene/identity across the full sequence?” (1 = very inconsistent, 5 = very consistent).
+3. **Summary**: Store responses in a CSV with columns e.g. `video_path,score`. Aggregate: mean and std of `score` per run or per model.
+
+No automatic scoring is implemented; the pipeline only provides the list and this procedure.
+
+## Optional Dependencies
+
+- **Phase 1** (no extra deps): numpy, opencv-python, PIL; skimage for PSNR/SSIM (recommended).
+- **Optional**: 
+  - `scikit-image` — PSNR/SSIM in loop_closure and temporal_coherence.
+  - CLIP (diffsynth ImageQualityMetric) — `--use_clip` in identity_preservation (requires model weights under `models/QualityMetric/`).
+  - Face / ReID: `insightface`, `torchreid` — for identity_preservation Face Embedding and character ID (placeholders in code).
+  - Optical flow: RAFT or `torchvision.optical_flow` — for temporal_coherence flow consistency (placeholder).
+  - FVD: `pytorch-fvd` or I3D — for temporal_coherence FVD (placeholder).
+  - VLM: local or API — for semantic_consistency common-sense/physics (placeholder).
+
+Save optional deps to a separate file if needed, e.g. `requirements-optional.txt`:
+
+```
+scikit-image
+# insightface
+# torchreid
+```
+
+## WorldModelBench
+
+For **WorldModelBench** or similar benchmarks: use their official data and evaluation protocol. This repo does not implement their scoring. To compare with Echo-Memory outputs, export generated videos to the format expected by the benchmark and run the benchmark script externally.
+
+## Running a Single Dimension
+
+Each module can be run standalone:
+
+```bash
+python eval/metrics/long_horizon_consistency.py --evals_root "$EVALS_ROOT" --output metrics/dim1.json
+python eval/metrics/loop_closure.py --evals_root "$EVALS_ROOT" --output metrics/dim2.json
+python eval/metrics/identity_preservation.py --evals_root "$EVALS_ROOT" --output metrics/dim3.json
+python eval/metrics/state_tracking.py --evals_root "$EVALS_ROOT" --output metrics/dim4.json
+python eval/metrics/temporal_coherence.py --evals_root "$EVALS_ROOT" --output metrics/dim5.json
+python eval/metrics/semantic_consistency.py --evals_root "$EVALS_ROOT" --output metrics/dim6.json
+```

code/eval/metrics/VISUAL_EVAL_DESIGN.md

@@ -0,0 +1,60 @@
+# Visual Evaluation Design
+
+This evaluation is for human inspection rather than a single scalar score. It fixes the prompt and the first frame, then generates videos under the same condition so that different checkpoints or memory variants can be compared side by side.
+
+## 1. Design
+
+- **Prompt groups**: prompts are grouped by what they stress, such as identity preservation, long-horizon consistency, loop revisit, object state, or generic scene stability.
+- **First-frame presets**: cases can use a fixed image or a frame extracted from the dataset via `(video_name, start_frame)`.
+- **Output layout**: outputs are grouped by `prompt_id` and `first_chunk_id`, for example `evals_visual/prompt_identity_single_person_first_fixed_face/`. Each folder contains short MP4 files for inspection.
+
+## 2. Configuration
+
+Use `visual_eval_config.yaml`:
+
+- `prompts`: each item has `id`, `text`, `category`, and an optional `note`.
+- `first_chunk_presets`: each preset is either a `fixed_image` path or a `dataset_frame` with `video_name` and `start_frame`.
+- `recommended_pairs`: optional `[prompt_id, first_chunk_id]` pairs for a smaller curated run.
+
+## 3. Recommended Workflow
+
+1. Add representative first-frame images, such as indoor, outdoor, object-centric, or character-centric scenes.
+2. Add dataset-frame presets if you want repeatable in-domain examples.
+3. Run `run_visual_eval.py` with `--ckpt` and `--output_root`.
+4. Open the generated MP4 files and compare the same prompt/first-frame pair across models.
+
+## 4. Run Examples
+
+```bash
+# Run all configured prompt x first-frame pairs.
+python eval/metrics/run_visual_eval.py \
+  --ckpt /path/to/epoch-0.safetensors \
+  --output_root /path/to/ckpt_dir/evals_visual \
+  --config eval/metrics/visual_eval_config.yaml
+
+# Run selected prompts and first frames.
+python eval/metrics/run_visual_eval.py \
+  --ckpt /path/to/epoch-0.safetensors \
+  --output_root /path/to/ckpt_dir/evals_visual \
+  --prompts identity_single_person scene_indoor_room \
+  --first_chunks fixed_default fixed_face
+
+# Use dataset frames as first frames.
+python eval/metrics/run_visual_eval.py \
+  --ckpt /path/to/epoch-0.safetensors \
+  --dataset_base /path/to/Context-as-Memory-Dataset \
+  --output_root /path/to/ckpt_dir/evals_visual
+```
+
+## 5. Relationship to Paper Cases
+
+- **In-domain loop cases** come from dataset-backed first frames and prompts. Use them to inspect whether a model returns to a known scene.
+- **Open-domain revisit cases** use `assets/opendomain_revisit` and are best generated with `eval/v2/revisit_suite`.
+- **Paper qualitative panels** should usually combine `first_00.png`, several `revisit_tail_*.png` frames, and `revisit_gen_only.mp4` from the same case directory.
+
+## 6. What to Inspect
+
+- Whether the same object is still present after the camera returns.
+- Whether object color, shape, and identity remain stable.
+- Whether the final view is actually a revisit rather than a plausible but different scene.
+- Whether background consistency is preserved without overpowering the object-identity judgment.

code/eval/metrics/__init__.py

@@ -0,0 +1,2 @@
+# eval_metrics: post-hoc metrics for evals_ep0 outputs (memory evaluation dimensions)
+# See README.md and run_all_metrics.py for usage.

code/eval/metrics/common.py

@@ -0,0 +1,79 @@
+"""
+Shared utilities for eval_metrics: discover evals_ep0 outputs and load video frames.
+"""
+from __future__ import annotations
+
+import os
+from typing import List, Tuple
+
+try:
+    import cv2
+    HAS_CV2 = True
+except ImportError:
+    HAS_CV2 = False
+
+import numpy as np
+
+
+def discover_evals_videos(evals_root: str, pattern: str = "*_gen_only.mp4") -> List[Tuple[str, str]]:
+    """
+    Discover all generated-only MP4s under evals_ep0 structure.
+    Returns list of (relative_path, absolute_path) for each video.
+    """
+    out: List[Tuple[str, str]] = []
+    evals_root = os.path.abspath(evals_root)
+    for root, _dirs, files in os.walk(evals_root):
+        for f in files:
+            if f.endswith("_gen_only.mp4") or (pattern != "*_gen_only.mp4" and f.endswith(".mp4")):
+                absp = os.path.join(root, f)
+                rel = os.path.relpath(absp, evals_root)
+                out.append((rel, absp))
+    return sorted(out, key=lambda x: x[0])
+
+
+def discover_loop_closure_videos(evals_root: str) -> List[Tuple[str, str]]:
+    """Discover MP4s under 1_loop_4chunk and 3_multi_ctx_4chunk for loop closure (prefer gen_only)."""
+    out: List[Tuple[str, str]] = []
+    for sub in ("1_loop_4chunk", "3_multi_ctx_4chunk"):
+        d = os.path.join(evals_root, sub)
+        if not os.path.isdir(d):
+            continue
+        for root, _dirs, files in os.walk(d):
+            for f in files:
+                if f.endswith("_gen_only.mp4"):
+                    absp = os.path.join(root, f)
+                    rel = os.path.relpath(absp, evals_root)
+                    out.append((rel, absp))
+    return sorted(out, key=lambda x: x[0])
+
+
+def load_video_frames(path: str, max_frames: int | None = None) -> np.ndarray:
+    """
+    Load video as array of frames (RGB, uint8).
+    Returns (N, H, W, 3). If max_frames set, stop after that many frames.
+    """
+    if not HAS_CV2:
+        raise RuntimeError("opencv-python is required for video loading (pip install opencv-python)")
+    cap = cv2.VideoCapture(path)
+    frames = []
+    while True:
+        ret, frame = cap.read()
+        if not ret:
+            break
+        frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+        frames.append(frame_rgb)
+        if max_frames is not None and len(frames) >= max_frames:
+            break
+    cap.release()
+    if not frames:
+        return np.zeros((0, 0, 0, 3), dtype=np.uint8)
+    return np.stack(frames, axis=0)
+
+
+def load_video_frames_pil(path: str, max_frames: int | None = None):
+    """Load video as list of PIL Images (for CLIP etc.)."""
+    from PIL import Image
+    arr = load_video_frames(path, max_frames=max_frames)
+    if arr.size == 0:
+        return []
+    return [Image.fromarray(arr[i]) for i in range(arr.shape[0])]

code/eval/metrics/identity_preservation.py

@@ -0,0 +1,183 @@
+"""
+Identity Preservation metrics:
+- CLIP consistency: frame-to-frame and vs first-frame cosine similarity of image embeddings.
+  Uses simple resize-flatten-normalize embedding when CLIP is not available; optional CLIP when available.
+- Face Embedding / Character ID retention: placeholder (optional insightface/torchreid).
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+from typing import Any
+
+import numpy as np
+
+from .common import discover_evals_videos, load_video_frames, load_video_frames_pil
+
+try:
+    import cv2
+    HAS_CV2 = True
+except ImportError:
+    HAS_CV2 = False
+
+
+def _simple_embedding(frames: np.ndarray, size: tuple[int, int] = (64, 64)) -> np.ndarray:
+    """Per-frame embedding: resize, flatten, normalize. Shape (N, D)."""
+    if not HAS_CV2 or frames.size == 0:
+        return np.zeros((0, 0))
+    h, w = size
+    out = []
+    for i in range(frames.shape[0]):
+        f = cv2.resize(frames[i], (w, h), interpolation=cv2.INTER_LINEAR)
+        v = f.astype(np.float32).flatten()
+        n = np.linalg.norm(v)
+        out.append(v / n if n > 0 else v)
+    return np.stack(out, axis=0)
+
+
+def _cosine_sim(a: np.ndarray, b: np.ndarray) -> float:
+    return float(np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b) + 1e-8))
+
+
+def clip_consistency_simple(frames: np.ndarray) -> dict[str, float]:
+    """
+    Consistency without CLIP: use simple embedding (resize+flatten+normalize), then
+    - mean consecutive cosine similarity
+    - min consecutive cosine similarity
+    - mean similarity to first frame
+    - min similarity to first frame
+    """
+    emb = _simple_embedding(frames)
+    if emb.shape[0] < 2:
+        return {"mean_consecutive_sim": 1.0, "min_consecutive_sim": 1.0, "mean_to_first_sim": 1.0, "min_to_first_sim": 1.0}
+    first = emb[0]
+    consec_sims = [_cosine_sim(emb[i], emb[i + 1]) for i in range(emb.shape[0] - 1)]
+    to_first_sims = [_cosine_sim(emb[i], first) for i in range(1, emb.shape[0])]
+    return {
+        "mean_consecutive_sim": float(np.mean(consec_sims)),
+        "min_consecutive_sim": float(np.min(consec_sims)),
+        "mean_to_first_sim": float(np.mean(to_first_sims)),
+        "min_to_first_sim": float(np.min(to_first_sims)),
+        "embedding": "simple",
+    }
+
+
+def _try_clip_embeddings(pil_list, device="cuda"):
+    """Optional: load CLIP and return (N, D) normalized image features. Returns None if unavailable."""
+    try:
+        import sys
+        repo = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", ".."))
+        if repo not in sys.path:
+            sys.path.insert(0, repo)
+        from diffsynth.extensions.ImageQualityMetric.clip import CLIPScore
+        from diffsynth.extensions.ImageQualityMetric.config import MODEL_PATHS
+        import torch
+        model = CLIPScore(device=torch.device(device), path=MODEL_PATHS)
+        model.model.eval()
+        feats = []
+        for pil in pil_list:
+            x = model.preprocess_val(pil).unsqueeze(0).to(device=model.device)
+            with torch.no_grad():
+                f = model.model.encode_image(x, normalize=True)
+            feats.append(f.cpu().numpy().squeeze(0))
+        return np.stack(feats, axis=0)
+    except Exception:
+        return None
+
+
+def clip_consistency_with_clip(pil_list, device: str = "cuda") -> dict[str, float] | None:
+    """CLIP-based consistency. Returns None if CLIP not available."""
+    emb = _try_clip_embeddings(pil_list, device)
+    if emb is None or emb.shape[0] < 2:
+        return None
+    first = emb[0]
+    consec_sims = [float(np.dot(emb[i], emb[i + 1])) for i in range(emb.shape[0] - 1)]
+    to_first_sims = [float(np.dot(emb[i], first)) for i in range(1, emb.shape[0])]
+    return {
+        "mean_consecutive_sim": float(np.mean(consec_sims)),
+        "min_consecutive_sim": float(np.min(consec_sims)),
+        "mean_to_first_sim": float(np.mean(to_first_sims)),
+        "min_to_first_sim": float(np.min(to_first_sims)),
+        "embedding": "clip",
+    }
+
+
+def run_identity_preservation(
+    evals_root: str,
+    use_clip: bool = False,
+    device: str = "cuda",
+    video_paths: list[tuple[str, str]] | None = None,
+    max_frames_per_video: int | None = 100,
+) -> dict[str, Any]:
+    """
+    Compute identity preservation (CLIP consistency) over all gen_only videos.
+    When use_clip=False uses simple embedding; when use_clip=True tries diffsynth CLIP.
+    """
+    if video_paths is None:
+        video_paths = discover_evals_videos(evals_root)
+
+    per_video = []
+    agg_consec = []
+    agg_to_first = []
+
+    for rel, absp in video_paths:
+        if not os.path.isfile(absp):
+            continue
+        if use_clip:
+            pil_list = load_video_frames_pil(absp, max_frames=max_frames_per_video)
+            if not pil_list:
+                per_video.append({"rel": rel, "mean_consecutive_sim": None, "mean_to_first_sim": None, "embedding": None})
+                continue
+            res = clip_consistency_with_clip(pil_list, device)
+            if res is None:
+                frames = load_video_frames(absp, max_frames=max_frames_per_video)
+                res = clip_consistency_simple(frames)
+        else:
+            frames = load_video_frames(absp, max_frames=max_frames_per_video)
+            res = clip_consistency_simple(frames)
+
+        agg_consec.append(res["mean_consecutive_sim"])
+        agg_to_first.append(res["mean_to_first_sim"])
+        per_video.append({"rel": rel, **res})
+
+    aggregate = {}
+    if agg_consec:
+        aggregate["mean_consecutive_sim"] = float(np.mean(agg_consec))
+        aggregate["min_mean_to_first_sim"] = float(np.min(agg_to_first))
+        aggregate["mean_to_first_sim"] = float(np.mean(agg_to_first))
+    aggregate["face_embedding_note"] = "Optional: install insightface/torchreid for Face Embedding / character ID retention."
+
+    return {
+        "dimension": "identity_preservation",
+        "params": {"use_clip": use_clip, "device": device},
+        "per_video": per_video,
+        "aggregate": aggregate,
+        "num_videos": len(per_video),
+    }
+
+
+def main():
+    p = argparse.ArgumentParser(description="Identity Preservation (CLIP consistency)")
+    p.add_argument("--evals_root", type=str, required=True)
+    p.add_argument("--use_clip", action="store_true", help="Use CLIP image encoder when available")
+    p.add_argument("--device", type=str, default="cuda")
+    p.add_argument("--max_frames", type=int, default=100)
+    p.add_argument("--output", type=str, default=None)
+    args = p.parse_args()
+
+    result = run_identity_preservation(
+        args.evals_root,
+        use_clip=args.use_clip,
+        device=args.device,
+        max_frames_per_video=args.max_frames,
+    )
+    out = json.dumps(result, indent=2)
+    print(out)
+    if args.output:
+        with open(args.output, "w") as f:
+            f.write(out)
+
+
+if __name__ == "__main__":
+    main()