diff --git a/.gitattributes b/.gitattributes index 3a4957f0dc5078394d10d47341989c787213ec9b..4b6f725d0e0fa8f794ba9932c3a5b4c3767cfa07 100644 --- a/.gitattributes +++ b/.gitattributes @@ -76,3 +76,27 @@ checkpoints/predictor_v4_timestep_rollout_detached_timestepfp32_blocks1-52_overf checkpoints/predictor_v4_timestep_rollout_legacy-timestepbf16_blocks1-52_overfit8_seed20260721_fp32master_8gpu_flr1e-4_blr1e-5_schedsteps200_swarmup8_bwarmup8/replay0/swanlab/run-20260721_173532-xqwuujlx/run-xqwuujlx.swanlab filter=lfs diff=lfs merge=lfs -text checkpoints/predictor_v4_timestep_rollout_legacy-timestepbf16_blocks1-52_overfit8_seed20260721_fp32master_8gpu_flr1e-4_blr1e-5_schedsteps200_swarmup8_bwarmup8/replay0p2/swanlab/run-20260721_180252-jskm98hf/run-jskm98hf.swanlab filter=lfs diff=lfs merge=lfs -text checkpoints/predictor_v4_timestep_rollout_legacy-timestepbf16_blocks1-52_overfit8_seed20260721_fp32master_8gpu_flr1e-4_blr1e-5_schedsteps200_swarmup8_bwarmup8/replay0p4/swanlab/run-20260721_183105-7bjwo4bv/run-7bjwo4bv.swanlab filter=lfs diff=lfs merge=lfs -text +confidence_validation/plots/risk_coverage.png filter=lfs diff=lfs merge=lfs -text +confidence_validation_wide4096x2_c1/plots/risk_coverage.png filter=lfs diff=lfs merge=lfs -text +confidence_validation_wide4096x2_c2/plots/risk_coverage.png filter=lfs diff=lfs merge=lfs -text +episode_000004/episode_000000.mp4 filter=lfs diff=lfs merge=lfs -text +episode_000004/episode_000001.mp4 filter=lfs diff=lfs merge=lfs -text +episode_000004/episode_000002.mp4 filter=lfs diff=lfs merge=lfs -text +episode_000004/episode_000003.mp4 filter=lfs diff=lfs merge=lfs -text +episode_000004/episode_000004.mp4 filter=lfs diff=lfs merge=lfs -text +episode_000004/episode_000005.mp4 filter=lfs diff=lfs merge=lfs -text +episode_000004/episode_000006.mp4 filter=lfs diff=lfs merge=lfs -text +episode_000004/episode_000007.mp4 filter=lfs diff=lfs merge=lfs -text +episode_000004/episode_000008.mp4 filter=lfs diff=lfs merge=lfs -text +episode_000004/episode_000009.mp4 filter=lfs diff=lfs merge=lfs -text +episode_000004/episode_000010.mp4 filter=lfs diff=lfs merge=lfs -text +episode_000004/episode_000011.mp4 filter=lfs diff=lfs merge=lfs -text +episode_000004/episode_000012.mp4 filter=lfs diff=lfs merge=lfs -text +episode_000004/episode_000013.mp4 filter=lfs diff=lfs merge=lfs -text +episode_000004/episode_000014.mp4 filter=lfs diff=lfs merge=lfs -text +episode_000004/hyworldplay_outputs/episode_000004_i2v_ffff_forward_then_backward_clamp_bottle_seed0/gen.mp4 filter=lfs diff=lfs merge=lfs -text +episode_000004/hyworldplay_outputs/episode_000004_i2v_ffff_static_clamp_bottle_seed0/gen.mp4 filter=lfs diff=lfs merge=lfs -text +episode_000004/hyworldplay_outputs/episode_000004_i2v_stage1_fp32master_fppf_forward_then_backward_clamp_bottle_seed0/predictor/case_0000_action_00.mp4 filter=lfs diff=lfs merge=lfs -text +episode_000004/i2v_input_000000/26-15/episode_000000_first_frame.png filter=lfs diff=lfs merge=lfs -text +outputs/c1_threshold_smoke/predictor/case_0000_action_00.mp4 filter=lfs diff=lfs merge=lfs -text +outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/assets/Arial.ttf filter=lfs diff=lfs merge=lfs -text diff --git a/checkpoints/smoke_predictor_v4_random_exit_dmd_p2/latest/predictor.safetensors b/checkpoints/smoke_predictor_v4_random_exit_dmd_p2/latest/predictor.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e470d3d9af7ba47a53dd6825312a054f18405729 --- /dev/null +++ b/checkpoints/smoke_predictor_v4_random_exit_dmd_p2/latest/predictor.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f13eb217226d3bcf37f1fdbb7599bf7acacb998e875a7265e9aa779750a0849d +size 696559936 diff --git a/checkpoints/smoke_predictor_v4_random_exit_dmd_p2/latest/predictor_training.pt b/checkpoints/smoke_predictor_v4_random_exit_dmd_p2/latest/predictor_training.pt new file mode 100644 index 0000000000000000000000000000000000000000..e9502f010302ea068f5fd478993d75efe8da5740 --- /dev/null +++ b/checkpoints/smoke_predictor_v4_random_exit_dmd_p2/latest/predictor_training.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61ce9cedd8445e08a9f2d561f4bf832f4e26f86500b75d5fd77a084978f08e79 +size 2668808903 diff --git a/checkpoints/smoke_predictor_v4_random_exit_dmd_p2/latest/state.json b/checkpoints/smoke_predictor_v4_random_exit_dmd_p2/latest/state.json new file mode 100644 index 0000000000000000000000000000000000000000..b715534a5978bc72ec8d192960030e01fd506041 --- /dev/null +++ b/checkpoints/smoke_predictor_v4_random_exit_dmd_p2/latest/state.json @@ -0,0 +1,11 @@ +{ + "critic_step": 1, + "exit_counts": [ + 0, + 0, + 1, + 0 + ], + "final": true, + "student_step": 1 +} diff --git a/checkpoints/smoke_predictor_v4_random_exit_dmd_p2/train_config.json b/checkpoints/smoke_predictor_v4_random_exit_dmd_p2/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..3163171bca42e422bcff3e913b68840388ea1c8a --- /dev/null +++ b/checkpoints/smoke_predictor_v4_random_exit_dmd_p2/train_config.json @@ -0,0 +1,48 @@ +{ + "base_model": "/mnt/local_nvme/cz/projects/HY-WorldPlay-DEV-Predictor/outputs/model_cache_repaired/HunyuanVideo-1.5-9b49404-v2", + "blocks_lr": 5e-07, + "cases_path": "/mnt/local_nvme/cz/projects/HY-WorldPlay-DEV-Predictor/datasets/hyworldplay_predictor_prefeature_v4/cases.jsonl", + "critic_updates_per_student": 1, + "data_root": "/mnt/local_nvme/cz/projects/HY-WorldPlay-DEV-Predictor/datasets/hyworldplay_predictor_prefeature_v4", + "fake_score_grad_clip": 10.0, + "fake_score_lr": 2e-07, + "fake_score_model": "50step_bidirectional_model_trainable", + "flow_shift": 5.0, + "forced_exit_step": 2, + "fusion_lr": 5e-06, + "keep_checkpoints": 1, + "latent_seed": 0, + "log_every": 1, + "max_steps": 1, + "max_trajectories": 8, + "output_dir": "/mnt/local_nvme/cz/projects/HY-WorldPlay-DEV-Predictor/checkpoints/smoke_predictor_v4_random_exit_dmd_p2", + "predictor_grad_clip": 1.0, + "predictor_trainable_parameters": 197218816, + "random_exits": [ + 1, + 2, + 3 + ], + "real_guidance_scale": 6.0, + "real_score_model": "50step_bidirectional_model_frozen", + "regression_loss": false, + "resume": null, + "rollout_model": "4step_ar_distilled_action_model", + "rollout_teacher_checkpoint": "/mnt/local_nvme/cz/checkpoints/hy_worldplay/huggingface/hub/models--tencent--HY-WorldPlay/snapshots/f4c29235647707b571479a69b569e4166f9f5bf8/ar_distilled_action_model/diffusion_pytorch_model.safetensors", + "save_every": 1, + "score_teacher_checkpoint": "/mnt/local_nvme/cz/checkpoints/hy_worldplay/huggingface/hub/models--tencent--HY-WorldPlay/snapshots/f4c29235647707b571479a69b569e4166f9f5bf8/bidirectional_model/diffusion_pytorch_model.safetensors", + "score_timestep_max": 0.98, + "score_timestep_min": 0.02, + "seed": 20260818, + "stage1_expected_step": 2000, + "stage1_training_state": "/mnt/local_nvme/cz/projects/HY-WorldPlay-DEV-Predictor/checkpoints/predictor_v4_prefeature_blocks1-52_fp32master_8gpu_flr1e-4_blr1e-5_schedsteps2000_swarmup100_bwarmup100_swanlab/training_latest.pt", + "swanlab_experiment": "predictor-v4-random-exit-dmd-predictor-only-stage1step2000-8gpu", + "swanlab_mode": "disabled", + "swanlab_project": "HY-WorldPlay-Predictor-v4", + "swanlab_workspace": null, + "training_mode": "predictor_only_random_exit_dmd", + "validate_config_only": false, + "warmup_steps": 100, + "weight_decay": 0.01, + "world_size": 8 +} diff --git a/checkpoints/smoke_predictor_v4_random_exit_dmd_p2/train_log.jsonl b/checkpoints/smoke_predictor_v4_random_exit_dmd_p2/train_log.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..1b0dc58ec64647949e274a25b9a31c29b90aa357 --- /dev/null +++ b/checkpoints/smoke_predictor_v4_random_exit_dmd_p2/train_log.jsonl @@ -0,0 +1 @@ +{"critic_grad_norm": 1.21875, "critic_loss": 0.05237160646356642, "critic_step": 1, "critic_timestep": 797.6681823730469, "dmd_gradient_norm": 0.47206708416342735, "dmd_loss": 0.2637263061478734, "dmd_normalizer": 0.1062138332054019, "dmd_score_timestep": 635.6212921142578, "exit_count_p1": 0, "exit_count_p2": 1, "exit_count_p3": 0, "exit_step": 2, "lr_fake_score": 2e-07, "lr_predictor_blocks": 1e-08, "lr_predictor_fusion": 1.0000000000000001e-07, "peak_memory_gib": 65.39037084579468, "predictor_grad_norm": 0.07057937979698181, "step_time_s": 202.15974524412377, "student_step": 1} diff --git a/checkpoints/smoke_predictor_v4_random_exit_dmd_p3.log b/checkpoints/smoke_predictor_v4_random_exit_dmd_p3.log new file mode 100644 index 0000000000000000000000000000000000000000..3b565f1b246eb74ff252a0eab506d0ff389c7aaa --- /dev/null +++ b/checkpoints/smoke_predictor_v4_random_exit_dmd_p3.log @@ -0,0 +1,33 @@ +HY-World 1.5 loading from: /mnt/local_nvme/cz/checkpoints/hy_worldplay/huggingface/hub/models--tencent--HY-WorldPlay/snapshots/f4c29235647707b571479a69b569e4166f9f5bf8/ar_distilled_action_model/diffusion_pytorch_model.safetensors +HY-World 1.5 loading from: /mnt/local_nvme/cz/checkpoints/hy_worldplay/huggingface/hub/models--tencent--HY-WorldPlay/snapshots/f4c29235647707b571479a69b569e4166f9f5bf8/ar_distilled_action_model/diffusion_pytorch_model.safetensors +HY-World 1.5 loading from: /mnt/local_nvme/cz/checkpoints/hy_worldplay/huggingface/hub/models--tencent--HY-WorldPlay/snapshots/f4c29235647707b571479a69b569e4166f9f5bf8/ar_distilled_action_model/diffusion_pytorch_model.safetensors +HY-World 1.5 loading from: /mnt/local_nvme/cz/checkpoints/hy_worldplay/huggingface/hub/models--tencent--HY-WorldPlay/snapshots/f4c29235647707b571479a69b569e4166f9f5bf8/ar_distilled_action_model/diffusion_pytorch_model.safetensors +HY-World 1.5 loading from: /mnt/local_nvme/cz/checkpoints/hy_worldplay/huggingface/hub/models--tencent--HY-WorldPlay/snapshots/f4c29235647707b571479a69b569e4166f9f5bf8/ar_distilled_action_model/diffusion_pytorch_model.safetensors +/mnt/local_nvme/cz/projects/HY-WorldPlay-DEV-Predictor/hyvideo/models/text_encoders/byT5/__init__.py:74: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + byt5_state_dict = torch.load(args["byT5_ckpt_path"], map_location=device) +HY-World 1.5 loading from: /mnt/local_nvme/cz/checkpoints/hy_worldplay/huggingface/hub/models--tencent--HY-WorldPlay/snapshots/f4c29235647707b571479a69b569e4166f9f5bf8/ar_distilled_action_model/diffusion_pytorch_model.safetensors +HY-World 1.5 loading from: /mnt/local_nvme/cz/checkpoints/hy_worldplay/huggingface/hub/models--tencent--HY-WorldPlay/snapshots/f4c29235647707b571479a69b569e4166f9f5bf8/ar_distilled_action_model/diffusion_pytorch_model.safetensors +/mnt/local_nvme/cz/projects/HY-WorldPlay-DEV-Predictor/hyvideo/models/text_encoders/byT5/__init__.py:74: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + byt5_state_dict = torch.load(args["byT5_ckpt_path"], map_location=device) +/mnt/local_nvme/cz/projects/HY-WorldPlay-DEV-Predictor/hyvideo/models/text_encoders/byT5/__init__.py:74: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + byt5_state_dict = torch.load(args["byT5_ckpt_path"], map_location=device) +HY-World 1.5 loading from: /mnt/local_nvme/cz/checkpoints/hy_worldplay/huggingface/hub/models--tencent--HY-WorldPlay/snapshots/f4c29235647707b571479a69b569e4166f9f5bf8/ar_distilled_action_model/diffusion_pytorch_model.safetensors +/mnt/local_nvme/cz/projects/HY-WorldPlay-DEV-Predictor/hyvideo/models/text_encoders/byT5/__init__.py:74: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + byt5_state_dict = torch.load(args["byT5_ckpt_path"], map_location=device) +/mnt/local_nvme/cz/projects/HY-WorldPlay-DEV-Predictor/hyvideo/models/text_encoders/byT5/__init__.py:74: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + byt5_state_dict = torch.load(args["byT5_ckpt_path"], map_location=device) + Loading checkpoint shards: 0%| | 0/5 [00:00 + +The HF token is required for downloading the vision encoder from FLUX.1-Redux-dev. +Request access at: https://huggingface.co/black-forest-labs/FLUX.1-Redux-dev +""" + +import argparse +import os +import shutil +import sys + +WORLDPLAY_ALLOW_PATTERNS = ["ar_distilled_action_model/*"] + + +def check_dependencies(): + """Check and install required dependencies.""" + try: + from huggingface_hub import snapshot_download + except ImportError: + print("Installing huggingface_hub...") + os.system("pip install -U 'huggingface_hub[cli]'") + + try: + import modelscope + except ImportError: + print("Installing modelscope...") + os.system("pip install modelscope") + + +def download_hy_worldplay(): + """Download HY-WorldPlay action models.""" + from huggingface_hub import snapshot_download + + print("\n" + "=" * 60) + print("[1/6] Downloading tencent/HY-WorldPlay distilled AR action model...") + print("=" * 60) + + worldplay_path = snapshot_download( + "tencent/HY-WorldPlay", + allow_patterns=WORLDPLAY_ALLOW_PATTERNS, + max_workers=1, + ) + print(f"Downloaded to: {worldplay_path}") + + # Fix: Rename model.safetensors to diffusion_pytorch_model.safetensors + # in ar_distilled_action_model (repo uses different filename) + ar_distill_dir = os.path.join(worldplay_path, "ar_distilled_action_model") + model_src = os.path.join(ar_distill_dir, "model.safetensors") + model_dst = os.path.join(ar_distill_dir, "diffusion_pytorch_model.safetensors") + + if os.path.exists(model_src) and not os.path.exists(model_dst): + real_src = os.path.realpath(model_src) + shutil.copy2(real_src, model_dst) + print( + "Fixed: Renamed model.safetensors -> diffusion_pytorch_model.safetensors in ar_distilled_action_model" + ) + + return worldplay_path + + +def download_hunyuan_video(): + """Download HunyuanVideo-1.5 base models (vae, scheduler, transformer).""" + from huggingface_hub import snapshot_download + + print("\n" + "=" * 60) + print("[2/6] Downloading tencent/HunyuanVideo-1.5 (vae, scheduler, transformer)...") + print("=" * 60) + + hunyuan_path = snapshot_download( + "tencent/HunyuanVideo-1.5", + allow_patterns=["vae/*", "scheduler/*", "transformer/480p_i2v/*"], + max_workers=1, + ) + print(f"Downloaded to: {hunyuan_path}") + return hunyuan_path + + +def download_llm_text_encoder(hunyuan_path): + """Download Qwen2.5-VL-7B-Instruct as the LLM text encoder.""" + from huggingface_hub import snapshot_download + + print("\n" + "=" * 60) + print("[3/6] Downloading LLM text encoder (Qwen2.5-VL-7B-Instruct)...") + print("=" * 60) + + text_encoder_base = os.path.join(hunyuan_path, "text_encoder") + os.makedirs(text_encoder_base, exist_ok=True) + + llm_target = os.path.join(text_encoder_base, "llm") + + if ( + os.path.exists(llm_target) + and os.path.isdir(llm_target) + and len(os.listdir(llm_target)) > 5 + ): + print(f"LLM text encoder already exists at: {llm_target}") + return + + # Clean up old/broken downloads + if os.path.islink(llm_target): + os.unlink(llm_target) + elif os.path.exists(llm_target): + shutil.rmtree(llm_target) + + print("Downloading Qwen/Qwen2.5-VL-7B-Instruct (~15GB)...") + qwen_cache = snapshot_download("Qwen/Qwen2.5-VL-7B-Instruct", max_workers=1) + + # Copy files (resolve symlinks) + os.makedirs(llm_target, exist_ok=True) + for item in os.listdir(qwen_cache): + src = os.path.realpath(os.path.join(qwen_cache, item)) + dst = os.path.join(llm_target, item) + if os.path.isdir(src): + shutil.copytree(src, dst, dirs_exist_ok=True) + else: + shutil.copy2(src, dst) + + print(f"Copied to: {llm_target}") + + +def download_byt5_encoders(hunyuan_path): + """Download ByT5 text encoders (byt5-small and Glyph-SDXL-v2).""" + from huggingface_hub import snapshot_download + from modelscope import snapshot_download as ms_snapshot_download + + print("\n" + "=" * 60) + print("[4/6] Downloading ByT5 text encoders...") + print("=" * 60) + + text_encoder_base = os.path.join(hunyuan_path, "text_encoder") + os.makedirs(text_encoder_base, exist_ok=True) + + # 1. Download google/byt5-small + byt5_target = os.path.join(text_encoder_base, "byt5-small") + if ( + os.path.exists(byt5_target) + and os.path.isdir(byt5_target) + and len(os.listdir(byt5_target)) > 3 + ): + print(f"byt5-small already exists at: {byt5_target}") + else: + if os.path.islink(byt5_target): + os.unlink(byt5_target) + elif os.path.exists(byt5_target): + shutil.rmtree(byt5_target) + + print("Downloading google/byt5-small...") + byt5_cache = snapshot_download("google/byt5-small", max_workers=1) + + os.makedirs(byt5_target, exist_ok=True) + for item in os.listdir(byt5_cache): + src = os.path.realpath(os.path.join(byt5_cache, item)) + dst = os.path.join(byt5_target, item) + if os.path.isdir(src): + shutil.copytree(src, dst, dirs_exist_ok=True) + else: + shutil.copy2(src, dst) + print(f"Copied to: {byt5_target}") + + # 2. Download Glyph-SDXL-v2 from ModelScope + glyph_target = os.path.join(text_encoder_base, "Glyph-SDXL-v2") + if os.path.exists(glyph_target) and os.path.exists( + os.path.join(glyph_target, "checkpoints", "byt5_model.pt") + ): + print(f"Glyph-SDXL-v2 already exists at: {glyph_target}") + else: + if os.path.exists(glyph_target): + shutil.rmtree(glyph_target) + + print("Downloading AI-ModelScope/Glyph-SDXL-v2 from ModelScope...") + glyph_cache = ms_snapshot_download( + "AI-ModelScope/Glyph-SDXL-v2", cache_dir="/tmp/glyph_cache" + ) + + os.makedirs(glyph_target, exist_ok=True) + for item in os.listdir(glyph_cache): + src = os.path.join(glyph_cache, item) + dst = os.path.join(glyph_target, item) + if os.path.isdir(src): + if os.path.exists(dst): + shutil.rmtree(dst) + shutil.copytree(src, dst) + else: + shutil.copy2(src, dst) + print(f"Copied to: {glyph_target}") + + +def download_vision_encoder(hunyuan_path, hf_token): + """Download SigLIP vision encoder from FLUX.1-Redux-dev.""" + from huggingface_hub import snapshot_download + + print("\n" + "=" * 60) + print("[5/6] Downloading Vision Encoder (SigLIP from FLUX.1-Redux-dev)...") + print("=" * 60) + + if not hf_token: + print("WARNING: No HF token provided!") + print( + "The vision encoder requires access to: https://huggingface.co/black-forest-labs/FLUX.1-Redux-dev" + ) + print("Skipping vision encoder download.") + print("\nYou can download it manually later.") + return + + vision_encoder_base = os.path.join(hunyuan_path, "vision_encoder") + os.makedirs(vision_encoder_base, exist_ok=True) + + siglip_target = os.path.join(vision_encoder_base, "siglip") + + if ( + os.path.exists(siglip_target) + and os.path.isdir(siglip_target) + and len(os.listdir(siglip_target)) > 3 + ): + print(f"siglip already exists at: {siglip_target}") + return + + # Clean up old/broken downloads + if os.path.islink(siglip_target): + os.unlink(siglip_target) + elif os.path.exists(siglip_target): + shutil.rmtree(siglip_target) + + print("Downloading black-forest-labs/FLUX.1-Redux-dev...") + try: + flux_cache = snapshot_download( + "black-forest-labs/FLUX.1-Redux-dev", token=hf_token, max_workers=1 + ) + + # Copy files (resolve symlinks) + os.makedirs(siglip_target, exist_ok=True) + for item in os.listdir(flux_cache): + src = os.path.realpath(os.path.join(flux_cache, item)) + dst = os.path.join(siglip_target, item) + if os.path.isdir(src): + shutil.copytree(src, dst, dirs_exist_ok=True) + else: + shutil.copy2(src, dst) + print(f"Copied to: {siglip_target}") + except Exception as e: + print(f"ERROR: Failed to download vision encoder: {e}") + print( + "Make sure you have requested access to FLUX.1-Redux-dev and your token is valid." + ) + + +def print_paths(): + """Print the model paths for run.sh configuration.""" + from huggingface_hub import snapshot_download + + print("\n" + "=" * 60) + print("[6/6] Verifying downloads...") + print("=" * 60) + + hunyuan_path = snapshot_download("tencent/HunyuanVideo-1.5", local_files_only=True) + worldplay_path = snapshot_download( + "tencent/HY-WorldPlay", + allow_patterns=WORLDPLAY_ALLOW_PATTERNS, + local_files_only=True, + ) + + print("\n" + "=" * 60) + print("ALL DOWNLOADS COMPLETE!") + print("=" * 60) + print("\nAdd these paths to your run.sh:\n") + print(f"MODEL_PATH={hunyuan_path}") + print( + f"AR_ACTION_MODEL_PATH={worldplay_path}/ar_model/diffusion_pytorch_model.safetensors" + ) + print( + f"AR_RL_ACTION_MODEL_PATH={worldplay_path}/ar_rl_model/diffusion_pytorch_model.safetensors" + ) + print( + f"BI_ACTION_MODEL_PATH={worldplay_path}/bidirectional_model/diffusion_pytorch_model.safetensors" + ) + print( + f"AR_DISTILL_ACTION_MODEL_PATH={worldplay_path}/ar_distilled_action_model/diffusion_pytorch_model.safetensors" + ) + print("\nYou can now run: bash run.sh") + + +def main(): + parser = argparse.ArgumentParser( + description="Download all required models for HY-WorldPlay", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Example: + python download_models.py --hf_token hf_xxxxxxxxxxxxx + +Note: + The HuggingFace token is required for downloading the vision encoder + from black-forest-labs/FLUX.1-Redux-dev. You need to: + 1. Request access at: https://huggingface.co/black-forest-labs/FLUX.1-Redux-dev + 2. Wait for approval (usually instant) + 3. Create a token at: https://huggingface.co/settings/tokens (select "Read" permission) + """, + ) + parser.add_argument( + "--hf_token", + type=str, + default=None, + help="HuggingFace token for downloading gated models (required for vision encoder)", + ) + parser.add_argument( + "--skip_vision_encoder", + action="store_true", + help="Skip downloading the vision encoder (if you don't have FLUX access yet)", + ) + + args = parser.parse_args() + + if not args.hf_token: + try: + from huggingface_hub import HfFolder + + args.hf_token = HfFolder.get_token() + except Exception: + args.hf_token = None + + print("=" * 60) + print("HY-WorldPlay Model Download Script") + print("=" * 60) + + # Check dependencies + check_dependencies() + + # Download models + worldplay_path = download_hy_worldplay() + hunyuan_path = download_hunyuan_video() + download_llm_text_encoder(hunyuan_path) + download_byt5_encoders(hunyuan_path) + + if not args.skip_vision_encoder: + download_vision_encoder(hunyuan_path, args.hf_token) + else: + print("\n[5/6] Skipping vision encoder download (--skip_vision_encoder flag)") + + # Print final paths + print_paths() + + +if __name__ == "__main__": + main() diff --git a/episode_000004/episode_000000.mp4 b/episode_000004/episode_000000.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9981fcc9e9125502023a7bf700ef282cfdf13a2a --- /dev/null +++ b/episode_000004/episode_000000.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:23113715278718006b69bcb5483c3c5bff2237d730e67f3b6c49cf5dbcd468cc +size 918381 diff --git a/episode_000004/episode_000001.mp4 b/episode_000004/episode_000001.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ceb19b14b740905b92c92c94ddc01d8972b5e233 --- /dev/null +++ b/episode_000004/episode_000001.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c31efa7d664a78f3ee318951a946e6a4ba502da5a81f3af2d33c876f75f751ba +size 629812 diff --git a/episode_000004/episode_000002.mp4 b/episode_000004/episode_000002.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b269c967d79795d84cc2ac910fc6126d01a265ff --- /dev/null +++ b/episode_000004/episode_000002.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e76cb826bb072f8f2418a78515abf9cd4bb31a2ec90c33de9c016febc0b3519a +size 578246 diff --git a/episode_000004/episode_000003.mp4 b/episode_000004/episode_000003.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..25609f208b6a75d7f7f13c43683192a96e2c6816 --- /dev/null +++ b/episode_000004/episode_000003.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e3bfd9994aeccf8c2abd85994829780329f84415a9c417b2d7e43ff80c2bee7c +size 770647 diff --git a/episode_000004/episode_000004.mp4 b/episode_000004/episode_000004.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..34d4b0cea8e3d1c934d66019b224ecd1b7a991c1 --- /dev/null +++ b/episode_000004/episode_000004.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9202aa26b8671fa43243487bc2b88fab811561aa04c6575b08b97f44f5c7a37 +size 813002 diff --git a/episode_000004/episode_000005.mp4 b/episode_000004/episode_000005.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..87e808baac8801caa5b23424bb24c831552ae7e1 --- /dev/null +++ b/episode_000004/episode_000005.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7a51a96142f5ece855788a6cf1f6281a118b9cb5b8a09556dbdd90f7af948213 +size 953924 diff --git a/episode_000004/episode_000006.mp4 b/episode_000004/episode_000006.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ca78c9342af667ca0bac25c67d975a39fad3c464 --- /dev/null +++ b/episode_000004/episode_000006.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e87b5ea0bb704213a380a7e8fe2b5ad0a7cfb558b71d1a83ad89efd871369333 +size 580863 diff --git a/episode_000004/episode_000007.mp4 b/episode_000004/episode_000007.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0d4f4d9d3380218ff9afb238be5250e817c18de6 --- /dev/null +++ b/episode_000004/episode_000007.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c3c58353ed5f6243ad998fa97838bb590b22f3249932965a11466c3894a0916 +size 636801 diff --git a/episode_000004/episode_000008.mp4 b/episode_000004/episode_000008.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c303eba25b28b3cecb2eda9f15fd0aae175bc4c2 --- /dev/null +++ b/episode_000004/episode_000008.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d694f7f0e1c6db0247e1e2077bfd62131836e66182081b5b117ea619ef8aca36 +size 795712 diff --git a/episode_000004/episode_000009.mp4 b/episode_000004/episode_000009.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d4f580f082e11862ca98b3135d90982c417eed27 --- /dev/null +++ b/episode_000004/episode_000009.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b1958c5dc6bf6b31934eb0d38dec928625234d18aab7a6740ce5ddca7032f736 +size 504082 diff --git a/episode_000004/episode_000010.mp4 b/episode_000004/episode_000010.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3b343d08dc5ff14f0daac42cf964bcb07d121c56 --- /dev/null +++ b/episode_000004/episode_000010.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:802d0123d20747eedfd47344050dc4194589a6c8f58c5093715ac1657151f39a +size 603153 diff --git a/episode_000004/episode_000011.mp4 b/episode_000004/episode_000011.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..870173396ea770e6e56480f1747f4138f2ca0596 --- /dev/null +++ b/episode_000004/episode_000011.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f916cb028223c492d741eb3fef19d22692e1aefd76aa850733426aa1d06a941e +size 1208398 diff --git a/episode_000004/episode_000012.mp4 b/episode_000004/episode_000012.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1008e4e35e6c4e90209a824c6182f86df440396a --- /dev/null +++ b/episode_000004/episode_000012.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:98aa476cdf0528e708fb29aca89e50e2eb181f1ca5ad352d2ab4e0c4cff7f8ab +size 408406 diff --git a/episode_000004/episode_000013.mp4 b/episode_000004/episode_000013.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fc648d0d622ee2d6c55d742e70968cd1fc22ec51 --- /dev/null +++ b/episode_000004/episode_000013.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f4c4475a017122204b57e0dd8222eb9a338f056e8fc1eafa42b0596a09db5ba6 +size 992262 diff --git a/episode_000004/episode_000014.mp4 b/episode_000004/episode_000014.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7287b1f769e399e52cfb6cc3a2976c8d0183f564 --- /dev/null +++ b/episode_000004/episode_000014.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e63c81b7336be3dd83f292802b0132ef556b928db46c4ae9ba4bce5c8e50a524 +size 616193 diff --git a/episode_000004/hyworldplay_custom_case/cases.jsonl b/episode_000004/hyworldplay_custom_case/cases.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..824181ae9d4845dc43b159d77054456ed069ed5b --- /dev/null +++ b/episode_000004/hyworldplay_custom_case/cases.jsonl @@ -0,0 +1 @@ +{"case_id": 0, "caption": "用夹子夹瓶子", "image_path": "/mnt/s3files/s3-us-west2-default/zoubin/cz/projects/HY-WorldPlay-DEV-Predictor/episode_000004/i2v_input_000000/26-15/episode_000000_first_frame.png"} diff --git a/episode_000004/hyworldplay_outputs/episode_000004_i2v_ffff_forward_then_backward_clamp_bottle_seed0/gen.mp4 b/episode_000004/hyworldplay_outputs/episode_000004_i2v_ffff_forward_then_backward_clamp_bottle_seed0/gen.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1affe65727034c92f64e14b18ab613d57ea4e5e7 --- /dev/null +++ b/episode_000004/hyworldplay_outputs/episode_000004_i2v_ffff_forward_then_backward_clamp_bottle_seed0/gen.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a794508acf360c7d994446027f1b12d9d2cd952a9a8501b78ffa7c5f961dcd62 +size 243946 diff --git a/episode_000004/hyworldplay_outputs/episode_000004_i2v_ffff_static_clamp_bottle_seed0/gen.mp4 b/episode_000004/hyworldplay_outputs/episode_000004_i2v_ffff_static_clamp_bottle_seed0/gen.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f766a5fe84b7a345d607c4e49224e8355636a839 --- /dev/null +++ b/episode_000004/hyworldplay_outputs/episode_000004_i2v_ffff_static_clamp_bottle_seed0/gen.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc1856e5d95989b0bb34494a657f027ae55fa9802888e8661d255ca5b412abc7 +size 108063 diff --git a/episode_000004/hyworldplay_outputs/episode_000004_i2v_stage1_fp32master_fppf_forward_then_backward_clamp_bottle_seed0/logs/worker_0.jsonl b/episode_000004/hyworldplay_outputs/episode_000004_i2v_stage1_fp32master_fppf_forward_then_backward_clamp_bottle_seed0/logs/worker_0.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..96c17cefd3bc4c6b10d49c645be8fab81b06329b --- /dev/null +++ b/episode_000004/hyworldplay_outputs/episode_000004_i2v_stage1_fp32master_fppf_forward_then_backward_clamp_bottle_seed0/logs/worker_0.jsonl @@ -0,0 +1 @@ +{"action_id": 0, "action_name": "w_s", "case_id": 0, "elapsed_s": 76.71275353385136, "end_to_end_s": 76.71275353385136, "mode": "predictor", "peak_memory_gib": 71.85594654083252, "predictor_variant": "v4", "replace_steps": [1, 2], "seed": 0, "source_blocks": [1, 52], "timing": {"ar_chunk_scheduler_timesteps": {"avg_s": 0.00020668881812265942, "count": 7, "max_s": 0.00024232384748756886, "min_s": 0.0001543019898235798, "total_s": 0.0014468217268586159}, "ar_chunk_total": {"avg_s": 4.289951821847353, "count": 8, "max_s": 6.595122660975903, "min_s": 1.7476372688543051, "total_s": 34.319614574778825}, "ar_context_setup": {"avg_s": 0.002706830156967044, "count": 1, "max_s": 0.002706830156967044, "min_s": 0.002706830156967044, "total_s": 0.002706830156967044}, "ar_history_kv_cache": {"avg_s": 2.68971249700657, "count": 7, "max_s": 4.136401807889342, "min_s": 0.4973818270955235, "total_s": 18.827987479045987}, "ar_history_prepare": {"avg_s": 0.00042369792104831764, "count": 7, "max_s": 0.0004667670000344515, "min_s": 0.0003934060223400593, "total_s": 0.0029658854473382235}, "ar_history_select": {"avg_s": 0.0314884513749608, "count": 7, "max_s": 0.17801832896657288, "min_s": 5.7841185480356216e-05, "total_s": 0.22041915962472558}, "ar_init_kv_cache": {"avg_s": 0.00010042102076113224, "count": 1, "max_s": 0.00010042102076113224, "min_s": 0.00010042102076113224, "total_s": 0.00010042102076113224}, "ar_rollout_total": {"avg_s": 47.51126848207787, "count": 1, "max_s": 47.51126848207787, "min_s": 47.51126848207787, "total_s": 47.51126848207787}, "ar_step_predictor": {"avg_s": 0.06730422407521733, "count": 14, "max_s": 0.08451879606582224, "min_s": 0.038312654942274094, "total_s": 0.9422591370530427}, "ar_step_scheduler": {"avg_s": 0.0007399889145744964, "count": 32, "max_s": 0.017267888877540827, "min_s": 0.00015821191482245922, "total_s": 0.023679645266383886}, "ar_step_transformer": {"avg_s": 0.7934716426922629, "count": 18, "max_s": 1.0640695649199188, "min_s": 0.4614686779677868, "total_s": 14.282489568460733}, "ar_text_kv_cache": {"avg_s": 0.18096599215641618, "count": 1, "max_s": 0.18096599215641618, "min_s": 0.18096599215641618, "total_s": 0.18096599215641618}, "byt5_encoder": {"avg_s": 0.2791885049082339, "count": 1, "max_s": 0.2791885049082339, "min_s": 0.2791885049082339, "total_s": 0.2791885049082339}, "pipeline_total": {"avg_s": 74.57681559817865, "count": 1, "max_s": 74.57681559817865, "min_s": 74.57681559817865, "total_s": 74.57681559817865}, "prepare_cond_latents": {"avg_s": 0.001255959039554, "count": 1, "max_s": 0.001255959039554, "min_s": 0.001255959039554, "total_s": 0.001255959039554}, "prepare_latents": {"avg_s": 0.0012153880670666695, "count": 1, "max_s": 0.0012153880670666695, "min_s": 0.0012153880670666695, "total_s": 0.0012153880670666695}, "scheduler_timesteps": {"avg_s": 0.00042407610453665257, "count": 1, "max_s": 0.00042407610453665257, "min_s": 0.00042407610453665257, "total_s": 0.00042407610453665257}, "text_encoder_1": {"avg_s": 11.269568706862628, "count": 1, "max_s": 11.269568706862628, "min_s": 11.269568706862628, "total_s": 11.269568706862628}, "vae_decode": {"avg_s": 10.258140043821186, "count": 1, "max_s": 10.258140043821186, "min_s": 10.258140043821186, "total_s": 10.258140043821186}, "vae_encode_image_condition": {"avg_s": 4.502021143911406, "count": 1, "max_s": 4.502021143911406, "min_s": 4.502021143911406, "total_s": 4.502021143911406}, "video_postprocess_to_cpu": {"avg_s": 0.24960487708449364, "count": 1, "max_s": 0.24960487708449364, "min_s": 0.24960487708449364, "total_s": 0.24960487708449364}, "vision_encoder": {"avg_s": 0.44101492199115455, "count": 1, "max_s": 0.44101492199115455, "min_s": 0.44101492199115455, "total_s": 0.44101492199115455}}, "video_encode_s": 1.990567458095029, "video_path": "/mnt/s3files/s3-us-west2-default/zoubin/cz/projects/HY-WorldPlay-DEV-Predictor/episode_000004/hyworldplay_outputs/episode_000004_i2v_stage1_fp32master_fppf_forward_then_backward_clamp_bottle_seed0/predictor/case_0000_action_00.mp4", "video_validate_s": 0.13354528695344925, "worker_id": 0} diff --git a/episode_000004/hyworldplay_outputs/episode_000004_i2v_stage1_fp32master_fppf_forward_then_backward_clamp_bottle_seed0/predictor/case_0000_action_00.mp4 b/episode_000004/hyworldplay_outputs/episode_000004_i2v_stage1_fp32master_fppf_forward_then_backward_clamp_bottle_seed0/predictor/case_0000_action_00.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a3f4a928714ee39265eb3de3ae56dc011b474172 --- /dev/null +++ b/episode_000004/hyworldplay_outputs/episode_000004_i2v_stage1_fp32master_fppf_forward_then_backward_clamp_bottle_seed0/predictor/case_0000_action_00.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7cb7495f4a368f37cbb3dd67da0e25207038cf0f465894313a2aeda7ec0fb82 +size 245159 diff --git a/episode_000004/i2v_input_000000/26-15/episode_000000_first_frame.png b/episode_000004/i2v_input_000000/26-15/episode_000000_first_frame.png new file mode 100644 index 0000000000000000000000000000000000000000..4098325455ce77997db82daa1786a77ef4a9194a --- /dev/null +++ b/episode_000004/i2v_input_000000/26-15/episode_000000_first_frame.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0633d87d1e4aecc23d62f2695fb76a8023ac9961825384c9146785ec7d0f4f2 +size 153742 diff --git a/episode_000004/i2v_input_000000/ffff_i2v_21latent.yaml b/episode_000004/i2v_input_000000/ffff_i2v_21latent.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c3e335ed3f91b9a3ddba91f4e54b5a30022d9b1f --- /dev/null +++ b/episode_000004/i2v_input_000000/ffff_i2v_21latent.yaml @@ -0,0 +1,12 @@ +# Self-Forcing DMD FFFF image-to-video inference. +# One independent conditioning latent + five 4-latent generated chunks = 21. +denoising_step_list: + - 1000 + - 750 + - 500 + - 250 +warp_denoising_step: true +num_frame_per_block: 4 +independent_first_frame: true +model_kwargs: + timestep_shift: 5.0 diff --git a/episode_000004/i2v_input_000000/frxf_fppf_i2v_22latent.yaml b/episode_000004/i2v_input_000000/frxf_fppf_i2v_22latent.yaml new file mode 100644 index 0000000000000000000000000000000000000000..926f7dd03b5f7f04865aacee5ce7327145bf22f5 --- /dev/null +++ b/episode_000004/i2v_input_000000/frxf_fppf_i2v_22latent.yaml @@ -0,0 +1,13 @@ +# I2V comparison layout for schedules whose Predictor is trained on +# regular three-latent chunks: one independent image latent followed by +# seven generated chunks (21 generated latents, 22 total latents). +denoising_step_list: + - 1000 + - 750 + - 500 + - 250 +warp_denoising_step: true +num_frame_per_block: 3 +independent_first_frame: true +model_kwargs: + timestep_shift: 5.0 diff --git a/episode_000004/i2v_input_000000/static_32_latents.json b/episode_000004/i2v_input_000000/static_32_latents.json new file mode 100644 index 0000000000000000000000000000000000000000..f57047b4d7afc91f2c788ea8997597dc514b7037 --- /dev/null +++ b/episode_000004/i2v_input_000000/static_32_latents.json @@ -0,0 +1,34 @@ +{ + "0": {"extrinsic": [[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]], "K": [[969.6969696969696, 0, 960], [0, 969.6969696969696, 540], [0, 0, 1]]}, + "1": {"extrinsic": [[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]], "K": [[969.6969696969696, 0, 960], [0, 969.6969696969696, 540], [0, 0, 1]]}, + "2": {"extrinsic": [[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]], "K": [[969.6969696969696, 0, 960], [0, 969.6969696969696, 540], [0, 0, 1]]}, + "3": {"extrinsic": [[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]], "K": [[969.6969696969696, 0, 960], [0, 969.6969696969696, 540], [0, 0, 1]]}, + "4": {"extrinsic": [[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]], "K": [[969.6969696969696, 0, 960], [0, 969.6969696969696, 540], [0, 0, 1]]}, + "5": {"extrinsic": [[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]], "K": [[969.6969696969696, 0, 960], [0, 969.6969696969696, 540], [0, 0, 1]]}, + "6": {"extrinsic": [[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]], "K": [[969.6969696969696, 0, 960], [0, 969.6969696969696, 540], [0, 0, 1]]}, + "7": {"extrinsic": [[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]], "K": [[969.6969696969696, 0, 960], [0, 969.6969696969696, 540], [0, 0, 1]]}, + "8": {"extrinsic": [[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]], "K": [[969.6969696969696, 0, 960], [0, 969.6969696969696, 540], [0, 0, 1]]}, + "9": {"extrinsic": [[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]], "K": [[969.6969696969696, 0, 960], [0, 969.6969696969696, 540], [0, 0, 1]]}, + "10": {"extrinsic": [[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]], "K": [[969.6969696969696, 0, 960], [0, 969.6969696969696, 540], [0, 0, 1]]}, + "11": {"extrinsic": [[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]], "K": [[969.6969696969696, 0, 960], [0, 969.6969696969696, 540], [0, 0, 1]]}, + "12": {"extrinsic": [[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]], "K": [[969.6969696969696, 0, 960], [0, 969.6969696969696, 540], [0, 0, 1]]}, + "13": {"extrinsic": [[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]], "K": [[969.6969696969696, 0, 960], [0, 969.6969696969696, 540], [0, 0, 1]]}, + "14": {"extrinsic": [[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]], "K": [[969.6969696969696, 0, 960], [0, 969.6969696969696, 540], [0, 0, 1]]}, + "15": {"extrinsic": [[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]], "K": [[969.6969696969696, 0, 960], [0, 969.6969696969696, 540], [0, 0, 1]]}, + "16": {"extrinsic": [[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]], "K": [[969.6969696969696, 0, 960], [0, 969.6969696969696, 540], [0, 0, 1]]}, + "17": {"extrinsic": [[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]], "K": [[969.6969696969696, 0, 960], [0, 969.6969696969696, 540], [0, 0, 1]]}, + "18": {"extrinsic": [[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]], "K": [[969.6969696969696, 0, 960], [0, 969.6969696969696, 540], [0, 0, 1]]}, + "19": {"extrinsic": [[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]], "K": [[969.6969696969696, 0, 960], [0, 969.6969696969696, 540], [0, 0, 1]]}, + "20": {"extrinsic": [[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]], "K": [[969.6969696969696, 0, 960], [0, 969.6969696969696, 540], [0, 0, 1]]}, + "21": {"extrinsic": [[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]], "K": [[969.6969696969696, 0, 960], [0, 969.6969696969696, 540], [0, 0, 1]]}, + "22": {"extrinsic": [[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]], "K": [[969.6969696969696, 0, 960], [0, 969.6969696969696, 540], [0, 0, 1]]}, + "23": {"extrinsic": [[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]], "K": [[969.6969696969696, 0, 960], [0, 969.6969696969696, 540], [0, 0, 1]]}, + "24": {"extrinsic": [[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]], "K": [[969.6969696969696, 0, 960], [0, 969.6969696969696, 540], [0, 0, 1]]}, + "25": {"extrinsic": [[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]], "K": [[969.6969696969696, 0, 960], [0, 969.6969696969696, 540], [0, 0, 1]]}, + "26": {"extrinsic": [[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]], "K": [[969.6969696969696, 0, 960], [0, 969.6969696969696, 540], [0, 0, 1]]}, + "27": {"extrinsic": [[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]], "K": [[969.6969696969696, 0, 960], [0, 969.6969696969696, 540], [0, 0, 1]]}, + "28": {"extrinsic": [[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]], "K": [[969.6969696969696, 0, 960], [0, 969.6969696969696, 540], [0, 0, 1]]}, + "29": {"extrinsic": [[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]], "K": [[969.6969696969696, 0, 960], [0, 969.6969696969696, 540], [0, 0, 1]]}, + "30": {"extrinsic": [[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]], "K": [[969.6969696969696, 0, 960], [0, 969.6969696969696, 540], [0, 0, 1]]}, + "31": {"extrinsic": [[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]], "K": [[969.6969696969696, 0, 960], [0, 969.6969696969696, 540], [0, 0, 1]]} +} diff --git a/episode_000004/i2v_input_000000/target_crop_info_26-15.json b/episode_000004/i2v_input_000000/target_crop_info_26-15.json new file mode 100644 index 0000000000000000000000000000000000000000..00efb2a54c7bb392e241812d6cf85e9b288ad2a5 --- /dev/null +++ b/episode_000004/i2v_input_000000/target_crop_info_26-15.json @@ -0,0 +1,18 @@ +[ + { + "file_name": "episode_000000_first_frame.png", + "caption": "用夹子夹瓶子", + "target_crop": { + "target_bbox": [ + 0, + 0, + 640, + 480 + ], + "target_ratio": "26-15" + }, + "type": "image", + "origin_width": 640, + "origin_height": 480 + } +] diff --git a/eval_predictor_offline.py b/eval_predictor_offline.py new file mode 100644 index 0000000000000000000000000000000000000000..c70712d01879afad93e4f1869e21ac7d92af3424 --- /dev/null +++ b/eval_predictor_offline.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +"""Offline Predictor vs Direct-Reuse evaluation on saved Teacher pairs.""" + +from __future__ import annotations + +import argparse +import json +import time +from collections import defaultdict +from pathlib import Path + +import torch +import torch.nn.functional as F +from torch.utils.data import DataLoader + +from hyvideo.commons.infer_state import initialize_infer_state +from models import HYWorldPlayPredictor +from predictor_training.checkpoint import load_predictor_weights +from predictor_training.dataset import ( + PredictorPairDataset, + move_pair_to_device, + predictor_collate, +) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--manifest", default="datasets/predictor_v1/train_eval_manifest.jsonl") + parser.add_argument("--weights", required=True) + parser.add_argument( + "--teacher_checkpoint", + default="models-ms/HY-WorldPlay/ar_distilled_action_model/diffusion_pytorch_model.safetensors", + ) + parser.add_argument("--output", default="outputs/predictor_v1/offline_metrics.json") + parser.add_argument("--max_items", type=int, default=None) + parser.add_argument("--max_records", type=int, default=None) + return parser.parse_args() + + +def init_state() -> None: + class Args: + sage_blocks_range = "0-53" + use_sageattn = False + enable_torch_compile = False + use_fp8_gemm = False + quant_type = "fp8-per-block" + include_patterns = "double_blocks" + use_vae_parallel = False + + initialize_infer_state(Args()) + + +def summarize(values: list[float]) -> dict[str, float]: + tensor = torch.tensor(values, dtype=torch.float64) + return { + "count": len(values), + "mean": float(tensor.mean()), + "median": float(tensor.median()), + "p90": float(torch.quantile(tensor, 0.9)), + "min": float(tensor.min()), + "max": float(tensor.max()), + } + + +def main() -> None: + args = parse_args() + init_state() + device = torch.device("cuda:0") + dtype = torch.bfloat16 + dataset = PredictorPairDataset(args.manifest, max_records=args.max_records) + loader = DataLoader(dataset, batch_size=1, shuffle=False, collate_fn=predictor_collate) + model = HYWorldPlayPredictor() + model.load_teacher_initialization(args.teacher_checkpoint) + load_predictor_weights(model, args.weights) + model.to(device=device, dtype=dtype).eval() + + metrics: dict[str, dict[str, list[float]]] = defaultdict(lambda: defaultdict(list)) + with torch.no_grad(): + for index, item in enumerate(loader): + if args.max_items is not None and index >= args.max_items: + break + inputs, target_hidden, target_velocity = move_pair_to_device( + item, device=device, dtype=dtype + ) + torch.cuda.synchronize() + started = time.perf_counter() + with torch.autocast(device_type="cuda", dtype=dtype): + output = model(**inputs) + torch.cuda.synchronize() + predictor_time = time.perf_counter() - started + + torch.cuda.synchronize() + started = time.perf_counter() + with torch.autocast(device_type="cuda", dtype=dtype): + reuse_velocity = model.hidden_to_velocity( + inputs["anchor_hidden"], inputs["target_frame_condition"] + ) + torch.cuda.synchronize() + reuse_time = time.perf_counter() - started + + group = item["pair_id"] + pred_hidden_mse = F.mse_loss(output["pred_hidden"].float(), target_hidden.float()) + pred_velocity_mse = F.mse_loss(output["pred_velocity"].float(), target_velocity.float()) + reuse_hidden_mse = F.mse_loss(inputs["anchor_hidden"].float(), target_hidden.float()) + reuse_velocity_mse = F.mse_loss(reuse_velocity.float(), target_velocity.float()) + hidden_cosine = F.cosine_similarity( + output["pred_hidden"].float().flatten(1), target_hidden.float().flatten(1) + ).mean() + values = { + "predictor_hidden_mse": float(pred_hidden_mse), + "predictor_velocity_mse": float(pred_velocity_mse), + "reuse_hidden_mse": float(reuse_hidden_mse), + "reuse_velocity_mse": float(reuse_velocity_mse), + "hidden_cosine": float(hidden_cosine), + "hidden_mse_ratio": float(pred_hidden_mse / reuse_hidden_mse.clamp_min(1e-12)), + "velocity_mse_ratio": float(pred_velocity_mse / reuse_velocity_mse.clamp_min(1e-12)), + "predictor_time_s": predictor_time, + "reuse_head_time_s": reuse_time, + } + for name, value in values.items(): + metrics[group][name].append(value) + metrics["all"][name].append(value) + + payload = { + "weights": str(Path(args.weights).resolve()), + "manifest": str(Path(args.manifest).resolve()), + "trainable_parameters": model.trainable_parameter_count(), + "groups": {}, + } + for group, group_metrics in metrics.items(): + payload["groups"][group] = {} + for name, values in group_metrics.items(): + summary = summarize(values) + if name.endswith("_ratio"): + summary["fraction_below_1"] = sum(value < 1.0 for value in values) / len(values) + payload["groups"][group][name] = summary + output_path = Path(args.output).resolve() + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps(payload, indent=2, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/eval_predictor_rollout.py b/eval_predictor_rollout.py new file mode 100644 index 0000000000000000000000000000000000000000..108f3726602686188cfcb59a46e6bdcf0fa6a5d9 --- /dev/null +++ b/eval_predictor_rollout.py @@ -0,0 +1,276 @@ +#!/usr/bin/env python3 +"""Online Full-DiT vs direct-reuse vs F-P-F-P Predictor evaluation.""" + +from __future__ import annotations + +import argparse +import csv +import gc +import json +import os +import time +from collections import defaultdict +from pathlib import Path +from types import SimpleNamespace + +import torch +import torch.nn.functional as F +from safetensors.torch import load_file, save_file + +from hyvideo.commons.infer_state import initialize_infer_state +from hyvideo.generate import pose_to_input +from hyvideo.pipelines.worldplay_video_pipeline import HunyuanVideo_1_5_Pipeline +from models import HYWorldPlayPredictor +from predictor_training.checkpoint import load_predictor_weights +from tools.build_predictor_dataset import configure_exact_teacher, parse_case_ids + + +DEFAULT_POSE = "w-7,s-8,a-8,d-8,left-8,right-8,up-8,down-8" + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--case_csv", default="assets/test_case.csv") + parser.add_argument("--case_ids", default="1-10") + parser.add_argument("--dataset_dir", default="datasets/predictor_v1") + parser.add_argument("--weights", required=True) + parser.add_argument("--output_dir", default="outputs/predictor_v1/online") + parser.add_argument("--modes", default="full,reuse,predictor") + parser.add_argument("--base_model", default="models-ms/HunyuanVideo-1.5") + parser.add_argument( + "--action_ckpt", + default="models-ms/HY-WorldPlay/ar_distilled_action_model/diffusion_pytorch_model.safetensors", + ) + parser.add_argument("--seed", type=int, default=0) + parser.add_argument("--height", type=int, default=480) + parser.add_argument("--width", type=int, default=832) + parser.add_argument("--video_frames", type=int, default=253) + parser.add_argument("--pose", default=DEFAULT_POSE) + parser.add_argument("--save_latents", action=argparse.BooleanOptionalAction, default=True) + return parser.parse_args() + + +def load_cases(csv_path: Path) -> dict[int, dict[str, str]]: + with csv_path.open("r", encoding="utf-8", newline="") as handle: + rows = list(csv.DictReader(handle)) + result = {} + for case_id, row in enumerate(rows, start=1): + image_path = (csv_path.parent.parent / row["image_name"]).resolve() + if not image_path.is_file(): + image_path = Path(row["image_name"]).resolve() + if not image_path.is_file(): + raise FileNotFoundError(f"Missing image for case {case_id}: {row['image_name']}") + result[case_id] = {"image_path": str(image_path), "caption": row["caption"]} + return result + + +def init_state() -> None: + initialize_infer_state( + SimpleNamespace( + sage_blocks_range="0-53", + use_sageattn=False, + enable_torch_compile=False, + use_fp8_gemm=False, + quant_type="fp8-per-block", + include_patterns="double_blocks", + use_vae_parallel=False, + ) + ) + + +def aggregate_timing(records: list[dict]) -> dict[str, float]: + totals: dict[str, float] = defaultdict(float) + counts: dict[str, int] = defaultdict(int) + for record in records: + name = str(record["name"]) + totals[name] += float(record["elapsed_s"]) + counts[name] += 1 + result = {} + for name in sorted(totals): + result[f"{name}_total_s"] = totals[name] + result[f"{name}_count"] = counts[name] + result[f"{name}_mean_s"] = totals[name] / counts[name] + return result + + +def latent_metrics(candidate: torch.Tensor, baseline: torch.Tensor) -> dict[str, float]: + candidate = candidate.float() + baseline = baseline.float() + mse = F.mse_loss(candidate, baseline) + baseline_energy = baseline.square().mean().clamp_min(1e-12) + return { + "latent_mse_vs_full": float(mse), + "latent_nrmse_vs_full": float(torch.sqrt(mse / baseline_energy)), + "latent_mae_vs_full": float(F.l1_loss(candidate, baseline)), + "latent_cosine_vs_full": float( + F.cosine_similarity(candidate.flatten(1), baseline.flatten(1)).mean() + ), + } + + +def write_json_atomic(path: Path, payload: dict) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_suffix(path.suffix + ".tmp") + tmp.write_text(json.dumps(payload, indent=2, ensure_ascii=False, sort_keys=True) + "\n") + os.replace(tmp, path) + + +def summarize(runs: list[dict]) -> dict[str, dict[str, float]]: + by_mode: dict[str, dict[str, list[float]]] = defaultdict(lambda: defaultdict(list)) + for run in runs: + mode = run["mode"] + for name, value in run.items(): + if name not in {"case_id", "mode"} and isinstance(value, (int, float)): + by_mode[mode][name].append(float(value)) + for name, value in run["timing"].items(): + if name.endswith("_s"): + by_mode[mode][f"timing.{name}"].append(float(value)) + return { + mode: { + name: sum(values) / len(values) + for name, values in sorted(metrics.items()) + } + for mode, metrics in by_mode.items() + } + + +def main() -> None: + args = parse_args() + modes = [item.strip() for item in args.modes.split(",") if item.strip()] + if any(mode not in {"full", "reuse", "predictor"} for mode in modes): + raise ValueError(f"Unsupported modes: {modes}") + if not modes or modes[0] != "full": + raise ValueError("Modes must start with full so alternatives use the same-run baseline") + if args.video_frames != 13 and args.video_frames != 253: + raise ValueError("Online evaluator currently supports the 13-frame smoke or 253-frame run") + + os.environ["HY_PROFILE_TIMING"] = "1" + init_state() + device = torch.device("cuda:0") + dtype = torch.bfloat16 + case_ids = parse_case_ids(args.case_ids) + cases = load_cases(Path(args.case_csv).resolve()) + latent_frames = (args.video_frames - 1) // 4 + 1 + viewmats, Ks, action = pose_to_input(args.pose, latent_frames) + + pipe = HunyuanVideo_1_5_Pipeline.create_pipeline( + pretrained_model_name_or_path=str(Path(args.base_model).resolve()), + transformer_version="480p_i2v", + enable_offloading=True, + enable_group_offloading=False, + create_sr_pipeline=False, + force_sparse_attn=False, + transformer_dtype=dtype, + action_ckpt=str(Path(args.action_ckpt).resolve()), + ) + configure_exact_teacher(pipe.transformer) + pipe.transformer.eval().requires_grad_(False) + + predictor = None + if "predictor" in modes: + predictor = HYWorldPlayPredictor() + predictor.load_teacher_initialization(args.action_ckpt) + load_predictor_weights(predictor, args.weights) + predictor.to(device="cpu", dtype=dtype).eval() + + output_dir = Path(args.output_dir).resolve() + latent_dir = output_dir / "latents" + output_path = output_dir / "metrics.json" + dataset_dir = Path(args.dataset_dir).resolve() + runs: list[dict] = [] + + for case_id in case_ids: + case = cases[case_id] + case_tensors = load_file( + str(dataset_dir / "cases" / f"case_{case_id:02d}.safetensors"), + device="cpu", + ) + baseline = None + for mode in modes: + runtime = None + if predictor is not None: + predictor.to(device=device if mode == "predictor" else "cpu") + if mode == "predictor": + runtime = { + "model": predictor, + "current_txt": case_tensors["current_txt"].to(device=device, dtype=dtype), + "cached_txt": case_tensors["cached_txt"].to(device=device, dtype=dtype), + "vec_txt": case_tensors["vec_txt"].to(device=device, dtype=dtype), + } + torch.manual_seed(args.seed) + torch.cuda.manual_seed_all(args.seed) + gc.collect() + torch.cuda.empty_cache() + torch.cuda.reset_peak_memory_stats(device) + torch.cuda.synchronize(device) + started = time.perf_counter() + output = pipe( + enable_sr=False, + prompt=case["caption"], + aspect_ratio="16:9", + num_inference_steps=4, + video_length=args.video_frames, + negative_prompt="", + seed=args.seed, + output_type="latent", + prompt_rewrite=False, + return_pre_sr_video=False, + reference_image=case["image_path"], + viewmats=viewmats.unsqueeze(0), + Ks=Ks.unsqueeze(0), + action=action.unsqueeze(0), + few_step=True, + chunk_latent_frames=4, + model_type="ar", + user_height=args.height, + user_width=args.width, + transformer_resident_ar_rollout=True, + predictor_mode=mode, + predictor_runtime=runtime if mode == "predictor" else None, + ) + torch.cuda.synchronize(device) + wall_time = time.perf_counter() - started + latent = output.videos.detach().to(device="cpu", dtype=torch.float32).contiguous() + run = { + "case_id": case_id, + "mode": mode, + "wall_time_s": wall_time, + "peak_memory_gib": torch.cuda.max_memory_allocated(device) / 1024**3, + "timing": aggregate_timing(pipe._timing_records), + } + if mode == "full": + baseline = latent + run.update( + { + "latent_mse_vs_full": 0.0, + "latent_nrmse_vs_full": 0.0, + "latent_mae_vs_full": 0.0, + "latent_cosine_vs_full": 1.0, + } + ) + else: + if baseline is None: + raise RuntimeError("Full baseline is unavailable") + run.update(latent_metrics(latent, baseline)) + runs.append(run) + + if args.save_latents: + latent_dir.mkdir(parents=True, exist_ok=True) + save_file( + {"latents": latent.to(torch.bfloat16)}, + str(latent_dir / f"case_{case_id:02d}_{mode}.safetensors"), + ) + payload = { + "config": vars(args), + "predictor_parameters": ( + predictor.trainable_parameter_count() if predictor is not None else 0 + ), + "runs": runs, + "summary_mean": summarize(runs), + } + write_json_atomic(output_path, payload) + print(json.dumps(run, indent=2, sort_keys=True), flush=True) + + +if __name__ == "__main__": + main() diff --git a/eval_predictor_v2_offline.py b/eval_predictor_v2_offline.py new file mode 100644 index 0000000000000000000000000000000000000000..8bc983ebd315eb92a4684b91939e9e1e5ad8f68b --- /dev/null +++ b/eval_predictor_v2_offline.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 +"""Offline Predictor-v2 versus direct-reuse evaluation on fresh saved targets.""" + +from __future__ import annotations + +import argparse +import json +import time +from collections import defaultdict +from pathlib import Path + +import torch +import torch.nn.functional as F +from torch.utils.data import DataLoader + +from hyvideo.commons.infer_state import initialize_infer_state +from models import HYWorldPlayPredictorV2, PredictorV2Config +from predictor_training.checkpoint import load_predictor_weights +from predictor_training.dataset_v2 import ( + PredictorV2PairDataset, + move_v2_pair_to_device, + predictor_v2_collate, +) +from train_predictor_v2 import parse_source_blocks + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--manifest", default="datasets/predictor_v2/train_eval_manifest.jsonl") + parser.add_argument("--source_blocks", type=parse_source_blocks, required=True) + parser.add_argument("--weights", required=True) + parser.add_argument( + "--teacher_checkpoint", + default="models-ms/HY-WorldPlay/ar_distilled_action_model/diffusion_pytorch_model.safetensors", + ) + parser.add_argument("--output", required=True) + parser.add_argument("--max_items", type=int, default=None) + parser.add_argument("--max_records", type=int, default=None) + return parser.parse_args() + + +def init_state() -> None: + class Args: + sage_blocks_range = "0-53" + use_sageattn = False + enable_torch_compile = False + use_fp8_gemm = False + quant_type = "fp8-per-block" + include_patterns = "double_blocks" + use_vae_parallel = False + + initialize_infer_state(Args()) + + +def summarize(values: list[float]) -> dict[str, float]: + tensor = torch.tensor(values, dtype=torch.float64) + return { + "count": len(values), + "mean": float(tensor.mean()), + "median": float(tensor.median()), + "p90": float(torch.quantile(tensor, 0.9)), + "min": float(tensor.min()), + "max": float(tensor.max()), + } + + +def main() -> None: + args = parse_args() + init_state() + device = torch.device("cuda:0") + dtype = torch.bfloat16 + dataset = PredictorV2PairDataset( + args.manifest, + source_block_ids=args.source_blocks, + max_records=args.max_records, + ) + loader = DataLoader(dataset, batch_size=1, shuffle=False, collate_fn=predictor_v2_collate) + model = HYWorldPlayPredictorV2(PredictorV2Config(source_block_ids=args.source_blocks)) + model.load_teacher_initialization(args.teacher_checkpoint) + load_predictor_weights(model, args.weights) + model.to(device=device, dtype=dtype).eval() + + metrics: dict[str, dict[str, list[float]]] = defaultdict(lambda: defaultdict(list)) + with torch.no_grad(): + for index, item in enumerate(loader): + if args.max_items is not None and index >= args.max_items: + break + inputs, target_hidden, target_velocity = move_v2_pair_to_device( + item, device=device, dtype=dtype + ) + torch.cuda.synchronize() + started = time.perf_counter() + with torch.autocast(device_type="cuda", dtype=dtype): + output = model(**inputs) + torch.cuda.synchronize() + predictor_time = time.perf_counter() - started + + with torch.autocast(device_type="cuda", dtype=dtype): + reuse_velocity = model.hidden_to_velocity( + inputs["anchor_hidden"], inputs["target_frame_condition"] + ) + group = item["pair_id"][0] + pred_hidden_mse = F.mse_loss(output["pred_hidden"].float(), target_hidden.float()) + pred_velocity_mse = F.mse_loss(output["pred_velocity"].float(), target_velocity.float()) + reuse_hidden_mse = F.mse_loss(inputs["anchor_hidden"].float(), target_hidden.float()) + reuse_velocity_mse = F.mse_loss(reuse_velocity.float(), target_velocity.float()) + values = { + "predictor_hidden_mse": float(pred_hidden_mse), + "predictor_velocity_mse": float(pred_velocity_mse), + "reuse_hidden_mse": float(reuse_hidden_mse), + "reuse_velocity_mse": float(reuse_velocity_mse), + "hidden_mse_ratio": float(pred_hidden_mse / reuse_hidden_mse.clamp_min(1e-12)), + "velocity_mse_ratio": float(pred_velocity_mse / reuse_velocity_mse.clamp_min(1e-12)), + "predictor_time_s": predictor_time, + } + for name, value in values.items(): + metrics[group][name].append(value) + metrics["all"][name].append(value) + del inputs, target_hidden, target_velocity, output, reuse_velocity, item + + payload = { + "weights": str(Path(args.weights).resolve()), + "manifest": str(Path(args.manifest).resolve()), + "source_blocks": list(args.source_blocks), + "trainable_parameters": model.trainable_parameter_count(), + "groups": {}, + } + for group, group_metrics in metrics.items(): + payload["groups"][group] = {} + for name, values in group_metrics.items(): + summary = summarize(values) + if name.endswith("_ratio"): + summary["fraction_below_1"] = sum(value < 1.0 for value in values) / len(values) + payload["groups"][group][name] = summary + output_path = Path(args.output).resolve() + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps(payload, indent=2, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/eval_predictor_v2_rollout.py b/eval_predictor_v2_rollout.py new file mode 100644 index 0000000000000000000000000000000000000000..8c9cb06234132d22276d590075035a4e258c1fce --- /dev/null +++ b/eval_predictor_v2_rollout.py @@ -0,0 +1,352 @@ +#!/usr/bin/env python3 +"""Online Full/Reuse/Predictor v2 or v3 evaluation with reference metrics.""" + +from __future__ import annotations + +import argparse +import gc +import json +import os +import time +from collections import defaultdict +from pathlib import Path +from types import SimpleNamespace + +import lpips +import torch + +from hyvideo.commons.infer_state import initialize_infer_state +from hyvideo.generate import pose_to_input, save_video +from hyvideo.pipelines.worldplay_video_pipeline import HunyuanVideo_1_5_Pipeline +from models import HYWorldPlayPredictorV2, HYWorldPlayPredictorV3, PredictorV2Config +from predictor_training.checkpoint import load_predictor_weights +from predictor_training.video_metrics import compute_video_metrics +from tools.build_predictor_dataset import ( + DEFAULT_POSE, + configure_exact_teacher, + load_cases, + parse_case_ids, +) + + +MODE_TO_BLOCKS = { + "predictor_0_53": (0, 53), + "predictor_1_52": (1, 52), +} + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--case_csv", default="assets/test_case.csv") + parser.add_argument("--case_ids", default="1-10") + parser.add_argument("--weights_0_53") + parser.add_argument("--weights_1_52") + parser.add_argument("--predictor_version", choices=("v2", "v3"), default="v2") + parser.add_argument("--output_dir", default="outputs/predictor_v2/online") + parser.add_argument("--modes", default="full,reuse,predictor_0_53,predictor_1_52") + parser.add_argument("--base_model", default="models-ms/HunyuanVideo-1.5") + parser.add_argument( + "--action_ckpt", + default="models-ms/HY-WorldPlay/ar_distilled_action_model/diffusion_pytorch_model.safetensors", + ) + parser.add_argument("--seed", type=int, default=0) + parser.add_argument("--height", type=int, default=480) + parser.add_argument("--width", type=int, default=832) + parser.add_argument("--video_frames", type=int, default=253) + parser.add_argument("--pose", default=DEFAULT_POSE) + parser.add_argument("--metric_batch_size", type=int, default=4) + return parser.parse_args() + + +def init_state() -> None: + initialize_infer_state( + SimpleNamespace( + sage_blocks_range="0-53", + use_sageattn=False, + enable_torch_compile=False, + use_fp8_gemm=False, + quant_type="fp8-per-block", + include_patterns="double_blocks", + use_vae_parallel=False, + ) + ) + + +def aggregate_timing(records: list[dict]) -> dict[str, float]: + totals: dict[str, float] = defaultdict(float) + counts: dict[str, int] = defaultdict(int) + for record in records: + name = str(record["name"]) + totals[name] += float(record["elapsed_s"]) + counts[name] += 1 + result = {} + for name in sorted(totals): + result[f"{name}_total_s"] = totals[name] + result[f"{name}_count"] = counts[name] + result[f"{name}_mean_s"] = totals[name] / counts[name] + return result + + +def write_json_atomic(path: Path, payload: dict) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_suffix(path.suffix + ".tmp") + tmp.write_text(json.dumps(payload, indent=2, ensure_ascii=False, sort_keys=True) + "\n") + os.replace(tmp, path) + + +def summarize(runs: list[dict]) -> dict[str, dict[str, float]]: + by_mode: dict[str, dict[str, list[float]]] = defaultdict(lambda: defaultdict(list)) + for run in runs: + mode = run["mode"] + for name, value in run.items(): + if name not in {"case_id", "mode", "timing", "video_path"} and isinstance(value, (int, float)): + by_mode[mode][name].append(float(value)) + for name, value in run["timing"].items(): + if name.endswith("_s"): + by_mode[mode][f"timing.{name}"].append(float(value)) + return { + mode: { + name: sum(values) / len(values) + for name, values in sorted(metrics.items()) + } + for mode, metrics in by_mode.items() + } + + +def identity_metrics() -> dict[str, float | None]: + return { + "psnr_db": None, + "frame_psnr_mean_db": None, + "ssim": 1.0, + "lpips_alex": 0.0, + "pixel_mse": 0.0, + "pixel_rmse": 0.0, + "pixel_mae": 0.0, + "temporal_delta_mse": 0.0, + } + + +def fit_pose_to_latent_frames(pose: str, latent_frames: int) -> str | dict: + """Return the prefix of a pose trajectory needed by a shorter rollout. + + A pose string describes camera *motions*, while ``pose_to_input`` consumes + the initial camera pose plus those motions. Consequently, N latent frames + require N-1 motion entries. Full 253-frame evaluation is unchanged; the + 29-frame v3 smoke test trims the default trajectory to its first 7 motions. + """ + if latent_frames < 1: + raise ValueError("latent_frames must be positive") + + if pose.endswith(".json"): + with Path(pose).open("r", encoding="utf-8") as handle: + pose_json = json.load(handle) + if len(pose_json) < latent_frames: + raise ValueError( + f"Pose JSON has {len(pose_json)} entries, but {latent_frames} are required" + ) + return dict(list(pose_json.items())[:latent_frames]) + + required_motions = latent_frames - 1 + remaining = required_motions + fitted_commands: list[str] = [] + available_motions = 0 + for raw_command in pose.split(","): + command = raw_command.strip() + if not command: + continue + parts = command.split("-") + if len(parts) != 2: + raise ValueError( + f"Invalid pose command: {command}. Expected 'action-duration'" + ) + action = parts[0].strip() + duration = int(float(parts[1].strip())) + if duration < 0: + raise ValueError(f"Pose duration must be non-negative: {command}") + available_motions += duration + if remaining > 0: + take = min(duration, remaining) + if take: + fitted_commands.append(f"{action}-{take}") + remaining -= take + + if remaining: + raise ValueError( + f"Pose has {available_motions + 1} latent entries, but {latent_frames} are required" + ) + return ",".join(fitted_commands) + + +def main() -> None: + args = parse_args() + modes = [item.strip() for item in args.modes.split(",") if item.strip()] + supported = {"full", "reuse", *MODE_TO_BLOCKS} + if any(mode not in supported for mode in modes): + raise ValueError(f"Unsupported modes: {modes}") + if not modes or modes[0] != "full": + raise ValueError("Modes must start with full") + latent_frames = (args.video_frames - 1) // 4 + 1 + if (args.video_frames - 1) % 4 or latent_frames % 4: + raise ValueError( + "video_frames must map to a latent-frame count divisible by four" + ) + + os.environ["HY_PROFILE_TIMING"] = "1" + init_state() + device = torch.device("cuda:0") + dtype = torch.bfloat16 + cases = load_cases(Path(args.case_csv).resolve()) + case_ids = parse_case_ids(args.case_ids) + fitted_pose = fit_pose_to_latent_frames(args.pose, latent_frames) + viewmats, Ks, action = pose_to_input(fitted_pose, latent_frames) + + pipe = HunyuanVideo_1_5_Pipeline.create_pipeline( + pretrained_model_name_or_path=str(Path(args.base_model).resolve()), + transformer_version="480p_i2v", + enable_offloading=True, + enable_group_offloading=False, + create_sr_pipeline=False, + force_sparse_attn=False, + transformer_dtype=dtype, + action_ckpt=str(Path(args.action_ckpt).resolve()), + ) + configure_exact_teacher(pipe.transformer) + pipe.transformer.eval().requires_grad_(False) + + weights = { + (0, 53): args.weights_0_53, + (1, 52): args.weights_1_52, + } + requested_blocks = { + MODE_TO_BLOCKS[mode] + for mode in modes + if mode in MODE_TO_BLOCKS + } + missing_weights = [ + source_blocks + for source_blocks in sorted(requested_blocks) + if not weights[source_blocks] + ] + if missing_weights: + raise ValueError(f"Missing Predictor weights for blocks: {missing_weights}") + model_class = ( + HYWorldPlayPredictorV3 + if args.predictor_version == "v3" + else HYWorldPlayPredictorV2 + ) + predictors: dict[tuple[int, int], HYWorldPlayPredictorV2] = {} + for source_blocks in sorted(requested_blocks): + weights_path = weights[source_blocks] + model = model_class(PredictorV2Config(source_block_ids=source_blocks)) + model.load_teacher_initialization(args.action_ckpt) + load_predictor_weights(model, weights_path) + predictors[source_blocks] = model.to(device="cpu", dtype=dtype).eval() + + lpips_model = lpips.LPIPS(net="alex", verbose=False).to(device=device).eval() + output_dir = Path(args.output_dir).resolve() + video_dir = output_dir / "videos" + output_path = output_dir / "metrics.json" + runs: list[dict] = [] + + for case_id in case_ids: + case = cases[case_id] + baseline: torch.Tensor | None = None + for mode in modes: + active_predictor = None + predictor_runtime = None + if mode in MODE_TO_BLOCKS: + source_blocks = MODE_TO_BLOCKS[mode] + active_predictor = predictors[source_blocks].to(device=device) + predictor_runtime = {"model": active_predictor} + + torch.manual_seed(args.seed) + torch.cuda.manual_seed_all(args.seed) + gc.collect() + torch.cuda.empty_cache() + torch.cuda.reset_peak_memory_stats(device) + torch.cuda.synchronize(device) + started = time.perf_counter() + output = pipe( + enable_sr=False, + prompt=case["caption"], + aspect_ratio="16:9", + num_inference_steps=4, + video_length=args.video_frames, + negative_prompt="", + seed=args.seed, + output_type="pt", + prompt_rewrite=False, + return_pre_sr_video=False, + reference_image=case["image_path"], + viewmats=viewmats.unsqueeze(0), + Ks=Ks.unsqueeze(0), + action=action.unsqueeze(0), + few_step=True, + chunk_latent_frames=4, + model_type="ar", + user_height=args.height, + user_width=args.width, + transformer_resident_ar_rollout=True, + predictor_mode="predictor" if mode in MODE_TO_BLOCKS else mode, + predictor_runtime=predictor_runtime, + ) + torch.cuda.synchronize(device) + wall_time = time.perf_counter() - started + peak_memory = torch.cuda.max_memory_allocated(device) / 1024**3 + video = output.videos.detach().to(device="cpu", dtype=torch.float32).contiguous() + if active_predictor is not None: + active_predictor.to(device="cpu") + torch.cuda.empty_cache() + + video_dir.mkdir(parents=True, exist_ok=True) + video_path = video_dir / f"case_{case_id:02d}_{mode}.mp4" + save_started = time.perf_counter() + save_video(video, str(video_path)) + save_time = time.perf_counter() - save_started + + metric_started = time.perf_counter() + if mode == "full": + baseline = video + quality = identity_metrics() + else: + if baseline is None: + raise RuntimeError("Full baseline is unavailable") + quality = compute_video_metrics( + video, + baseline, + lpips_model=lpips_model, + device=device, + batch_size=args.metric_batch_size, + ) + metric_time = time.perf_counter() - metric_started + run = { + "case_id": case_id, + "mode": mode, + "wall_time_s": wall_time, + "video_save_time_s": save_time, + "metric_time_s": metric_time, + "peak_memory_gib": peak_memory, + "video_path": str(video_path), + "timing": aggregate_timing(pipe._timing_records), + **quality, + } + runs.append(run) + payload = { + "config": vars(args), + "weights": { + mode: str(Path(weights[MODE_TO_BLOCKS[mode]]).resolve()) + for mode in modes + if mode in MODE_TO_BLOCKS + }, + "runs": runs, + "summary_mean": summarize(runs), + } + write_json_atomic(output_path, payload) + print(json.dumps(run, indent=2, sort_keys=True), flush=True) + if mode != "full": + del video + gc.collect() + + +if __name__ == "__main__": + main() diff --git a/evaluation_pipelines/validation25_offline_ffff_targets_step2000/launcher.log b/evaluation_pipelines/validation25_offline_ffff_targets_step2000/launcher.log new file mode 100644 index 0000000000000000000000000000000000000000..dc7a1955f5ae1a12fd4df1fe0cb91d57f1ae31c4 --- /dev/null +++ b/evaluation_pipelines/validation25_offline_ffff_targets_step2000/launcher.log @@ -0,0 +1,30 @@ +Validation evaluation complete: /mnt/local_nvme/zoubin/cz/hyworldplay_predictor_vbench_val25_test50/evaluations/eval_validation25_detached-chunk-timestep-rollout-offline-ffff-targets-fppp-fulltrain100_schedule-fppf_blocks1-52_ckptstep2000_flr5e-5_blr5e-6_schedsteps2000_swarmup100_bwarmup100linear_bs1 +Durable copy: s3://s3-us-west2-default/zoubin/cz/projects/HY-WorldPlay-DEV-Predictor/datasets/hyworldplay_predictor_vbench_val25_test50/evaluations/eval_validation25_detached-chunk-timestep-rollout-offline-ffff-targets-fppp-fulltrain100_schedule-fppf_blocks1-52_ckptstep2000_flr5e-5_blr5e-6_schedsteps2000_swarmup100_bwarmup100linear_bs1 +Validation evaluation complete: /mnt/local_nvme/zoubin/cz/hyworldplay_predictor_vbench_val25_test50/evaluations/eval_validation25_detached-chunk-timestep-rollout-offline-ffff-targets-fppp-fulltrain100_schedule-fppp_blocks1-52_ckptstep2000_flr5e-5_blr5e-6_schedsteps2000_swarmup100_bwarmup100linear_bs1 +Durable copy: s3://s3-us-west2-default/zoubin/cz/projects/HY-WorldPlay-DEV-Predictor/datasets/hyworldplay_predictor_vbench_val25_test50/evaluations/eval_validation25_detached-chunk-timestep-rollout-offline-ffff-targets-fppp-fulltrain100_schedule-fppp_blocks1-52_ckptstep2000_flr5e-5_blr5e-6_schedsteps2000_swarmup100_bwarmup100linear_bs1 +Wrote 200 prompts to /mnt/local_nvme/zoubin/cz/hyworldplay_predictor_vbench_val25_test50/vbench/validation25_offline_ffff_targets_step2000/validation25_prompt_map.json +[vbench] subject_consistency start 2026-07-25T11:51:20Z +subject_consistency 200 0.9138530379828187 +[vbench] subject_consistency complete 2026-07-25T11:52:41Z +[vbench] background_consistency start 2026-07-25T11:52:41Z +background_consistency 200 0.927283202140562 +[vbench] background_consistency complete 2026-07-25T11:53:49Z +[vbench] temporal_flickering start 2026-07-25T11:53:49Z +temporal_flickering 200 0.965473731071341 +[vbench] temporal_flickering complete 2026-07-25T11:54:36Z +[vbench] motion_smoothness start 2026-07-25T11:54:36Z +motion_smoothness 200 0.9876062893505951 +[vbench] motion_smoothness complete 2026-07-25T11:56:05Z +[vbench] dynamic_degree start 2026-07-25T11:56:06Z +dynamic_degree 200 0.8 +[vbench] dynamic_degree complete 2026-07-25T11:58:11Z +[vbench] aesthetic_quality start 2026-07-25T11:58:11Z +aesthetic_quality 200 0.5568837748467922 +[vbench] aesthetic_quality complete 2026-07-25T11:59:21Z +[vbench] imaging_quality start 2026-07-25T11:59:21Z +imaging_quality 200 0.652922448000336 +[vbench] imaging_quality complete 2026-07-25T12:01:17Z +[vbench] overall_consistency start 2026-07-25T12:01:17Z +overall_consistency 200 0.2455315287411213 +[vbench] overall_consistency complete 2026-07-25T12:02:08Z +[validation] all standard and VBench metrics complete: /mnt/local_nvme/zoubin/cz/hyworldplay_predictor_vbench_val25_test50/vbench/validation25_offline_ffff_targets_step2000 diff --git a/evaluation_pipelines/validation25_offline_ffff_targets_step2000/pipeline.pid b/evaluation_pipelines/validation25_offline_ffff_targets_step2000/pipeline.pid new file mode 100644 index 0000000000000000000000000000000000000000..4e5f4e89b8546e50e2ef596e751d940bb57202e1 --- /dev/null +++ b/evaluation_pipelines/validation25_offline_ffff_targets_step2000/pipeline.pid @@ -0,0 +1 @@ +2497648 diff --git a/evaluation_pipelines/validation25_stage1_continued_offline_tf_step2000/launcher.log b/evaluation_pipelines/validation25_stage1_continued_offline_tf_step2000/launcher.log new file mode 100644 index 0000000000000000000000000000000000000000..ce01637608a9774f2b74b89198ea5c623d17b9e4 --- /dev/null +++ b/evaluation_pipelines/validation25_stage1_continued_offline_tf_step2000/launcher.log @@ -0,0 +1,30 @@ +Validation evaluation complete: /mnt/local_nvme/zoubin/cz/hyworldplay_predictor_vbench_val25_test50/evaluations/eval_validation25_stage1-continued-offline-tf_schedule-fppf_blocks1-52_ckptstep2000-initstage1step2000_flr5e-5_blr5e-6_schedsteps2000_swarmup100_bwarmup100linear_bs16 +Durable copy: s3://s3-us-west2-default/zoubin/cz/projects/HY-WorldPlay-DEV-Predictor/datasets/hyworldplay_predictor_vbench_val25_test50/evaluations/eval_validation25_stage1-continued-offline-tf_schedule-fppf_blocks1-52_ckptstep2000-initstage1step2000_flr5e-5_blr5e-6_schedsteps2000_swarmup100_bwarmup100linear_bs16 +Validation evaluation complete: /mnt/local_nvme/zoubin/cz/hyworldplay_predictor_vbench_val25_test50/evaluations/eval_validation25_stage1-continued-offline-tf_schedule-fppp_blocks1-52_ckptstep2000-initstage1step2000_flr5e-5_blr5e-6_schedsteps2000_swarmup100_bwarmup100linear_bs16 +Durable copy: s3://s3-us-west2-default/zoubin/cz/projects/HY-WorldPlay-DEV-Predictor/datasets/hyworldplay_predictor_vbench_val25_test50/evaluations/eval_validation25_stage1-continued-offline-tf_schedule-fppp_blocks1-52_ckptstep2000-initstage1step2000_flr5e-5_blr5e-6_schedsteps2000_swarmup100_bwarmup100linear_bs16 +Wrote 200 prompts to /mnt/local_nvme/zoubin/cz/hyworldplay_predictor_vbench_val25_test50/vbench/validation25_stage1_continued_offline_tf_step2000/validation25_prompt_map.json +[vbench] subject_consistency start 2026-07-24T15:14:03Z +subject_consistency 200 0.9374588255506126 +[vbench] subject_consistency complete 2026-07-24T15:15:22Z +[vbench] background_consistency start 2026-07-24T15:15:22Z +background_consistency 200 0.9374834909746724 +[vbench] background_consistency complete 2026-07-24T15:16:27Z +[vbench] temporal_flickering start 2026-07-24T15:16:27Z +temporal_flickering 200 0.9627548102602064 +[vbench] temporal_flickering complete 2026-07-24T15:17:09Z +[vbench] motion_smoothness start 2026-07-24T15:17:10Z +motion_smoothness 200 0.988383442837023 +[vbench] motion_smoothness complete 2026-07-24T15:18:35Z +[vbench] dynamic_degree start 2026-07-24T15:18:35Z +dynamic_degree 200 0.805 +[vbench] dynamic_degree complete 2026-07-24T15:20:39Z +[vbench] aesthetic_quality start 2026-07-24T15:20:39Z +aesthetic_quality 200 0.5817374125123024 +[vbench] aesthetic_quality complete 2026-07-24T15:21:45Z +[vbench] imaging_quality start 2026-07-24T15:21:45Z +imaging_quality 200 0.7113044539459229 +[vbench] imaging_quality complete 2026-07-24T15:23:33Z +[vbench] overall_consistency start 2026-07-24T15:23:33Z +overall_consistency 200 0.2481275948137045 +[vbench] overall_consistency complete 2026-07-24T15:24:17Z +[validation] all standard and VBench metrics complete: /mnt/local_nvme/zoubin/cz/hyworldplay_predictor_vbench_val25_test50/vbench/validation25_stage1_continued_offline_tf_step2000 diff --git a/evaluation_pipelines/validation25_stage1_continued_offline_tf_step2000/pipeline.pid b/evaluation_pipelines/validation25_stage1_continued_offline_tf_step2000/pipeline.pid new file mode 100644 index 0000000000000000000000000000000000000000..3689b9e665c87566104324d74f74268142c75f67 --- /dev/null +++ b/evaluation_pipelines/validation25_stage1_continued_offline_tf_step2000/pipeline.pid @@ -0,0 +1 @@ +1334327 diff --git a/experiment.md b/experiment.md new file mode 100644 index 0000000000000000000000000000000000000000..ba45ba92977aeeb0f8b45e93adc2404a767c78d0 --- /dev/null +++ b/experiment.md @@ -0,0 +1,1166 @@ +# Experiment + +## 1. 范围与当前状态 + +本文只汇总当前目录中能够由 checkpoint、日志、视频和 `metrics.json` 直接核验的实验。旧机器上的 +v1/v2/v3 实验不再纳入报告。 + +checkpoint、日志、SwanLab experiment 与评测输出采用同一套显式超参数命名；字段含义和当前配置 +索引见 [train.md](train.md#实验命名规范)。 + +| 实验 | 数据规模 | 状态 | +| --- | ---: | --- | +| v4 blocks `[0,53]` 训练 | 8400 transitions | 完成 2000 steps | +| v4 blocks `[1,52]` BF16 master 训练 | 8400 transitions | 完成 2000 steps | +| v4 blocks `[1,52]` FP32 master 训练 | 8400 transitions | 完成 2000 steps | +| predictor-disca blocks `[0,53]` 训练 | 8400 transitions | 完成 2000 steps | +| 训练集 Full/Reuse/v4/disca 视频评测 | 100 pairs × 4 actions | 完成 | +| 留出集 Full/Reuse 基线 | 75 pairs × 4 actions | 完成 | +| validation v4/DisCa Predictor 评测 | 25 pairs × 4 actions × 4 checkpoints | 完成 | +| Stage-1 previous-condition 输入消融 | 8400 transitions × 3 modes；validation25 | Torch 2.6 无 FlashAttention 重训；8 组 reference 与 VBench 评测全部完成 | +| test v4/DisCa Predictor 评测 | — | 尚未运行 | +| detached timestep on-policy，完整训练集 | 100 pairs × 4 actions | 保存至 step 1000；validation25 FPPF/FPPP 已评测 | +| detached chunk+timestep on-policy FPPP，完整训练集 | 100 pairs × 4 actions | 从 step 1000 恢复并完成 2000 steps；step 1000/2000 validation25 FPPF/FPPP 均已评测 | +| Stage-1 continued offline teacher-forcing，完整训练集 | 8400 transitions | 从 Stage-1 step 2000 模型参数继续训练 2000 steps；validation25 FPPF/FPPP 已评测 | +| detached chunk+timestep rollout + offline FFFF targets，完整训练集 | 100 pairs × 4 actions | 完成 2000 steps；validation25 FPPF/FPPP 已评测 | +| C0/C1/C2 confidence head 离线训练 | train 8400 + validation 2100 transitions | 完成；独立 validation25，无 validation tuning | +| Stage-1 C1 threshold 动态 fallback | validation25 × 4 thresholds × 100 videos | reference 8 项与 VBench 8 项均已完成 | +| Torch 2.6 固定环境复评 | validation25 × 7 个 Confidence 组 × 100 videos | 完成；reference 8 项已汇总并同步 S3 | +| VBench 5 项 | validation25，14 组 × 100 videos | 完成 | +| ATC Predictor 五组架构消融训练 | 8400 transitions × 5 variants | 全部完成 2000 steps | +| ATC Predictor FPPF/FPPP 评测 | validation25 × 5 variants × 2 schedules × 100 videos | reference 8 项与 VBench 8 项全部完成 | +| ATC（Causal-Forcing 对齐版全局 Transport）Stage-1 重训 + on-policy rollout | 8400 transitions；100 pairs × 4 actions | 4 卡各 2000 steps 完成；rollout step 500/800/1000/1500/2000 的 validation25 FPPF/FPPP 与 VBench 8 项全部完成（第 12 节） | + +当前已完成四个 Stage-1 Predictor checkpoint 在 25-pair validation split 上的视频评测；完整训练集 +detached timestep on-policy 的 step 1000，以及 detached chunk+timestep on-policy 的 step 1000/2000， +以及 Stage-1 continued offline teacher-forcing 的 step 2000，均已完成 validation25 FPPF/FPPP +评测；detached chunk+timestep rollout + offline FFFF targets 的 step 2000 也已完成相同协议。 +14 组固定 schedule 视频均已完成并汇总 VBench 5 项。Stage-1 C1 confidence 另外完成四个训练集 +分位阈值的 validation25 动态 fallback，共生成 400 个视频，reference 8 项与完整 VBench 8 项均已 +汇总；previous-condition 的 8 组共 800 个视频也已完成完整 VBench 8 项评测；50-pair test split +的 Predictor 评测尚未运行。 + +2026-08-21～23 另外完成 Anchor–Transport–Correct（ATC）Predictor 的五组完整架构消融。五个 +checkpoint 均训练 2000 steps，并在固定 Torch 2.6 环境下分别生成 FPPF/FPPP 各 100 个视频； +10 组视频的 Full-reference 8 项指标与 VBench 8 项指标均已完成，详见第 11 节。 + +2026-08-21 环境审计确认，2026-08-14 以后新增的一批实验曾使用 Torch `2.5.1+cu121` 和 +FlashAttention 2，而 7 月正式基线使用 Torch `2.6.0+cu124` 且没有安装 FlashAttention。固定环境 +`/mnt/local_nvme/cz/envs/hy_worldplay_july20260720` 重跑 Stage-1 FPPF 后，与 7 月原始 100 个视频 +达到 `100/100` SHA256 完全一致；Torch 2.5 环境重跑则为 `0/100` 一致。2026-08-23 已删除 +previous-condition 消融中 `none/previous_step/both` 的错误 checkpoint、日志、生成视频和指标， +并在正确环境中从零重训。三个新 checkpoint 均已完成 2000 steps，并完成同环境 validation25 +FPPF reference 8 项评测。2026-08-24 进一步发现原始 `none` 与其余版本的首 chunk 调度不同，随后 +补跑 `none` step 300/500/1000/2000 的首 chunk FFFF 公平对照；正式结果和修正后的解释见第 6.4 节。 +Confidence 历史实验另见第 9～10 节。 + +## 2. 统一设置 + +训练数据使用 100 个图文 pair，每个 pair 运行四种双向 action： + +```text +(w, s), (a, d), (up, down), (left, right) +``` + +每个视频为 8 chunks、125 frames、480×832。每个 chunk 保存四步 Full-DiT Teacher 结果，训练全部 +相邻转换 `0→1`、`1→2`、`2→3`，共 `2800 records × 3 = 8400 pairs`。 + +主线训练集与 validation 评测以 Full-DiT 为 reference： + +```text +Full-DiT: F-F-F-F +Direct Reuse: F-R-R-F +Predictor: F-P-P-F +``` + +即 step 0、3 由 Full-DiT 计算，step 1、2 使用 Direct Reuse 或 Predictor。指标包括 full-resolution +PSNR、逐帧 PSNR 均值、SSIM、AlexNet LPIPS、pixel MSE/RMSE/MAE 和 temporal-delta MSE。 +第 7 节的 rollout 实验另外评测 F-P-P-P，即只保留 step 0 Full-DiT，steps 1、2、3 均由 +Predictor 替代。 + +## 3. 训练结果 + +四版初始 Stage-1 训练与 continued offline teacher-forcing 均使用 8 卡、每卡 batch size 16、 +全局 batch size 128，每阶段训练 2000 optimizer steps；checkpoint 每 100 steps 保存一次。 +continued 版本只加载 Stage-1 step 2000 的模型参数，重新初始化 optimizer、scheduler 和局部 step， +因此表中的 continued step 2000 相当于累计训练 4000 steps。表中 loss 和耗时是各自 step 2000 +日志窗口均值，不是独立 validation set 的全量 loss。 + +| 版本 | blocks | total loss | hidden MSE | velocity MSE | 峰值显存/GPU | micro-step 均时 | +| --- | --- | ---: | ---: | ---: | ---: | ---: | +| v4 | `[0,53]` | 0.011895 | 0.011744 | 0.010721 | 62.82 GiB | 4.47 s | +| v4 | `[1,52]` | 0.011537 | 0.011732 | 0.010364 | 62.82 GiB | 4.49 s | +| v4 FP32 master | `[1,52]` | 0.010910 | 0.011501 | 0.009760 | 64.43 GiB | 4.50 s | +| v4 FP32 master + offline TF continuation | `[1,52]` | **0.009355** | **0.009983** | **0.008357** | 64.43 GiB | 4.91 s | +| predictor-disca | `[0,53]` | 0.011053 | 0.011121 | 0.009941 | **31.18 GiB** | **1.66 s** | + +FP32 master 版避免了 BF16 参数更新被量化，step-2000 日志窗口中的 total/velocity loss 低于旧 +BF16 `[1,52]`；在四版初始 Stage-1 中，predictor-disca 的 hidden loss 最低且训练 micro-step +明显更快，但下面的连续视频 rollout 结果低于 v4。continued offline teacher-forcing 的单步 loss +进一步降低，但其 FPPP 感知指标仍未超过 on-policy rollout；这再次说明 Teacher-forcing 单步 loss +不能代替在线连续替代的视频质量评测。 + +## 4. 训练集视频质量 + +评测覆盖 100 个训练 pair × 4 actions，共 400 个视频/方法。箭头表示指标优劣方向。 + +| 方法 | blocks | PSNR ↑ | frame PSNR ↑ | SSIM ↑ | LPIPS ↓ | MSE ↓ | RMSE ↓ | MAE ↓ | temporal MSE ↓ | +| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| Direct Reuse | — | 18.4639 | 19.6970 | 0.6429 | 0.2083 | 0.01891 | 0.12964 | 0.07145 | 0.01212 | +| v4 | `[0,53]` | 21.9337 | 24.4274 | 0.7606 | 0.1204 | 0.00918 | 0.08881 | 0.04341 | 0.00865 | +| v4 | `[1,52]` | **22.1090** | **24.5848** | **0.7670** | **0.1168** | **0.00876** | **0.08677** | **0.04220** | **0.00832** | +| predictor-disca | `[0,53]` | 19.4084 | 20.6095 | 0.6748 | 0.1816 | 0.01513 | 0.11606 | 0.06208 | 0.01061 | + +主要结论： + +- v4 `[1,52]` 在所有已记录的视频质量指标上都是当前最优 Predictor。相对 v4 `[0,53]`，PSNR + 提高 0.1754 dB，SSIM 提高 0.0064，LPIPS 降低 3.0%，temporal MSE 降低 3.7%。 +- v4 `[0,53]` 相对 Direct Reuse 的 PSNR 提高 3.4698 dB、SSIM 提高 0.1177，LPIPS 降低 + 42.2%，pixel MSE 降低 51.4%，temporal MSE 降低 28.7%。 +- 在相同 blocks `[0,53]` 下，v4 相对 predictor-disca 的 PSNR 提高 2.5253 dB、SSIM 提高 + 0.0858，LPIPS 降低 33.7%，pixel MSE 降低 39.3%，temporal MSE 降低 18.5%。这支持 + previous-chunk hidden 与 Context Vision K/V 对连续 WorldPlay rollout 有明显贡献。 +- predictor-disca 仍优于 Direct Reuse：PSNR 提高 0.9445 dB，LPIPS 降低 12.8%，pixel MSE + 降低 20.0%。即便不注入上述 AR Context，双 block Predictor 仍学到了有效修正。 + +## 5. 训练集生成耗时 + +以下为各次评测日志中的端到端单视频 wall time。不同 Predictor 是分次启动，受机器负载、模型 +offload 和缓存状态影响，因此不能把跨次运行的几秒差异直接解释成架构速度差异。 + +| 方法/运行 | 视频数 | mean | median | P90 | +| --- | ---: | ---: | ---: | ---: | +| Direct Reuse，与 v4 `[0,53]` 同次运行 | 400 | 88.60 s | 84.25 s | 111.37 s | +| v4 `[0,53]` | 400 | 89.84 s | 85.79 s | 111.34 s | +| v4 `[1,52]` | 400 | 85.04 s | 79.92 s | 102.55 s | +| predictor-disca `[0,53]` | 400 | 90.10 s | 89.79 s | 106.56 s | + +同次运行的 v4 `[0,53]` 比 Direct Reuse 平均多 1.25 秒/视频，约 1.4%。端到端时间还包含文本编码、 +history K/V、VAE 和视频编码，所以它不是 Predictor block 的纯 forward benchmark。 + +## 6. 留出集 Full/Reuse 基线 + +留出集从 VBench 中随机选择且与训练集互斥：25 个 validation pair 和 50 个 test pair，两者也互斥。 +每个 pair 运行四种 action，共生成 300 个 Full 视频和 300 个 Direct Reuse 视频。 + +### 6.1 视频质量 + +Full/Reuse 基线覆盖 validation 和 test；四个现存 Predictor checkpoint 当前均补齐了 validation。 + +| split | 视频数 | PSNR ↑ | frame PSNR ↑ | SSIM ↑ | LPIPS ↓ | MSE ↓ | RMSE ↓ | MAE ↓ | temporal MSE ↓ | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| validation | 100 | 17.9333 | 19.0671 | 0.6107 | 0.2154 | 0.02030 | 0.13614 | 0.07513 | 0.01347 | +| test | 200 | 18.0988 | 19.2316 | 0.5934 | 0.2042 | 0.02079 | 0.13522 | 0.07543 | 0.01495 | +| overall | 300 | 18.0436 | 19.1767 | 0.5991 | 0.2080 | 0.02063 | 0.13552 | 0.07533 | 0.01446 | + +### 6.2 组件耗时 + +| 模式 | 视频数 | 端到端 mean | 端到端 median | P90 | pipeline | AR rollout | history K/V | step transformer | VAE decode | 峰值显存 | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| Full | 300 | 101.11 s | 100.05 s | 113.31 s | 98.89 s | 64.90 s | 19.06 s | 27.24 s | 10.97 s | 71.06 GiB | +| Reuse | 300 | 87.41 s | 86.89 s | 100.17 s | 85.19 s | 51.23 s | 19.09 s | 13.65 s | 10.82 s | 71.11 GiB | + +Direct Reuse 将端到端平均耗时降低 13.55%，AR rollout 降低 21.06%，其中 step transformer +耗时降低 49.90%。峰值显存基本不变，因为 Full Teacher、history cache 和其他 pipeline 组件仍然驻留。 + +### 6.3 validation Predictor 对比 + +四个 checkpoint 均在线替代 steps 1、2，以同 seed 的 Full-DiT validation 视频为 reference。每版 +评测 25 pairs × 4 actions = 100 个视频。 + +| 方法 | PSNR ↑ | frame PSNR ↑ | SSIM ↑ | LPIPS ↓ | MSE ↓ | RMSE ↓ | MAE ↓ | temporal MSE ↓ | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| Direct Reuse | 17.9333 | 19.0671 | 0.6107 | 0.2154 | 0.02030 | 0.13614 | 0.07513 | 0.01347 | +| predictor-disca BF16 `[0,53]` | 18.9360 | 20.0334 | 0.6463 | 0.1863 | 0.01658 | 0.12184 | 0.06543 | 0.01179 | +| v4 BF16 `[0,53]` | 21.2107 | 23.6721 | 0.7349 | 0.1271 | 0.01023 | 0.09515 | 0.04698 | 0.00940 | +| v4 BF16 `[1,52]` | 21.3348 | 23.7490 | 0.7384 | 0.1252 | 0.01011 | 0.09409 | 0.04645 | 0.00907 | +| v4 FP32-master `[1,52]` | **21.5933** | **23.9745** | **0.7467** | **0.1210** | **0.00969** | **0.09162** | **0.04511** | **0.00882** | + +相对 Direct Reuse，v4 的 PSNR 提高 3.6601 dB、SSIM 提高 0.1360，LPIPS 降低 43.84%，pixel +MSE 降低 52.26%，temporal MSE 降低 34.51%。Predictor 端到端生成耗时均值 84.08 秒、中位数 +84.42 秒、P90 91.16 秒；不同方法来自分次运行，因此该耗时只作为运行记录，不作为严格速度对照。 + +在相同 blocks `[1,52]` 下，FP32-master 相对旧 BF16 checkpoint 的 PSNR 提高 0.2586 dB、SSIM +提高 0.0083，LPIPS 降低 3.39%，pixel MSE 降低 4.12%，temporal MSE 降低 2.66%。这与训练时 +保留 FP32 master 参数、避免小更新被 BF16 量化的预期一致。 + +新补跑三版使用相同组件计时实现： + +| 版本 | 端到端 mean | median | P90 | pipeline | AR rollout | history K/V | Full transformer | Predictor | VAE decode | 峰值显存 | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| v4 BF16 `[0,53]` | 87.51 s | 86.45 s | 92.40 s | 85.38 s | 52.18 s | 19.14 s | 14.55 s | 0.97 s | 10.59 s | 71.86 GiB | +| v4 BF16 `[1,52]` | 87.34 s | 85.47 s | 96.58 s | 85.21 s | 52.50 s | 19.15 s | 14.65 s | 1.00 s | 10.62 s | 71.86 GiB | +| predictor-disca BF16 `[0,53]` | **80.89 s** | **79.50 s** | **88.37 s** | **78.76 s** | **49.47 s** | **19.03 s** | **13.62 s** | **0.61 s** | **9.96 s** | **71.77 GiB** | + +v4 在 Chunk 0 必须在线生成 previous-chunk hidden，因此每视频实际执行 18 次 Full transformer、14 次 +Predictor；DisCa 不需要 previous-chunk hidden，执行 16 次 Full transformer、16 次 Predictor。这是 +DisCa 端到端比 v4 快约 6.5 秒的主要原因之一，但其视频质量明显低于两版 v4。 + +### 6.4 previous-condition 输入消融（Torch 2.6 重训） + +旧 `none/previous_step/both` 实验使用了错误的 Torch 2.5 + FlashAttention 2 环境，其 checkpoint、 +训练日志、生成视频和指标均已删除，旧数值不再保留。2026-08-23 使用以下固定环境从随机初始化 +顺序重训三个分支： + +```text +Python: /mnt/local_nvme/cz/envs/hy_worldplay_july20260720/bin/python +PyTorch: 2.6.0+cu124 +CUDA runtime: 12.4 +FlashAttention: 未安装 +``` + +重训保持原消融协议不变：Predictor v4、source blocks `[1,52]`、8400 条离线 transition、8 GPU、 +global batch 128、FP32 trainable parameters、seed 0 和 2000 optimizer steps。三个分支分别是不使用 +上一 chunk hidden（`none`）、使用前一 denoise step（`previous_step`）和同时使用 same/previous +step（`both`）。正确的正式 `same_step` Stage-1 checkpoint 保留作为基线，不重复训练。 + +三个分支分别于 2026-08-23 完成 2000 steps，并于 2026-08-24 在同一 Torch 2.6 环境中使用 +validation25、seed 0 各生成 100 个视频；reference 是已通过 `100/100` SHA256 复现的同环境 FFFF +视频。事后审计逐视频 timing record 时发现，原始评测虽然都把 chunk 内 schedule 写成 FPPF，实际 +Full/Predictor 调用预算并不一致：需要 previous hidden 的 `same_step/previous_step/both` 会把首 +chunk 强制为 FFFF，每视频执行 `18 Full + 14 Predictor`；`none` 不受这一约束，首 chunk 也执行 +FPPF，每视频是 `16 Full + 16 Predictor`。 + +#### 6.4.1 Full-reference 与 VBench Core5 统一对比 + +下表合并原始调度和首 chunk FFFF 公平复评。Full-reference 只保留 PSNR、Frame PSNR、SSIM、 +LPIPS 四项，并在同一行加入 VBench Core5。Core5 是 Subject、Background、Motion、Aesthetic、 +Imaging 的简单平均，不是官方加权总分。 + +| 输入条件 / checkpoint | 首 chunk | Full / Predictor 调用 | PSNR ↑ | Frame PSNR ↑ | SSIM ↑ | LPIPS ↓ | VBench Core5 ↑ | +| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | +| `same_step` 正式基线，step 2000 | FFFF | 18 / 14 | 21.5933 | 23.9745 | 0.74668 | 0.12100 | 0.842022 | +| `previous_step`，step 2000 | FFFF | 18 / 14 | 21.5329 | 23.9052 | 0.74597 | 0.12127 | **0.842153** | +| `both`，step 2000 | FFFF | 18 / 14 | 21.4970 | 23.8920 | 0.74411 | 0.12231 | 0.841854 | +| `none`，step 2000，原始调度 | FPPF | 16 / 16 | 19.2348 | 20.3065 | 0.65579 | 0.17864 | 0.840606 | +| `none`，step 300 | FFFF | 18 / 14 | 20.7686 | 23.2580 | 0.71924 | 0.13494 | 0.842060 | +| `none`，step 500 | FFFF | 18 / 14 | 21.1103 | 23.5607 | 0.73249 | 0.12854 | 0.842129 | +| `none`，step 1000 | FFFF | 18 / 14 | 21.5026 | 23.8763 | 0.74372 | 0.12321 | 0.841753 | +| **`none`，step 2000** | FFFF | 18 / 14 | **21.6985** | **24.0849** | **0.75098** | **0.11861** | 0.841869 | + +#### 6.4.2 调度公平性与训练步数分析 + +为隔离 previous-condition 输入，`none` 的公平复评显式固定首 chunk 为 FFFF，后续 chunk 仍为 +FPPF。step 300、500、1000、2000 四组均为 validation25 的 100 个视频；逐视频日志确认全部执行 +`18 Full + 14 Predictor` 且 `force_first_chunk_full=true`，与三个 previous-condition 版本一致。 + +- 在相同 `18/14` 预算的 `same_step/previous_step/both` 中，表内四项 reference 指标一致为 + `same_step > previous_step > both`。`previous_step` 与 `same_step` 最接近，PSNR、Frame PSNR、 + SSIM 分别低 `0.0605 dB/0.0693 dB/0.00071`，LPIPS 高约 `0.22%`；其 Core5 则高 + `0.000131`，但 paired Core5 仅 `48/100` 胜出。 +- `both` 没有从额外输入获益，相对 `same_step` 的 PSNR/SSIM 为 + `-0.0963 dB/-0.00257`，LPIPS 高约 `1.08%`，Core5 低 `0.000168`。当前 concat fusion 无法 + 有效区分两路 previous condition，反而引入轻微干扰。 +- 原始 `none` 的 PSNR 比 `same_step` 低 `2.3586 dB`，但它的首 chunk 也是 FPPF，每视频少两次 + Full、多两次 Predictor。首 chunk 误差会通过 AR Context K/V 传播到后续七个 chunk，因此该行 + 不能用于判断移除 previous-chunk hidden 本身的影响。 +- 固定首 chunk FFFF 后，`none` 的四项 reference 指标从 step 300 到 500、1000、再到 2000 全部 + 单调改善。step 300→500 的 PSNR 提升 `0.3417 dB`、LPIPS 降低 `0.00640`；step 500→1000 的 + PSNR 提升 `0.3923 dB`、LPIPS 降低 `0.00533`；step 1000→2000 的 PSNR继续提升 + `0.1959 dB`、LPIPS继续降低 `0.00460`。 +- 在完全相同的 `18/14` 预算下，`none` step 2000 的四项 reference 指标全部略优于 + `same_step`：PSNR/Frame PSNR 分别高 `0.1051/0.1104 dB`，SSIM 高 `0.00430`，LPIPS 降低 + 约 `1.98%`；Core5 则低 `0.000154`。它也与 ATC Anchor-only 基本处于同一档，说明首 chunk + 是否进行 Full warmup 是此前结果中的主导混杂变量。 + +#### 6.4.3 VBench 8 项与 Core5 + +对本节全部 8 组现有视频按相同协议计算 VBench，每组 100 个视频、每个维度共 800 条有效记录。Core5 +沿用本文定义，是 Subject、Background、Motion、Aesthetic、Imaging 的简单平均，不是官方加权 +总分。原始 `none` 的首 chunk 为 FPPF；其余标注 FFFF 的 `none` 与三个 previous-condition 版本 +均使用首 chunk Full warmup。 + +| 版本 | Subject | Background | Flickering | Motion | Dynamic | Aesthetic | Imaging | Overall | Core5 ↑ | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| `same_step` step 2000 | 0.947461 | 0.943822 | 0.960451 | 0.989683 | 0.800 | 0.598215 | 0.730930 | 0.249178 | 0.842022 | +| **`previous_step` step 2000** | **0.947814** | **0.943840** | 0.960425 | 0.989658 | 0.800 | 0.598349 | 0.731104 | 0.248914 | **0.842153** | +| `both` step 2000 | 0.947802 | 0.942704 | 0.960404 | 0.989690 | 0.800 | **0.598393** | 0.730683 | 0.248555 | 0.841854 | +| `none` step 2000，首 chunk FPPF | 0.947438 | 0.943414 | **0.960857** | 0.989679 | **0.810** | 0.595039 | 0.727459 | **0.249854** | 0.840606 | +| `none` step 300，首 chunk FFFF | 0.947615 | 0.942934 | 0.960277 | 0.989700 | 0.800 | 0.598111 | 0.731941 | 0.248790 | 0.842060 | +| `none` step 500，首 chunk FFFF | 0.947657 | 0.943116 | 0.960430 | 0.989618 | 0.800 | 0.598174 | **0.732081** | 0.248830 | 0.842129 | +| `none` step 1000，首 chunk FFFF | 0.947806 | 0.943152 | 0.960720 | **0.989733** | 0.800 | 0.597812 | 0.730261 | 0.249078 | 0.841753 | +| `none` step 2000，首 chunk FFFF | 0.947618 | 0.943388 | 0.960367 | 0.989626 | 0.790 | 0.597700 | 0.731010 | 0.248968 | 0.841869 | + +VBench 给出的排序与 Full-reference 指标不同： + +- `previous_step` 的 Core5 最高，但只比 `same_step` 高 `0.000131`，paired Core5 为 `48/100` + 胜出，说明均值优势并不具有逐视频一致性。`both` 比 `same_step` 低 `0.000168`。 +- 公平预算下的 `none` step 2000 比 `same_step` 低 `0.000154`，paired Core5 为 `41/100` 胜出； + 差距很小，但 reference 八项的全面小幅优势没有转化为 Core5 优势。 +- `none` 的 Core5 不随训练单调提高：step 300/500/1000/2000 分别为 + `0.842060/0.842129/0.841753/0.841869`。step 2000 最贴近 Full reference，而 step 500 的无参考 + Core5 最高，进一步说明离线拟合收敛和无参考观感不是同一目标。 +- 首 chunk FFFF 仍有明确作用：同为 `none` step 2000，Core5 从首 chunk FPPF 的 `0.840606` + 提升到 `0.841869`，增加 `0.001263`。 + +修正后的结论是：当前实验不能证明显式 previous-chunk hidden 是 v4 保持 FPPF 质量的关键输入； +在已有 AR Context K/V 的条件下，`none` 可以恢复 reference 质量，且其 VBench Core5 与 +`same_step/previous_step/both` 处于非常接近的区间。`previous_step` 虽取得最高 Core5，但幅度仅 +`+0.000131` 且 paired 胜率不足一半；简单 concat 的 `both` 仍没有收益。由于尚未按 chunk、运动 +强度或场景切换分桶，暂不把结论扩大为“previous hidden 在所有条件下都无用”；但后续输入消融 +必须统一首 chunk FFFF 和 Full/Predictor 调用次数，不能再使用原始 `none` 行支持 previous hidden +必要性的论断。 + + +## 7. 完整训练集 Stage-2 实验 + +本节只保留使用全部 100 个训练 pair（`fulltrain100`）的 Stage-2 实验，并统一在独立的 +validation25 上评测。其中 7.1～7.2 比较两种 on-policy rollout，7.3 是相同低学习率预算的离线 +teacher-forcing 对照，7.4 保留 student rollout 输入但改用离线 FFFF target。 + +### 7.1 detached timestep on-policy + +随后将 detached timestep-only on-policy 扩展到全部 100 个训练 pair。每步每 rank 采样一条 +`case/action/chunk` record，完成 P1→P2→P3 forward；两个 timestep 边界都 detach。训练使用 +`fusion_lr=5e-5`、`blocks_lr=5e-6`、两个 warmup 均为 100 steps，block 线性 warmup， +BF16 activation、FP32 trainable parameters 和每卡 batch size 1。训练保存到 step 1000 后，在完整 +validation25 上评测。它和 Stage-1、chunk+timestep 版本的统一数值对比见 7.2，避免重复列出同一组 +结果。 + +FPPF 下 candidate 的 PSNR 下降 0.1829 dB、LPIPS 变差 1.51%、temporal MSE 增加 2.72%。FPPP +下 PSNR 下降 0.1397 dB、temporal MSE 增加 4.30%，但 LPIPS 降低 7.06%，并在 89/100 个配对视频 +上胜出。该冲突正是增加无参考 VBench 指标的主要原因：on-policy 版本可能偏离 Full-DiT 的逐帧轨迹， +但不一定具有更差的主体稳定性、运动平滑性或感知画质。 + +### 7.2 detached chunk+timestep on-policy + +当前实验进一步同时模拟 chunk 与 timestep 两个维度的 FPPP 推理误差： + +- Chunk 0 使用 FFFF，只建立初始历史； +- Chunk 1～7 使用 FPPP，P1/P2/P3 都训练； +- timestep 边界和 chunk 边界全部 detach，后续 loss 不反传到前序状态； +- Teacher 在当前 student latent 上在线产生 target； +- 每个 rank 每步完成一条 8-chunk trajectory，8 卡全局每步 8 条 trajectory； +- 学习率、warmup、精度和 7.1 的 timestep-only 实验一致。 + +训练第一次在记录 step 1005 后停止；由于 step 1001～1005 未形成完整训练状态，随后从完整保存的 +step 1000 `training_latest.pt` 恢复 optimizer、scheduler、epoch 和 batch 位置，继续训练到 step 2000。 +恢复后 cosine scheduler 延续原进度而没有重复 warmup，step 2000 时两个 parameter group 的 LR 均降至 +0。下表中的 Stage-1、detached timestep 和 detached chunk+timestep checkpoint 均使用完整的 +100-pair 训练集；评测统一使用独立的 validation25，每种 schedule 均覆盖 +25 pairs × 4 actions = 100 个视频。VBench 五项平均来自 8.3 的同一批视频： + +| schedule | 版本 | PSNR ↑ | SSIM ↑ | LPIPS ↓ | VBench 5项平均 ↑ | +| --- | --- | ---: | ---: | ---: | ---: | +| FPPF | Stage-1 FP32-master | 21.5933 | 0.74668 | 0.12100 | 0.842022 | +| FPPF | detached timestep on-policy step 1000 | 21.4104 | 0.74233 | 0.12282 | 0.841987 | +| FPPF | detached chunk+timestep on-policy step 1000 | 21.5753 | 0.74806 | 0.12041 | **0.842026** | +| FPPF | detached chunk+timestep on-policy step 2000 | 21.7289 | 0.75234 | 0.11818 | 0.841941 | +| FPPF | Stage-1 continued offline TF step 2000 | 21.8472 | 0.75308 | **0.11673** | 0.842012 | +| FPPF | detached rollout + offline FFFF targets step 2000 | **21.9107** | **0.75435** | 0.12432 | 0.838527 | +| | | | | | | +| FPPP | Stage-1 FP32-master | 20.3085 | 0.69143 | 0.23528 | 0.819219 | +| FPPP | detached timestep on-policy step 1000 | 20.1688 | 0.69018 | 0.21867 | 0.825319 | +| FPPP | detached chunk+timestep on-policy step 1000 | 20.3330 | **0.69592** | 0.21495 | 0.826423 | +| FPPP | detached chunk+timestep on-policy step 2000 | 20.3006 | 0.69430 | **0.21146** | **0.827286** | +| FPPP | Stage-1 continued offline TF step 2000 | 20.3381 | 0.69498 | 0.22958 | 0.820535 | +| FPPP | detached rollout + offline FFFF targets step 2000 | **20.8005** | 0.68960 | 0.32174 | 0.776892 | + +在 FPPF 上，chunk+timestep step 1000 相对 Stage-1 的 PSNR 仅低 0.0181 dB，SSIM 提高 0.00138， +LPIPS 降低 0.49%；三项分别在 `55/100、59/100、59/100` 个逐视频配对中胜出，整体可视为基本持平。 +它又一致优于 detached timestep-only：PSNR 提高 0.1649 dB、SSIM 提高 0.00573、LPIPS 降低 +1.97%，三项各在 `66/100、64/100、64/100` 个视频上胜出。 + +在目标 FPPP schedule 上，chunk+timestep step 1000 相对 Stage-1 的 PSNR 提高 0.0245 dB、SSIM 提高 +0.00448、LPIPS 降低 8.65%；三项分别在 `52/100、62/100、94/100` 个视频上胜出。相对 +detached timestep-only，PSNR 提高 0.1642 dB、SSIM 提高 0.00573、LPIPS 降低 1.70%。这说明同时 +模拟 chunk 与 timestep 两个维度的推理误差，主要改善了三步 Predictor 替代时的感知质量和稳定性， +且没有造成 FPPF 的明显退化。 + +继续训练到 step 2000 后，FPPF 相对 step 1000 的 PSNR 提高 0.1536 dB、SSIM 提高 0.00428、 +LPIPS 降低 1.85%、MSE 降低 4.30%、temporal MSE 降低 2.23%；五项逐视频胜出数依次为 +`57、60、62、57、54/100`。它相对 Stage-1 也在五项聚合指标上全部改善，其中 PSNR 提高 +0.1356 dB、LPIPS 降低 2.33%，说明延长训练显著改善了 FPPF 下的 Full step 接管兼容性。 + +FPPP 并未随训练步数单调改善。step 2000 相对 step 1000 的 LPIPS 继续降低 1.62%，并在 79/100 个 +配对视频上胜出；但 PSNR 下降 0.0325 dB、SSIM 下降 0.00161、MSE 增加 1.10%、temporal MSE +增加 2.66%。相对 Stage-1，step 2000 的 PSNR 基本持平（-0.0080 dB），SSIM 提高 0.00287， +LPIPS 降低 10.13% 且在 93/100 个视频上胜出，MSE 降低 0.34%，但 temporal MSE 增加 4.02%。 +因此 step 2000 更偏向感知质量和 FPPF 兼容性；若优先考虑 FPPP 的 reference fidelity 与时序误差， +step 1000 仍是更均衡的 checkpoint。 + +### 7.3 Stage-1 continued offline teacher-forcing 公平对照 + +为区分“继续训练本身”和“detached chunk+timestep on-policy 数据分布”的作用，新增一条与 rollout +版本使用相同学习率预算的离线对照： + +- 从正式 Stage-1 FP32-master `[1,52]` step 2000 的 `training_latest.pt` 只加载 Predictor 模型参数； +- 不继承原 optimizer、scheduler 和 global step，重新执行 2000 个局部 optimizer steps； +- 继续使用 Stage-1 的 grouped offline teacher-forcing：8400 个 FFFF 相邻 transition pair、 + context bucket sampler、每卡 batch size 16、8 卡全局 batch size 128； +- 不执行 timestep/chunk rollout，不加载在线 Teacher target； +- 使用与完整训练集 rollout 相同的 `fusion_lr=5e-5`、`blocks_lr=5e-6`、scheduler/block warmup + 各 100 steps，两个 parameter group 均从头线性 warmup 后 cosine decay 到 0； +- 使用 BF16 forward activation、FP32 trainable parameters、FP32 AdamW moments，训练 checkpoint + 每 100 steps 保存一次。 + +因此它与 rollout 的模型初始化、学习率峰值、warmup、局部 optimizer steps 和 FP32 master 设置一致， +但每个 update 的样本单位不同：离线对照是 128 个独立 transition pair，而 chunk+timestep rollout +是 8 条 trajectory、每条 7 个训练 chunk × 3 个 Predictor step。该实验控制了“再训练 2000 steps” +的影响，但不等价控制总 forward/loss 数或 FLOPs。 + +validation25 结果已并入 7.2 的统一表。相对原 Stage-1，这条离线 continuation 在 FPPF 的 PSNR、 +SSIM、LPIPS、MSE、temporal MSE 五项都改善，分别为 `+0.2539 dB、+0.00640、-0.00427、 +-0.000597、-0.000182`；相对 chunk+timestep step 2000 也五项全部更好，其中 PSNR 再提高 +0.1182 dB、LPIPS 再降低 1.22%。 + +FPPP 下，离线 continuation 相对原 Stage-1 的 PSNR 提高 0.0296 dB、SSIM 提高 0.00355、LPIPS +降低 2.42%、temporal MSE 降低 0.82%，MSE 基本持平（增加 0.15%）。与 chunk+timestep step 2000 +相比，它的 PSNR、SSIM 和 temporal MSE 更好，但 LPIPS 从 `0.21146` 退化到 `0.22958`。这说明 +continued offline teacher-forcing 更擅长保持 Full-DiT reference fidelity，而 on-policy rollout +在三步连续替代时取得了更明显的感知距离收益。 + +### 7.4 detached chunk+timestep rollout + offline FFFF targets + +这版实验保留 detached FPPP self-rollout 产生 Predictor 输入，但把监督改为固定的离线 FFFF +Teacher 轨迹： + +- Chunk 0 直接从离线数据恢复 FFFF final latent 和 previous-chunk hidden，不再在线重复生成； +- Chunk 1～7 仍根据当前 student history 在线计算 exact context K/V，并由 Full Teacher 计算 step 0； +- P1/P2/P3 递归输入来自 Predictor 自己产生的 hidden、velocity 和 scheduler latent； +- 三步的 hidden/velocity target 固定读取对应 chunk 的离线 FFFF trajectory，而不是在当前 student + latent 上运行 same-state Teacher； +- timestep 与 chunk 边界都 detach，后序 loss 只优化当前 Predictor 调用； +- 初始化、`fusion_lr=5e-5`、`blocks_lr=5e-6`、两个 100-step linear warmup、2000 optimizer + steps、BF16 activation/FP32 master 和 8 卡 batch 设定均与 on-policy 完整训练集版本一致。 + +因为 Chunk 0 使用离线缓存，且 Chunk 1～7 每个 chunk 只需一次在线 Full step 0，这版每条训练 +trajectory 的 step 2000 日志均时为 `31.30 s`，比同配置 on-policy 版本的 `51.57 s` 低 +`39.3%`；峰值显存基本相同（`50.59 GiB` 对 `50.57 GiB`）。step 2000 日志窗口的 total loss 为 +`0.07554`，hidden MSE 为 `0.04137/0.04143/0.04417`，velocity MSE 为 +`0.06771/0.07089/0.07531`。 + +validation25 的 reference 指标已并入 7.2。FPPF 相对 Stage-1 的 PSNR 提高 `0.3173 dB`，SSIM +提高 `0.00766`，MSE 和 temporal MSE 分别降低 `10.89%`、`6.38%`；对应逐视频胜出数为 +`62、54、62、71/100`。但 LPIPS 从 `0.12100` 升至 `0.12432`，仅 `39/100` 个视频胜出。 + +FPPP 的冲突更明显：相对 Stage-1，PSNR 提高 `0.4920 dB`，MSE 和 temporal MSE 分别降低 +`10.93%`、`12.57%`，逐视频胜出 `74、74、88/100`；但 SSIM 降低 `0.00183`，LPIPS 从 +`0.23528` 大幅升至 `0.32174`，LPIPS 仅 `1/100` 个视频胜出。相对 on-policy step 2000，它的 +PSNR 高 `0.4999 dB`、temporal MSE 低 `15.95%`，但 LPIPS 在 `0/100` 个配对视频上胜出。 + +这说明“student rollout 状态 + 固定 FFFF target”存在明显的 state-label mismatch：优化可以把像素 +轨迹拉向 Full reference 并压低 MSE/temporal MSE，却未必给当前偏移状态提供语义和感知上合理的 +局部速度方向。仅看 PSNR 会把这版判断为最好；LPIPS 与下节 VBench 均显示其 FPPP 感知质量明显 +退化，因此它不适合作为当前最终模型。 + +## 8. VBench 无参考视频质量评测 + +### 8.1 指标与统一输入 + +本节只展示以下 5 个视频质量维度： + +```text +subject_consistency +background_consistency +motion_smoothness +aesthetic_quality +imaging_quality +``` + +5 项均使用 VBench `custom_input`，不使用 Full-DiT reference。所有版本统一使用 validation25 的 +25 pairs × 4 actions = 100 个视频、125 frames、480×832、相同 seed/FPS； +`imaging_quality_preprocessing_mode` 固定为官方默认 `longer`。当前共 14 组、每项评测 1400 个 +视频，5 项合计汇总 7000 条逐视频结果，同时保存总体均值、paired win rate 和四种 action 分组结果。 + +模型权重持久化到： + +```text +/mnt/s3files/s3-us-west2-default/zoubin/cz/.cache/vbench +/mnt/s3files/s3-us-west2-default/zoubin/cz/.cache/clip +/mnt/s3files/s3-us-west2-default/zoubin/cz/.cache/torch +``` + +### 8.2 核心版本与视频状态 + +| 优先级 | 视频版本 | schedule | 视频数 | 当前视频状态 | 目的 | +| --- | --- | --- | ---: | --- | --- | +| P0 | Full-DiT | FFFF | 100 | 已评测 | Full Teacher 基准；无参考分数不预设为上限 | +| P0 | Direct Reuse | FRRF | 100 | 已评测 | 无训练加速基线 | +| P0 | Stage-1 v4 FP32-master `[1,52]` step 2000 | FPPF | 100 | 已评测 | 正式 Stage-1 基线 | +| P0 | Stage-1 v4 FP32-master `[1,52]` step 2000 | FPPP | 100 | 已评测 | 三步替代基线 | +| P0 | detached timestep on-policy fulltrain100 step 1000 | FPPF | 100 | 已评测 | timestep-only rollout | +| P0 | detached timestep on-policy fulltrain100 step 1000 | FPPP | 100 | 已评测 | timestep-only rollout | +| P0 | detached chunk+timestep on-policy fulltrain100 step 1000 | FPPF | 100 | 已评测 | 检查 Full 接管兼容性 | +| P0 | detached chunk+timestep on-policy fulltrain100 step 1000 | FPPP | 100 | 已评测 | 中间 checkpoint | +| P0 | detached chunk+timestep on-policy fulltrain100 step 2000 | FPPF | 100 | 已评测 | 最终 checkpoint 的 Full 接管兼容性 | +| P0 | detached chunk+timestep on-policy fulltrain100 step 2000 | FPPP | 100 | 已评测 | 最终 checkpoint 的目标 schedule | +| P0 | Stage-1 continued offline TF step 2000 | FPPF | 100 | 已评测 | 相同低学习率预算的离线继续训练对照 | +| P0 | Stage-1 continued offline TF step 2000 | FPPP | 100 | 已评测 | 区分继续训练与 on-policy rollout 的收益 | +| P0 | detached rollout + offline FFFF targets step 2000 | FPPF | 100 | 已评测 | 固定离线监督下的 Full 接管兼容性 | +| P0 | detached rollout + offline FFFF targets step 2000 | FPPP | 100 | 已评测 | 检查固定离线监督与 student 状态错配 | + +### 8.3 五项 VBench 结果 + +所有分数均按 VBench 输出方向统一为越高越好。粗体分别标记 Full/Reuse/FPPF 范围和 FPPP 范围内的 +最高值。最后一列是表中 5 个原始单项的简单算术平均，用于本项目内部排序。这是项目自定义的五项 +集合，且未执行官方归一化加权，因此不是官方 VBench-I2V Quality Score。 + +| 版本 | schedule | Subject ↑ | Background ↑ | Motion ↑ | Aesthetic ↑ | Imaging ↑ | 5项平均 ↑ | +| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | +| Full-DiT | FFFF | 0.948075 | 0.943076 | 0.989400 | 0.598486 | 0.734028 | 0.842613 | +| Direct Reuse | FRRF | **0.948163** | **0.944339** | **0.989935** | 0.597758 | **0.734619** | **0.842963** | +| Stage-1 FP32-master | FPPF | 0.947461 | 0.943822 | 0.989683 | 0.598215 | 0.730930 | 0.842022 | +| detached timestep on-policy step 1000 | FPPF | 0.947665 | 0.943096 | 0.989678 | 0.598344 | 0.731153 | 0.841987 | +| detached chunk+timestep step 1000 | FPPF | 0.947636 | 0.943356 | 0.989713 | **0.598682** | 0.730743 | 0.842026 | +| detached chunk+timestep step 2000 | FPPF | 0.947705 | 0.943808 | 0.989733 | 0.597985 | 0.730474 | 0.841941 | +| Stage-1 continued offline TF step 2000 | FPPF | 0.947929 | 0.943915 | 0.989614 | 0.598109 | 0.730493 | 0.842012 | +| detached rollout + offline FFFF targets step 2000 | FPPF | 0.941993 | 0.940776 | 0.989453 | 0.594622 | 0.725793 | 0.838527 | +| | | | | | | | | +| Stage-1 FP32-master | FPPP | 0.925574 | 0.930874 | 0.987025 | 0.563811 | 0.688810 | 0.819219 | +| detached timestep on-policy step 1000 | FPPP | 0.931156 | 0.933912 | 0.987361 | 0.572131 | 0.702035 | 0.825319 | +| detached chunk+timestep step 1000 | FPPP | 0.932699 | 0.934256 | 0.987628 | 0.573063 | 0.704467 | 0.826423 | +| detached chunk+timestep step 2000 | FPPP | **0.933535** | **0.934409** | **0.987653** | **0.574861** | **0.705972** | **0.827286** | +| Stage-1 continued offline TF step 2000 | FPPP | 0.926989 | 0.931052 | 0.987153 | 0.565366 | 0.692116 | 0.820535 | +| detached rollout + offline FFFF targets step 2000 | FPPP | 0.885713 | 0.913790 | 0.985759 | 0.519146 | 0.580052 | 0.776892 | + +FPPF 下，chunk+timestep step 2000 相对 step 1000 的 Subject、Background、Motion、Aesthetic 和 +Imaging 差异分别为 `+0.000068、+0.000452、+0.000020、-0.000697、-0.000269`，逐视频胜出数为 +`51、56、50、45、45/100`。五项平均只降低 `0.000085`，结合 +第 7.2 节全面改善的 reference 指标，可以判断其 FPPF 收益主要是更接近 Full-DiT trajectory，而 +五维无参考质量总体保持稳定。 + +FPPP 下继续训练带来更清晰的无参考收益。step 2000 相对 step 1000 的 Subject、Background、Motion、 +Aesthetic 和 Imaging 分别提高 `0.000836、0.000153、0.000025、0.001798、0.001505`，逐视频胜出 +`67、52、57、68、61/100`；五项平均由 `0.826423` 提高到 `0.827286`。相对 Stage-1， +上述五项分别提高 `0.007961、0.003535、0.000628、0.011050、0.017162`，逐视频胜出 +`97、72、88、90、94/100`。因此 step 2000 虽未改善 FPPP 的 reference PSNR 和 temporal MSE， +却改善了五维无参考主体稳定性、运动和画质。 + +continued offline teacher-forcing 的 FPPF 五维结果与原 Stage-1 和 chunk+timestep step 2000 +基本持平，单项绝对差异都不超过 `0.0005`。FPPP 下，它相对原 Stage-1 的 Subject、Background、 +Motion、Aesthetic 和 Imaging 分别提高 `0.001415、0.000178、0.000128、0.001555、0.003306`， +五项平均由 `0.819219` 提高至 `0.820535`，但仍显著低于 chunk+timestep step 2000 的五个单项和 +五项平均 `0.827286`。结合 7.3 的 reference 指标，公平对照支持：额外离线训练主要提升对 Full-DiT +trajectory 的拟合；on-policy rollout 的独立收益主要体现在 FPPP 的无参考主体稳定性和感知画质。 + +offline FFFF targets 版本进一步揭示了 reference fidelity 与无参考观感的冲突。FPPF 五项平均为 +`0.838527`，相对 on-policy step 2000 和 continued offline TF 分别低 `0.003414`、`0.003485`。 +FPPP 的退化更大：五项平均为 `0.776892`，比 Stage-1 的 `0.819219` 低 `0.042327`，比 on-policy +step 2000 的 `0.827286` 低 `0.050394`（相对降低 `6.09%`）；五个单项均低于 on-policy step 2000。 +结合其更低的 temporal MSE，这更像是固定 FFFF target 把偏移轨迹推向感知内容失真的解，而不是带来 +更好的综合视频质量。 + +### 8.4 可选消融 + +| 优先级 | 视频版本 | 是否建议 | 原因 | +| --- | --- | --- | --- | +| P1 | predictor-disca BF16 `[0,53]` FPPF | 建议 | 判断缺少 previous-chunk hidden/Context Vision K/V 是否也损害无参考稳定性 | +| P1 | v4 BF16 `[1,52]` FPPF | 可选 | 与 FP32-master 对比参数更新精度，但现有 reference 指标差异较小 | +| P2 | v4 BF16 `[0,53]` FPPF | 可选 | 只用于 block 初始化位置消融 | + +### 8.5 解释边界 + +`subject_consistency` 和 `background_consistency` 比较视频内部当前帧、前一帧与首帧的特征相似度； +WorldPlay 的合法相机移动也可能造成分数下降。`motion_smoothness` 衡量插帧可预测性，不判断 action +方向是否正确；`aesthetic_quality` 和 `imaging_quality` 主要评价逐帧画质。因此这五项用于补充 +PSNR/SSIM/LPIPS，不能替代 reference fidelity、caption 一致性或后续的 action/相机轨迹指标。 + +## 9. Stage-1 C1 confidence threshold 动态 fallback + +> 历史环境提示：本节离线 head、原始 threshold 在线视频及其 VBench 均在 Torch 2.5 环境完成。 +> 数值继续保留，但不代表固定运行环境下的正式结果。现有 head 不重训、保持原阈值，在 Torch 2.6 +> 环境重跑的 reference 指标见第 10.2～10.3 节；对应固定环境 VBench 尚未重跑。 + +### 9.1 协议与路由 + +C1 confidence head 使用冻结的正式 Stage-1 v4 FP32-master `[1,52]` step 2000 提取特征：训练集为 +`fulltrain100` 的 8400 个离线 FFFF transition，验证集为独立 validation25 的 2100 个 transition， +训练过程不使用 validation 调参。head 输入为 Predictor `pred_hidden` 的 LayerNorm token 均值、 +denoise-step embedding，以及 `pred_hidden-anchor_hidden` 的 log residual；输出为预测的 `log(C1)`。 +独立 validation25 上，C1 的 overall Spearman/Pearson/log-MAE 为 +`0.952918/0.965393/0.159999`，P1/P2/P3 内的 Spearman 分别为 +`0.859072/0.863169/0.870032`。overall 相关性包含不同 timestep 风险量级的可分性，实际动态 +routing 仍需由下面的 rollout 视频指标判断。 + +另使用完全相同的离线特征和 C1 标签，将 head 从 `2081 -> 256 -> 1`（537,441 参数）扩大为 +`2081 -> 4096 -> 4096 -> 1`（25,317,473 参数），只训练 C1，训练设置仍为 40 epochs、AdamW +`1e-4`。最终训练 Smooth-L1 从 `0.022912` 降至 `0.011734`；validation25 对比如下： + +| C1 head | 参数量 | Spearman | Pearson | log-MAE | 33.3% fallback 剩余 C1 risk | +| --- | ---: | ---: | ---: | ---: | ---: | +| 256 | 537,441 | 0.952918 | 0.965393 | 0.159999 | **0.00487898** | +| 4096 × 2 | 25,317,473 | **0.970702** | **0.978563** | **0.131041** | 0.00488749 | + +宽 head 的整体及分 step 回归明显更好，并在 5%～25% 和 40%～75% 的已测 fallback 预算下降低 +剩余风险；但恰好 33.3% 预算时比小 head 高 `0.17%`。宽 head 的全局 r33 在线验证结果也列在 +下表；其余 C1-r10/r25/r33/r50 仍使用原 256 维 C1 head。 + +同样复用现有离线特征中的 `true_c2`，训练了 `2081 -> 4096 -> 4096 -> 1` 的宽 C2 head。 +C2 是 Predictor 与 Full-DiT 从同一 `x_t` 更新所得两个下一 noisy latent 的相对 MAE；本次也修正 +risk-coverage 为按真实 C2 risk 评估，而不是用 C2 score 排序 C1 risk。结果如下： + +| C2 head | 参数量 | 训练 Smooth-L1 | Spearman | P1 / P2 / P3 Spearman | log-MAE | 33.3% fallback 剩余 C2 risk | +| --- | ---: | ---: | ---: | --- | ---: | ---: | +| 256 | 537,441 | 0.022536 | 0.947227 | 0.862289 / 0.869345 / 0.829100 | 0.161864 | 0.00890321 | +| 4096 × 2 | 25,317,473 | **0.011930** | **0.967081** | **0.903906 / 0.925671 / 0.934346** | **0.134602** | **0.00884546** | + +在离线模拟的 `P3 固定 Full` hybrid 中，对 P1、P2 分别 fallback 10%（总 fallback 40%）时，宽 +C2/宽 C1 的剩余 C2 risk 为 `0.00757638/0.00758285`；分别 fallback 25%（总 fallback 50%）时为 +`0.00574437/0.00575903`。宽 C2 的下一状态目标在两个预算下都略优于宽 C1，但差距只有约 +`0.09%/0.25%`，因此需要在线视频指标确认是否能转化成可见收益。 + +在线验证统一使用 validation25、seed 0、100 个视频。Chunk 0 因没有 previous-chunk hidden 而固定 +FFFF；Chunk 1～7 的 P0 固定 Full，P1/P2/P3 先运行 Stage-1 Predictor 和 C1 head。当 +`predicted log(C1) > threshold` 时，再运行同一步 Full-DiT 并采用 Full 输出，否则采用 Predictor +输出。每个视频产生 `7 chunks × 3 steps = 21` 次决策，每组各有 2100 次决策。阈值来自对应 head +的训练集 C1 score 分位点，不使用 validation score 选阈值。 + +| 策略 | log-risk 阈值 | 训练目标 fallback | 在线实际 fallback | fallback 次数 | P1 / P2 / P3 fallback | +| --- | ---: | ---: | ---: | ---: | --- | +| C1-r10 | -3.523589 | 10.0% | 14.38% | 302/2100 | 0.0% / 0.0% / 43.14% | +| C1-r25 | -3.915276 | 25.0% | 28.62% | 601/2100 | 0.0% / 0.0% / 85.86% | +| C1-r33 | -4.329443 | 33.3% | 33.81% | 710/2100 | 1.43% / 5.29% / 94.71% | +| Wide-C1-r33 | -4.308431 | 33.3% | 34.43% | 723/2100 | 3.14% / 6.86% / 93.29% | +| C1-r50 | -4.798927 | 50.0% | 51.71% | 1086/2100 | 13.14% / 42.00% / 100.0% | +| Wide-C2-Hybrid-r10 | P1 -4.136674 / P2 -3.798281 | 40.0% | 41.24% | 866/2100 | 11.86% / 11.86% / 100.0% | +| Wide-C2-Hybrid-r25 | P1 -4.400577 / P2 -4.049408 | 50.0% | 51.71% | 1086/2100 | 29.00% / 26.14% / 100.0% | + +训练分位点与在线比例存在分布偏移，低 fallback 端最明显：r10 从目标 10% 变成 14.38%。全局 +C1 排序仍主要是 timestep 排序；r10/r25 只 fallback P3，r33 也把 93.38% 的 fallback 预算分给 +P3。r50 才在保证全部 P3 Full 的基础上，进一步 fallback 部分 P1/P2。 + +### 9.2 Full-reference 指标 + +FFFF 是配对 reference，因此其 PSNR/SSIM/误差是恒等结果，不代表无参考质量上界。其余方法使用 +同一批 FFFF 视频作为 reference。PSNR、frame PSNR、SSIM 越高越好，LPIPS 与四项误差越低越好。 + +| 策略 | fallback | PSNR ↑ | frame PSNR ↑ | SSIM ↑ | LPIPS ↓ | MAE ↓ | MSE ↓ | RMSE ↓ | temporal MSE ↓ | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| FFFF | 100% | ∞ | ∞ | 1.00000 | 0 | 0 | 0 | 0 | 0 | +| FPPP | 0% | 20.3085 | 22.4461 | 0.69143 | 0.23528 | 0.05425 | 0.011888 | 0.10329 | 0.008931 | +| C1-r10 | 14.38% | 20.6740 | 22.6026 | 0.71893 | 0.16237 | 0.05034 | 0.011000 | 0.09901 | 0.008912 | +| C1-r25 | 28.62% | 21.0259 | 23.0004 | 0.73270 | 0.13290 | 0.04774 | 0.010374 | 0.09575 | 0.009166 | +| C1-r33 | 33.81% | 21.1834 | 23.1924 | 0.73752 | 0.12620 | 0.04692 | 0.010172 | 0.09456 | 0.009169 | +| Wide-C1-r33 | 34.43% | 21.2130 | 23.2123 | 0.73878 | 0.12558 | 0.04678 | 0.010158 | 0.09435 | 0.009126 | +| FPPF | 33.33% | **21.5933** | **23.9745** | **0.74668** | **0.12100** | **0.04511** | **0.009692** | **0.09162** | **0.008824** | +| Wide-C2-Hybrid-r10 | 41.24% | 21.6800 | 23.6780 | 0.75244 | 0.11492 | 0.04367 | 0.009040 | 0.08955 | 0.008539 | +| C1-r50 | 51.71% | **22.1161** | **24.1236** | **0.76838** | **0.10507** | **0.04096** | **0.008328** | **0.08546** | **0.008148** | +| Wide-C2-Hybrid-r25 | 51.71% | **22.3228** | **24.3398** | **0.77588** | **0.10172** | **0.03964** | **0.007909** | **0.08337** | **0.007927** | + +同为约 1/3 fallback，C1-r33 没有超过固定 FPPF：PSNR 低 `0.4100 dB`，SSIM 低 `0.00917`， +LPIPS、pixel MSE、temporal MSE 分别变差 `4.30%、4.96%、3.92%`。r33 平均每视频 fallback +7.1 次，和 FPPF 固定 fallback 7 个 P3 基本相同，但它把 47/710 次预算分给 P1/P2，并留下 +37/700 个 P3 使用 Predictor。结果说明 P3 的结构性重要性高于当前 C1 对跨 step 样本风险的细排。 + +Wide-C1-r33 相比小 head 的 C1-r33，八项 reference 指标全部小幅改善：PSNR 增加 `0.0296 dB`、 +SSIM 增加 `0.00127`、LPIPS 降低 `0.49%`；但实际 fallback 多 13 次，且只改变少量视频的路由。 +相比固定 FPPF，它仍低 `0.3804 dB` PSNR 和 `0.00790` SSIM，LPIPS、pixel MSE、temporal MSE +分别差 `3.79%/4.81%/3.43%`。扩大 head 容量改善了风险拟合，但没有解决跨 step 全局 threshold +错误交换 P3 与 P1/P2 预算的问题；宽 head 尚未运行 VBench。 + +固定 P3 Full 后，Wide-C2-Hybrid-r10 相比 FPPF 的 PSNR/SSIM 增加 `0.0867 dB/0.00575`， +LPIPS、pixel MSE、temporal MSE 分别改善 `5.02%/6.73%/3.22%`，但 frame PSNR 低 `0.2965 dB`； +paired median 也显示部分常规样本略退，整体均值收益主要来自修复高风险尾部。Wide-C2-Hybrid-r25 +则八项全面超过 FPPF，PSNR/SSIM 增加 `0.7294 dB/0.02920`，LPIPS 和 pixel MSE 改善 +`15.93%/18.39%`，六项主要 paired 指标均有 53～62/100 个视频胜出。 + +Wide-C2-Hybrid-r25 与 C1-r50 的 fallback 次数完全相同，都是 `1086/2100`。C2 hybrid 将早期预算 +从 C1-r50 的 P1/P2 `92/294` 调整为 `203/183`，八项指标全部更好：PSNR 增加 `0.2066 dB`、 +SSIM 增加 `0.00751`、LPIPS 降低 `3.20%`、pixel MSE 降低 `5.04%`。这证明固定 P3 再按分 step +下一状态风险选择 P1/P2，比跨 step 全局 C1 threshold 更有效；两组 C2 hybrid 尚未运行 VBench。 + +C1-r50 使用更高预算后全面超过 FPPF：PSNR 提高 `0.5228 dB`、SSIM 提高 `0.02169`、LPIPS +降低 `13.16%`、pixel MSE 降低 `14.06%`、temporal MSE 降低 `7.65%`。这组路由固定 fallback +全部 P3，再选择 42% P2 和 13.14% P1，本质上是 FPPF 加 early-step confidence fallback。 + +### 9.3 VBench 8 项 + +四组 C1 threshold 视频完成 Subject、Background、Temporal Flickering、Motion Smoothness、Dynamic +Degree、Aesthetic、Imaging、Overall Consistency 全 8 项 VBench；每项含 400 条逐视频结果。下表 +所有指标均为越高越好。Core5 沿用 8.3 的项目内部定义，即 Subject、Background、Motion、Aesthetic、 +Imaging 的简单平均，不是官方 VBench 总分。 + +| 策略 | Subject | Background | Flicker | Motion | Dynamic | Aesthetic | Imaging | Overall | Core5 | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| FFFF | 0.948075 | 0.943076 | 0.958449 | 0.989400 | 0.82 | 0.598486 | 0.734028 | 0.248095 | **0.842613** | +| FPPP | 0.925574 | 0.930874 | **0.965478** | 0.987025 | 0.82 | 0.563811 | 0.688810 | 0.247289 | 0.819219 | +| C1-r10 | 0.940300 | 0.938889 | 0.961824 | 0.988701 | 0.82 | 0.586908 | 0.717366 | 0.249135 | 0.834433 | +| C1-r25 | 0.946081 | 0.942168 | 0.960483 | 0.989495 | 0.80 | 0.595933 | 0.727648 | 0.249157 | 0.840265 | +| C1-r33 | 0.947570 | 0.942669 | 0.959909 | 0.989582 | 0.80 | 0.597482 | 0.729476 | 0.249080 | 0.841356 | +| FPPF | 0.947461 | **0.943822** | 0.960451 | **0.989683** | 0.80 | 0.598215 | 0.730930 | **0.249178** | 0.842022 | +| C1-r50 | **0.947762** | 0.943007 | 0.958938 | 0.989504 | 0.80 | **0.599139** | **0.732626** | 0.248683 | **0.842408** | + +C1-r33 只有 Subject 比 FPPF 高 `0.000109`；Background、Flicker、Motion、Aesthetic、Imaging、 +Overall 均略低，Core5 低 `0.000666`。C1-r50 的 Subject、Aesthetic、Imaging 和 Core5 超过 +FPPF，Core5 高 `0.000385`，但 Background、Flicker、Motion、Overall 仍略低。r50 Core5 距离 +FFFF 仅 `-0.000205`，但 Dynamic Degree 为 `0.80`，低于 FFFF 的 `0.82`。 + +### 9.4 计算成本与结论 + +下表是生成日志的端到端均时。FPPP/FPPF 来自此前独立评测，四组 C1 threshold 同时运行且共享 +主机 I/O，因此跨次的几秒差异只能作为近似参考。 + +| 策略 | 端到端均时 | +| --- | ---: | +| FPPP | 81.97 s | +| FPPF | 84.08 s | +| C1-r10 | 84.43 s | +| C1-r25 | 83.51 s | +| C1-r33 | 84.58 s | +| Wide-C1-r33 | 93.05 s | +| C1-r50 | 99.26 s | +| Wide-C2-Hybrid-r10 | 88.50 s | +| Wide-C2-Hybrid-r25 | 90.94 s | + +当前 C1 特征依赖 Predictor `pred_hidden` 与 residual，所以必须先运行 Predictor 才能决定是否 +fallback。发生 fallback 时同一步又运行 Full-DiT，不能按 fallback 比例直接节省计算。C1-r50 虽然 +取得最佳非 FFFF reference 指标和略高于 FPPF 的 Core5，但均时比 FPPF 长约 `18.05%`，因此不是 +FPPF 的 Pareto 替代。 + +当前结论是：全局 C1 threshold 不应直接替代 FPPF。下一版应固定 P3 Full，以 FPPF 为基础，仅用 +C1 决定 P1/P2 是否额外 fallback，并为 P1、P2 分别校准阈值。优先测试总 fallback 40%～50% 的 +hybrid schedule；如果目标是实际降低计算，还需要改用 Predictor 前即可获得的 pre-routing feature， +否则 fallback step 会重复支付 Predictor 和 Full-DiT 成本。 + +## 10. Torch 2.6 固定环境复评（2026-08-21） + +### 10.1 协议与结果边界 + +本节统一使用固定环境： + +```text +Python: /mnt/local_nvme/cz/envs/hy_worldplay_july20260720/bin/python +PyTorch: 2.6.0+cu124 +CUDA runtime: 12.4 +FlashAttention: 未安装 +``` + +评测继续使用 validation25、seed 0、每组 25 pairs × 4 actions = 100 个视频，并与 7 月固定环境生成 +的 Full-DiT 视频比较。所有新目录均带 `oldenv_reval_20260821` 后缀，没有覆盖历史结果。原先共完成 +13 组；其中三个错误环境训练的 previous-condition 结果已于 2026-08-23 删除，三个错误环境训练且 +没有继续研究价值的旧 Gate 结果已于 2026-08-24 删除。本节现在只保留四个小 C1 阈值、 +Wide-C1-r33 和两组 Wide-C2 hybrid。random-exit DMD 按计划跳过。 + +本次只重跑验证，不重训 checkpoint。也没有重建约 318 GiB 的 confidence 离线 prefeature；现有 +confidence head 和训练集阈值保持不变，以固定环境下的实际 fallback 比例衡量 score 分布漂移。 +因此本节只用于判断现有 Confidence checkpoint 在正式环境中的实际表现；其离线 head 与阈值实验 +仍来自 Torch 2.5 环境，相关限制见第 9 节。 + +### 10.2 Confidence 实际 fallback + +阈值保持第 9 节训练集阈值不变。C1 每组有 2100 个动态决策；Wide-C2 hybrid 的 P1/P2 有 1400 +个 confidence 决策，另有 700 个 P3 固定 Full，因此总 fallback 需要把 P3 的 700 次计入。 + +| 策略 | 阈值 | 实际总 fallback | fallback 次数 | P1 / P2 / P3 fallback | +| --- | --- | ---: | ---: | --- | +| C1-r10 | `-3.523589` | 14.48% | 304/2100 | 0.00% / 0.00% / 43.43% | +| C1-r25 | `-3.915276` | 28.67% | 602/2100 | 0.00% / 0.00% / 86.00% | +| C1-r33 | `-4.329443` | 34.00% | 714/2100 | 1.86% / 5.43% / 94.71% | +| Wide-C1-r33 | `-4.308431` | 34.86% | 732/2100 | 3.43% / 7.71% / 93.43% | +| C1-r50 | `-4.798927` | 51.48% | 1081/2100 | 12.86% / 41.57% / 100.00% | +| Wide-C2-Hybrid-r10 | P1 `-4.136674` / P2 `-3.798281` | 41.19% | 865/2100 | 11.86% / 11.71% / 100.00% | +| Wide-C2-Hybrid-r25 | P1 `-4.400577` / P2 `-4.049408` | 51.52% | 1082/2100 | 28.57% / 26.00% / 100.00% | + +固定环境下 fallback 比例与 Torch 2.5 原运行非常接近，最大差异只有约 `0.43` 个百分点。因此下面 +reference 指标的大幅变化不是简单由 fallback 预算增加造成，而是候选视频与 Full reference 终于 +处于相同数值运行环境。 + +### 10.3 Confidence full-reference 指标 + +| 策略 | fallback | PSNR ↑ | Frame PSNR ↑ | SSIM ↑ | LPIPS ↓ | MAE ↓ | MSE ↓ | RMSE ↓ | Temporal MSE ↓ | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| FPPF | 33.33% | 21.5933 | 23.9745 | 0.74668 | 0.12100 | 0.04511 | 0.009692 | 0.09162 | 0.008824 | +| C1-r10 | 14.48% | 20.9305 | 23.2116 | 0.72750 | 0.15512 | 0.04812 | 0.010232 | 0.09574 | 0.008719 | +| C1-r25 | 28.67% | 21.3102 | 23.6701 | 0.74247 | 0.12580 | 0.04566 | 0.009778 | 0.09263 | 0.008837 | +| C1-r33 | 34.00% | 21.5423 | 23.9262 | 0.74944 | 0.11739 | 0.04440 | 0.009421 | 0.09074 | 0.008742 | +| Wide-C1-r33 | 34.86% | **21.7675** | **24.8766** | **0.75477** | **0.11358** | **0.04330** | **0.009114** | **0.08889** | **0.008497** | +| Wide-C2-Hybrid-r10 | 41.19% | ∞ | 31.1778 | 0.79024 | 0.09382 | 0.03703 | 0.007501 | 0.07813 | 0.007187 | +| C1-r50 | 51.48% | ∞ | 32.8428 | 0.81382 | 0.07868 | 0.03233 | 0.006107 | 0.06994 | 0.006150 | +| Wide-C2-Hybrid-r25 | 51.52% | ∞ | **38.7995** | **0.84716** | **0.06542** | **0.02709** | **0.004946** | **0.06060** | **0.004813** | + +`∞` 不是指标失败。C1-r50、Wide-C2-Hybrid-r10、Wide-C2-Hybrid-r25 分别有 `5/5/7` 个视频 +所有动态 step 都采用 Full 输出，因运行环境与 reference 完全一致而得到逐像素零误差。三组的 +median PSNR 分别为 `23.0252/21.9159/23.8008 dB`；固定 FPPF median 为 `21.1858 dB`。含有完全 +匹配样本时，不再使用 arithmetic mean PSNR 排名，应优先看 MSE、LPIPS、SSIM、temporal MSE 和 +median PSNR。 + +在约 1/3 fallback 的近似同预算比较中： + +- C1-r33 的 PSNR 比 FPPF 低 `0.0510 dB`、Frame PSNR 低 `0.0483 dB`，但 SSIM 高 `0.00276`， + LPIPS、MSE、temporal MSE 分别改善 `2.98%/2.79%/0.93%`，已变成混合但接近的结果。 +- Wide-C1-r33 以 `34.86%` fallback 全面超过 FPPF：PSNR `+0.1742 dB`、Frame PSNR + `+0.9021 dB`、SSIM `+0.00809`，LPIPS、MSE、temporal MSE 改善 `6.13%/5.95%/3.71%`。 + 它比固定 FPPF 多 32 次 fallback，因此后续速度比较必须使用本次独立运行的端到端计时。 + +更高预算下，Wide-C2-Hybrid-r10 相对 FPPF 的 SSIM 增加 `0.04356`，LPIPS、MSE、temporal MSE +改善 `22.46%/22.60%/18.55%`。在几乎相同的约 51.5% fallback 下，Wide-C2-Hybrid-r25 明显优于 +C1-r50：SSIM 为 `0.84716` 对 `0.81382`，LPIPS 为 `0.06542` 对 `0.07868`，MSE 为 `0.004946` +对 `0.006107`，temporal MSE 为 `0.004813` 对 `0.006150`。这进一步支持固定 P3 Full、再用 C2 +按 step 选择 P1/P2 的路由策略。 + +本次没有重跑 VBench。第 9.3 节 VBench 对应 Torch 2.5 生成的视频，不能与本节固定环境 reference +结果组合成正式结论。下一步只需要对 Wide-C1-r33 和 Wide-C2-Hybrid-r25 等候选方案补固定环境 +VBench，无需对所有阈值再次全量评测。 + +## 11. Anchor–Transport–Correct（ATC）Predictor 完整消融（2026-08-21～23） + +### 11.1 各版本方法差异 + +这一批实验没有改变 4-step sampler，也没有跳过任何 denoising update；`FPPF/FPPP` 中的 `P` +仍表示用 Predictor 替代一次 Full DiT 计算。所有 ATC 版本均使用 v4 的 `[1,52]` source blocks、 +`previous_condition_mode=same_step`，区别只在 Predictor 内部如何组合当前输入、当前 chunk anchor +和前一 chunk 的同 timestep hidden。 + +| 版本 | Anchor Evolution | 前一 chunk 同 step 信息 | Transport | Gate | Anchor distance | 方法说明 | +| --- | --- | --- | --- | --- | --- | --- | +| Concat Stage-1 | 无显式分解 | 直接使用 | 无 | 无 | 无 | 分别归一化 `e_t/h_a/h_p` 后 concat，再由 MLP 一次性融合，是原始 v4 基线。 | +| ATC Anchor-only | 有 | 不使用 | 无 | 无 | 有 | 只学习从最新 Full anchor 到目标 step 的残差，测量仅靠 denoising 轴演化能恢复多少信息。 | +| ATC Raw-prev | 有 | 用于 Causal Correction | 无，直接输入 `h_p` | token 标量 gate | 有 | 检验同 timestep 跨 chunk 信息本身是否有用，不处理运动造成的空间错位。 | +| ATC Full | 有 | 用于 Causal Correction | 3×3 local cross-attention | token 标量 gate | 有 | 先将前一 chunk 的同 step hidden 对齐到当前 token，再按可靠性门控注入 correction。 | +| ATC No-gate | 有 | 用于 Causal Correction | 3×3 local cross-attention | 固定为 1 | 有 | 强制完整使用 correction，用于检验学习到的 gate 是否过度抑制跨 chunk 信息。 | +| ATC No-distance | 有 | 用于 Causal Correction | 3×3 local cross-attention | token 标量 gate | 无 | 去掉 anchor 到目标 step 的距离条件，用于检验显式距离编码是否必要。 | + +Full ATC 的计算可以用文字概括为：先从当前 chunk 的 Full anchor 预测一个 denoising 演化残差； +再用当前 noisy-state token 查询前一 chunk 的同 timestep token，在局部窗口内完成对齐；随后预测 +跨 chunk correction，并通过每个 token 一个标量 gate 控制 correction 强度。最终 hidden 为： + +```text +预测 hidden = 当前 chunk anchor + anchor 演化残差 + gate × 跨 chunk correction +``` + +因此 ATC 并不是把原来的三路 concat MLP 单纯做大，而是把“denoising 轴演化”和“跨 chunk +同 timestep 信息”显式拆成两个残差来源。原 Concat Predictor 有 `197,218,816` 个可训练参数， +Full ATC 有 `291,649,226` 个，增加 `94,430,410`（`+47.9%`）。 + +### 11.2 训练设置与收敛结果 + +五个版本均在固定的 Torch 2.6 环境 +`/mnt/local_nvme/cz/envs/hy_worldplay_july20260720/bin/python` 中独立训练；使用相同的 8400 条离线 +transition、8 GPU、每卡 batch 8、gradient accumulation 2（global batch 128）、FP32 trainable +parameters、gradient checkpointing 和 2000 optimizer steps。训练与后续 FFFF/Concat 对照使用同一 +conda 环境。 + +最后约 100 step 的训练均值如下： + +| 版本 | Total loss ↓ | Hidden loss ↓ | Velocity loss ↓ | Gate mean | +| --- | ---: | ---: | ---: | ---: | +| Concat Stage-1 | 0.010459 | 0.011231 | 0.009336 | — | +| ATC Full | 0.009881 | 0.010231 | 0.008858 | 0.05484 | +| ATC Anchor-only | 0.010005 | 0.010533 | 0.008952 | — | +| ATC Raw-prev | 0.009900 | 0.010253 | 0.008874 | 0.05593 | +| ATC No-gate | 0.010068 | **0.009951** | 0.009073 | 1.00000 | +| ATC No-distance | **0.009848** | 0.010463 | **0.008801** | 0.07333 | + +Full ATC 相对 Concat 的训练尾段 total/hidden/velocity loss 分别降低约 +`5.5%/8.9%/5.1%`，说明新增分支确实提升了离线拟合能力；但验证结果表明，更低的训练 loss +并不总是对应更高的无参考感知分数。 + +### 11.3 Full-reference 与 VBench 平均分统一对比 + +下表把 Full-reference 指标和 VBench 平均分放在同一行。`VBench Core5` 是本项目一直采用的 +五项算术平均：`Subject Consistency`、`Background Consistency`、`Motion Smoothness`、 +`Aesthetic Quality`、`Imaging Quality`；它不是 VBench 官方加权总分。除 LPIPS/MSE/Temporal MSE +越低越好外，其余均越高越好。FFFF 是 Full reference 本身，因此像素指标没有比较意义，只列 +VBench 分数。 + +| Schedule | 方法 | PSNR ↑ | Frame PSNR ↑ | SSIM ↑ | LPIPS ↓ | MSE ↓ | Temporal MSE ↓ | VBench Core5 ↑ | +| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| FFFF | Full DiT reference | — | — | — | — | — | — | **0.842613** | +| FPPF | Concat Stage-1 | 21.5933 | 23.9745 | 0.74668 | 0.12100 | 0.009692 | 0.008824 | 0.842022 | +| FPPF | ATC Anchor-only | 21.7135 | 24.0851 | 0.75252 | 0.11828 | 0.009485 | 0.008601 | 0.842017 | +| FPPF | ATC Raw-prev | 21.7833 | 24.1671 | 0.75568 | 0.11757 | 0.009425 | 0.008821 | 0.841989 | +| FPPF | ATC No-gate | 21.7322 | 24.0900 | 0.75228 | 0.11860 | 0.009456 | 0.008672 | 0.841963 | +| FPPF | ATC Full | 21.8753 | 24.2164 | 0.75733 | 0.11629 | **0.009084** | 0.008665 | **0.842149** | +| FPPF | ATC No-distance | **21.8834** | **24.2312** | **0.75761** | **0.11620** | 0.009220 | **0.008595** | 0.842001 | +| FPPP | Concat Stage-1 | 20.3085 | 22.4461 | 0.69143 | 0.23528 | 0.011888 | 0.008931 | 0.819219 | +| FPPP | ATC Anchor-only | 20.4469 | 22.5692 | 0.69587 | 0.22804 | 0.011555 | 0.008857 | 0.820577 | +| FPPP | ATC Raw-prev | 20.4316 | 22.5527 | 0.69641 | 0.22727 | 0.011595 | 0.008927 | 0.820556 | +| FPPP | ATC Full | 20.3921 | 22.5169 | 0.69594 | 0.22734 | 0.011778 | 0.008952 | 0.820680 | +| FPPP | ATC No-gate | 20.4386 | 22.5584 | 0.69668 | 0.22636 | 0.011630 | 0.008826 | **0.820939** | +| FPPP | ATC No-distance | **20.4551** | **22.5866** | **0.69758** | **0.22587** | **0.011427** | **0.008767** | 0.820629 | + +注：FPPF 的 Concat 行是固定 Torch 2.6 环境 SHA 复评结果；FPPP 的 Concat 行沿用已有 Stage-1 +正式结果。ATC 的 10 组生成、Full-reference 评测和 VBench 均在上述固定环境下执行，每组 100 个 +视频，候选视频与 FFFF reference 的 prompt、action 和 seed 一致。 + +主要结论： + +- FPPF 下，Full ATC 相对 Concat 的 PSNR/SSIM 分别提高 `0.2820 dB/0.01064`，LPIPS、MSE、 + Temporal MSE 分别降低 `3.89%/6.27%/1.79%`；VBench Core5 小幅提高 `0.000127`，100 个视频中 + 有 54 个视频的 paired Core5 更高。其 Core5 与 FFFF 只差 `0.000464`。 +- FPPP 下，所有 ATC 消融的主要指标都整体优于 Concat。Full ATC 的 Core5 提高 `0.001461`， + paired Core5 胜出 68/100；No-gate 提高最多，为 `+0.001720`，胜出 74/100。 +- No-distance 在 FPPF 和 FPPP 都取得最佳或接近最佳的 Full-reference 指标，但 Core5 不是最高。 + 这说明当前显式 anchor-distance embedding 没有带来净收益，也说明像素/感知 reference 指标与 + 无参考 VBench 的排序并不完全一致。 +- FPPF 中 Full ATC 优于 Raw-prev，说明局部 transport 有小幅价值；但在连续三个 Predictor step + 的 FPPP 中，这个优势不稳定。Transport attention entropy 约为 `3.520`，接近 36 个候选位置的 + 最大熵 `log(36)=3.584`，说明当前 attention 更像平滑混合，还没有学到很尖锐的局部对应关系。 +- Full ATC 的推理 gate mean 在 FPPF/FPPP 分别只有 `0.0813/0.0656`，近零 gate 占比约 + `61.7%/68.5%`。这与 No-gate 在 FPPP 上反而得到最高 Core5 一致：当前 gate 偏保守，容易把 + correction 过度压低。 + +### 11.4 VBench 8 项明细 + +`Flickering` 一列数值越高表示闪烁越少。为了便于判断平均分由哪些维度贡献，下表同时保留 8 个 +原始维度；Core5 与上一张统一对比表完全相同。 + +| Schedule | 方法 | Subject | Background | Flickering | Motion | Dynamic | Aesthetic | Imaging | Overall | Core5 | +| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| FFFF | Full DiT | 0.94807 | 0.94308 | 0.95845 | 0.98940 | 0.82000 | 0.59849 | 0.73403 | 0.24809 | 0.842613 | +| FPPF | Concat Stage-1 | 0.94746 | 0.94382 | 0.96045 | 0.98968 | 0.80000 | 0.59822 | 0.73093 | 0.24918 | 0.842022 | +| FPPF | ATC Anchor-only | 0.94779 | 0.94363 | 0.96047 | 0.98969 | 0.80000 | 0.59832 | 0.73064 | 0.24892 | 0.842017 | +| FPPF | ATC Raw-prev | 0.94785 | 0.94383 | 0.96042 | 0.98966 | 0.80000 | 0.59808 | 0.73052 | 0.24906 | 0.841989 | +| FPPF | ATC No-gate | 0.94795 | 0.94343 | 0.96034 | 0.98965 | 0.80000 | 0.59755 | 0.73123 | 0.24921 | 0.841963 | +| FPPF | ATC Full | 0.94803 | 0.94292 | 0.96032 | 0.98964 | 0.80000 | 0.59904 | 0.73112 | 0.24917 | **0.842149** | +| FPPF | ATC No-distance | 0.94823 | 0.94378 | 0.96038 | 0.98966 | 0.80000 | 0.59761 | 0.73072 | 0.24885 | 0.842001 | +| FPPP | Concat Stage-1 | 0.92557 | 0.93087 | 0.96548 | 0.98702 | 0.82000 | 0.56381 | 0.68881 | 0.24729 | 0.819219 | +| FPPP | ATC Anchor-only | 0.92683 | 0.93146 | 0.96515 | 0.98700 | 0.82000 | 0.56524 | 0.69235 | 0.24710 | 0.820577 | +| FPPP | ATC Raw-prev | 0.92710 | 0.93184 | 0.96513 | 0.98696 | 0.81000 | 0.56590 | 0.69098 | 0.24712 | 0.820556 | +| FPPP | ATC Full | 0.92722 | 0.93200 | 0.96507 | 0.98697 | 0.81000 | 0.56580 | 0.69141 | 0.24731 | 0.820680 | +| FPPP | ATC No-gate | 0.92716 | 0.93161 | 0.96500 | 0.98695 | 0.82000 | 0.56629 | 0.69269 | 0.24794 | **0.820939** | +| FPPP | ATC No-distance | 0.92731 | 0.93120 | 0.96503 | 0.98697 | 0.82000 | 0.56609 | 0.69158 | 0.24720 | 0.820629 | + +ATC 虽然将 Stage-1 Concat FPPP 的 Core5 从 `0.819219` 提升到最高 `0.820939`，但仍低于此前 +detached chunk+timestep on-policy step 2000 的 `0.827286`。因此目前最清楚的研究结论是：ATC +架构提高了离线拟合和 reference fidelity，也小幅改善了连续替代的 VBench，但仅靠架构还不能 +解决 rollout distribution shift。下一轮最有价值的组合实验是用 `No-gate` 或 `No-distance` ATC +初始化，再进行 same-state on-policy rollout 训练；无需继续扩大离线 concat 容量。 + +## 12. ATC（Causal-Forcing 对齐版全局 Transport）Stage-1 重训 + on-policy rollout 的 checkpoint 扫描（2026-09-06～10） + +2026-09-06 将 `models/predictor_atc.py` 的 State Transport 对齐到 Causal-Forcing-a 的实现（全局 RoPE cross-chunk +attention，见 model.md），并在 4 卡上重训：Stage-1（bs 8×2，global batch 64，2000 steps；训练尾段 total +0.009805，与 8 月局部窗口版 0.009881 持平）→ on-policy detached chunk+timestep FPPP rollout（每卡 1 条轨迹 × +累积 2，flr 5e-5 / blr 5e-6，2000 steps）。下表为 validation25 上 rollout step 500/800/1000/1500/2000 的 +FPPF/FPPP，reference 为同环境 FFFF 视频；Stage-1 行为本次重训的 step 2000。VBench 8 维与 Core5 在同一次 +VBench 运行中得到（`vbench/validation25_atc_global_rollout_ckpt_sweep`）。 + +| Schedule | 版本 | PSNR ↑ | Frame PSNR ↑ | SSIM ↑ | LPIPS ↓ | MSE ↓ | Temporal MSE ↓ | VBench Core5 ↑ | +| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| FPPF | Stage-1 global-transport step 2000 | 21.5008 | 23.5230 | 0.74390 | 0.12568 | 0.009872 | 0.008910 | 0.841845 | +| FPPF | rollout step 500 | 21.6325 | 24.0124 | 0.74799 | 0.12052 | 0.009588 | 0.008963 | 0.842119 | +| FPPF | rollout step 800 | 21.8962 | 24.2536 | 0.75700 | 0.11631 | 0.009000 | 0.008527 | 0.842323 | +| FPPF | rollout step 1000 | 21.7462 | 24.1287 | 0.75372 | 0.11760 | 0.009407 | 0.008894 | 0.842230 | +| FPPF | rollout step 1500 | **22.0260** | **24.3491** | **0.76285** | **0.11372** | **0.008747** | **0.008182** | 0.842264 | +| FPPF | rollout step 2000 | 21.9842 | 24.3144 | 0.76105 | 0.11441 | 0.008873 | 0.008360 | **0.842402** | +| FPPP | Stage-1 global-transport step 2000 | 20.3408 | 22.2056 | 0.69305 | 0.23281 | 0.011831 | **0.008907** | 0.819771 | +| FPPP | rollout step 500 | 20.2365 | 22.4140 | 0.69556 | **0.20630** | 0.012013 | 0.009418 | **0.828711** | +| FPPP | rollout step 800 | **20.4208** | 22.5659 | **0.69912** | 0.20867 | **0.011610** | 0.009209 | 0.827394 | +| FPPP | rollout step 1000 | 20.3430 | 22.4965 | 0.69521 | 0.20968 | 0.011891 | 0.009393 | 0.827097 | +| FPPP | rollout step 1500 | 20.3851 | 22.5502 | 0.69888 | 0.20689 | 0.011702 | 0.009269 | 0.827710 | +| FPPP | rollout step 2000 | 20.4148 | **22.5719** | 0.69893 | 0.20775 | 0.011652 | 0.009287 | 0.827682 | + +对照 7.2 的 Concat 版 detached chunk+timestep on-policy step 2000（FPPF 21.7289 / 0.75234 / 0.11818 / 0.841941； +FPPP 20.3006 / 0.69430 / 0.21146 / 0.827286）： + +- FPPF：ATC rollout 从 step 800 起全面超过 Concat rollout；step 1500/2000 的 PSNR 高 `0.26～0.30 dB`、 + LPIPS 低 `3.2%～3.8%`，Core5 0.8424 也高于 Concat 的 0.8419 并接近 FFFF 的 0.8426。 +- FPPP：ATC rollout 的 LPIPS（0.206～0.210）低于 Concat rollout 的 0.211，Core5 持平略高（0.8277～0.8287 对 + 0.8273）；PSNR/SSIM 与 Concat 在噪声范围内。FPPP 的 Core5 提升主要来自 imaging_quality（0.689→0.705～0.709） + 和 aesthetic_quality（0.565→0.575～0.576），dynamic_degree 保持 0.82。 +- checkpoint 选择：FPPF 在 step 1500 达到像素指标最优、step 2000 达到 Core5 最优，两者差异极小；FPPP 各 + checkpoint 在 500 步之后基本饱和（Core5 0.8271～0.8287，LPIPS 0.206～0.210）。综合取 **step 2000** 作为 + 该版本的正式权重（`predictor_step_02000.safetensors`），step 1500 可作等价备选。 +- rollout 训练期间 ATC 的 token gate 均值保持 0.03～0.04，Transport 注意力熵 8.6～8.7（接近均匀）， + 与 8 月局部窗口版一致：在 HY 上跨 chunk correction 分支贡献有限，提升主要来自 Anchor Evolution 与 + rollout 训练本身。 + +## 13. 总结与下一步 + +当前结果支持以下判断： + +1. v4 `[1,52]` 是当前训练集连续替代 step 1、2 时质量最好的版本。 +2. 在相同 `[0,53]` 初始化下，v4 明显优于 predictor-disca，但该对比同时改变了 explicit + previous-chunk hidden、Context Vision K/V 和首 chunk 调度，不能单独归因于 previous hidden。 + 新的等预算 `none` 结果表明 explicit previous hidden 并非当前 v4 保持 FPPF reference fidelity + 的必要条件，AR Context K/V 和首 chunk Full warmup 很可能是更主要的因素。 +3. predictor-disca 的训练速度和显存占用更优，但较低的单步训练 loss 没有转化成最佳视频质量。 +4. Direct Reuse 的主要收益来自减少 step transformer 计算；当前留出集上端到端加速约 13.6%。 +5. v4 FP32-master `[1,52]` 是正式 Stage-1 基线；detached chunk+timestep step 2000 已在 validation25 + FPPF 的五项 reference 指标上全部超过它。下一项关键实验是用相同 action、seed 和 Full reference + 补跑 50-pair test。 +6. 完整训练集 detached timestep on-policy 在 validation25 FPPP 上使 LPIPS 改善 7.06%，同时已测 + VBench 五维平均优于 Stage-1 FPPP；这支持“reference trajectory 偏移不等于无参考观感退化” + 的判断。 +7. detached chunk+timestep step 1000 在 FPPP 的 PSNR、SSIM、MSE 和 temporal MSE 上比 step 2000 + 更均衡；step 2000 则取得更低的 FPPP LPIPS，并使 FPPF 五项 reference 指标全部超过 step 1000 + 和 Stage-1。其 FPPP VBench 五维均高于 step 1000，说明继续训练主要强化了无参考感知质量与 + Full step 接管兼容性，但没有让 FPPP 的 reference 和时序指标全部单调改善。 +8. Stage-1 continued offline teacher-forcing 的公平对照在 FPPF 五项 reference 指标上进一步超过 + chunk+timestep step 2000，在 FPPP 也取得更高 PSNR/SSIM 和更低 temporal MSE；但其 FPPP LPIPS + 及 Subject、Background、Aesthetic、Imaging 均明显不如 on-policy rollout。这把两类收益区分得 + 更清楚：离线继续训练更贴近 Full reference，on-policy rollout 更有利于三步连续替代时的无参考 + 感知质量。 +9. detached rollout + offline FFFF targets 取得了当前最高的 FPPF/FPPP PSNR 和最低的 + MSE/temporal MSE，但 FPPP LPIPS 升至 `0.32174`，VBench 五项平均降至 `0.776892`。固定 + FFFF target 与 student rollout 状态发生错配，说明不能用更好的 reference 像素误差单独判断这类 + rollout 监督有效；当前最终模型仍应优先选择 same-state on-policy step 2000。 +10. 固定环境复评推翻了原先“约 1/3 fallback 的 Wide-C1 不如 FPPF”的在线视频结论。 + Wide-C1-r33 在 `34.86%` fallback 下 PSNR、SSIM、LPIPS、MSE 和 temporal MSE 全部优于 FPPF； + C1-r33 则是 PSNR 略低、其余主要误差指标略优。第 9.3 节旧 VBench 尚不能用于该固定环境结论。 +11. ATC 五组架构消融均已完成 2000-step 训练以及 FPPF/FPPP 的 reference 和 VBench 评测。Full + ATC 在 FPPF reference 上稳定超过 Concat；FPPP 中 No-gate 的 VBench Core5 最高，No-distance + 的 reference 指标最佳。当前 gate 过度保守、distance embedding 无净收益、local transport + 对 FPPP 的增益不稳定，下一步应优先把 No-gate/No-distance 与 same-state on-policy 训练结合。 +12. previous-condition 输入消融已在 Torch 2.6 无 FlashAttention 环境中从零重训并重新评测。 + 原始 `none` 使用 `16 Full + 16 Predictor`，其余版本使用 `18 Full + 14 Predictor`，因此原先 + `same_step > previous_step > both > none` 不能作为纯输入消融结论。固定 `none` 首 chunk FFFF + 后，step 2000 在八项 reference 指标上全部略优于 `same_step`，且 step 300/500/1000/2000 全部 + 单调改善；VBench Core5 则不单调，step 500 最高。`previous_step` 的 Core5 比 `same_step` 仅高 + `0.000131`，等预算 `none` step 2000 仅低 `0.000154`，显式 previous hidden 的必要性仍没有 + 得到支持，但需要分 chunk/运动分桶继续验证。 +13. 约 51.5% fallback 下，Wide-C2-Hybrid-r25 明显优于同预算 C1-r50，继续支持“P3 固定 Full， + C2 分 step 路由 P1/P2”。下一步优先为 Wide-C1-r33 和 Wide-C2-Hybrid-r25 补固定环境 VBench， + 并在需要真实加速时改用 Predictor 前即可获得的 pre-routing feature。 + +## 14. 结果与产物路径 + +训练 checkpoint： + +```text +checkpoints/predictor_v4_atc-full_transport-global-chunk_blocks1-52_fulltrain100_fp32master_4gpu_bs8x2_flr1e-4_blr1e-5_steps2000 +checkpoints/predictor_v4_atc-global-chunk_detached-chunk-timestep-rollout-onpolicy-fppp_blocks1-52_fulltrain100_seed20260721_fp32master_4gpu_flr5e-5_blr5e-6_schedsteps2000_swarmup100_bwarmup100linear_bs1x2 +checkpoints/predictor_v4_prefeature_blocks0-53_bf16_8gpu_flr1e-4_blr1e-5_schedsteps2000_swarmup100_bwarmup100_swanlab +checkpoints/predictor_v4_prefeature_blocks1-52_bf16_8gpu_flr1e-4_blr1e-5_schedsteps2000_swarmup100_bwarmup100_swanlab +checkpoints/predictor_v4_prefeature_blocks1-52_fp32master_8gpu_flr1e-4_blr1e-5_schedsteps2000_swarmup100_bwarmup100_swanlab +checkpoints/predictor_v4_prefeature-continued-from-stage1step2000_blocks1-52_fulltrain100_fp32master_8gpu_flr5e-5_blr5e-6_schedsteps2000_swarmup100_bwarmup100linear_bs16_swanlab +checkpoints/predictor_disca_blocks0-53_bf16_8gpu_flr1e-4_blr1e-5_schedsteps2000_swarmup100_bwarmup100_swanlab +``` + +第 12 节（对齐版 ATC rollout checkpoint 扫描）的评测产物： + +```text +datasets/hyworldplay_predictor_vbench_val25_test50/evaluations/eval_validation25_atc-global-chunk-rollout_schedule-{fppf,fppp}_blocks1-52_ckptstep{500,800,1000,1500,2000} +datasets/hyworldplay_predictor_vbench_val25_test50/evaluations/eval_validation25_atc-full_transport-global-chunk_4gpu_bs8x2_schedule-{fppf,fppp}_blocks1-52_step2000 +datasets/hyworldplay_predictor_vbench_val25_test50/vbench/validation25_atc_global_rollout_ckpt_sweep +checkpoints/predictor_v4_atc-global-chunk_detached-chunk-timestep-rollout-onpolicy-fppp_.../validation25_sweep_summary.{md,json} +``` + +对应脚本：`tools/launch_predictor_v4_atc_stage1_8gpu.sh`（Stage-1，`PREDICTOR_NPROC/BATCH_SIZE/GRAD_ACCUM`）、 +`tools/launch_predictor_v4_atc_detached_chunk_timestep_rollout.sh`（rollout）、 +`tools/run_atc_global_rollout_ckpt_sweep_validation25.sh`（生成 + 全参考指标 + VBench + 汇总）。 + +训练集视频指标与日志： + +```text +outputs/predictor_v4/eval_train100_schedule-fppf_blocks0-53_ckptstep2000_flr1e-4_blr1e-5_schedsteps2000_swarmup100_bwarmup100 +outputs/predictor_v4/eval_train100_schedule-fppf_blocks1-52_ckptstep2000_flr1e-4_blr1e-5_schedsteps2000_swarmup100_bwarmup100 +outputs/predictor_disca/eval_train100_schedule-fppf_blocks0-53_ckptstep2000_flr1e-4_blr1e-5_schedsteps2000_swarmup100_bwarmup100 +``` + +留出集清单、视频、日志和指标： + +```text +datasets/hyworldplay_predictor_vbench_val25_test50 +datasets/hyworldplay_predictor_vbench_val25_test50/metrics.json +datasets/hyworldplay_predictor_vbench_val25_test50/logs +datasets/hyworldplay_predictor_vbench_val25_test50/evaluations/eval_validation25_v4_blocks1-52_fp32master_ckptstep2000_flr1e-4_blr1e-5_schedsteps2000_swarmup100_bwarmup100 +``` + +detached timestep on-policy： + +```text +checkpoints/predictor_v4_detached-timestep-rollout-onpolicy_blocks1-52_fulltrain100_seed20260721_fp32master_8gpu_flr5e-5_blr5e-6_schedsteps2000_swarmup100_bwarmup100linear_bs1_swanlab +datasets/hyworldplay_predictor_vbench_val25_test50/evaluations/eval_validation25_detached-timestep-rollout-onpolicy-fulltrain100_schedule-fppf_blocks1-52_ckptstep1000_flr5e-5_blr5e-6_schedsteps2000_swarmup100_bwarmup100linear_bs1 +datasets/hyworldplay_predictor_vbench_val25_test50/evaluations/eval_validation25_detached-timestep-rollout-onpolicy-fulltrain100_schedule-fppp_blocks1-52_ckptstep1000_flr5e-5_blr5e-6_schedsteps2000_swarmup100_bwarmup100linear_bs1 +``` + +detached chunk+timestep on-policy： + +```text +checkpoints/predictor_v4_detached-chunk-timestep-rollout-onpolicy-fppp_blocks1-52_fulltrain100_seed20260721_fp32master_8gpu_flr5e-5_blr5e-6_schedsteps2000_swarmup100_bwarmup100linear_bs1_swanlab +datasets/hyworldplay_predictor_vbench_val25_test50/evaluations/eval_validation25_detached-chunk-timestep-rollout-onpolicy-fppp-fulltrain100_schedule-fppf_blocks1-52_ckptstep1000_flr5e-5_blr5e-6_schedsteps2000_swarmup100_bwarmup100linear_bs1 +datasets/hyworldplay_predictor_vbench_val25_test50/evaluations/eval_validation25_detached-chunk-timestep-rollout-onpolicy-fppp-fulltrain100_schedule-fppp_blocks1-52_ckptstep1000_flr5e-5_blr5e-6_schedsteps2000_swarmup100_bwarmup100linear_bs1 +datasets/hyworldplay_predictor_vbench_val25_test50/evaluations/eval_validation25_detached-chunk-timestep-rollout-onpolicy-fppp-fulltrain100_schedule-fppf_blocks1-52_ckptstep2000_flr5e-5_blr5e-6_schedsteps2000_swarmup100_bwarmup100linear_bs1 +datasets/hyworldplay_predictor_vbench_val25_test50/evaluations/eval_validation25_detached-chunk-timestep-rollout-onpolicy-fppp-fulltrain100_schedule-fppp_blocks1-52_ckptstep2000_flr5e-5_blr5e-6_schedsteps2000_swarmup100_bwarmup100linear_bs1 +``` + +detached chunk+timestep rollout + offline FFFF targets： + +```text +checkpoints/predictor_v4_detached-chunk-timestep-rollout-offline-ffff-targets-fppp_blocks1-52_fulltrain100_seed20260721_fp32master_8gpu_flr5e-5_blr5e-6_schedsteps2000_swarmup100_bwarmup100linear_bs1_swanlab +datasets/hyworldplay_predictor_vbench_val25_test50/evaluations/eval_validation25_detached-chunk-timestep-rollout-offline-ffff-targets-fppp-fulltrain100_schedule-fppf_blocks1-52_ckptstep2000_flr5e-5_blr5e-6_schedsteps2000_swarmup100_bwarmup100linear_bs1 +datasets/hyworldplay_predictor_vbench_val25_test50/evaluations/eval_validation25_detached-chunk-timestep-rollout-offline-ffff-targets-fppp-fulltrain100_schedule-fppp_blocks1-52_ckptstep2000_flr5e-5_blr5e-6_schedsteps2000_swarmup100_bwarmup100linear_bs1 +``` + +Stage-1 continued offline teacher-forcing： + +```text +datasets/hyworldplay_predictor_vbench_val25_test50/evaluations/eval_validation25_stage1-continued-offline-tf_schedule-fppf_blocks1-52_ckptstep2000-initstage1step2000_flr5e-5_blr5e-6_schedsteps2000_swarmup100_bwarmup100linear_bs16 +datasets/hyworldplay_predictor_vbench_val25_test50/evaluations/eval_validation25_stage1-continued-offline-tf_schedule-fppp_blocks1-52_ckptstep2000-initstage1step2000_flr5e-5_blr5e-6_schedsteps2000_swarmup100_bwarmup100linear_bs16 +``` + +VBench 五项采用的原始结果、日志、逐组汇总和 paired comparison： + +```text +datasets/hyworldplay_predictor_vbench_val25_test50/vbench/validation25_core8 +datasets/hyworldplay_predictor_vbench_val25_test50/vbench/validation25_chunk_timestep_step2000 +datasets/hyworldplay_predictor_vbench_val25_test50/vbench/validation25_stage1_continued_offline_tf_step2000 +datasets/hyworldplay_predictor_vbench_val25_test50/vbench/validation25_offline_ffff_targets_step2000 +``` + +C1 confidence head、动态 threshold 视频、reference 指标与 VBench 8 项： + +```text +confidence_validation/checkpoints/confidence_c1.pt +confidence_validation_wide4096x2_c1/checkpoints/confidence_c1.pt +confidence_validation_wide4096x2_c1/summary.json +confidence_validation_wide4096x2_c2/checkpoints/confidence_c2.pt +confidence_validation_wide4096x2_c2/summary.json +confidence_validation/c1_threshold_validation25/report.md +confidence_validation/c1_threshold_validation25/summary.csv +confidence_validation/c1_threshold_validation25/summary.json +datasets/hyworldplay_predictor_vbench_val25_test50/evaluations/eval_validation25_stage1-c1-threshold-r10_blocks1-52_fp32master_step2000 +datasets/hyworldplay_predictor_vbench_val25_test50/evaluations/eval_validation25_stage1-c1-threshold-r25_blocks1-52_fp32master_step2000 +datasets/hyworldplay_predictor_vbench_val25_test50/evaluations/eval_validation25_stage1-c1-threshold-r33_blocks1-52_fp32master_step2000 +datasets/hyworldplay_predictor_vbench_val25_test50/evaluations/eval_validation25_stage1-wide4096x2-c1-threshold-r33_blocks1-52_fp32master_step2000 +datasets/hyworldplay_predictor_vbench_val25_test50/evaluations/eval_validation25_stage1-c1-threshold-r50_blocks1-52_fp32master_step2000 +datasets/hyworldplay_predictor_vbench_val25_test50/evaluations/eval_validation25_stage1-wide4096x2-c2-hybrid-p3full-p12r10_blocks1-52_fp32master_step2000 +datasets/hyworldplay_predictor_vbench_val25_test50/evaluations/eval_validation25_stage1-wide4096x2-c2-hybrid-p3full-p12r25_blocks1-52_fp32master_step2000 +datasets/hyworldplay_predictor_vbench_val25_test50/vbench/validation25_c1_thresholds +``` + +Stage-1 previous-condition Torch 2.6 无 FlashAttention 重训与 validation25 FPPF 评测： + +```text +checkpoints/predictor_v4_stage1_previous_condition_ablation_seed0/none +checkpoints/predictor_v4_stage1_previous_condition_ablation_seed0/previous_step +checkpoints/predictor_v4_stage1_previous_condition_ablation_seed0/both +logs/predictor_v4_stage1_previous_condition_ablation_seed0 +logs/stage1_previous_condition_validation_torch26-retrain_20260824 +datasets/hyworldplay_predictor_vbench_val25_test50/evaluations/eval_validation25_previous-condition-none_torch26-retrain_20260824 +datasets/hyworldplay_predictor_vbench_val25_test50/evaluations/eval_validation25_previous-condition-previous-step_torch26-retrain_20260824 +datasets/hyworldplay_predictor_vbench_val25_test50/evaluations/eval_validation25_previous-condition-both_torch26-retrain_20260824 +datasets/hyworldplay_predictor_vbench_val25_test50/evaluations/eval_validation25_previous-condition-none-step0300_force-first-chunk-ffff_torch26_20260824 +datasets/hyworldplay_predictor_vbench_val25_test50/evaluations/eval_validation25_previous-condition-none-step0500_force-first-chunk-ffff_torch26_20260824 +datasets/hyworldplay_predictor_vbench_val25_test50/evaluations/eval_validation25_previous-condition-none-step1000_force-first-chunk-ffff_torch26_20260824 +datasets/hyworldplay_predictor_vbench_val25_test50/evaluations/eval_validation25_previous-condition-none_force-first-chunk-ffff_torch26_20260824 +datasets/hyworldplay_predictor_vbench_val25_test50/vbench/validation25_previous_condition_torch26_20260824 +datasets/hyworldplay_predictor_vbench_val25_test50/vbench/validation25_previous_condition_none_step0300_torch26_20260825 +logs/none_force_first_chunk_ffff_checkpoint_validation_20260824 +tools/run_stage1_previous_condition_ablation.sh +tools/run_stage1_previous_condition_retrain_validation.sh +tools/run_vbench_previous_condition_validation25.sh +tools/run_vbench_previous_condition_none_step0300_validation25.sh +``` + +Torch 2.6 固定环境复评脚本、状态和结果： + +```text +tools/run_oldenv_validation_reruns_sequential.sh +tools/run_confidence_validation25_reference_single.sh +logs/oldenv_reval_20260821/status.tsv +logs/oldenv_reval_20260821/sequential.log +datasets/hyworldplay_predictor_vbench_val25_test50/evaluations/*_oldenv_reval_20260821 +``` + +ATC 五组训练 checkpoint、FPPF/FPPP 评测、VBench 汇总和运行脚本： + +```text +checkpoints/predictor_v4_atc-full_blocks1-52_fulltrain100_fp32master_8gpu_flr1e-4_blr1e-5_steps2000 +checkpoints/predictor_v4_atc-anchor_only_blocks1-52_fulltrain100_fp32master_8gpu_flr1e-4_blr1e-5_steps2000 +checkpoints/predictor_v4_atc-raw_prev_blocks1-52_fulltrain100_fp32master_8gpu_flr1e-4_blr1e-5_steps2000 +checkpoints/predictor_v4_atc-no_gate_blocks1-52_fulltrain100_fp32master_8gpu_flr1e-4_blr1e-5_steps2000 +checkpoints/predictor_v4_atc-no_distance_blocks1-52_fulltrain100_fp32master_8gpu_flr1e-4_blr1e-5_steps2000 +datasets/hyworldplay_predictor_vbench_val25_test50/evaluations/eval_validation25_atc-*_schedule-fppf_blocks1-52_step2000_oldenv +datasets/hyworldplay_predictor_vbench_val25_test50/evaluations/eval_validation25_atc-*_schedule-fppp_blocks1-52_step2000_oldenv +datasets/hyworldplay_predictor_vbench_val25_test50/vbench/validation25_atc_ablation_oldenv/summary.json +datasets/hyworldplay_predictor_vbench_val25_test50/vbench/validation25_atc_ablation_oldenv/summary.csv +datasets/hyworldplay_predictor_vbench_val25_test50/vbench/validation25_atc_ablation_oldenv/summary.md +logs/atc_full_experiment_20260821.log +logs/atc_validation_20260822.log +tools/run_predictor_v4_atc_full_experiment.sh +tools/run_vbench_atc_validation25.sh +``` diff --git a/hyvideo/__init__.py b/hyvideo/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..90903539cb122cde708922f06fd279eede277517 --- /dev/null +++ b/hyvideo/__init__.py @@ -0,0 +1,46 @@ +# Licensed under the TENCENT HUNYUAN COMMUNITY LICENSE AGREEMENT (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://github.com/Tencent-Hunyuan/HunyuanVideo-1.5/blob/main/LICENSE +# +# Unless and only to the extent required by applicable law, the Tencent Hunyuan works and any +# output and results therefrom are provided "AS IS" without any express or implied warranties of +# any kind including any warranties of title, merchantability, noninfringement, course of dealing, +# usage of trade, or fitness for a particular purpose. You are solely responsible for determining the +# appropriateness of using, reproducing, modifying, performing, displaying or distributing any of +# the Tencent Hunyuan works or outputs and assume any and all risks associated with your or a +# third party's use or distribution of any of the Tencent Hunyuan works or outputs and your exercise +# of rights and permissions under this agreement. +# See the License for the specific language governing permissions and limitations under the License. + +import os +import socket +from .commons import get_gpu_memory + +if "TOKENIZERS_PARALLELISM" not in os.environ: + os.environ["TOKENIZERS_PARALLELISM"] = "false" + + +def find_free_port(): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.bind(("localhost", 0)) + port = sock.getsockname()[1] + sock.close() + return port + + +def _init_dist_env(): + if "RANK" not in os.environ: + os.environ["RANK"] = "0" + if "WORLD_SIZE" not in os.environ: + os.environ["WORLD_SIZE"] = "1" + if "LOCAL_RANK" not in os.environ: + os.environ["LOCAL_RANK"] = "0" + if "MASTER_ADDR" not in os.environ: + os.environ["MASTER_ADDR"] = "localhost" + if "MASTER_PORT" not in os.environ: + os.environ["MASTER_PORT"] = str(find_free_port()) + + +_init_dist_env() diff --git a/hyvideo/commons/__init__.py b/hyvideo/commons/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b55a2a19d36d80f3138686b0afda3648e567ea24 --- /dev/null +++ b/hyvideo/commons/__init__.py @@ -0,0 +1,266 @@ +# Licensed under the TENCENT HUNYUAN COMMUNITY LICENSE AGREEMENT (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://github.com/Tencent-Hunyuan/HunyuanVideo-1.5/blob/main/LICENSE +# +# Unless and only to the extent required by applicable law, the Tencent Hunyuan works and any +# output and results therefrom are provided "AS IS" without any express or implied warranties of +# any kind including any warranties of title, merchantability, noninfringement, course of dealing, +# usage of trade, or fitness for a particular purpose. You are solely responsible for determining the +# appropriateness of using, reproducing, modifying, performing, displaying or distributing any of +# the Tencent Hunyuan works or outputs and assume any and all risks associated with your or a +# third party's use or distribution of any of the Tencent Hunyuan works or outputs and your exercise +# of rights and permissions under this agreement. +# See the License for the specific language governing permissions and limitations under the License. + +import os +import torch +from itertools import repeat +from contextlib import contextmanager +from torch import nn +import collections.abc + + +def _ntuple(n): + """Create a function that converts input to n-tuple.""" + + def parse(x): + if isinstance(x, collections.abc.Iterable) and not isinstance(x, str): + x = tuple(x) + if len(x) == 1: + x = tuple(repeat(x[0], n)) + return x + return tuple(repeat(x, n)) + + return parse + + +# Convenience functions for common tuple sizes +to_1tuple = _ntuple(1) +to_2tuple = _ntuple(2) +to_3tuple = _ntuple(3) +to_4tuple = _ntuple(4) + +PRECISION_TO_TYPE = { + "fp32": torch.float32, + "fp16": torch.float16, + "bf16": torch.bfloat16, +} + +# Default generation pipeline configurations +PIPELINE_CONFIGS = { + "480p_t2v": { + "guidance_scale": 6.0, + "embedded_guidance_scale": None, + "flow_shift": 5.0, + }, + "480p_i2v": { + "guidance_scale": 6.0, + "embedded_guidance_scale": None, + "flow_shift": 5.0, + }, + "720p_t2v": { + "guidance_scale": 6.0, + "embedded_guidance_scale": None, + "flow_shift": 9.0, + }, + "720p_i2v": { + "guidance_scale": 6.0, + "embedded_guidance_scale": None, + "flow_shift": 7.0, + }, + "480p_t2v_distilled": { + "guidance_scale": 1.0, + "embedded_guidance_scale": None, + "flow_shift": 5.0, + }, + "480p_i2v_distilled": { + "guidance_scale": 1.0, + "embedded_guidance_scale": None, + "flow_shift": 5.0, + }, + "720p_t2v_distilled": { + "guidance_scale": 1.0, + "embedded_guidance_scale": None, + "flow_shift": 9.0, + }, + "720p_i2v_distilled": { + "guidance_scale": 1.0, + "embedded_guidance_scale": None, + "flow_shift": 7.0, + }, + "720p_t2v_distilled_sparse": { + "guidance_scale": 1.0, + "embedded_guidance_scale": None, + "flow_shift": 7.0, + }, + "720p_i2v_distilled_sparse": { + "guidance_scale": 1.0, + "embedded_guidance_scale": None, + "flow_shift": 9.0, + }, +} + +# Default super-resolution pipeline configurations +SR_PIPELINE_CONFIGS = { + "720p_sr_distilled": { + "flow_shift": 2.0, + "base_resolution": "480p", + "guidance_scale": 1.0, + "embedded_guidance_scale": None, + "num_inference_steps": 6, + }, + "1080p_sr_distilled": { + "flow_shift": 2.0, + "base_resolution": "720p", + "guidance_scale": 1.0, + "embedded_guidance_scale": None, + "num_inference_steps": 8, + }, +} + +TRANSFORMER_VERSION_TO_SR_VERSION = { + "480p_t2v": "720p_sr_distilled", + "720p_t2v": "1080p_sr_distilled", + "480p_i2v": "720p_sr_distilled", + "720p_i2v": "1080p_sr_distilled", + "480p_t2v_distilled": "720p_sr_distilled", + "720p_t2v_distilled": "1080p_sr_distilled", + "480p_i2v_distilled": "720p_sr_distilled", + "720p_i2v_distilled": "1080p_sr_distilled", + "480p_t2v_distilled_sparse": "720p_sr_distilled", + "720p_t2v_distilled_sparse": "1080p_sr_distilled", + "480p_i2v_distilled_sparse": "720p_sr_distilled", + "720p_i2v_distilled_sparse": "1080p_sr_distilled", +} + + +def is_flash2_available(): + try: + from flash_attn import flash_attn_varlen_qkvpacked_func + + return True + except Exception: + return False + + +def is_flash3_available(): + try: + from flash_attn_interface import ( + flash_attn_varlen_func as flash_attn_varlen_func_v3, + ) # noqa: F401 + + return True + except Exception: + return False + + +def is_flash_available(): + return is_flash2_available() or is_flash3_available() + + +def is_sparse_attn_supported(): + return "nvidia h" in torch.cuda.get_device_properties(0).name.lower() + + +def is_sparse_attn_available(): + if not is_sparse_attn_supported(): + return False + try: + from flex_block_attn import flex_block_attn_func # noqa: F401 + + return True + except Exception: + return False + + +def maybe_fallback_attn_mode(attn_mode, infer_state=None, block_idx=None): + """ + Determine the final attention mode based on configuration and availability. + + Args: + attn_mode: Requested attention mode + infer_state: Inference configuration object (optional) + block_idx: Current block index (optional) + + Returns: + Final attention mode to use + """ + import warnings + + # Check for sageattn and flex-block-attn conflict + enable_sageattn = False + if infer_state is not None: + enable_sageattn = ( + infer_state.enable_sageattn and block_idx in infer_state.sage_blocks_range + ) + + assert not (enable_sageattn and attn_mode == "flex-block-attn"), ( + "SageAttention cannot be used with flex-block-attn mode. " + "Please disable enable_sageattn or use a different attention mode." + ) + + # Use SageAttention if configured + if enable_sageattn: + attn_mode = "sageattn" + return attn_mode + + # Handle flash attention modes + if attn_mode == "flash": + if is_flash3_available(): + attn_mode = "flash3" + elif is_flash2_available(): + attn_mode = "flash2" + else: + warnings.warn("flash is not available. Falling back to torch attention.") + attn_mode = "torch" + elif attn_mode == "flash3": + if not is_flash3_available(): + warnings.warn("flash3 is not available. Falling back to torch attention.") + attn_mode = "torch" + elif attn_mode == "flash2": + if not is_flash2_available(): + warnings.warn("flash2 is not available. Falling back to torch attention.") + attn_mode = "torch" + if attn_mode in ("flex-block-attn"): + from hyvideo.commons import is_sparse_attn_available + + if not is_sparse_attn_available(): + raise ValueError( + f"{attn_mode} is not available for your GPU or flex-block-attn is not properly installed." + ) + return attn_mode + + +@contextmanager +def auto_offload_model(models, device, enabled=True): + from diffusers.hooks.group_offloading import _is_group_offload_enabled + + if enabled: + if isinstance(models, nn.Module): + models = [models] + for model in models: + if model is not None: + model.to(device) + yield + if enabled: + for model in models: + if model is not None: + model.to(torch.device("cpu")) + + +def get_gpu_memory(device=None): + if not torch.cuda.is_available(): + return 0 + device = device if device is not None else torch.cuda.current_device() + props = torch.cuda.get_device_properties(device) + if hasattr(torch.cuda, "get_per_process_memory_fraction"): + memory_fraction = torch.cuda.get_per_process_memory_fraction() + else: + memory_fraction = 1.0 + return props.total_memory * memory_fraction + + +def get_rank(): + return int(os.environ.get("RANK", "0")) diff --git a/hyvideo/commons/infer_state.py b/hyvideo/commons/infer_state.py new file mode 100644 index 0000000000000000000000000000000000000000..44af02b64db78161655b2c2877931f2052da8619 --- /dev/null +++ b/hyvideo/commons/infer_state.py @@ -0,0 +1,76 @@ +# Licensed under the TENCENT HUNYUAN COMMUNITY LICENSE AGREEMENT (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://github.com/Tencent-Hunyuan/HunyuanVideo-1.5/blob/main/LICENSE +# +# Unless and only to the extent required by applicable law, the Tencent Hunyuan works and any +# output and results therefrom are provided "AS IS" without any express or implied warranties of +# any kind including any warranties of title, merchantability, noninfringement, course of dealing, +# usage of trade, or fitness for a particular purpose. You are solely responsible for determining the +# appropriateness of using, reproducing, modifying, performing, displaying or distributing any of +# the Tencent Hunyuan works or outputs and assume any and all risks associated with your or a +# third party's use or distribution of any of the Tencent Hunyuan works or outputs and your exercise +# of rights and permissions under this agreement. +# See the License for the specific language governing permissions and limitations under the License. + +from typing import Optional +from dataclasses import dataclass, field + + +@dataclass +class InferState: + enable_sageattn: bool = False # whether to use SageAttention + sage_blocks_range: Optional[range] = None # block range to use SageAttention + enable_torch_compile: bool = False # whether to use torch compile + + # fp8 gemm related + use_fp8_gemm: bool = False # whether to use fp8 gemm + quant_type: str = "fp8-per-block" # fp8 quantization type + include_patterns: list = field( + default_factory=lambda: ["double_blocks"] + ) # include patterns for fp8 gemm + + # vae related + use_vae_parallel: bool = False # whether to use vae parallel + + +__infer_state = None + + +def parse_range(value): + if "-" in value: + start, end = map(int, value.split("-")) + return list(range(start, end + 1)) + else: + return [int(x) for x in value.split(",")] + + +def initialize_infer_state(args): + global __infer_state + sage_blocks_range = parse_range(args.sage_blocks_range) + # Map CLI argument use_sageattn to internal enable_sageattn field + use_sageattn = getattr(args, "use_sageattn", False) + + # Parse include_patterns from args + include_patterns = getattr(args, "include_patterns", "double_blocks") + if isinstance(include_patterns, str): + # Split by comma and strip whitespace + include_patterns = [p.strip() for p in include_patterns.split(",") if p.strip()] + + __infer_state = InferState( + enable_sageattn=use_sageattn, + sage_blocks_range=sage_blocks_range, + enable_torch_compile=args.enable_torch_compile, + # fp8 gemm related + use_fp8_gemm=args.use_fp8_gemm, + quant_type=args.quant_type, + include_patterns=include_patterns, + # vae related + use_vae_parallel=args.use_vae_parallel, + ) + return __infer_state + + +def get_infer_state(): + return __infer_state diff --git a/hyvideo/commons/parallel_states.py b/hyvideo/commons/parallel_states.py new file mode 100644 index 0000000000000000000000000000000000000000..420096de852869ccc7f8b013aad4eab5572efd68 --- /dev/null +++ b/hyvideo/commons/parallel_states.py @@ -0,0 +1,86 @@ +# Licensed under the TENCENT HUNYUAN COMMUNITY LICENSE AGREEMENT (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://github.com/Tencent-Hunyuan/HunyuanVideo-1.5/blob/main/LICENSE +# +# Unless and only to the extent required by applicable law, the Tencent Hunyuan works and any +# output and results therefrom are provided "AS IS" without any express or implied warranties of +# any kind including any warranties of title, merchantability, noninfringement, course of dealing, +# usage of trade, or fitness for a particular purpose. You are solely responsible for determining the +# appropriateness of using, reproducing, modifying, performing, displaying or distributing any of +# the Tencent Hunyuan works or outputs and assume any and all risks associated with your or a +# third party's use or distribution of any of the Tencent Hunyuan works or outputs and your exercise +# of rights and permissions under this agreement. +# See the License for the specific language governing permissions and limitations under the License. + +import os +from dataclasses import dataclass + +import torch.distributed as dist +from torch.distributed.device_mesh import init_device_mesh + + +@dataclass +class ParallelDims: + sp: int = 1 + world_size: int = -1 + + def __post_init__(self): + if self.world_size == -1: + if dist.is_initialized(): + self.world_size = dist.get_world_size() + else: + self.world_size = int(os.getenv("WORLD_SIZE", "1")) + self.build_mesh("cuda") + + def build_mesh(self, device_type): + assert self.world_size % self.sp == 0, "world_size must be divisible by sp" + mesh = init_device_mesh( + device_type, + [self.world_size // self.sp, self.sp], + mesh_dim_names=["dp", "sp"], + ) + self.world_mesh = mesh + return mesh + + @property + def sp_enabled(self): + return self.sp > 1 + + @property + def sp_group(self): + return self.world_mesh["sp"].get_group() + + @property + def sp_mesh(self): + return self.world_mesh["sp"] + + @property + def sp_rank(self): + if self.sp_enabled: + return self.world_mesh["sp"].get_local_rank() + else: + return dist.get_rank() + + @property + def dp_enabled(self): + return self.sp > 1 + + +__parallel_dims = None + + +def initialize_parallel_state( + sp: int = 1, +): + global __parallel_dims + __parallel_dims = ParallelDims(sp=sp) + return __parallel_dims + + +def get_parallel_state(): + if __parallel_dims is None: + # create default parallel states (without enabling any parallelism) + initialize_parallel_state() + return __parallel_dims diff --git a/hyvideo/generate.py b/hyvideo/generate.py new file mode 100644 index 0000000000000000000000000000000000000000..7e15770e56b0df4e3ba8a6ea165520e78b324fa2 --- /dev/null +++ b/hyvideo/generate.py @@ -0,0 +1,1172 @@ +# Licensed under the TENCENT HUNYUAN COMMUNITY LICENSE AGREEMENT (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://github.com/Tencent-Hunyuan/HunyuanVideo-1.5/blob/main/LICENSE +# +# Unless and only to the extent required by applicable law, the Tencent Hunyuan works and any +# output and results therefrom are provided "AS IS" without any express or implied warranties of +# any kind including any warranties of title, merchantability, noninfringement, course of dealing, +# usage of trade, or fitness for a particular purpose. You are solely responsible for determining the +# appropriateness of using, reproducing, modifying, performing, displaying or distributing any of +# the Tencent Hunyuan works or outputs and assume any and all risks associated with your or a +# third party's use or distribution of any of the Tencent Hunyuan works or outputs and your exercise +# of rights and permissions under this agreement. +# See the License for the specific language governing permissions and limitations under the License. + +import os + +if "PYTORCH_CUDA_ALLOC_CONF" not in os.environ: + os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True" + +import loguru +import torch +import argparse +import einops +import imageio +import json +import time +import numpy as np +from scipy.spatial.transform import Rotation as R +from PIL import Image, ImageDraw, ImageFont +try: + from moviepy.editor import VideoFileClip, VideoClip +except ImportError: # Optional: only needed by the keyboard-overlay helpers. + VideoFileClip = None + VideoClip = None + +from hyvideo.pipelines.worldplay_video_pipeline import HunyuanVideo_1_5_Pipeline +from hyvideo.commons.parallel_states import initialize_parallel_state +from hyvideo.commons.infer_state import initialize_infer_state +from hyvideo.generate_custom_trajectory import generate_camera_trajectory_local + +parallel_dims = initialize_parallel_state(sp=int(os.environ.get("WORLD_SIZE", "1"))) +torch.cuda.set_device(int(os.environ.get("LOCAL_RANK", "0"))) + +mapping = { + (0, 0, 0, 0): 0, + (1, 0, 0, 0): 1, + (0, 1, 0, 0): 2, + (0, 0, 1, 0): 3, + (0, 0, 0, 1): 4, + (1, 0, 1, 0): 5, + (1, 0, 0, 1): 6, + (0, 1, 1, 0): 7, + (0, 1, 0, 1): 8, +} + + +def one_hot_to_one_dimension(one_hot): + y = torch.tensor([mapping[tuple(row.tolist())] for row in one_hot]) + return y + + +def parse_pose_string(pose_string): + """ + Parse pose string to motions list. + Format: "w-3, right-0.5, d-4" + - w: forward movement + - s: backward movement + - a: left movement + - d: right movement + - up: pitch up rotation + - down: pitch down rotation + - left: yaw left rotation + - right: yaw right rotation + - number after dash: duration in latents + + Args: + pose_string: str, comma-separated pose commands + + Returns: + list of dict: motions for generate_camera_trajectory_local + """ + # Movement amount per frame + forward_speed = 0.08 # units per frame + yaw_speed = np.deg2rad(3) # radians per frame + pitch_speed = np.deg2rad(3) # radians per frame + + motions = [] + commands = [cmd.strip() for cmd in pose_string.split(",")] + + for cmd in commands: + if not cmd: + continue + + parts = cmd.split("-") + if len(parts) != 2: + raise ValueError( + f"Invalid pose command: {cmd}. Expected format: 'action-duration'" + ) + + action = parts[0].strip() + try: + duration = float(parts[1].strip()) + except ValueError: + raise ValueError(f"Invalid duration in command: {cmd}") + + num_frames = int(duration) + + # Parse action and create motion dict + if action == "w": + # Forward + for _ in range(num_frames): + motions.append({"forward": forward_speed}) + elif action == "s": + # Backward + for _ in range(num_frames): + motions.append({"forward": -forward_speed}) + elif action == "a": + # Left + for _ in range(num_frames): + motions.append({"right": -forward_speed}) + elif action == "d": + # Right + for _ in range(num_frames): + motions.append({"right": forward_speed}) + elif action == "up": + # Pitch up + for _ in range(num_frames): + motions.append({"pitch": pitch_speed}) + elif action == "down": + # Pitch down + for _ in range(num_frames): + motions.append({"pitch": -pitch_speed}) + elif action == "left": + # Yaw left + for _ in range(num_frames): + motions.append({"yaw": -yaw_speed}) + elif action == "right": + # Yaw right + for _ in range(num_frames): + motions.append({"yaw": yaw_speed}) + else: + raise ValueError( + f"Unknown action: {action}. Supported actions: w, s, a, d, up, down, left, right" + ) + + return motions + + +def pose_string_to_json(pose_string): + """ + Convert pose string to pose JSON format. + + Args: + pose_string: str, comma-separated pose commands + + Returns: + dict: pose JSON with extrinsic and intrinsic parameters + """ + motions = parse_pose_string(pose_string) + poses = generate_camera_trajectory_local(motions) + + # Default intrinsic matrix (from generate_custom_trajectory.py) + intrinsic = [ + [969.6969696969696, 0.0, 960.0], + [0.0, 969.6969696969696, 540.0], + [0.0, 0.0, 1.0], + ] + + pose_json = {} + for i, p in enumerate(poses): + pose_json[str(i)] = {"extrinsic": p.tolist(), "K": intrinsic} + + return pose_json + + +def pose_to_input(pose_data, latent_num, tps=False): + """ + Convert pose data to input tensors. + + Args: + pose_data: str or dict + - If str ending with '.json': path to JSON file + - If str: pose string (e.g., "w-3, right-0.5, d-4") + - If dict: pose JSON data + latent_num: int, number of latents + tps: bool, third person mode + + Returns: + tuple: (w2c_list, intrinsic_list, action_one_label) + """ + # Handle different input types + if isinstance(pose_data, str): + if pose_data.endswith(".json"): + # Load from JSON file + pose_json = json.load(open(pose_data, "r")) + else: + # Parse pose string + pose_json = pose_string_to_json(pose_data) + elif isinstance(pose_data, dict): + pose_json = pose_data + else: + raise ValueError( + f"Invalid pose_data type: {type(pose_data)}. Expected str or dict." + ) + + pose_keys = list(pose_json.keys()) + latent_num_from_pose = len(pose_keys) + assert latent_num_from_pose == latent_num, ( + f"pose corresponds to {latent_num_from_pose * 4 - 3} frames, num_frames " + f"must be set to {latent_num_from_pose * 4 - 3} to ensure alignment." + ) + + intrinsic_list = [] + w2c_list = [] + for i in range(latent_num): + t_key = pose_keys[i] + c2w = np.array(pose_json[t_key]["extrinsic"]) + w2c = np.linalg.inv(c2w) + w2c_list.append(w2c) + intrinsic = np.array(pose_json[t_key]["K"]) + intrinsic[0, 0] /= intrinsic[0, 2] * 2 + intrinsic[1, 1] /= intrinsic[1, 2] * 2 + intrinsic[0, 2] = 0.5 + intrinsic[1, 2] = 0.5 + intrinsic_list.append(intrinsic) + + w2c_list = np.array(w2c_list) + intrinsic_list = torch.tensor(np.array(intrinsic_list)) + + c2ws = np.linalg.inv(w2c_list) + C_inv = np.linalg.inv(c2ws[:-1]) + relative_c2w = np.zeros_like(c2ws) + relative_c2w[0, ...] = c2ws[0, ...] + relative_c2w[1:, ...] = C_inv @ c2ws[1:, ...] + trans_one_hot = np.zeros((relative_c2w.shape[0], 4), dtype=np.int32) + rotate_one_hot = np.zeros((relative_c2w.shape[0], 4), dtype=np.int32) + + move_norm_valid = 0.0001 + for i in range(1, relative_c2w.shape[0]): + move_dirs = relative_c2w[i, :3, 3] # direction vector + move_norms = np.linalg.norm(move_dirs) + if move_norms > move_norm_valid: # threshold for movement + move_norm_dirs = move_dirs / move_norms + angles_rad = np.arccos(move_norm_dirs.clip(-1.0, 1.0)) + trans_angles_deg = angles_rad * (180.0 / torch.pi) # convert to degrees + else: + trans_angles_deg = torch.zeros(3) + + R_rel = relative_c2w[i, :3, :3] + r = R.from_matrix(R_rel) + rot_angles_deg = r.as_euler("xyz", degrees=True) + + # Determine movement and rotation actions + if move_norms > move_norm_valid: # threshold for movement + if (not tps) or ( + tps == True + and abs(rot_angles_deg[1]) < 5e-2 + and abs(rot_angles_deg[0]) < 5e-2 + ): + if trans_angles_deg[2] < 60: + trans_one_hot[i, 0] = 1 # forward + elif trans_angles_deg[2] > 120: + trans_one_hot[i, 1] = 1 # backward + + if trans_angles_deg[0] < 60: + trans_one_hot[i, 2] = 1 # right + elif trans_angles_deg[0] > 120: + trans_one_hot[i, 3] = 1 # left + + if rot_angles_deg[1] > 5e-2: + rotate_one_hot[i, 0] = 1 # right + elif rot_angles_deg[1] < -5e-2: + rotate_one_hot[i, 1] = 1 # left + + if rot_angles_deg[0] > 5e-2: + rotate_one_hot[i, 2] = 1 # up + elif rot_angles_deg[0] < -5e-2: + rotate_one_hot[i, 3] = 1 # down + trans_one_hot = torch.tensor(trans_one_hot) + rotate_one_hot = torch.tensor(rotate_one_hot) + + trans_one_label = one_hot_to_one_dimension(trans_one_hot) + rotate_one_label = one_hot_to_one_dimension(rotate_one_hot) + action_one_label = trans_one_label * 9 + rotate_one_label + + return torch.as_tensor(w2c_list), torch.as_tensor(intrinsic_list), action_one_label + + +def save_video(video, path): + if video.ndim == 5: + assert video.shape[0] == 1 + video = video[0] + vid = (video * 255).clamp(0, 255).to(torch.uint8) + vid = einops.rearrange(vid, "c f h w -> f h w c") + imageio.mimwrite(path, vid, fps=24) + + +def summarize_timing_records(records): + summary = {} + for record in records: + elapsed_s = record.get("elapsed_s") + if elapsed_s is None: + continue + scope = record.get("scope", "") + name = record.get("name", "") + key = f"{scope}.{name}" if scope else name + item = summary.setdefault( + key, + { + "count": 0, + "total_s": 0.0, + "avg_s": 0.0, + "min_s": None, + "max_s": None, + }, + ) + item["count"] += 1 + item["total_s"] += elapsed_s + item["min_s"] = elapsed_s if item["min_s"] is None else min(item["min_s"], elapsed_s) + item["max_s"] = elapsed_s if item["max_s"] is None else max(item["max_s"], elapsed_s) + + for item in summary.values(): + item["avg_s"] = item["total_s"] / item["count"] + return dict(sorted(summary.items(), key=lambda kv: kv[1]["total_s"], reverse=True)) + + +def rank0_log(message, level): + if int(os.environ.get("RANK", "0")) == 0: + loguru.logger.log(level, message) + + +def str_to_bool(value): + """Convert string to boolean, supporting true/false, 1/0, yes/no. + If value is None (when flag is provided without value), returns True.""" + if value is None: + return True # When --flag is provided without value, enable it + if isinstance(value, bool): + return value + if isinstance(value, str): + value = value.lower().strip() + if value in ("true", "1", "yes", "on"): + return True + elif value in ("false", "0", "no", "off"): + return False + raise argparse.ArgumentTypeError(f"Boolean value expected, got: {value}") + + +def camera_center_normalization(w2c): + c2w = np.linalg.inv(w2c) + C0_inv = np.linalg.inv(c2w[0]) + c2w_aligned = np.array([C0_inv @ C for C in c2w]) + return np.linalg.inv(c2w_aligned) + + +def parse_pose_string_to_actions(pose_string, fps=24): + """ + Parse pose string to frame-level action timeline. + + Format: pose string uses latent counts, where: + - 1 latent = 4 frames + - Special rule: first frame of entire video is extra (frame 0) + - Example: "w-4,d-4" means: + - w-4: forward for frames 0-16 (17 frames total: 1 extra + 4*4) + - d-4: right for frames 17-32 (16 frames total: 4*4) + + Args: + pose_string: str, comma-separated pose commands (e.g., "w-4,d-4") + fps: int, frames per second for video (default: 24) + + Returns: + list of dict with frame_idx and actions for each frame + """ + commands = [cmd.strip() for cmd in pose_string.split(",")] + + # Build frame-level actions list + frame_actions = [] + is_first_command = True + + for cmd in commands: + if not cmd: + continue + + parts = cmd.split("-") + if len(parts) != 2: + raise ValueError( + f"Invalid pose command: {cmd}. Expected format: 'action-duration'" + ) + + action = parts[0].strip() + try: + num_latents = int(parts[1].strip()) + except ValueError: + raise ValueError(f"Invalid duration in command: {cmd}") + + # Convert latents to frames + # First command gets 1 extra frame (the special frame 0) + # Formula: first command = 1 + num_latents * 4, others = num_latents * 4 + if is_first_command: + num_frames = 1 + num_latents * 4 + is_first_command = False + else: + num_frames = num_latents * 4 + + # Map action to action values + action_values = {"forward": 0, "left": 0, "yaw": 0, "pitch": 0} + + if action == "w": + action_values["forward"] = 1 + elif action == "s": + action_values["forward"] = -1 + elif action == "a": + action_values["left"] = 1 + elif action == "d": + action_values["left"] = -1 + elif action == "up": + action_values["pitch"] = 1 + elif action == "down": + action_values["pitch"] = -1 + elif action == "left": + action_values["yaw"] = -1 + elif action == "right": + action_values["yaw"] = 1 + else: + raise ValueError(f"Unknown action: {action}") + + # Add frame-level actions + for _ in range(num_frames): + frame_actions.append(action_values.copy()) + + # Return frame-level timeline (each entry represents one frame) + return frame_actions + + +def draw_rounded_rectangle(draw, xy, radius, fill=None, outline=None, width=1): + """Draw a rounded rectangle.""" + x1, y1, x2, y2 = xy + diameter = radius * 2 + + # Draw four corners (circles) + draw.ellipse( + [x1, y1, x1 + diameter, y1 + diameter], fill=fill, outline=outline, width=width + ) + draw.ellipse( + [x2 - diameter, y1, x2, y1 + diameter], fill=fill, outline=outline, width=width + ) + draw.ellipse( + [x1, y2 - diameter, x1 + diameter, y2], fill=fill, outline=outline, width=width + ) + draw.ellipse( + [x2 - diameter, y2 - diameter, x2, y2], fill=fill, outline=outline, width=width + ) + + # Draw two rectangles to fill the middle + draw.rectangle([x1 + radius, y1, x2 - radius, y2], fill=fill) + draw.rectangle([x1, y1 + radius, x2, y2 - radius], fill=fill) + + # Draw border if outline is specified + if outline: + # Top and bottom lines + draw.line([x1 + radius, y1, x2 - radius, y1], fill=outline, width=width) + draw.line([x1 + radius, y2, x2 - radius, y2], fill=outline, width=width) + # Left and right lines + draw.line([x1, y1 + radius, x1, y2 - radius], fill=outline, width=width) + draw.line([x2, y1 + radius, x2, y2 - radius], fill=outline, width=width) + + +def create_wasd_keyboard(actions, key_size=70, key_spacing=6, corner_radius=14): + """Create WASD keyboard overlay.""" + keyboard_width = 3 * key_size + 2 * key_spacing + keyboard_height = 2 * key_size + key_spacing + + img = Image.new("RGBA", (keyboard_width, keyboard_height), (0, 0, 0, 0)) + draw = ImageDraw.Draw(img) + + bg_normal = (0, 0, 0, 128) + bg_active = (30, 120, 255, 220) + text_color = (255, 255, 255, 255) + font_size = 28 + + try: + font = ImageFont.truetype( + "/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf", font_size + ) + except (IOError, OSError): + try: + font = ImageFont.truetype( + "/usr/share/fonts/TTF/DejaVuSans-Bold.ttf", font_size + ) + except (IOError, OSError): + font = ImageFont.load_default() + + def draw_key(x, y, label, is_active): + bg_color = bg_active if is_active else bg_normal + draw_rounded_rectangle( + draw, + [x, y, x + key_size, y + key_size], + radius=corner_radius, + fill=bg_color, + ) + bbox = draw.textbbox((0, 0), label, font=font) + text_width = bbox[2] - bbox[0] + text_height = bbox[3] - bbox[1] + text_x = x + (key_size - text_width) // 2 + text_y = y + (key_size - text_height) // 2 + draw.text((text_x, text_y), label, fill=text_color, font=font) + + forward_val = actions.get("forward", 0) + left_val = actions.get("left", 0) + + w_active = forward_val > 0 + s_active = forward_val < 0 + a_active = left_val > 0 + d_active = left_val < 0 + + wasd_keys = [ + ("W", 1, 0, w_active), + ("A", 0, 1, a_active), + ("S", 1, 1, s_active), + ("D", 2, 1, d_active), + ] + + for label, col, row, is_active in wasd_keys: + x = col * (key_size + key_spacing) + y = row * (key_size + key_spacing) + draw_key(x, y, label, is_active) + + return img + + +def create_arrow_keyboard(actions, key_size=70, key_spacing=6, corner_radius=14): + """Create arrow keys keyboard overlay with triangle symbols.""" + keyboard_width = 3 * key_size + 2 * key_spacing + keyboard_height = 2 * key_size + key_spacing + + img = Image.new("RGBA", (keyboard_width, keyboard_height), (0, 0, 0, 0)) + draw = ImageDraw.Draw(img) + + bg_normal = (0, 0, 0, 128) + bg_active = (30, 120, 255, 220) + text_color = (255, 255, 255, 255) + + def draw_key_with_triangle(x, y, direction, is_active): + bg_color = bg_active if is_active else bg_normal + draw_rounded_rectangle( + draw, + [x, y, x + key_size, y + key_size], + radius=corner_radius, + fill=bg_color, + ) + cx = x + key_size // 2 + cy = y + key_size // 2 + size = key_size // 8 + if direction == "up": + points = [ + (cx, cy - size), + (cx - size, cy + size // 2), + (cx + size, cy + size // 2), + ] + elif direction == "down": + points = [ + (cx, cy + size), + (cx - size, cy - size // 2), + (cx + size, cy - size // 2), + ] + elif direction == "left": + points = [ + (cx - size, cy), + (cx + size // 2, cy - size), + (cx + size // 2, cy + size), + ] + elif direction == "right": + points = [ + (cx + size, cy), + (cx - size // 2, cy - size), + (cx - size // 2, cy + size), + ] + draw.polygon(points, fill=text_color) + + yaw_val = actions.get("yaw", 0) + pitch_val = actions.get("pitch", 0) + + up_active = pitch_val > 0 + down_active = pitch_val < 0 + left_active = yaw_val < 0 + right_active = yaw_val > 0 + + arrow_keys = [ + ("up", 1, 0, up_active), + ("left", 0, 1, left_active), + ("down", 1, 1, down_active), + ("right", 2, 1, right_active), + ] + + for direction, col, row, is_active in arrow_keys: + kx = col * (key_size + key_spacing) + ky = row * (key_size + key_spacing) + draw_key_with_triangle(kx, ky, direction, is_active) + + return img + + +def blend_overlay(base_frame, overlay, position): + """Blend an RGBA overlay onto a RGB frame.""" + x, y = position + overlay_array = np.array(overlay) + + oh, ow = overlay_array.shape[:2] + bh, bw = base_frame.shape[:2] + + x = max(0, min(x, bw - 1)) + y = max(0, min(y, bh - 1)) + + if x + ow > bw: + ow = bw - x + overlay_array = overlay_array[:, :ow] + if y + oh > bh: + oh = bh - y + overlay_array = overlay_array[:oh, :] + + if ow <= 0 or oh <= 0: + return base_frame + + overlay_rgb = overlay_array[:, :, :3].astype(np.float32) + overlay_alpha = overlay_array[:, :, 3:4].astype(np.float32) / 255.0 + + base_region = base_frame[y : y + oh, x : x + ow].astype(np.float32) + + blended = (overlay_rgb * overlay_alpha + base_region * (1 - overlay_alpha)).astype( + np.uint8 + ) + + result = base_frame.copy() + result[y : y + oh, x : x + ow] = blended + + return result + + +def add_keyboard_overlay_to_video(video_path, output_path, actions_timeline): + """Add keyboard overlay to an existing video. + + Args: + video_path: path to input video + output_path: path to output video + actions_timeline: list of action dicts, one per frame + """ + try: + video = VideoFileClip(video_path) + width, height = video.size + fps = video.fps + duration = video.duration + + key_size = 70 + key_spacing = 6 + corner_radius = 14 + margin = 40 + + keyboard_width = 3 * key_size + 2 * key_spacing + keyboard_height = 2 * key_size + key_spacing + + wasd_pos = (margin, height - margin - keyboard_height) + arrow_pos = (width - margin - keyboard_width, height - margin - keyboard_height) + + wasd_cache = {} + arrow_cache = {} + + def get_actions_at_time(t): + # Convert time to frame index + frame_idx = int(t * fps) + if 0 <= frame_idx < len(actions_timeline): + return actions_timeline[frame_idx] + return {"forward": 0, "left": 0, "yaw": 0, "pitch": 0} + + def make_frame(t): + frame = video.get_frame(t) + actions = get_actions_at_time(t) + + def sign(x): + if x > 0: + return 1 + elif x < 0: + return -1 + return 0 + + wasd_key = (sign(actions.get("forward", 0)), sign(actions.get("left", 0))) + arrow_key = (sign(actions.get("yaw", 0)), sign(actions.get("pitch", 0))) + + if wasd_key not in wasd_cache: + wasd_cache[wasd_key] = create_wasd_keyboard( + actions, key_size, key_spacing, corner_radius + ) + wasd_img = wasd_cache[wasd_key] + + if arrow_key not in arrow_cache: + arrow_cache[arrow_key] = create_arrow_keyboard( + actions, key_size, key_spacing, corner_radius + ) + arrow_img = arrow_cache[arrow_key] + + frame = blend_overlay(frame, wasd_img, wasd_pos) + frame = blend_overlay(frame, arrow_img, arrow_pos) + + return frame + + output_clip = VideoClip(make_frame, duration=duration) + output_clip = output_clip.set_fps(fps) + + if video.audio is not None: + output_clip = output_clip.set_audio(video.audio) + + output_clip.write_videofile( + output_path, + codec="libx264", + audio_codec="aac", + temp_audiofile="temp-audio.m4a", + remove_temp=True, + logger=None, + ) + + video.close() + output_clip.close() + + return True + + except Exception as e: + import traceback + + print(f"Error adding keyboard overlay: {e}") + traceback.print_exc() + return False + + +def generate_video(args): + profile_timing = os.environ.get("HY_PROFILE_TIMING", "0").lower() in { + "1", + "true", + "yes", + "on", + } + timing_records = [] + + def timer_start(): + if profile_timing and torch.cuda.is_available(): + torch.cuda.synchronize() + return time.perf_counter() + + def timer_stop(name, start, **metadata): + if not profile_timing: + return + if torch.cuda.is_available(): + torch.cuda.synchronize() + if int(os.environ.get("RANK", "0")) != 0: + return + record = { + "scope": "generate", + "name": name, + "elapsed_s": time.perf_counter() - start, + } + record.update(metadata) + timing_records.append(record) + + total_start = timer_start() + + assert ( + (args.video_length - 1) // 4 + 1 + ) % 4 == 0, "number of latents must be divisible by 4" + initialize_infer_state(args) + + task = "i2v" if args.image_path else "t2v" + + enable_sr = args.sr + + # Build transformer_version based on flags + transformer_version = f"{args.resolution}_{task}" + assert transformer_version == "480p_i2v" + + if args.dtype == "bf16": + transformer_dtype = torch.bfloat16 + elif args.dtype == "fp32": + transformer_dtype = torch.float32 + else: + raise ValueError(f"Unsupported dtype: {args.dtype}. Must be 'bf16' or 'fp32'") + + model_load_start = timer_start() + pipe = HunyuanVideo_1_5_Pipeline.create_pipeline( + pretrained_model_name_or_path=args.model_path, + transformer_version=transformer_version, + enable_offloading=args.offloading, + enable_group_offloading=args.group_offloading, + create_sr_pipeline=enable_sr, + force_sparse_attn=False, + transformer_dtype=transformer_dtype, + action_ckpt=args.action_ckpt, + ) + timer_stop("create_pipeline", model_load_start) + + extra_kwargs = {} + if task == "i2v": + extra_kwargs["reference_image"] = args.image_path + + enable_rewrite = args.rewrite + if not args.rewrite: + rank0_log( + "Warning: Prompt rewriting is disabled. This may affect the quality of generated videos.", + "WARNING", + ) + + pose_start = timer_start() + viewmats, Ks, action = pose_to_input(args.pose, (args.video_length - 1) // 4 + 1) + timer_stop("pose_to_input", pose_start) + + if task == "i2v": + extra_kwargs["reference_image"] = args.image_path + + pipeline_start = timer_start() + out = pipe( + enable_sr=enable_sr, + prompt=args.prompt, + aspect_ratio=args.aspect_ratio, + num_inference_steps=args.num_inference_steps, + sr_num_inference_steps=None, + video_length=args.video_length, + negative_prompt=args.negative_prompt, + seed=args.seed, + output_type="pt", + prompt_rewrite=enable_rewrite, + return_pre_sr_video=args.save_pre_sr_video, + viewmats=viewmats.unsqueeze(0), + Ks=Ks.unsqueeze(0), + action=action.unsqueeze(0), + few_step=args.few_step, + chunk_latent_frames=4 if args.model_type == "ar" else 16, + model_type=args.model_type, + user_height=args.height, + user_width=args.width, + transformer_resident_ar_rollout=args.transformer_resident_ar_rollout, + **extra_kwargs, + ) + timer_stop("pipeline_call", pipeline_start) + if profile_timing and int(os.environ.get("RANK", "0")) == 0: + timing_records.extend( + {"scope": "pipeline", **record} + for record in getattr(pipe, "_timing_records", []) + ) + + # save video + if int(os.environ.get("RANK", "0")) == 0: + output_path = args.output_path + os.makedirs(output_path, exist_ok=True) + + save_video_path = os.path.join(output_path, "gen.mp4") + save_video_sr_path = os.path.join(output_path, "gen_sr.mp4") + + # Determine which video to process for UI overlay + video_to_process = None + final_video_path = None + + save_outputs_start = timer_start() + if enable_sr and hasattr(out, "sr_videos"): + save_start = timer_start() + save_video(out.sr_videos, save_video_sr_path) + timer_stop("save_sr_video", save_start) + print(f"Saved SR video to: {save_video_sr_path}") + video_to_process = save_video_sr_path + final_video_path = save_video_sr_path + + if args.save_pre_sr_video: + save_start = timer_start() + save_video(out.videos, save_video_path) + timer_stop("save_pre_sr_video", save_start) + print(f"Saved original video (before SR) to: {save_video_path}") + else: + save_start = timer_start() + save_video(out.videos, save_video_path) + timer_stop("save_video", save_start) + print(f"Saved video to: {save_video_path}") + video_to_process = save_video_path + final_video_path = save_video_path + timer_stop("save_outputs", save_outputs_start) + + # Add keyboard overlay if --with-ui is enabled and pose is a string + if ( + args.with_ui + and isinstance(args.pose, str) + and not args.pose.endswith(".json") + ): + print(f"Adding keyboard overlay to video...") + try: + overlay_start = timer_start() + actions_timeline = parse_pose_string_to_actions(args.pose) + + # Create temporary output path for video with UI + video_with_ui_path = os.path.join(output_path, "gen_with_ui_temp.mp4") + + if add_keyboard_overlay_to_video( + video_to_process, video_with_ui_path, actions_timeline + ): + # Replace original video with UI version + os.replace(video_with_ui_path, final_video_path) + print(f"Successfully added keyboard overlay to: {final_video_path}") + else: + print(f"Failed to add keyboard overlay, keeping original video") + if os.path.exists(video_with_ui_path): + os.remove(video_with_ui_path) + timer_stop("add_keyboard_overlay", overlay_start) + except Exception as e: + print(f"Error processing keyboard overlay: {e}") + import traceback + + traceback.print_exc() + + timer_stop("generate_video_total", total_start) + if profile_timing: + timing_path = os.path.join(output_path, "timings.json") + timing_payload = { + "metadata": { + "video_length": args.video_length, + "pose": args.pose, + "seed": args.seed, + "num_inference_steps": args.num_inference_steps, + "model_type": args.model_type, + "resolution": args.resolution, + "width": args.width, + "height": args.height, + "cuda_visible_devices": os.environ.get("CUDA_VISIBLE_DEVICES"), + }, + "summary": summarize_timing_records(timing_records), + "records": timing_records, + } + with open(timing_path, "w", encoding="utf-8") as f: + json.dump(timing_payload, f, indent=2) + print(f"[Timing] Saved timing report to: {timing_path}") + + +def main(): + parser = argparse.ArgumentParser( + description="Generate video using HunyuanWorld-1.5" + ) + + parser.add_argument( + "--pose", + type=str, + default="./assets/pose/test_forward_32_latents.json", + help="Path to pose JSON file or pose string (e.g., 'w-3, right-0.5, d-4')", + ) + parser.add_argument( + "--prompt", type=str, required=True, help="Text prompt for video generation" + ) + parser.add_argument( + "--negative_prompt", + type=str, + default="", + help="Negative prompt for video generation (default: empty string)", + ) + parser.add_argument( + "--resolution", + type=str, + required=True, + choices=["480p", "720p"], + help="Video resolution (480p or 720p)", + ) + parser.add_argument( + "--model_path", type=str, required=True, help="Path to pretrained model" + ) + parser.add_argument( + "--action_ckpt", type=str, required=True, help="Path to pretrained action model" + ) + parser.add_argument( + "--aspect_ratio", type=str, default="16:9", help="Aspect ratio (default: 16:9)" + ) + parser.add_argument( + "--num_inference_steps", + type=int, + default=50, + help="Number of inference steps (default: 50)", + ) + parser.add_argument( + "--video_length", + type=int, + default=127, + help="Number of frames to generate (default: 127)", + ) + parser.add_argument( + "--sr", + type=str_to_bool, + nargs="?", + const=True, + default=True, + help="Enable super resolution (default: true). " + "Use --sr or --sr true/1 to enable, --sr false/0 to disable", + ) + parser.add_argument( + "--save_pre_sr_video", + type=str_to_bool, + nargs="?", + const=True, + default=False, + help="Save original video before super resolution (default: false). " + "Use --save_pre_sr_video or --save_pre_sr_video true/1 to enable, " + "--save_pre_sr_video false/0 to disable", + ) + parser.add_argument( + "--rewrite", + type=str_to_bool, + nargs="?", + const=True, + default=False, + help="Enable prompt rewriting (default: true). " + "Use --rewrite or --rewrite true/1 to enable, --rewrite false/0 to disable", + ) + parser.add_argument( + "--offloading", + type=str_to_bool, + nargs="?", + const=True, + default=True, + help="Enable offloading (default: true). " + "Use --offloading or --offloading true/1 to enable, " + "--offloading false/0 to disable", + ) + parser.add_argument( + "--group_offloading", + type=str_to_bool, + nargs="?", + const=True, + default=None, + help="Enable group offloading (default: None, automatically enabled if offloading is enabled). " + "Use --group_offloading or --group_offloading true/1 to enable, " + "--group_offloading false/0 to disable", + ) + parser.add_argument( + "--dtype", + type=str, + default="bf16", + choices=["bf16", "fp32"], + help="Data type for transformer (default: bf16). " + "bf16: faster, lower memory; fp32: better quality, slower, higher memory", + ) + parser.add_argument( + "--seed", type=int, default=123, help="Random seed (default: 123)" + ) + parser.add_argument( + "--image_path", + type=str, + default=None, + help="Path to reference image for i2v (if provided, uses i2v mode)", + ) + parser.add_argument( + "--output_path", + type=str, + default=None, + help="Output file path for generated video (if not provided, saves to ./outputs/output.mp4)", + ) + parser.add_argument( + "--enable_torch_compile", + type=str_to_bool, + nargs="?", + const=True, + default=False, + help="Enable torch compile for transformer (default: false). " + "Use --enable_torch_compile or --enable_torch_compile true/1 to enable, " + "--enable_torch_compile false/0 to disable", + ) + parser.add_argument( + "--few_step", + type=str_to_bool, + nargs="?", + const=False, + default=False, + help="Enable super resolution (default: true). " + "Use --few_step or --few_step true/1 to enable, --few_step false/0 to disable", + ) + parser.add_argument( + "--model_type", + type=str, + required=True, + choices=["bi", "ar"], + help="inference bidirectional or autoregressive model. ", + ) + parser.add_argument( + "--height", + type=int, + default=None, + help="height for generation (recommended to set as 480)", + ) + parser.add_argument( + "--width", + type=int, + default=None, + help="width for generation (recommended to set as 832)", + ) + parser.add_argument( + "--with-ui", + type=str_to_bool, + nargs="?", + const=True, + default=False, + help="Add keyboard overlay to generated video (default: false). " + "Only works with pose string input, not JSON files. " + "Use --with-ui or --with-ui true/1 to enable, --with-ui false/0 to disable", + ) + + parser.add_argument( + "--use_sageattn", + type=str_to_bool, + nargs="?", + const=True, + default=False, + help="Enable sageattn (default: false). " + "Use --use_sageattn or --use_sageattn true/1 to enable, " + "--use_sageattn false/0 to disable", + ) + parser.add_argument( + "--sage_blocks_range", + type=str, + default="0-53", + help="Sageattn blocks range (e.g., 0-5 or 0,1,2,3,4,5)", + ) + parser.add_argument( + "--use_vae_parallel", + type=str_to_bool, + nargs="?", + const=True, + default=False, + help="Enable vae parallel (default: false). " + "Use --use_vae_parallel or --use_vae_parallel true/1 to enable, " + "--use_vae_parallel false/0 to disable", + ) + # fp8 gemm related + parser.add_argument( + "--use_fp8_gemm", + type=str_to_bool, + nargs="?", + const=True, + default=False, + help="Enable fp8 gemm for transformer (default: false). " + "Use --use_fp8_gemm or --use_fp8_gemm true/1 to enable, " + "--use_fp8_gemm false/0 to disable", + ) + parser.add_argument( + "--quant_type", + type=str, + default="fp8-per-block", + help="Quantization type for fp8 gemm (e.g., fp8-per-tensor-weight-only, fp8-per-tensor, fp8-per-block)", + ) + parser.add_argument( + "--include_patterns", + type=str, + default="double_blocks", + help="Include patterns for fp8 gemm (default: double_blocks)", + ) + parser.add_argument( + "--transformer_resident_ar_rollout", + type=str_to_bool, + nargs="?", + const=True, + default=False, + help="Keep transformer on GPU for entire AR rollout instead of per-chunk offloading (default: false). " + "Reduces inference time without increasing peak VRAM. Only affects AR model_type with offloading enabled. " + "Use --transformer_resident_ar_rollout or --transformer_resident_ar_rollout true to enable.", + ) + + args = parser.parse_args() + + assert args.image_path is not None + + generate_video(args) + + +if __name__ == "__main__": + main() diff --git a/hyvideo/generate_custom_trajectory.py b/hyvideo/generate_custom_trajectory.py new file mode 100644 index 0000000000000000000000000000000000000000..481a262293436866def81bf10ed319e9fd8513a4 --- /dev/null +++ b/hyvideo/generate_custom_trajectory.py @@ -0,0 +1,110 @@ +import numpy as np +import json + + +def rot_x(theta): + c, s = np.cos(theta), np.sin(theta) + return np.array([[1, 0, 0], [0, c, -s], [0, s, c]]) + + +def rot_y(theta): + c, s = np.cos(theta), np.sin(theta) + return np.array([[c, 0, s], [0, 1, 0], [-s, 0, c]]) + + +def rot_z(theta): + c, s = np.cos(theta), np.sin(theta) + return np.array([[c, -s, 0], [s, c, 0], [0, 0, 1]]) + + +def generate_camera_trajectory_local(motions): + """ + motions: list of dict + {"forward": 1.0}, {"yaw": np.pi/2}, {"pitch": np.pi/6}, {"right": 1.0} + - forward: Translation (Forward or Backward) + - yaw: Rotate (Left or Right) + - pitch: Rotate (Up or Down) + - right: Translation (Right or Left) + - third_yaw: Third Perspective Rotate (Left or Right) + """ + + poses = [] + T = np.eye(4) + poses.append(T.copy()) + + for move in motions: + # Rotate (Left or Right) + if "yaw" in move: + R = rot_y(move["yaw"]) + T[:3, :3] = T[:3, :3] @ R + + # Rotate (Up or Down) + if "pitch" in move: + R = rot_x(move["pitch"]) + T[:3, :3] = T[:3, :3] @ R + + # Translation (Z-direction of the camera's local coordinate system) + forward = move.get("forward", 0.0) + if forward != 0: + local_t = np.array([0, 0, forward]) + world_t = T[:3, :3] @ local_t + T[:3, 3] += world_t + + # Translation (Z-direction of the camera's local coordinate system) + right = move.get("right", 0.0) + if right != 0: + local_t = np.array([right, 0, 0]) + world_t = T[:3, :3] @ local_t + T[:3, 3] += world_t + + # Third Perspective Rotate (Left or Right) + third_yaw = move.get("third_yaw", 0.0) + if third_yaw != 0: + theta = -third_yaw + C = np.array([[1, 0.0, 0, 0], [0, 1, 0, 0], [0, 0, 1, -1.0], [0, 0, 0, 1]]) + c_origin = C.copy() + # Rotation around the Y-axis + R_y = np.array( + [ + [np.cos(theta), 0, np.sin(theta)], + [0, 1, 0], + [-np.sin(theta), 0, np.cos(theta)], + ] + ) + # Translation + C[:3, :3] = C[:3, :3] @ R_y + C[:3, 3] = R_y @ C[:3, 3] + c_inv = np.linalg.inv(c_origin) + c_relative = c_inv @ C + T = T @ c_relative + + poses.append(T.copy()) + + return poses + + +if __name__ == "__main__": + # Examples: Forward 0.08 * 16 -> Right Rotate 3 degree * 16 + motions = [] + for i in range(15): + motions.append({"forward": 0.08}) + + for i in range(16): + motions.append({"yaw": np.deg2rad(3)}) + + intrinsic = [ + [969.6969696969696, 0.0, 960.0], + [0.0, 969.6969696969696, 540.0], + [0.0, 0.0, 1.0], + ] + + poses = generate_camera_trajectory_local(motions) + custom_c2w = {} + for i, p in enumerate(poses): + custom_c2w[str(i)] = {"extrinsic": p.tolist(), "K": intrinsic} + json.dump( + custom_c2w, + open("./assets/pose/pose.json", "w"), + indent=4, + ensure_ascii=False, + ) diff --git a/hyvideo/models/__init__.py b/hyvideo/models/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6a429aae5c9bed641d117863ba11ac863103c860 --- /dev/null +++ b/hyvideo/models/__init__.py @@ -0,0 +1,15 @@ +# Licensed under the TENCENT HUNYUAN COMMUNITY LICENSE AGREEMENT (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://github.com/Tencent-Hunyuan/HunyuanVideo-1.5/blob/main/LICENSE +# +# Unless and only to the extent required by applicable law, the Tencent Hunyuan works and any +# output and results therefrom are provided "AS IS" without any express or implied warranties of +# any kind including any warranties of title, merchantability, noninfringement, course of dealing, +# usage of trade, or fitness for a particular purpose. You are solely responsible for determining the +# appropriateness of using, reproducing, modifying, performing, displaying or distributing any of +# the Tencent Hunyuan works or outputs and assume any and all risks associated with your or a +# third party's use or distribution of any of the Tencent Hunyuan works or outputs and your exercise +# of rights and permissions under this agreement. +# See the License for the specific language governing permissions and limitations under the License. diff --git a/hyvideo/models/autoencoders/__init__.py b/hyvideo/models/autoencoders/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6a429aae5c9bed641d117863ba11ac863103c860 --- /dev/null +++ b/hyvideo/models/autoencoders/__init__.py @@ -0,0 +1,15 @@ +# Licensed under the TENCENT HUNYUAN COMMUNITY LICENSE AGREEMENT (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://github.com/Tencent-Hunyuan/HunyuanVideo-1.5/blob/main/LICENSE +# +# Unless and only to the extent required by applicable law, the Tencent Hunyuan works and any +# output and results therefrom are provided "AS IS" without any express or implied warranties of +# any kind including any warranties of title, merchantability, noninfringement, course of dealing, +# usage of trade, or fitness for a particular purpose. You are solely responsible for determining the +# appropriateness of using, reproducing, modifying, performing, displaying or distributing any of +# the Tencent Hunyuan works or outputs and assume any and all risks associated with your or a +# third party's use or distribution of any of the Tencent Hunyuan works or outputs and your exercise +# of rights and permissions under this agreement. +# See the License for the specific language governing permissions and limitations under the License. diff --git a/hyvideo/models/autoencoders/hunyuanvideo_15_vae_w_cache.py b/hyvideo/models/autoencoders/hunyuanvideo_15_vae_w_cache.py new file mode 100644 index 0000000000000000000000000000000000000000..971becab4d102e147fba2be6e8feb5675ba85114 --- /dev/null +++ b/hyvideo/models/autoencoders/hunyuanvideo_15_vae_w_cache.py @@ -0,0 +1,1409 @@ +# Licensed under the TENCENT HUNYUAN COMMUNITY LICENSE AGREEMENT (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://github.com/Tencent-Hunyuan/HunyuanVideo-1.5/blob/main/LICENSE +# +# Unless and only to the extent required by applicable law, the Tencent Hunyuan works and any +# output and results therefrom are provided "AS IS" without any express or implied warranties of +# any kind including any warranties of title, merchantability, noninfringement, course of dealing, +# usage of trade, or fitness for a particular purpose. You are solely responsible for determining the +# appropriateness of using, reproducing, modifying, performing, displaying or distributing any of +# the Tencent Hunyuan works or outputs and assume any and all risks associated with your or a +# third party's use or distribution of any of the Tencent Hunyuan works or outputs and your exercise +# of rights and permissions under this agreement. +# See the License for the specific language governing permissions and limitations under the License. + +import math +from dataclasses import dataclass +from typing import Optional, Tuple, Union + +import loguru +import numpy as np +import torch +import torch.nn.functional as F +from einops import rearrange +from torch import Tensor, nn +from torch.nn import Conv3d +from torch import distributed as dist + +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.models.autoencoders.vae import BaseOutput, DiagonalGaussianDistribution +from diffusers.models.modeling_outputs import AutoencoderKLOutput +from diffusers.models.modeling_utils import ModelMixin + +from hyvideo.commons.parallel_states import get_parallel_state + +CACHE_T = 2 + + +@dataclass +class DecoderOutput(BaseOutput): + sample: torch.FloatTensor + posterior: Optional[DiagonalGaussianDistribution] = None + + +def swish(x: Tensor, inplace=False) -> Tensor: + """Applies the swish activation function (SiLU) with optional inplace support.""" + return F.silu(x, inplace=inplace) + + +def forward_with_checkpointing(module, *inputs, use_checkpointing=False): + """ + Executes the forward pass of a module, optionally using gradient checkpointing. + + Gradient checkpointing trades compute for memory by not storing intermediate + activations during the forward pass, but re-computing them during the backward pass. + + Args: + module (nn.Module): The module to run. + *inputs: The inputs to the module. + use_checkpointing (bool): If True, wraps execution in torch.utils.checkpoint. + """ + + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + + if use_checkpointing: + # If checkpointing is enabled, wrap the custom forward function. + # use_reentrant=False is generally recommended for newer PyTorch versions + # to avoid issues with backward hooks and distributed training. + return torch.utils.checkpoint.checkpoint( + create_custom_forward(module), *inputs, use_reentrant=False + ) + else: + # Standard forward pass without checkpointing overhead + return module(*inputs) + + +# Optimized implementation of CogVideoXSafeConv3d +# https://github.com/huggingface/diffusers/blob/c9ff360966327ace3faad3807dc871a4e5447501/src/diffusers/models/autoencoders/autoencoder_kl_cogvideox.py#L38 +class PatchCausalConv3d(nn.Conv3d): + r"""Causal Conv3d with efficient patch processing for large tensors.""" + + def find_split_indices(self, seq_len, part_num): + ideal_interval = seq_len / part_num + possible_indices = list(range(0, seq_len, self.stride[0])) + selected_indices = [] + + for i in range(1, part_num): + closest = min( + possible_indices, key=lambda x: abs(x - round(i * ideal_interval)) + ) + if closest not in selected_indices: + selected_indices.append(closest) + + merged_indices = [] + prev_idx = 0 + for idx in selected_indices: + if idx - prev_idx >= self.kernel_size[0]: + merged_indices.append(idx) + prev_idx = idx + + return merged_indices + + def forward(self, input): + T = input.shape[2] # input: NCTHW + memory_count = torch.prod(torch.tensor(input.shape)).item() * 2 / 1024**3 + + if T > self.kernel_size[0] and memory_count > 0.6: + kernel_size = self.kernel_size[0] + part_num = int(memory_count / 2) + 1 + split_indices = self.find_split_indices(T, part_num) + input_chunks = ( + torch.tensor_split(input, split_indices, dim=2) + if len(split_indices) > 0 + else [input] + ) + if kernel_size > 1: + input_chunks = [input_chunks[0]] + [ + torch.cat( + ( + input_chunks[i - 1][:, :, -kernel_size + 1 :], + input_chunks[i], + ), + dim=2, + ) + for i in range(1, len(input_chunks)) + ] + output_chunks = [] + for input_chunk in input_chunks: + output_chunks.append(super().forward(input_chunk)) + output = torch.cat(output_chunks, dim=2) + return output + else: + return super().forward(input) + + +class RMS_norm(nn.Module): + """Root Mean Square Layer Normalization for Channel-First or Last""" + + def __init__(self, dim, channel_first=True, images=True, bias=False): + super().__init__() + broadcastable_dims = (1, 1, 1) if not images else (1, 1) + shape = (dim, *broadcastable_dims) if channel_first else (dim,) + + self.channel_first = channel_first + self.scale = dim**0.5 + self.gamma = nn.Parameter(torch.ones(shape)) + self.bias = nn.Parameter(torch.zeros(shape)) if bias else 0.0 + + def forward(self, x): + return ( + F.normalize(x, dim=(1 if self.channel_first else -1)) + * self.scale + * self.gamma + + self.bias + ) + + +class CausalConv3d(nn.Module): + """Causal Conv3d with configurable padding for temporal axis.""" + + def __init__( + self, + chan_in, + chan_out, + kernel_size: Union[int, Tuple[int, int, int]], + stride: Union[int, Tuple[int, int, int]] = 1, + dilation: Union[int, Tuple[int, int, int]] = 1, + pad_mode="replicate", + disable_causal=False, + enable_patch_conv=False, + **kwargs + ): + super().__init__() + + self.pad_mode = pad_mode + if disable_causal: + padding = ( + kernel_size // 2, + kernel_size // 2, + kernel_size // 2, + kernel_size // 2, + kernel_size // 2, + kernel_size // 2, + ) + else: + padding = ( + kernel_size // 2, + kernel_size // 2, + kernel_size // 2, + kernel_size // 2, + kernel_size - 1, + 0, + ) + self.time_causal_padding = padding + + if enable_patch_conv: + self.conv = PatchCausalConv3d( + chan_in, + chan_out, + kernel_size, + stride=stride, + dilation=dilation, + **kwargs + ) + else: + self.conv = nn.Conv3d( + chan_in, + chan_out, + kernel_size, + stride=stride, + dilation=dilation, + **kwargs + ) + + def forward(self, x, cache_x=None): + padding = list(self.time_causal_padding) + if ( + cache_x is not None and self.time_causal_padding[4] > 0 + ): # not the first frame + cache_x = cache_x.to(x.device) + x = torch.cat( + [cache_x, x], dim=2 + ) # concat on temporal dimension (concat one) + padding[4] -= cache_x.shape[ + 2 + ] # from [1,1,1,1,2,0] to [1,1,1,1,1,0] (padding one) + x = F.pad(x, padding, mode=self.pad_mode) + return self.conv(x) + + +def prepare_causal_attention_mask( + n_frame: int, n_hw: int, dtype, device, batch_size: int = None +): + """Prepare a causal attention mask for 3D videos. + + Args: + n_frame (int): Number of frames (temporal length). + n_hw (int): Product of height and width. + dtype: Desired mask dtype. + device: Device for the mask. + batch_size (int, optional): If set, expands for batch. + + Returns: + torch.Tensor: Causal attention mask. + """ + seq_len = n_frame * n_hw + mask = torch.full((seq_len, seq_len), float("-inf"), dtype=dtype, device=device) + for i in range(seq_len): + i_frame = i // n_hw + mask[i, : (i_frame + 1) * n_hw] = 0 + if batch_size is not None: + mask = mask.unsqueeze(0).expand(batch_size, -1, -1) + return mask + + +class AttnBlock(nn.Module): + """Self-attention block for 3D video tensors.""" + + def __init__(self, in_channels: int): + super().__init__() + self.in_channels = in_channels + + self.norm = RMS_norm(in_channels, images=False) + + self.q = Conv3d(in_channels, in_channels, kernel_size=1) + self.k = Conv3d(in_channels, in_channels, kernel_size=1) + self.v = Conv3d(in_channels, in_channels, kernel_size=1) + self.proj_out = Conv3d(in_channels, in_channels, kernel_size=1) + + def attention(self, h_: Tensor) -> Tensor: + h_ = self.norm(h_) + q = self.q(h_) + k = self.k(h_) + v = self.v(h_) + + b, c, f, h, w = q.shape + q = rearrange(q, "b c f h w -> b 1 (f h w) c").contiguous() + k = rearrange(k, "b c f h w -> b 1 (f h w) c").contiguous() + v = rearrange(v, "b c f h w -> b 1 (f h w) c").contiguous() + attention_mask = prepare_causal_attention_mask( + f, h * w, h_.dtype, h_.device, batch_size=b + ) + h_ = nn.functional.scaled_dot_product_attention( + q, k, v, attn_mask=attention_mask.unsqueeze(1) + ) + + return rearrange(h_, "b 1 (f h w) c -> b c f h w", f=f, h=h, w=w, c=c, b=b) + + def forward(self, x: Tensor) -> Tensor: + return x + self.proj_out(self.attention(x)) + + +class ResnetBlock(nn.Module): + """ResNet-style block for 3D video tensors.""" + + def __init__(self, in_channels: int, out_channels: int): + super().__init__() + self.in_channels = in_channels + out_channels = in_channels if out_channels is None else out_channels + self.out_channels = out_channels + + self.norm1 = RMS_norm(in_channels, images=False) + self.conv1 = CausalConv3d(in_channels, out_channels, kernel_size=3) + + self.norm2 = RMS_norm(out_channels, images=False) + self.conv2 = CausalConv3d(out_channels, out_channels, kernel_size=3) + if self.in_channels != self.out_channels: + self.nin_shortcut = Conv3d( + in_channels, out_channels, kernel_size=1, stride=1, padding=0 + ) + + def forward(self, x, feat_cache=None, feat_idx=[0]): + h = x + h = self.norm1(h) + h = swish(h, inplace=True) + + # apply the feature cacheing mechanism + if feat_cache is not None: + # Retrieve the current layer index + idx = feat_idx[0] + + # Clone the last CACHE_T frames from the current input to store for the next step. + cache_x = h[:, :, -CACHE_T:, :, :].clone() + + # Handle boundary conditions: if the current temporal chunk is too short (< 2 frames) + # and we have a previous cache, prepend the last frame of the previous cache. + # This ensures sufficient temporal context for the convolution kernel. + if ( + cache_x.shape[2] < 2 and feat_cache[idx] is not None + ): # actually this means chunk 1 after chunk 0 + cache_x = torch.cat( + [ + feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to(cache_x.device), + cache_x, + ], + dim=2, + ) + + # Apply the convolution layer using the input 'h' and the *previous* cached state. + h = self.conv1(h, feat_cache[idx]) + + # Update the cache for this layer with the newly prepared context ('cache_x'). + feat_cache[idx] = cache_x + + # Increment the global layer index. + feat_idx[0] += 1 + else: + h = self.conv1(h) + + h = self.norm2(h) + h = swish(h, inplace=True) + + if feat_cache is not None: + idx = feat_idx[0] + cache_x = h[:, :, -CACHE_T:, :, :].clone() + if cache_x.shape[2] < 2 and feat_cache[idx] is not None: + cache_x = torch.cat( + [ + feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to(cache_x.device), + cache_x, + ], + dim=2, + ) + h = self.conv2(h, feat_cache[idx]) + feat_cache[idx] = cache_x + feat_idx[0] += 1 + else: + h = self.conv2(h) + + if self.in_channels != self.out_channels: + x = self.nin_shortcut(x) + return x + h + + +class Downsample(nn.Module): + """Hierarchical downsampling with temporal/ spatial support.""" + + def __init__( + self, in_channels: int, out_channels: int, add_temporal_downsample: bool = True + ): + super().__init__() + factor = 2 * 2 * 2 if add_temporal_downsample else 1 * 2 * 2 + assert out_channels % factor == 0 + self.conv = CausalConv3d(in_channels, out_channels // factor, kernel_size=3) + self.add_temporal_downsample = add_temporal_downsample + self.group_size = factor * in_channels // out_channels + + def forward(self, x, feat_cache=None, feat_idx=[0]): + r1 = 2 if self.add_temporal_downsample else 1 + if feat_cache is not None: + idx = feat_idx[0] + cache_x = x[:, :, -CACHE_T:, :, :].clone() + if cache_x.shape[2] < 2 and feat_cache[idx] is not None: + cache_x = torch.cat( + [ + feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to(cache_x.device), + cache_x, + ], + dim=2, + ) + h = self.conv(x, feat_cache[idx]) + feat_cache[idx] = cache_x + feat_idx[0] += 1 + else: + h = self.conv( + x + ) # change the channel, ready for saptial or temporal downsample + + if self.add_temporal_downsample: + if x.shape[2] == 1: + h = rearrange(h, "b c f (h r2) (w r3) -> b (r2 r3 c) f h w", r2=2, r3=2) + h = torch.cat([h, h], dim=1) + # compute the shortcut part + shortcut = rearrange( + x, "b c f (h r2) (w r3) -> b (r2 r3 c) f h w", r2=2, r3=2 + ) + B, C, T, H, W = shortcut.shape + shortcut = shortcut.view( + B, h.shape[1], self.group_size // 2, T, H, W + ).mean(dim=2) + else: + h = rearrange( + h, + "b c (f r1) (h r2) (w r3) -> b (r1 r2 r3 c) f h w", + r1=r1, + r2=2, + r3=2, + ) + # compute the shortcut part + shortcut = rearrange( + x, + "b c (f r1) (h r2) (w r3) -> b (r1 r2 r3 c) f h w", + r1=r1, + r2=2, + r3=2, + ) + B, C, T, H, W = shortcut.shape + shortcut = shortcut.view(B, h.shape[1], self.group_size, T, H, W).mean( + dim=2 + ) + else: + h = rearrange( + h, "b c (f r1) (h r2) (w r3) -> b (r1 r2 r3 c) f h w", r1=r1, r2=2, r3=2 + ) + shortcut = rearrange( + x, "b c (f r1) (h r2) (w r3) -> b (r1 r2 r3 c) f h w", r1=r1, r2=2, r3=2 + ) + B, C, T, H, W = shortcut.shape + shortcut = shortcut.view(B, h.shape[1], self.group_size, T, H, W).mean( + dim=2 + ) + + return h + shortcut + + +class Upsample(nn.Module): + """Hierarchical upsampling with temporal/ spatial support.""" + + def __init__( + self, in_channels: int, out_channels: int, add_temporal_upsample: bool = True + ): + super().__init__() + factor = 2 * 2 * 2 if add_temporal_upsample else 1 * 2 * 2 + self.conv = CausalConv3d(in_channels, out_channels * factor, kernel_size=3) + self.add_temporal_upsample = add_temporal_upsample + self.repeats = factor * out_channels // in_channels + + def forward(self, x, feat_cache=None, feat_idx=[0], first_chunk=False): + r1 = 2 if self.add_temporal_upsample else 1 + if feat_cache is not None: + idx = feat_idx[0] + cache_x = x[:, :, -CACHE_T:, :, :].clone() + if cache_x.shape[2] < 2 and feat_cache[idx] is not None: + cache_x = torch.cat( + [ + feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to(cache_x.device), + cache_x, + ], + dim=2, + ) + h = self.conv(x, feat_cache[idx]) + feat_cache[idx] = cache_x + feat_idx[0] += 1 + else: + h = self.conv( + x + ) # change the channel, ready for saptial or temporal downsample + + if self.add_temporal_upsample: + if first_chunk: + h = rearrange(h, "b (r2 r3 c) f h w -> b c f (h r2) (w r3)", r2=2, r3=2) + h = h[:, : h.shape[1] // 2] + # compute the shortcut part + shortcut = rearrange( + x, "b (r2 r3 c) f h w -> b c f (h r2) (w r3)", r2=2, r3=2 + ) + shortcut = shortcut.repeat_interleave(repeats=self.repeats // 2, dim=1) + elif feat_cache is None and x.shape[2] > 1: + # Separate first frame and remaining frames + h_first = h[:, :, :1, :, :] # first frame + h_rest = h[:, :, 1:, :, :] # remaining frames + x_first = x[:, :, :1, :, :] + x_rest = x[:, :, 1:, :, :] + + # First frame: only spatial upsample + h_first = rearrange( + h_first, "b (r2 r3 c) f h w -> b c f (h r2) (w r3)", r2=2, r3=2 + ) + h_first = h_first[:, : h_first.shape[1] // 2] + shortcut_first = rearrange( + x_first, "b (r2 r3 c) f h w -> b c f (h r2) (w r3)", r2=2, r3=2 + ) + shortcut_first = shortcut_first.repeat_interleave( + repeats=self.repeats // 2, dim=1 + ) + out_first = h_first + shortcut_first + + # Remaining frames: spatio-temporal upsample + h_rest = rearrange( + h_rest, + "b (r1 r2 r3 c) f h w -> b c (f r1) (h r2) (w r3)", + r1=r1, + r2=2, + r3=2, + ) + shortcut_rest = rearrange( + x_rest, + "b (r1 r2 r3 c) f h w -> b c (f r1) (h r2) (w r3)", + r1=r1, + r2=2, + r3=2, + ) + shortcut_rest = shortcut_rest.repeat_interleave( + repeats=self.repeats, dim=1 + ) + out_rest = h_rest + shortcut_rest + + return torch.cat([out_first, out_rest], dim=2) + else: + h = rearrange( + h, + "b (r1 r2 r3 c) f h w -> b c (f r1) (h r2) (w r3)", + r1=r1, + r2=2, + r3=2, + ) + # compute the shortcut part + shortcut = rearrange( + x, + "b (r1 r2 r3 c) f h w -> b c (f r1) (h r2) (w r3)", + r1=r1, + r2=2, + r3=2, + ) + shortcut = shortcut.repeat_interleave(repeats=self.repeats, dim=1) + else: + h = rearrange( + h, "b (r1 r2 r3 c) f h w -> b c (f r1) (h r2) (w r3)", r1=r1, r2=2, r3=2 + ) + shortcut = x.repeat_interleave(repeats=self.repeats, dim=1) + shortcut = rearrange( + shortcut, + "b (r1 r2 r3 c) f h w -> b c (f r1) (h r2) (w r3)", + r1=r1, + r2=2, + r3=2, + ) + + return h + shortcut + + +class Encoder(nn.Module): + """Hierarchical video encoder with temporal and spatial factorization.""" + + def __init__( + self, + in_channels: int, + z_channels: int, + block_out_channels: Tuple[int, ...], + num_res_blocks: int, + ffactor_spatial: int, + ffactor_temporal: int, + downsample_match_channel: bool = True, + ): + super().__init__() + assert block_out_channels[-1] % (2 * z_channels) == 0 + + self.z_channels = z_channels + self.block_out_channels = block_out_channels + self.num_res_blocks = num_res_blocks + + # downsampling + self.conv_in = CausalConv3d(in_channels, block_out_channels[0], kernel_size=3) + + self.down = nn.ModuleList() + block_in = block_out_channels[0] + for i_level, ch in enumerate(block_out_channels): + block = nn.ModuleList() + block_out = ch + for _ in range(self.num_res_blocks): + block.append(ResnetBlock(in_channels=block_in, out_channels=block_out)) + block_in = block_out + down = nn.Module() + down.block = block + + add_spatial_downsample = bool(i_level < np.log2(ffactor_spatial)) + add_temporal_downsample = add_spatial_downsample and bool( + i_level >= np.log2(ffactor_spatial // ffactor_temporal) + ) + if add_spatial_downsample or add_temporal_downsample: + assert i_level < len(block_out_channels) - 1 + block_out = ( + block_out_channels[i_level + 1] + if downsample_match_channel + else block_in + ) + down.downsample = Downsample( + block_in, block_out, add_temporal_downsample + ) + block_in = block_out + self.down.append(down) + + # middle + self.mid = nn.Module() + self.mid.block_1 = ResnetBlock(in_channels=block_in, out_channels=block_in) + self.mid.attn_1 = AttnBlock(block_in) + self.mid.block_2 = ResnetBlock(in_channels=block_in, out_channels=block_in) + + # end + self.norm_out = RMS_norm(block_in, images=False) + self.conv_out = CausalConv3d(block_in, 2 * z_channels, kernel_size=3) + + self.gradient_checkpointing = False + + def forward(self, x, feat_cache=None, feat_idx=[0]): + """Forward pass through the encoder.""" + use_checkpointing = bool(self.training and self.gradient_checkpointing) + + if feat_cache is not None: + idx = feat_idx[0] + cache_x = x[:, :, -CACHE_T:, :, :].clone() # cache x before layers + if cache_x.shape[2] < 2 and feat_cache[idx] is not None: + # cache last frame of last two chunk + cache_x = torch.cat( + [ + feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to(cache_x.device), + cache_x, + ], + dim=2, + ) + h = self.conv_in(x, feat_cache[idx]) # [B, 160, T, H, W] with padding + feat_cache[idx] = cache_x + feat_idx[0] += 1 # move to next cache + else: + h = self.conv_in(x) + + for i_level in range(len(self.block_out_channels)): + for i_block in range(self.num_res_blocks): + h = forward_with_checkpointing( + self.down[i_level].block[i_block], + h, + feat_cache, + feat_idx, + use_checkpointing=use_checkpointing, + ) + if hasattr(self.down[i_level], "downsample"): + h = forward_with_checkpointing( + self.down[i_level].downsample, + h, + feat_cache, + feat_idx, + use_checkpointing=use_checkpointing, + ) + + # middle + h = forward_with_checkpointing( + self.mid.block_1, + h, + feat_cache, + feat_idx, + use_checkpointing=use_checkpointing, + ) + h = forward_with_checkpointing( + self.mid.attn_1, h, use_checkpointing=use_checkpointing + ) + h = forward_with_checkpointing( + self.mid.block_2, + h, + feat_cache, + feat_idx, + use_checkpointing=use_checkpointing, + ) + + # end + group_size = self.block_out_channels[-1] // (2 * self.z_channels) + shortcut = rearrange(h, "b (c r) f h w -> b c r f h w", r=group_size).mean( + dim=2 + ) + h = self.norm_out(h) + h = swish(h, inplace=True) + + if feat_cache is not None: + idx = feat_idx[0] + cache_x = h[:, :, -CACHE_T:, :, :].clone() + if cache_x.shape[2] < 2 and feat_cache[idx] is not None: + cache_x = torch.cat( + [ + feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to(cache_x.device), + cache_x, + ], + dim=2, + ) + + h = self.conv_out(h, feat_cache[idx]) + feat_cache[idx] = cache_x + feat_idx[0] += 1 + else: + h = self.conv_out(h) + h += shortcut + return h + + +class Decoder(nn.Module): + """Hierarchical video decoder with upsampling factories.""" + + def __init__( + self, + z_channels: int, + out_channels: int, + block_out_channels: Tuple[int, ...], + num_res_blocks: int, + ffactor_spatial: int, + ffactor_temporal: int, + upsample_match_channel: bool = True, + ): + super().__init__() + assert block_out_channels[0] % z_channels == 0 + + self.z_channels = z_channels + self.block_out_channels = block_out_channels + self.num_res_blocks = num_res_blocks + + block_in = block_out_channels[0] + self.conv_in = CausalConv3d(z_channels, block_in, kernel_size=3) + + # middle + self.mid = nn.Module() + self.mid.block_1 = ResnetBlock(in_channels=block_in, out_channels=block_in) + self.mid.attn_1 = AttnBlock(block_in) + self.mid.block_2 = ResnetBlock(in_channels=block_in, out_channels=block_in) + + # upsampling + self.up = nn.ModuleList() + for i_level, ch in enumerate(block_out_channels): + block = nn.ModuleList() + block_out = ch + for _ in range(self.num_res_blocks + 1): + block.append(ResnetBlock(in_channels=block_in, out_channels=block_out)) + block_in = block_out + up = nn.Module() + up.block = block + + add_spatial_upsample = bool(i_level < np.log2(ffactor_spatial)) + add_temporal_upsample = bool(i_level < np.log2(ffactor_temporal)) + if add_spatial_upsample or add_temporal_upsample: + assert i_level < len(block_out_channels) - 1 + block_out = ( + block_out_channels[i_level + 1] + if upsample_match_channel + else block_in + ) + up.upsample = Upsample(block_in, block_out, add_temporal_upsample) + block_in = block_out + self.up.append(up) + + # end + self.norm_out = RMS_norm(block_in, images=False) + self.conv_out = CausalConv3d(block_in, out_channels, kernel_size=3) + + self.gradient_checkpointing = False + + def forward(self, z, feat_cache=None, feat_idx=[0], first_chunk=False): + """Forward pass through the decoder.""" + use_checkpointing = bool(self.training and self.gradient_checkpointing) + + repeats = self.block_out_channels[0] // (self.z_channels) + if feat_cache is not None: + idx = feat_idx[0] + cache_x = z[:, :, -CACHE_T:, :, :].clone() + if cache_x.shape[2] < 2 and feat_cache[idx] is not None: + cache_x = torch.cat( + [ + feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to(cache_x.device), + cache_x, + ], + dim=2, + ) + h = self.conv_in(z, feat_cache[idx]) + z.repeat_interleave( + repeats=repeats, dim=1 + ) + feat_cache[idx] = cache_x + feat_idx[0] += 1 + else: + # z to block_in + h = self.conv_in(z) + z.repeat_interleave(repeats=repeats, dim=1) + + # middle + h = forward_with_checkpointing( + self.mid.block_1, + h, + feat_cache, + feat_idx, + use_checkpointing=use_checkpointing, + ) + h = forward_with_checkpointing( + self.mid.attn_1, h, use_checkpointing=use_checkpointing + ) + h = forward_with_checkpointing( + self.mid.block_2, + h, + feat_cache, + feat_idx, + use_checkpointing=use_checkpointing, + ) + + # upsampling + for i_level in range(len(self.block_out_channels)): + for i_block in range(self.num_res_blocks + 1): + h = forward_with_checkpointing( + self.up[i_level].block[i_block], + h, + feat_cache, + feat_idx, + use_checkpointing=use_checkpointing, + ) + if hasattr(self.up[i_level], "upsample"): + h = forward_with_checkpointing( + self.up[i_level].upsample, + h, + feat_cache, + feat_idx, + first_chunk, + use_checkpointing=use_checkpointing, + ) + + # end + h = self.norm_out(h) + h = swish(h, inplace=True) + + if feat_cache is not None: + idx = feat_idx[0] + cache_x = h[:, :, -CACHE_T:, :, :].clone() + if cache_x.shape[2] < 2 and feat_cache[idx] is not None: + cache_x = torch.cat( + [ + feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to(cache_x.device), + cache_x, + ], + dim=2, + ) + h = self.conv_out(h, feat_cache[idx]) + feat_cache[idx] = cache_x + feat_idx[0] += 1 + else: + h = self.conv_out(h) + return h + + +class AutoencoderKLConv3D(ModelMixin, ConfigMixin): + """KL regularized 3D Conv VAE with advanced tiling and slicing strategies.""" + + _supports_gradient_checkpointing = True + + @register_to_config + def __init__( + self, + in_channels: int, + out_channels: int, + latent_channels: int, + block_out_channels: Tuple[int, ...], + layers_per_block: int, + ffactor_spatial: int, + ffactor_temporal: int, + sample_size: int, + sample_tsize: int, + scaling_factor: float = None, + shift_factor: Optional[float] = None, + downsample_match_channel: bool = True, + upsample_match_channel: bool = True, + ): + super().__init__() + self.ffactor_spatial = ffactor_spatial + self.ffactor_temporal = ffactor_temporal + self.scaling_factor = scaling_factor + self.shift_factor = shift_factor + + self.encoder = Encoder( + in_channels=in_channels, + z_channels=latent_channels, + block_out_channels=block_out_channels, + num_res_blocks=layers_per_block, + ffactor_spatial=ffactor_spatial, + ffactor_temporal=ffactor_temporal, + downsample_match_channel=downsample_match_channel, + ) + self.decoder = Decoder( + z_channels=latent_channels, + out_channels=out_channels, + block_out_channels=list(reversed(block_out_channels)), + num_res_blocks=layers_per_block, + ffactor_spatial=ffactor_spatial, + ffactor_temporal=ffactor_temporal, + upsample_match_channel=upsample_match_channel, + ) + + self.use_slicing = False + self.use_spatial_tiling = False + self.use_temporal_tiling = False + + # only relevant if vae tiling is enabled + self.tile_sample_min_size = sample_size + self.tile_latent_min_size = sample_size // ffactor_spatial + self.tile_sample_min_tsize = sample_tsize + self.tile_latent_min_tsize = sample_tsize // ffactor_temporal + self.tile_overlap_factor = 0.25 + + self._tile_parallelism_enabled = False + + # Precompute and cache conv counts for encoder and decoder for clear_cache speedup + self._cached_conv_counts = { + "decoder": ( + sum(isinstance(m, CausalConv3d) for m in self.decoder.modules()) + if self.decoder is not None + else 0 + ), + "encoder": ( + sum(isinstance(m, CausalConv3d) for m in self.encoder.modules()) + if self.encoder is not None + else 0 + ), + } + + def set_tile_sample_min_size( + self, sample_size: int, tile_overlap_factor: float = 0.2 + ): + self.tile_sample_min_size = sample_size + self.tile_latent_min_size = sample_size // self.ffactor_spatial + self.tile_overlap_factor = tile_overlap_factor + + assert ( + self.tile_latent_min_size * self.tile_overlap_factor + ).is_integer(), "self.tile_latent_min_size multiplied by tile_overlap_factor must be an integer" + + def _set_gradient_checkpointing(self, module, value=False): + """Enable or disable gradient checkpointing on encoder and decoder.""" + if isinstance(module, (Encoder, Decoder)): + module.gradient_checkpointing = value + + def enable_temporal_tiling(self, use_tiling: bool = True): + raise RuntimeError("Temporal tiling is not supported for this VAE.") + self.use_temporal_tiling = use_tiling + + def disable_temporal_tiling(self): + self.enable_temporal_tiling(False) + + def enable_spatial_tiling(self, use_tiling: bool = True): + self.use_spatial_tiling = use_tiling + + def disable_spatial_tiling(self): + self.enable_spatial_tiling(False) + + def enable_tiling(self, use_tiling: bool = True): + self.enable_spatial_tiling(use_tiling) + + def disable_tiling(self): + self.disable_spatial_tiling() + + def enable_slicing(self): + self.use_slicing = True + + def disable_slicing(self): + self.use_slicing = False + + def blend_h(self, a: torch.Tensor, b: torch.Tensor, blend_extent: int): + """Blend tensor b horizontally into a at blend_extent region.""" + blend_extent = min(a.shape[-1], b.shape[-1], blend_extent) + for x in range(blend_extent): + b[:, :, :, :, x] = a[:, :, :, :, -blend_extent + x] * ( + 1 - x / blend_extent + ) + b[:, :, :, :, x] * (x / blend_extent) + return b + + def blend_v(self, a: torch.Tensor, b: torch.Tensor, blend_extent: int): + """Blend tensor b vertically into a at blend_extent region.""" + blend_extent = min(a.shape[-2], b.shape[-2], blend_extent) + for y in range(blend_extent): + b[:, :, :, y, :] = a[:, :, :, -blend_extent + y, :] * ( + 1 - y / blend_extent + ) + b[:, :, :, y, :] * (y / blend_extent) + return b + + def blend_t(self, a: torch.Tensor, b: torch.Tensor, blend_extent: int): + """Blend tensor b temporally into a at blend_extent region.""" + blend_extent = min(a.shape[-3], b.shape[-3], blend_extent) + for x in range(blend_extent): + b[:, :, x, :, :] = a[:, :, -blend_extent + x, :, :] * ( + 1 - x / blend_extent + ) + b[:, :, x, :, :] * (x / blend_extent) + return b + + def spatial_tiled_encode(self, x: torch.Tensor): + """Tiled spatial encoding for large inputs via overlapping.""" + B, C, T, H, W = x.shape + overlap_size = int(self.tile_sample_min_size * (1 - self.tile_overlap_factor)) + blend_extent = int(self.tile_latent_min_size * self.tile_overlap_factor) + row_limit = self.tile_latent_min_size - blend_extent + + rows = [] + for i in range(0, H, overlap_size): + row = [] + for j in range(0, W, overlap_size): + tile = x[ + :, + :, + :, + i : i + self.tile_sample_min_size, + j : j + self.tile_sample_min_size, + ] + tile = self.encoder(tile) + row.append(tile) + rows.append(row) + result_rows = [] + for i, row in enumerate(rows): + result_row = [] + for j, tile in enumerate(row): + if i > 0: + tile = self.blend_v(rows[i - 1][j], tile, blend_extent) + if j > 0: + tile = self.blend_h(row[j - 1], tile, blend_extent) + result_row.append(tile[:, :, :, :row_limit, :row_limit]) + result_rows.append(torch.cat(result_row, dim=-1)) + moments = torch.cat(result_rows, dim=-2) + return moments + + def temporal_tiled_encode(self, x: torch.Tensor): + """Tiled temporal encoding for large video sequences.""" + raise RuntimeError("Temporal tiling is not supported for this VAE.") + B, C, T, H, W = x.shape + overlap_size = int(self.tile_sample_min_tsize * (1 - self.tile_overlap_factor)) + blend_extent = int(self.tile_latent_min_tsize * self.tile_overlap_factor) + t_limit = self.tile_latent_min_tsize - blend_extent + + row = [] + for i in range(0, T, overlap_size): + tile = x[:, :, i : i + self.tile_sample_min_tsize + 1, :, :] + if self.use_spatial_tiling and ( + tile.shape[-1] > self.tile_sample_min_size + or tile.shape[-2] > self.tile_sample_min_size + ): + tile = self.spatial_tiled_encode(tile) + else: + tile = self.encoder(tile) + if i > 0: + tile = tile[:, :, 1:, :, :] + row.append(tile) + result_row = [] + for i, tile in enumerate(row): + if i > 0: + tile = self.blend_t(row[i - 1], tile, blend_extent) + result_row.append(tile[:, :, :t_limit, :, :]) + else: + result_row.append(tile[:, :, : t_limit + 1, :, :]) + moments = torch.cat(result_row, dim=-3) + return moments + + def enable_tile_parallelism(self): + self._tile_parallelism_enabled = True + + def disable_tile_parallelism(self): + self._tile_parallelism_enabled = False + + def tile_parallel_spatial_tiled_decode(self, z: torch.Tensor): + B, C, T, H, W = z.shape + overlap_size = int(self.tile_latent_min_size * (1 - self.tile_overlap_factor)) + blend_extent = int(self.tile_sample_min_size * self.tile_overlap_factor) + row_limit = self.tile_sample_min_size - blend_extent + + rank = get_parallel_state().sp_rank + world_size = get_parallel_state().sp + + num_rows = math.ceil(H / overlap_size) + num_cols = math.ceil(W / overlap_size) + total_tiles = num_rows * num_cols + tiles_per_rank = math.ceil(total_tiles / world_size) + + my_linear_indices = list(range(rank, total_tiles, world_size)) + decoded_tiles = [] + decoded_metas = [] + H_out_std = self.tile_sample_min_size + W_out_std = self.tile_sample_min_size + for lin_idx in my_linear_indices: + ri = lin_idx // num_cols + rj = lin_idx % num_cols + i = ri * overlap_size + j = rj * overlap_size + tile = z[ + :, + :, + :, + i : i + self.tile_latent_min_size, + j : j + self.tile_latent_min_size, + ] + dec = self.decoder(tile) + + pad_h = max(0, H_out_std - dec.shape[-2]) + pad_w = max(0, W_out_std - dec.shape[-1]) + if pad_h > 0 or pad_w > 0: + dec = F.pad(dec, (0, pad_w, 0, pad_h, 0, 0), "constant", 0) + decoded_tiles.append(dec) + decoded_metas.append( + torch.tensor([ri, rj, pad_w, pad_h], device=z.device, dtype=torch.int64) + ) + + while len(decoded_tiles) < tiles_per_rank: + T_out = ( + decoded_tiles[0].shape[2] + if len(decoded_tiles) > 0 + else (T - 1) * self.ffactor_temporal + 1 + ) + zero_tile = torch.zeros( + [1, 3, T_out, self.tile_sample_min_size, self.tile_sample_min_size], + device=dec.device, + dtype=dec.dtype, + ) + decoded_tiles.append(zero_tile) + meta_tensor = torch.tensor( + [-1, -1, self.tile_sample_min_size, self.tile_sample_min_size], + device=z.device, + dtype=torch.int64, + ) + decoded_metas.append(meta_tensor) + + decoded_tiles = torch.stack(decoded_tiles, dim=0) + decoded_metas = torch.stack(decoded_metas, dim=0) + + tiles_gather_list = [torch.empty_like(decoded_tiles) for _ in range(world_size)] + metas_gather_list = [torch.empty_like(decoded_metas) for _ in range(world_size)] + + dist.all_gather( + tiles_gather_list, decoded_tiles, group=get_parallel_state().sp_group + ) + dist.all_gather( + metas_gather_list, decoded_metas, group=get_parallel_state().sp_group + ) + dist.barrier() + + if rank != 0: + return torch.empty(0, device=z.device) + + rows = [[None for _ in range(num_cols)] for _ in range(num_rows)] + for r in range(world_size): + gathered_tiles_r = tiles_gather_list[r] # [tiles_per_rank, B, C, T, H, W] + gathered_metas_r = metas_gather_list[r] # [tiles_per_rank, 4] + for k in range(gathered_tiles_r.shape[0]): + ri = int(gathered_metas_r[k][0]) + rj = int(gathered_metas_r[k][1]) + if ri < 0 or rj < 0: + continue + if ri < num_rows and rj < num_cols: + # remove padding + pad_w = int(gathered_metas_r[k][2]) + pad_h = int(gathered_metas_r[k][3]) + h_end = None if pad_h == 0 else -pad_h + w_end = None if pad_w == 0 else -pad_w + rows[ri][rj] = gathered_tiles_r[k][:, :, :, :h_end, :w_end] + + result_rows = [] + for i, row in enumerate(rows): + result_row = [] + for j, tile in enumerate(row): + if tile is None: + continue + if i > 0: + tile = self.blend_v(rows[i - 1][j], tile, blend_extent) + if j > 0: + tile = self.blend_h(row[j - 1], tile, blend_extent) + result_row.append(tile[:, :, :, :row_limit, :row_limit]) + result_rows.append(torch.cat(result_row, dim=-1)) + + dec = torch.cat(result_rows, dim=-2) + return dec + + def spatial_tiled_decode(self, z: torch.Tensor): + if self._tile_parallelism_enabled: + return self.tile_parallel_spatial_tiled_decode(z) + + B, C, T, H, W = z.shape + overlap_size = int(self.tile_latent_min_size * (1 - self.tile_overlap_factor)) + blend_extent = int(self.tile_sample_min_size * self.tile_overlap_factor) + row_limit = self.tile_sample_min_size - blend_extent + + rows = [] + for i in range(0, H, overlap_size): + row = [] + for j in range(0, W, overlap_size): + tile = z[ + :, + :, + :, + i : i + self.tile_latent_min_size, + j : j + self.tile_latent_min_size, + ] + decoded = self.decoder(tile) + row.append(decoded) + rows.append(row) + + result_rows = [] + for i, row in enumerate(rows): + result_row = [] + for j, tile in enumerate(row): + if i > 0: + tile = self.blend_v(rows[i - 1][j], tile, blend_extent) + if j > 0: + tile = self.blend_h(row[j - 1], tile, blend_extent) + result_row.append(tile[:, :, :, :row_limit, :row_limit]) + result_rows.append(torch.cat(result_row, dim=-1)) + dec = torch.cat(result_rows, dim=-2) + return dec + + def temporal_tiled_decode(self, z: torch.Tensor): + """Tiled temporal decoding for long sequence latents.""" + raise RuntimeError("Temporal tiling is not supported for this VAE.") + B, C, T, H, W = z.shape + overlap_size = int(self.tile_latent_min_tsize * (1 - self.tile_overlap_factor)) + blend_extent = int(self.tile_sample_min_tsize * self.tile_overlap_factor) + t_limit = self.tile_sample_min_tsize - blend_extent + assert 0 < overlap_size < self.tile_latent_min_tsize + + row = [] + for i in range(0, T, overlap_size): + tile = z[:, :, i : i + self.tile_latent_min_tsize + 1, :, :] + if self.use_spatial_tiling and ( + tile.shape[-1] > self.tile_latent_min_size + or tile.shape[-2] > self.tile_latent_min_size + ): + decoded = self.spatial_tiled_decode(tile) + else: + decoded = self.decoder(tile) + if i > 0: + decoded = decoded[:, :, 1:, :, :] + row.append(decoded) + + result_row = [] + for i, tile in enumerate(row): + if i > 0: + tile = self.blend_t(row[i - 1], tile, blend_extent) + result_row.append(tile[:, :, :t_limit, :, :]) + else: + result_row.append(tile[:, :, : t_limit + 1, :, :]) + dec = torch.cat(result_row, dim=-3) + return dec + + def clear_cache(self): + # Use cached conv counts for decoder and encoder to avoid re-iterating modules each call + self._conv_num = self._cached_conv_counts["decoder"] + self._conv_idx = [0] + self._feat_map = [None] * self._conv_num + # cache encode + self._enc_conv_num = self._cached_conv_counts["encoder"] + self._enc_conv_idx = [0] + self._enc_feat_map = [None] * self._enc_conv_num + + def encode(self, x: Tensor, return_dict: bool = True): + + def _encode(x): + """slice encoding video latents""" + _, _, num_frame, height, width = x.shape + + if self.use_temporal_tiling and x.shape[-3] > self.tile_sample_min_tsize: + return self.temporal_tiled_encode(x) + if self.use_spatial_tiling and ( + x.shape[-1] > self.tile_sample_min_size + or x.shape[-2] > self.tile_sample_min_size + ): + return self.spatial_tiled_encode(x) + + self.clear_cache() + iter_ = 1 + (num_frame - 1) // 4 + for i in range(iter_): + self._enc_conv_idx = [0] + if i == 0: + out = self.encoder( + x[:, :, :1, :, :], + feat_cache=self._enc_feat_map, + feat_idx=self._enc_conv_idx, + ) + else: + out_ = self.encoder( + x[:, :, 1 + 4 * (i - 1) : 1 + 4 * i, :, :], + feat_cache=self._enc_feat_map, + feat_idx=self._enc_conv_idx, + ) # [B, C, 4, H, W] after patchify + out = torch.cat([out, out_], 2) + + self.clear_cache() + return out + + assert len(x.shape) == 5 # (B, C, T, H, W) + + if self.use_slicing and x.shape[0] > 1: + encoded_slices = [_encode(x_slice) for x_slice in x.split(1)] + h = torch.cat(encoded_slices) + else: + h = _encode(x) + posterior = DiagonalGaussianDistribution(h) + + if not return_dict: + return (posterior,) + + return AutoencoderKLOutput(latent_dist=posterior) + + def decode(self, z: Tensor, return_dict: bool = True, generator=None): + + def _decode(z): + """slice decoding video latents""" + _, _, num_frame, height, width = z.shape + + if self.use_temporal_tiling and z.shape[-3] > self.tile_latent_min_tsize: + return self.temporal_tiled_decode(z) + if self.use_spatial_tiling and ( + z.shape[-1] > self.tile_latent_min_size + or z.shape[-2] > self.tile_latent_min_size + ): + return self.spatial_tiled_decode(z) + + self.clear_cache() + for i in range(num_frame): + self._conv_idx = [0] + if i == 0: + out = self.decoder( + z[:, :, i : i + 1, :, :], + feat_cache=self._feat_map, + feat_idx=self._conv_idx, + first_chunk=True, + ) + else: + out_ = self.decoder( + z[:, :, i : i + 1, :, :], + feat_cache=self._feat_map, + feat_idx=self._conv_idx, + ) + out = torch.cat([out, out_], 2) + + self.clear_cache() + + return out + + if self.use_slicing and z.shape[0] > 1: + decoded_slices = [_decode(z_slice) for z_slice in z.split(1)] + decoded = torch.cat(decoded_slices) + else: + decoded = _decode(z) + + if not return_dict: + return (decoded,) + + return DecoderOutput(sample=decoded) + + def forward( + self, + sample: torch.Tensor, + sample_posterior: bool = False, + return_posterior: bool = True, + return_dict: bool = True, + ): + """ + Performs the full forward pass of the Autoencoder. + + It encodes the input into a latent distribution, samples (or takes the mode) from it, + and then decodes it back to the image space. + + Args: + sample (torch.Tensor): Input image tensor (B, C, H, W). + sample_posterior (bool): If True, samples from the latent distribution (stochastic). + If False, uses the mode (deterministic). + return_posterior (bool): Whether to include the posterior distribution in the output. + return_dict (bool): If True, returns a structured DecoderOutput object; + otherwise, returns a standard tuple. + """ + # 1. Encode: Map input 'sample' to a latent distribution (Gaussian). + posterior = self.encode(sample).latent_dist + + # 2. Sample: Reparameterization trick. + # Use random sampling for training (variational) or mode for deterministic inference. + z = posterior.sample() if sample_posterior else posterior.mode() + + # 3. Decode: Reconstruct the image from the latent vector 'z'. + dec = self.decode(z).sample + + # 4. Return: Format the output based on 'return_dict' preference. + if return_dict: + return DecoderOutput(sample=dec, posterior=posterior) + else: + return (dec, posterior) diff --git a/hyvideo/models/text_encoders/__init__.py b/hyvideo/models/text_encoders/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..88a562761a53a9d13ed8d81e68f882c75c89e49a --- /dev/null +++ b/hyvideo/models/text_encoders/__init__.py @@ -0,0 +1,522 @@ +# Licensed under the TENCENT HUNYUAN COMMUNITY LICENSE AGREEMENT (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://github.com/Tencent-Hunyuan/HunyuanVideo-1.5/blob/main/LICENSE +# +# Unless and only to the extent required by applicable law, the Tencent Hunyuan works and any +# output and results therefrom are provided "AS IS" without any express or implied warranties of +# any kind including any warranties of title, merchantability, noninfringement, course of dealing, +# usage of trade, or fitness for a particular purpose. You are solely responsible for determining the +# appropriateness of using, reproducing, modifying, performing, displaying or distributing any of +# the Tencent Hunyuan works or outputs and assume any and all risks associated with your or a +# third party's use or distribution of any of the Tencent Hunyuan works or outputs and your exercise +# of rights and permissions under this agreement. +# See the License for the specific language governing permissions and limitations under the License. + +import os +from copy import deepcopy +from dataclasses import dataclass +from typing import Optional, Tuple + +import torch +import torch.nn as nn + +from transformers import ( + AutoTokenizer, + AutoModel, +) +from transformers.utils import ModelOutput + + +def use_default(value, default): + """Utility: return value if not None, else default.""" + return value if value is not None else default + + +# Prompt templates for different models and tasks + + +__all__ = [ + "C_SCALE", + "PROMPT_TEMPLATE", + "MODEL_BASE", +] + +# =================== Constant Values ===================== +# Computation scale factor, 1P = 1_000_000_000_000_000. Tensorboard will display the value in PetaFLOPS to avoid +# overflow error when tensorboard logging values. +C_SCALE = 1_000_000_000_000_000 + +PROMPT_TEMPLATE_ENCODE_IMAGE_JSON = [ + { + "role": "system", + "content": "You are a helpful assistant. Describe the image by detailing the following aspects: \ + 1. The main content and theme of the image. \ + 2. The color, shape, size, texture, quantity, text, and spatial relationships of the objects. \ + 3. The background environment, light, style and atmosphere.", + }, + {"role": "user", "content": "{}"}, +] + +PROMPT_TEMPLATE_ENCODE_VIDEO_JSON = [ + { + "role": "system", + "content": "You are a helpful assistant. Describe the video by detailing the following aspects: \ + 1. The main content and theme of the video. \ + 2. The color, shape, size, texture, quantity, text, and spatial relationships of the objects. \ + 3. Actions, events, behaviors temporal relationships, physical movement changes of the objects. \ + 4. background environment, light, style and atmosphere. \ + 5. camera angles, movements, and transitions used in the video.", + }, + {"role": "user", "content": "{}"}, +] + +PROMPT_TEMPLATE = { + "li-dit-encode-image-json": { + "template": PROMPT_TEMPLATE_ENCODE_IMAGE_JSON, + "crop": -1, + }, # auto-calculate crop_start + "li-dit-encode-video-json": { + "template": PROMPT_TEMPLATE_ENCODE_VIDEO_JSON, + "crop_start": -1, + }, # auto-calculate crop_start +} + + +MODEL_BASE = os.getenv("MODEL_BASE", "") +TEXT_ENCODER_PATH = {} +TOKENIZER_PATH = {} + +PRECISION_TO_TYPE = { + "fp32": torch.float32, + "fp16": torch.float16, + "bf16": torch.bfloat16, +} + + +def load_text_encoder( + text_encoder_type, + text_encoder_precision=None, + text_encoder_path=None, + logger=None, + device=None, +): + if text_encoder_path is None: + if text_encoder_type not in TEXT_ENCODER_PATH: + raise ValueError(f"Unsupported text encoder type: {text_encoder_type}") + text_encoder_path = TEXT_ENCODER_PATH[text_encoder_type] + + text_encoder = AutoModel.from_pretrained(text_encoder_path, low_cpu_mem_usage=True) + + if hasattr(text_encoder, "language_model"): + text_encoder = text_encoder.language_model + text_encoder.final_layer_norm = text_encoder.norm + + # from_pretrained will ensure that the model is in eval mode. + if text_encoder_precision is not None: + text_encoder = text_encoder.to(dtype=PRECISION_TO_TYPE[text_encoder_precision]) + + text_encoder.requires_grad_(False) + + if device is not None: + text_encoder = text_encoder.to(device) + + return text_encoder, text_encoder_path + + +def load_tokenizer( + tokenizer_type, tokenizer_path=None, padding_side="right", logger=None +): + processor = None + if tokenizer_path is None: + if tokenizer_type not in TOKENIZER_PATH: + raise ValueError(f"Unsupported tokenizer type: {tokenizer_type}") + tokenizer_path = TOKENIZER_PATH[tokenizer_type] + + tokenizer = AutoTokenizer.from_pretrained(tokenizer_path, padding_side=padding_side) + + return tokenizer, tokenizer_path, processor + + +@dataclass +class TextEncoderModelOutput(ModelOutput): + """ + Base class for model's outputs that also contains a pooling of the last hidden states. + + Args: + hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): + Sequence of hidden-states at the output of the last layer of the model. + attention_mask (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in ``[0, 1]``: + hidden_states_list (`tuple(torch.FloatTensor)`, *optional*: + Returned when `output_hidden_states=True` is passed). + Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. + Hidden-states of the model at the output of each layer plus the optional initial embedding outputs. + text_outputs (`list`, *optional*, returned when `return_texts=True` is passed): + List of decoded texts. + """ + + hidden_state: torch.FloatTensor = None + attention_mask: Optional[torch.LongTensor] = None + hidden_states_list: Optional[Tuple[torch.FloatTensor, ...]] = None + text_outputs: Optional[list] = None + image_features: Optional[list] = None + + +class TextEncoder(nn.Module): + def __init__( + self, + text_encoder_type: str, + max_length: int, + text_encoder_precision: Optional[str] = None, + text_encoder_path: Optional[str] = None, + tokenizer_type: Optional[str] = None, + tokenizer_path: Optional[str] = None, + output_key: Optional[str] = None, + use_attention_mask: bool = True, + prompt_template: Optional[dict] = None, + prompt_template_video: Optional[dict] = None, + hidden_state_skip_layer: Optional[int] = None, + apply_final_norm: bool = False, + reproduce: bool = False, + logger=None, + device=None, + ): + super().__init__() + self.text_encoder_type = text_encoder_type + self.max_length = max_length + self.precision = text_encoder_precision + self.model_path = text_encoder_path + self.tokenizer_type = ( + tokenizer_type if tokenizer_type is not None else text_encoder_type + ) + self.tokenizer_path = ( + tokenizer_path if tokenizer_path is not None else text_encoder_path + ) + self.use_attention_mask = use_attention_mask + if prompt_template_video is not None: + assert ( + use_attention_mask is True + ), "Attention mask is True required when training videos." + self.prompt_template = prompt_template + self.prompt_template_video = prompt_template_video + self.hidden_state_skip_layer = hidden_state_skip_layer + self.apply_final_norm = apply_final_norm + self.reproduce = reproduce + self.logger = logger + + self.use_template = self.prompt_template is not None + if self.use_template: + assert ( + isinstance(self.prompt_template, dict) + and "template" in self.prompt_template + ), f"`prompt_template` must be a dictionary with a key 'template', got {self.prompt_template}" + assert "{}" in str(self.prompt_template["template"]), ( + "`prompt_template['template']` must contain a placeholder `{}` for the input text, " + f"got {self.prompt_template['template']}" + ) + + self.use_video_template = self.prompt_template_video is not None + if self.use_video_template: + if self.prompt_template_video is not None: + assert ( + isinstance(self.prompt_template_video, dict) + and "template" in self.prompt_template_video + ), f"`prompt_template_video` must be a with a key 'template', " + f"got {self.prompt_template_video}" + assert "{}" in str(self.prompt_template_video["template"]), ( + "`prompt_template_video['template']` must contain a placeholder `{}` for the input text, " + f"got {self.prompt_template_video['template']}" + ) + + if text_encoder_type != "llm": + raise ValueError(f"Unsupported text encoder type: {text_encoder_type}") + self.output_key = output_key or "last_hidden_state" + + self.model, self.model_path = load_text_encoder( + text_encoder_type=self.text_encoder_type, + text_encoder_precision=self.precision, + text_encoder_path=self.model_path, + logger=self.logger, + device=device, + ) + + self.tokenizer, self.tokenizer_path, self.processor = load_tokenizer( + tokenizer_type=self.tokenizer_type, + tokenizer_path=self.tokenizer_path, + padding_side="right", + logger=self.logger, + ) + + # pre-calculate crop_start for image and video + if self.use_template and self.prompt_template is not None: + self.text2tokens("a photo of a cat", data_type="image") + if self.use_video_template and self.prompt_template_video is not None: + self.text2tokens("a photo of a cat", data_type="video") + + @property + def dtype(self): + return self.model.dtype + + @property + def device(self): + return self.model.device + + def __repr__(self): + return f"{self.text_encoder_type} ({self.precision} - {self.model_path})" + + @staticmethod + def apply_text_to_template(text, template, prevent_empty_text=True): + """ + Apply text to template. + + Args: + text (str): Input text. + template (str or list): Template string or list of chat conversation. + prevent_empty_text (bool): If Ture, we will prevent the user text from being empty + by adding a space. Defaults to True. + """ + if isinstance(template, str): + # Will send string to tokenizer. Used for llm + return template.format(text) + elif isinstance(template, list): + # For JSON list template format (chat conversation) + # Create a deep copy to avoid modifying the original template + template_copy = deepcopy(template) + for item in template_copy: + if isinstance(item, dict) and "content" in item: + # Replace placeholder with text in the content field + item["content"] = item["content"].format( + text if text else (" " if prevent_empty_text else "") + ) + return template_copy + else: + raise TypeError(f"Unsupported template type: {type(template)}") + + def calculate_crop_start(self, tokenized_input): + """ + Automatically calculate the crop_start position based on identifying user tokens. + + Args: + tokenized_input: The output from the tokenizer containing input_ids + + Returns: + int: The position where the actual prompt content begins (after user markers) + """ + input_ids = tokenized_input["input_ids"][ + 0 + ].tolist() # Get the first example's tokens + + marker = "<|im_start|>user\n" + + # Tokenize just the marker to get its token IDs + marker_tokens = self.tokenizer(marker, add_special_tokens=False)["input_ids"] + + # Find the end position of the marker in the input sequence + for i in range(len(input_ids) - len(marker_tokens) + 1): + if input_ids[i : i + len(marker_tokens)] == marker_tokens: + # Return the position after the marker + return i + len(marker_tokens) + + # If marker not found, try to find based on special tokens + if hasattr(self.tokenizer, "special_tokens_map"): + # Check for user token or any other special token that might indicate user input start + for token_name, token_value in self.tokenizer.special_tokens_map.items(): + if "user" in token_name.lower(): + user_token_id = self.tokenizer.convert_tokens_to_ids(token_value) + if user_token_id in input_ids: + return input_ids.index(user_token_id) + 1 + + # Default fallback: return 0 (no cropping) + return 0 + + def text2tokens(self, text, data_type="image", max_length=300): + """ + Tokenize the input text. + + Args: + text (str or list): Input text. + """ + tokenize_input_type = "str" + if self.use_template or self.use_video_template: + if data_type == "image": + prompt_template = self.prompt_template["template"] + crop_start = self.prompt_template.get("crop_start", -1) + elif data_type == "video": + prompt_template = self.prompt_template_video["template"] + crop_start = self.prompt_template_video.get("crop_start", -1) + else: + raise ValueError(f"Unsupported data type: {data_type}") + if isinstance(text, (list, tuple)): + text = [ + self.apply_text_to_template(one_text, prompt_template) + for one_text in text + ] + if isinstance(text[0], list): + tokenize_input_type = "list" + elif isinstance(text, str): + text = self.apply_text_to_template(text, prompt_template) + if isinstance(text, list): + tokenize_input_type = "list" + else: + raise TypeError(f"Unsupported text type: {type(text)}") + + # First pass: tokenize with arbitrary max_length to find crop_start + if crop_start == -1: + # Use temporary max_length for the first pass (large enough) + temp_kwargs = dict( + truncation=True, + max_length=256, # Temporary large value + padding="max_length", + return_tensors="pt", + ) + + # First tokenization pass to calculate crop_start + if tokenize_input_type == "str": + temp_tokenized = self.tokenizer( + text, + return_length=False, + return_overflowing_tokens=False, + return_attention_mask=True, + **temp_kwargs, + ) + elif tokenize_input_type == "list": + temp_tokenized = self.tokenizer.apply_chat_template( + text, + add_generation_prompt=True, + tokenize=True, + return_dict=True, + **temp_kwargs, + ) + + # Calculate the crop_start from this first pass + crop_start = self.calculate_crop_start(temp_tokenized) + + # Store the calculated crop_start for future use + if data_type == "image": + self.prompt_template["crop_start"] = crop_start + else: + self.prompt_template_video["crop_start"] = crop_start + else: + crop_start = 0 + + # Second pass: tokenize with the proper max_length using the found crop_start + kwargs = dict( + truncation=True, + max_length=max_length + (crop_start if crop_start > 0 else 0), + padding="max_length", + return_tensors="pt", + ) + + if tokenize_input_type == "str": + tokenized_output = self.tokenizer( + text, + return_length=False, + return_overflowing_tokens=False, + return_attention_mask=True, + **kwargs, + ) + elif tokenize_input_type == "list": + tokenized_output = self.tokenizer.apply_chat_template( + text, + add_generation_prompt=True, + tokenize=True, + return_dict=True, + **kwargs, + ) + else: + raise ValueError(f"Unsupported tokenize_input_type: {tokenize_input_type}") + + return tokenized_output + + def encode( + self, + batch_encoding, + use_attention_mask=None, + output_hidden_states=False, + do_sample=None, + hidden_state_skip_layer=None, + return_texts=False, + data_type="image", + device=None, + is_uncond=False, + ): + """ + Args: + batch_encoding (dict): Batch encoding from tokenizer. + use_attention_mask (bool): Whether to use attention mask. If None, use self.use_attention_mask. + Defaults to None. + output_hidden_states (bool): Whether to output hidden states. If False, return the value of + self.output_key. If True, return the entire output. If set self.hidden_state_skip_layer, + output_hidden_states will be set True. Defaults to False. + do_sample (bool): Whether to sample from the model. Used for Decoder-Only LLMs. Defaults to None. + When self.produce is False, do_sample is set to True by default. + hidden_state_skip_layer (int): Number of hidden states to hidden_state_skip_layer. 0 means the last layer. + If None, self.output_key will be used. Defaults to None. + return_texts (bool): Whether to return the decoded texts. Defaults to False. + """ + device = self.model.device if device is None else device + use_attention_mask = use_default(use_attention_mask, self.use_attention_mask) + hidden_state_skip_layer = use_default( + hidden_state_skip_layer, self.hidden_state_skip_layer + ) + do_sample = use_default(do_sample, not self.reproduce) + + attention_mask = ( + batch_encoding["attention_mask"].to(device) if use_attention_mask else None + ) + outputs = self.model( + input_ids=batch_encoding["input_ids"].to(device), + attention_mask=attention_mask, + output_hidden_states=output_hidden_states + or hidden_state_skip_layer is not None, + ) + if hidden_state_skip_layer is not None: + last_hidden_state = outputs.hidden_states[-(hidden_state_skip_layer + 1)] + # Real last hidden state already has layer norm applied. So here we only apply it + # for intermediate layers. + if hidden_state_skip_layer > 0 and self.apply_final_norm: + last_hidden_state = self.model.final_layer_norm(last_hidden_state) + else: + last_hidden_state = outputs[self.output_key] + + # Remove hidden states of instruction tokens, only keep prompt tokens. + if self.use_template: + if data_type == "image": + crop_start = self.prompt_template.get("crop_start", 0) + elif data_type == "video": + crop_start = self.prompt_template_video.get("crop_start", 0) + else: + raise ValueError(f"Unsupported data type: {data_type}") + if crop_start > 0: + last_hidden_state = last_hidden_state[:, crop_start:] + attention_mask = ( + attention_mask[:, crop_start:] if use_attention_mask else None + ) + + if output_hidden_states: + return TextEncoderModelOutput( + last_hidden_state, attention_mask, outputs.hidden_states + ) + return TextEncoderModelOutput(last_hidden_state, attention_mask) + + def forward( + self, + text, + use_attention_mask=None, + output_hidden_states=False, + do_sample=False, + hidden_state_skip_layer=None, + return_texts=False, + ): + batch_encoding = self.text2tokens(text, max_length=self.max_length) + return self.encode( + batch_encoding, + use_attention_mask=use_attention_mask, + output_hidden_states=output_hidden_states, + do_sample=do_sample, + hidden_state_skip_layer=hidden_state_skip_layer, + return_texts=return_texts, + ) diff --git a/hyvideo/models/text_encoders/byT5/__init__.py b/hyvideo/models/text_encoders/byT5/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..58b07c0d6720409f32f482cd26699b731ed7b76f --- /dev/null +++ b/hyvideo/models/text_encoders/byT5/__init__.py @@ -0,0 +1,232 @@ +# Licensed under the TENCENT HUNYUAN COMMUNITY LICENSE AGREEMENT (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://github.com/Tencent-Hunyuan/HunyuanVideo-1.5/blob/main/LICENSE +# +# Unless and only to the extent required by applicable law, the Tencent Hunyuan works and any +# output and results therefrom are provided "AS IS" without any express or implied warranties of +# any kind including any warranties of title, merchantability, noninfringement, course of dealing, +# usage of trade, or fitness for a particular purpose. You are solely responsible for determining the +# appropriateness of using, reproducing, modifying, performing, displaying or distributing any of +# the Tencent Hunyuan works or outputs and assume any and all risks associated with your or a +# third party's use or distribution of any of the Tencent Hunyuan works or outputs and your exercise +# of rights and permissions under this agreement. +# See the License for the specific language governing permissions and limitations under the License. + +import json +import torch +import torch.nn as nn +from transformers import AutoTokenizer, T5ForConditionalGeneration + + +def load_glyph_byT5_v2(args, device): + """ + Loads ByT5 tokenizer and encoder model for glyph encoding. + + Args: + args (dict): Configuration dictionary containing paths and settings. + device (str or torch.device): Device to load the model onto. + + Returns: + dict: Dictionary with keys 'byt5_tokenizer', 'byt5_model', 'byt5_max_length'. + """ + byt5_tokenizer, byt5_model, byt5_max_length = create_byt5(args, device) + byt5_model = byt5_model.to(device=device) + return { + "byt5_tokenizer": byt5_tokenizer, + "byt5_model": byt5_model, + "byt5_max_length": byt5_max_length, + } + + +def create_byt5(args, device): + """ + Create ByT5 tokenizer and encoder, load weights if provided. + + Args: + args (dict): Configuration dictionary. + device (str or torch.device): Device to load the model onto. + + Returns: + tuple: (byt5_tokenizer, byt5_model, byt5_max_length) + """ + byt5_max_length = args["byt5_max_length"] + byt5_config = dict( + byt5_name=args["byT5_google_path"], + special_token=True, + color_special_token=True, + font_special_token=True, + color_ann_path=args["multilingual_prompt_format_color_path"], + font_ann_path=args["multilingual_prompt_format_font_path"], + multilingual=True, + ) + huggingface_cache_dir = None + byt5_model, byt5_tokenizer = load_byt5_and_byt5_tokenizer( + **byt5_config, + huggingface_cache_dir=huggingface_cache_dir, + device=device, + ) + + # Load custom checkpoint if provided + if args["byT5_ckpt_path"] is not None: + if "cuda" not in str(device): + byt5_state_dict = torch.load(args["byT5_ckpt_path"], map_location=device) + else: + byt5_state_dict = torch.load(args["byT5_ckpt_path"], map_location=device) + if "state_dict" in byt5_state_dict: + sd = byt5_state_dict["state_dict"] + newsd = {} + for k, v in sd.items(): + if k.startswith("module.text_tower.encoder."): + newsd[k[len("module.text_tower.encoder.") :]] = v + byt5_state_dict = newsd + byt5_model.load_state_dict(byt5_state_dict) + byt5_model.requires_grad_(False) + return byt5_tokenizer, byt5_model, byt5_max_length + + +def add_special_token( + tokenizer, + text_encoder, + add_color, + add_font, + color_ann_path, + font_ann_path, + multilingual=False, +): + """ + Add special tokens for color and font to tokenizer and text encoder. + + Args: + tokenizer: Huggingface tokenizer. + text_encoder: Huggingface T5 encoder. + add_color (bool): Whether to add color tokens. + add_font (bool): Whether to add font tokens. + color_ann_path (str): Path to color annotation JSON. + font_ann_path (str): Path to font annotation JSON. + multilingual (bool): Whether to use multilingual font tokens. + """ + with open(font_ann_path, "r") as f: + idx_font_dict = json.load(f) + with open(color_ann_path, "r") as f: + idx_color_dict = json.load(f) + + if multilingual: + font_token = [ + f"<{font_code[:2]}-font-{idx_font_dict[font_code]}>" + for font_code in idx_font_dict + ] + else: + font_token = [f"" for i in range(len(idx_font_dict))] + color_token = [f"" for i in range(len(idx_color_dict))] + additional_special_tokens = [] + if add_color: + additional_special_tokens += color_token + if add_font: + additional_special_tokens += font_token + + tokenizer.add_tokens(additional_special_tokens, special_tokens=True) + # Set mean_resizing=False to avoid PyTorch LAPACK dependency + text_encoder.resize_token_embeddings(len(tokenizer), mean_resizing=False) + + +def load_byt5_and_byt5_tokenizer( + byt5_name="google/byt5-small", + special_token=False, + color_special_token=False, + font_special_token=False, + color_ann_path="assets/color_idx.json", + font_ann_path="assets/font_idx_512.json", + huggingface_cache_dir=None, + multilingual=False, + device=None, +): + """ + Load ByT5 encoder and tokenizer from Huggingface, and add special tokens if needed. + + Args: + byt5_name (str): Model name or path. + special_token (bool): Whether to add special tokens. + color_special_token (bool): Whether to add color tokens. + font_special_token (bool): Whether to add font tokens. + color_ann_path (str): Path to color annotation JSON. + font_ann_path (str): Path to font annotation JSON. + huggingface_cache_dir (str): Huggingface cache directory. + multilingual (bool): Whether to use multilingual font tokens. + device (str or torch.device): Device to load the model onto. + + Returns: + tuple: (byt5_text_encoder, byt5_tokenizer) + """ + byt5_tokenizer = AutoTokenizer.from_pretrained( + byt5_name, + cache_dir=huggingface_cache_dir, + ) + byt5_text_encoder = T5ForConditionalGeneration.from_pretrained( + byt5_name, + cache_dir=huggingface_cache_dir, + ).get_encoder() + + if "cuda" not in str(device): + device = torch.device(device) + else: + device = torch.device(device) + byt5_text_encoder = byt5_text_encoder.to(device) + + if special_token: + add_special_token( + byt5_tokenizer, + byt5_text_encoder, + add_color=color_special_token, + add_font=font_special_token, + color_ann_path=color_ann_path, + font_ann_path=font_ann_path, + multilingual=multilingual, + ) + return byt5_text_encoder, byt5_tokenizer + + +class ByT5Mapper(nn.Module): + """ + ByT5Mapper: Maps ByT5 encoder outputs to a new space, with optional residual connection. + + Args: + in_dim (int): Input dimension (must equal out_dim if use_residual). + out_dim (int): Output dimension after second linear layer. + hidden_dim (int): Hidden dimension for intermediate layer. + out_dim1 (int): Final output dimension. + use_residual (bool): Whether to use residual connection (default: True). + """ + + def __init__(self, in_dim, out_dim, hidden_dim, out_dim1, use_residual=True): + super().__init__() + if use_residual: + assert in_dim == out_dim + self.layernorm = nn.LayerNorm(in_dim) + self.fc1 = nn.Linear(in_dim, hidden_dim) + self.fc2 = nn.Linear(hidden_dim, out_dim) + self.fc3 = nn.Linear(out_dim, out_dim1) + self.use_residual = use_residual + self.act_fn = nn.GELU() + + def forward(self, x): + """ + Forward pass for ByT5Mapper. + + Args: + x (Tensor): Input tensor of shape (..., in_dim). + + Returns: + Tensor: Output tensor of shape (..., out_dim1). + """ + residual = x + x = self.layernorm(x) + x = self.fc1(x) + x = self.act_fn(x) + x = self.fc2(x) + x2 = self.act_fn(x) + x2 = self.fc3(x2) + if self.use_residual: + x2 = x2 + residual + return x2 diff --git a/hyvideo/models/text_encoders/byT5/format_prompt.py b/hyvideo/models/text_encoders/byT5/format_prompt.py new file mode 100644 index 0000000000000000000000000000000000000000..88913de85a39c60a581aca29ee896d58d3cb3366 --- /dev/null +++ b/hyvideo/models/text_encoders/byT5/format_prompt.py @@ -0,0 +1,88 @@ +# Licensed under the TENCENT HUNYUAN COMMUNITY LICENSE AGREEMENT (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://github.com/Tencent-Hunyuan/HunyuanVideo-1.5/blob/main/LICENSE +# +# Unless and only to the extent required by applicable law, the Tencent Hunyuan works and any +# output and results therefrom are provided "AS IS" without any express or implied warranties of +# any kind including any warranties of title, merchantability, noninfringement, course of dealing, +# usage of trade, or fitness for a particular purpose. You are solely responsible for determining the +# appropriateness of using, reproducing, modifying, performing, displaying or distributing any of +# the Tencent Hunyuan works or outputs and assume any and all risks associated with your or a +# third party's use or distribution of any of the Tencent Hunyuan works or outputs and your exercise +# of rights and permissions under this agreement. +# See the License for the specific language governing permissions and limitations under the License. + +import json + + +def closest_color(requested_color): + import webcolors + + min_colors = {} + for key, name in webcolors.CSS3_HEX_TO_NAMES.items(): + + r_c, g_c, b_c = webcolors.hex_to_rgb(key) + rd = (r_c - requested_color[0]) ** 2 + gd = (g_c - requested_color[1]) ** 2 + bd = (b_c - requested_color[2]) ** 2 + min_colors[(rd + gd + bd)] = name + return min_colors[min(min_colors.keys())] + + +def convert_rgb_to_names(rgb_tuple): + try: + import webcolors + + color_name = webcolors.rgb_to_name(rgb_tuple) + except ValueError: + color_name = closest_color(rgb_tuple) + return color_name + + +class MultilingualPromptFormat: + + def __init__( + self, + font_path: str = "assets/glyph_sdxl_assets/multilingual_10-lang_idx.json", + color_path: str = "assets/glyph_sdxl_assets/color_idx.json", + ): + with open(font_path, "r") as f: + self.font_dict = json.load(f) + with open(color_path, "r") as f: + self.color_dict = json.load(f) + + def format_prompt(self, texts, styles): + """ + Text "{text}" in {color}, {type}. + """ + + prompt = "" + for text, style in zip(texts, styles): + text_prompt = f'Text "{text}"' + + attr_list = [] + + # format color + if style["color"] is not None: + import webcolors + + hex_color = style["color"] + rgb_color = webcolors.hex_to_rgb(hex_color) + color_name = convert_rgb_to_names(rgb_color) + attr_list.append(f"") + + # format font + if style["font-family"] is not None: + attr_list.append( + f"<{style['font-family'][:2]}-font-{self.font_dict[style['font-family']]}>" + ) + attr_suffix = ", ".join(attr_list) + text_prompt += " in " + attr_suffix + text_prompt += ". " + else: + text_prompt += ". " + + prompt = prompt + text_prompt + return prompt diff --git a/hyvideo/models/transformers/modules/activation_layers.py b/hyvideo/models/transformers/modules/activation_layers.py new file mode 100644 index 0000000000000000000000000000000000000000..b3f3fa95d91191c85bd24180aaec3e6b5c7d590f --- /dev/null +++ b/hyvideo/models/transformers/modules/activation_layers.py @@ -0,0 +1,39 @@ +# Licensed under the TENCENT HUNYUAN COMMUNITY LICENSE AGREEMENT (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://github.com/Tencent-Hunyuan/HunyuanVideo-1.5/blob/main/LICENSE +# +# Unless and only to the extent required by applicable law, the Tencent Hunyuan works and any +# output and results therefrom are provided "AS IS" without any express or implied warranties of +# any kind including any warranties of title, merchantability, noninfringement, course of dealing, +# usage of trade, or fitness for a particular purpose. You are solely responsible for determining the +# appropriateness of using, reproducing, modifying, performing, displaying or distributing any of +# the Tencent Hunyuan works or outputs and assume any and all risks associated with your or a +# third party's use or distribution of any of the Tencent Hunyuan works or outputs and your exercise +# of rights and permissions under this agreement. +# See the License for the specific language governing permissions and limitations under the License. + +import torch.nn as nn + + +def get_activation_layer(act_type): + """get activation layer + + Args: + act_type (str): the activation type + + Returns: + torch.nn.functional: the activation layer + """ + if act_type == "gelu": + return lambda: nn.GELU() + elif act_type == "gelu_tanh": + # Approximate `tanh` requires torch >= 1.13 + return lambda: nn.GELU(approximate="tanh") + elif act_type == "relu": + return nn.ReLU + elif act_type == "silu": + return nn.SiLU + else: + raise ValueError(f"Unknown activation type: {act_type}") diff --git a/hyvideo/models/transformers/modules/ar_sparse_operation.py b/hyvideo/models/transformers/modules/ar_sparse_operation.py new file mode 100644 index 0000000000000000000000000000000000000000..3cef1feed9d85ea203ec7ff117bd9244d9035676 --- /dev/null +++ b/hyvideo/models/transformers/modules/ar_sparse_operation.py @@ -0,0 +1,362 @@ +# Copyright (c) 2026 Jiacheng Lu and contributors. +# Licensed under the MIT License. +# See the LICENSE file in the repository root for details. + +import torch +import torch.nn.functional as F +import math + +try: + import triton + import triton.language as tl + from .longcat_kernel import longcat_flash_attn + HAS_TRITON = True +except ImportError: + HAS_TRITON = False + print("[SparseAttn] Triton not found. Falling back to SDPA.") + + +# ============================================================================== +# Triton Kernels +# ============================================================================== +if HAS_TRITON: + # -------------------------------------------------------------------------- + # Kernel 1: Q Preparation (Tiling + Mean Pooling) + # -------------------------------------------------------------------------- + @triton.jit + def _fused_q_prep_kernel( + Src_ptr, Dst_ptr, Mean_ptr, + s_b, s_h, s_l, s_d, d_b, d_h, d_n, d_blk, d_d, m_b, m_h, m_n, m_d, + NUM_HEADS, T, H_img, W_img, tt, th, tw, nh, nw, + BLOCK_SIZE: tl.constexpr, HEAD_DIM: tl.constexpr, BLOCK_SIZE_FLOAT: tl.constexpr + ): + blk_idx, pid_bh = tl.program_id(0), tl.program_id(1) + cur_head, cur_batch = pid_bh % NUM_HEADS, pid_bh // NUM_HEADS + + # Pointers + src_base = Src_ptr + cur_batch * s_b + cur_head * s_h + dst_base = Dst_ptr + cur_batch * d_b + cur_head * d_h + mean_base = Mean_ptr + cur_batch * m_b + cur_head * m_h + + # Grid Logic + cur_nw = blk_idx % nw + rem = blk_idx // nw + cur_nh, cur_nt = rem % nh, rem // nh + + base_t, base_h, base_w = cur_nt * tt, cur_nh * th, cur_nw * tw + + # Inner Loop + offs_flat = tl.arange(0, BLOCK_SIZE) + offs_tw = offs_flat % tw + rem_t = offs_flat // tw + offs_th, offs_tt = rem_t % th, rem_t // th + + coords_t = base_t + offs_tt + coords_h = base_h + offs_th + coords_w = base_w + offs_tw + + mask = (coords_t < T) & (coords_h < H_img) & (coords_w < W_img) + idx_l = coords_t * (H_img * W_img) + coords_h * W_img + coords_w + offs_d = tl.arange(0, HEAD_DIM) + + # Load & Store + val = tl.load(src_base + idx_l[:, None] * s_l + offs_d[None, :] * s_d, mask=mask[:, None], other=0.0) + + dst_off = blk_idx * d_n + offs_flat[:, None] * d_blk + offs_d[None, :] * d_d + tl.store(dst_base + dst_off, val) + + # Mean Pooling + mean_val = tl.sum(val.to(tl.float32), axis=0) / BLOCK_SIZE_FLOAT + tl.store(mean_base + blk_idx * m_n + offs_d * m_d, mean_val.to(val.dtype)) + + # -------------------------------------------------------------------------- + # Kernel 2: KV Preparation (Double Stream Fused) + # -------------------------------------------------------------------------- + @triton.jit + def _fused_kv_prep_kernel( + SK_ptr, SV_ptr, DK_ptr, DV_ptr, MK_ptr, + sk_b, sk_h, sk_l, sk_d, sv_b, sv_h, sv_l, sv_d, + dk_b, dk_h, dk_n, dk_blk, dk_d, dv_b, dv_h, dv_n, dv_blk, dv_d, + m_b, m_h, m_n, m_d, + NUM_HEADS, T, H_img, W_img, tt, th, tw, nh, nw, + dst_offset_blk, pool_offset_blk, + BLOCK_SIZE: tl.constexpr, HEAD_DIM: tl.constexpr, BLOCK_SIZE_FLOAT: tl.constexpr + ): + blk_idx, pid_bh = tl.program_id(0), tl.program_id(1) + cur_head, cur_batch = pid_bh % NUM_HEADS, pid_bh // NUM_HEADS + + # Base Pointers + sk_base, sv_base = SK_ptr + cur_batch * sk_b + cur_head * sk_h, SV_ptr + cur_batch * sv_b + cur_head * sv_h + dk_base, dv_base = DK_ptr + cur_batch * dk_b + cur_head * dk_h, DV_ptr + cur_batch * dv_b + cur_head * dv_h + mk_base = MK_ptr + cur_batch * m_b + cur_head * m_h + + # Grid Logic + cur_nw = blk_idx % nw + rem = blk_idx // nw + cur_nh, cur_nt = rem % nh, rem // nh + + offs_flat = tl.arange(0, BLOCK_SIZE) + offs_tw = offs_flat % tw + rem_t = offs_flat // tw + offs_th, offs_tt = rem_t % th, rem_t // th + + coords_t = cur_nt * tt + offs_tt + coords_h = cur_nh * th + offs_th + coords_w = cur_nw * tw + offs_tw + + mask = (coords_t < T) & (coords_h < H_img) & (coords_w < W_img) + idx_l = coords_t * (H_img * W_img) + coords_h * W_img + coords_w + offs_d = tl.arange(0, HEAD_DIM) + + # Load + val_k = tl.load(sk_base + idx_l[:, None] * sk_l + offs_d[None, :] * sk_d, mask=mask[:, None], other=0.0) + val_v = tl.load(sv_base + idx_l[:, None] * sv_l + offs_d[None, :] * sv_d, mask=mask[:, None], other=0.0) + + # Store Tiled + dst_off = (blk_idx + dst_offset_blk) * dk_n + offs_flat[:, None] * dk_blk + offs_d[None, :] * dk_d + tl.store(dk_base + dst_off, val_k) + tl.store(dv_base + dst_off, val_v) + + # Store K Mean + mean_val = tl.sum(val_k.to(tl.float32), axis=0) / BLOCK_SIZE_FLOAT + tl.store(mk_base + (blk_idx + pool_offset_blk) * m_n + offs_d * m_d, mean_val.to(val_k.dtype)) + + # -------------------------------------------------------------------------- + # Kernel 3: Untile (Scatter back to linear) + # -------------------------------------------------------------------------- + @triton.jit + def _fused_untile_kernel( + Src_ptr, Dst_ptr, + s_b, s_h, s_n, s_blk, s_d, d_b, d_h, d_l, d_d, + NUM_HEADS, T, H_img, W_img, tt, th, tw, nh, nw, + BLOCK_SIZE: tl.constexpr, HEAD_DIM: tl.constexpr + ): + blk_idx, pid_bh = tl.program_id(0), tl.program_id(1) + cur_head, cur_batch = pid_bh % NUM_HEADS, pid_bh // NUM_HEADS + + src_base = Src_ptr + cur_batch * s_b + cur_head * s_h + dst_base = Dst_ptr + cur_batch * d_b + cur_head * d_h + + # Grid Logic + cur_nw = blk_idx % nw + rem = blk_idx // nw + cur_nh, cur_nt = rem % nh, rem // nh + + offs_flat = tl.arange(0, BLOCK_SIZE) + offs_tw = offs_flat % tw + rem_t = offs_flat // tw + offs_th, offs_tt = rem_t % th, rem_t // th + + coords_t = cur_nt * tt + offs_tt + coords_h = cur_nh * th + offs_th + coords_w = cur_nw * tw + offs_tw + + mask = (coords_t < T) & (coords_h < H_img) & (coords_w < W_img) + offs_d = tl.arange(0, HEAD_DIM) + + # Load Tiled & Store Linear + val = tl.load(src_base + blk_idx * s_n + offs_flat[:, None] * s_blk + offs_d[None, :] * s_d) + + idx_l = coords_t * (H_img * W_img) + coords_h * W_img + coords_w + tl.store(dst_base + idx_l[:, None] * d_l + offs_d[None, :] * d_d, val, mask=mask[:, None]) + + # -------------------------------------------------------------------------- + # Launchers + # -------------------------------------------------------------------------- + def launch_q_prep(src, dst, mean, T, H, W, tt, th, tw, nh, nw): + B, NH, _, D = src.shape + sb, sh, sl, sd = src.stride() + db, dh, dn, dblk, dd = dst.stride() + mb, mh, mn, md = mean.stride() + + num_blocks = ((T + tt - 1) // tt) * nh * nw + BLOCK_SIZE = tt * th * tw + + _fused_q_prep_kernel[(num_blocks, B * NH)]( + src, dst, mean, + sb, sh, sl, sd, db, dh, dn, dblk, dd, mb, mh, mn, md, + NH, T, H, W, tt, th, tw, nh, nw, + BLOCK_SIZE=BLOCK_SIZE, HEAD_DIM=D, BLOCK_SIZE_FLOAT=float(BLOCK_SIZE), + num_warps=8, num_stages=3 + ) + + def launch_kv_prep(k_src, v_src, k_dst, v_dst, k_mean, dst_off, pool_off, T, H, W, tt, th, tw, nh, nw): + B, NH, _, D = k_src.shape + skb, skh, skl, skd = k_src.stride() + svb, svh, svl, svd = v_src.stride() + dkb, dkh, dkn, dkblk, dkd = k_dst.stride() + dvb, dvh, dvn, dvblk, dvd = v_dst.stride() + mb, mh, mn, md = k_mean.stride() + + num_blocks = ((T + tt - 1) // tt) * nh * nw + BLOCK_SIZE = tt * th * tw + + _fused_kv_prep_kernel[(num_blocks, B * NH)]( + k_src, v_src, k_dst, v_dst, k_mean, + skb, skh, skl, skd, svb, svh, svl, svd, + dkb, dkh, dkn, dkblk, dkd, dvb, dvh, dvn, dvblk, dvd, + mb, mh, mn, md, + NH, T, H, W, tt, th, tw, nh, nw, dst_off, pool_off, + BLOCK_SIZE=BLOCK_SIZE, HEAD_DIM=D, BLOCK_SIZE_FLOAT=float(BLOCK_SIZE), + num_warps=8, num_stages=3 + ) + + def launch_untile(src, dst, T, H, W, tt, th, tw, nh, nw): + B, NH, _, D = dst.shape + sb, sh, sn, sblk, sd = src.stride() + db, dh, dl, dd = dst.stride() + + num_blocks = ((T + tt - 1) // tt) * nh * nw + BLOCK_SIZE = tt * th * tw + + _fused_untile_kernel[(num_blocks, B * NH)]( + src, dst, + sb, sh, sn, sblk, sd, db, dh, dl, dd, + NH, T, H, W, tt, th, tw, nh, nw, + BLOCK_SIZE=BLOCK_SIZE, HEAD_DIM=D, + num_warps=4 + ) + +# ============================================================================== +# Indexing (Logic) +# ============================================================================== +def fast_generate_indices(q_pool, k_pool, text_len, grid_thw_q, grid_thw_k, params, device): + """Build block-sparse attention indices for the autoregressive decode path. + + Computes block-level similarity between query and key pools, protects + the most recent visual blocks from pruning, and selects the top-k + historical blocks. Text blocks are always prepended (never pruned). + + Args: + q_pool: Pooled query features ``(B, H, Nq_blk, D)``. + k_pool: Pooled key features ``(B, H, Nk_blk, D)``. + text_len: Number of text tokens (used to offset visual block indices). + grid_thw_q: Query grid ``(T_chunks, H_blks, W_blks)``. + grid_thw_k: Key grid ``(T_chunks, H_blks, W_blks)``. + params: Dict with keys ``tile_size``, ``sim_metric``, + ``recent_protect``, ``topk_ratio``. + device: Target device for output tensors. + + Returns: + Tuple of ``(final_indices, final_counts)`` where *final_indices* is + ``(B, H, Nq_blk, K)`` int32 and *final_counts* gives the number of + selected blocks per query block. + """ + tt, th, tw = params['tile_size'] + BLOCK_SIZE = tt * th * tw + b, h, nq, d = q_pool.shape + nk = k_pool.shape[2] + + metric = params.get('sim_metric', 'cosine') + if metric == 'cosine': + sim = torch.matmul(F.normalize(q_pool, dim=-1), F.normalize(k_pool, dim=-1).transpose(-2, -1)) + elif metric == 'scaled_dot_product': + sim = torch.matmul(q_pool, k_pool.transpose(-2, -1)) * (1.0 / math.sqrt(d)) + else: + sim = torch.matmul(q_pool, k_pool.transpose(-2, -1)) + + score_for_ranking = sim.clone() + + n_prot_blk = 0 + Nt_k, Nh, Nw = grid_thw_k[0], grid_thw_q[1], grid_thw_q[2] + + if params.get('recent_protect', 0) > 0: + n_prot_blk = min(((params['recent_protect'] + tt - 1) // tt) * Nh * Nw, nk) + if n_prot_blk > 0: + score_for_ranking[..., -n_prot_blk:] += 1e6 + + n_history_blk = max(0, nk - n_prot_blk) + history_ratio = params.get('topk_ratio', 1.0) + k_budget_history = int(n_history_blk * history_ratio) + k_budget = n_prot_blk + k_budget_history + k_budget = min(max(k_budget, 1 if nk > 0 else 0), nk) + + _, vis_indices = torch.topk(score_for_ranking, k=k_budget, dim=-1) + + vis_indices, _ = torch.sort(vis_indices, dim=-1) + num_txt_blk = (text_len + BLOCK_SIZE - 1) // BLOCK_SIZE + vis_indices = vis_indices + num_txt_blk + + txt_indices = torch.arange(num_txt_blk, device=device, dtype=torch.int32).view(1, 1, 1, -1).expand(b, h, vis_indices.shape[2], -1) + final_indices = torch.cat([txt_indices, vis_indices.int()], dim=-1).contiguous() + final_counts = torch.full((b, h, vis_indices.shape[2]), final_indices.shape[-1], dtype=torch.int32, device=device) + + return final_indices, final_counts + +# ============================================================================== +# Main Pipeline +# ============================================================================== +def run_sparse_attention(q, k_txt, v_txt, k_vis, v_vis, block_idx, latent_hw, sparse_params): + """Execute a single autoregressive sparse-attention forward pass. + + When Triton is available, tiles Q/K/V into 3D blocks, generates a + block-sparse index, and runs the fused LongCat-style attention kernel. + Otherwise, falls back to full SDPA. + + Args: + q: Query tensor ``(B, H, Nq, D)``. + k_txt, v_txt: Text KV ``(B, H, N_txt, D)``. + k_vis, v_vis: Visual KV ``(B, H, N_vis, D)``. + block_idx: Current chunk index (informational, used for early-chunk + dense gating at the caller level). + latent_hw: ``(H_lat, W_lat)`` of the latent grid. + sparse_params: Dict with keys ``tile_size``, ``topk_ratio``, + ``recent_protect``, ``sim_metric``. + + Returns: + Attention output ``(B, H, Nq, D)``. + """ + LATENT_H, LATENT_W = latent_hw + dense_k = torch.cat([k_txt, k_vis], dim=2) + dense_v = torch.cat([v_txt, v_vis], dim=2) + + text_len = k_txt.shape[2] + vis_len_q, vis_len_k = q.shape[2], k_vis.shape[2] + + if not HAS_TRITON or vis_len_q % (LATENT_H * LATENT_W) != 0: + return F.scaled_dot_product_attention(q, dense_k, dense_v, is_causal=False) + + tt, th, tw = sparse_params.get('tile_size', (4, 8, 4)) + BLOCK_SIZE = tt * th * tw + + nh, nw = (LATENT_H + (th - LATENT_H % th) % th) // th, (LATENT_W + (tw - LATENT_W % tw) % tw) // tw + T_q, T_k = vis_len_q // (LATENT_H * LATENT_W), vis_len_k // (LATENT_H * LATENT_W) + nt_q, nt_k = (T_q + (tt - T_q % tt) % tt) // tt, (T_k + (tt - T_k % tt) % tt) // tt + + num_txt_blk = (text_len + (BLOCK_SIZE - text_len % BLOCK_SIZE) % BLOCK_SIZE) // BLOCK_SIZE + num_vis_blk_q, num_vis_blk_k = nt_q * nh * nw, nt_k * nh * nw + + B, H, _, D = q.shape + q_tiled = torch.empty((B, H, num_vis_blk_q, BLOCK_SIZE, D), device=q.device, dtype=q.dtype) + k_all = torch.empty((B, H, num_txt_blk + num_vis_blk_k, BLOCK_SIZE, D), device=q.device, dtype=q.dtype) + v_all = torch.empty((B, H, num_txt_blk + num_vis_blk_k, BLOCK_SIZE, D), device=q.device, dtype=q.dtype) + + q_pool_vis = torch.empty((B, H, num_vis_blk_q, D), device=q.device, dtype=q.dtype) + k_pool_vis = torch.empty((B, H, num_vis_blk_k, D), device=q.device, dtype=q.dtype) + + pad_txt = (BLOCK_SIZE - text_len % BLOCK_SIZE) % BLOCK_SIZE + k_txt_padded, v_txt_padded = k_txt, v_txt + if pad_txt > 0: + k_txt_padded = F.pad(k_txt, (0, 0, 0, pad_txt)) + v_txt_padded = F.pad(v_txt, (0, 0, 0, pad_txt)) + + k_all[:, :, :num_txt_blk].copy_(k_txt_padded.view(B, H, -1, BLOCK_SIZE, D)) + v_all[:, :, :num_txt_blk].copy_(v_txt_padded.view(B, H, -1, BLOCK_SIZE, D)) + + launch_q_prep(q, q_tiled, q_pool_vis, T_q, LATENT_H, LATENT_W, tt, th, tw, nh, nw) + launch_kv_prep(k_vis, v_vis, k_all, v_all, k_pool_vis, num_txt_blk, 0, + T_k, LATENT_H, LATENT_W, tt, th, tw, nh, nw) + + indices, counts = fast_generate_indices( + q_pool_vis, k_pool_vis, text_len, + (nt_q, nh, nw), (nt_k, nh, nw), sparse_params, q.device + ) + + o_flat = longcat_flash_attn( + q_tiled.flatten(2, 3), k_all.flatten(2, 3), v_all.flatten(2, 3), + indices, counts, sm_scale=1.0/(D**0.5), logical_block_size=BLOCK_SIZE + ) + + o_out = torch.empty_like(q) + launch_untile(o_flat.view(B, H, num_vis_blk_q, BLOCK_SIZE, D), o_out, T_q, LATENT_H, LATENT_W, tt, th, tw, nh, nw) + return o_out diff --git a/hyvideo/models/transformers/modules/attention.py b/hyvideo/models/transformers/modules/attention.py new file mode 100644 index 0000000000000000000000000000000000000000..45d54572bca4793fa220e3bfaf5d400a102d49fe --- /dev/null +++ b/hyvideo/models/transformers/modules/attention.py @@ -0,0 +1,715 @@ +# Licensed under the TENCENT HUNYUAN COMMUNITY LICENSE AGREEMENT (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://github.com/Tencent-Hunyuan/HunyuanVideo-1.5/blob/main/LICENSE +# +# Unless and only to the extent required by applicable law, the Tencent Hunyuan works and any +# output and results therefrom are provided "AS IS" without any express or implied warranties of +# any kind including any warranties of title, merchantability, noninfringement, course of dealing, +# usage of trade, or fitness for a particular purpose. You are solely responsible for determining the +# appropriateness of using, reproducing, modifying, performing, displaying or distributing any of +# the Tencent Hunyuan works or outputs and assume any and all risks associated with your or a +# third party's use or distribution of any of the Tencent Hunyuan works or outputs and your exercise +# of rights and permissions under this agreement. +# See the License for the specific language governing permissions and limitations under the License. + +import einops +import torch +from typing import Optional +from loguru import logger +import numpy as np +import torch.nn.functional as F +from einops import repeat + +from hyvideo.commons.parallel_states import get_parallel_state +from hyvideo.utils.communications import ( + all_gather, + all_to_all_4D, +) +from hyvideo.utils.flash_attn_no_pad import ( + flash_attn_no_pad, + flash_attn_no_pad_v3, +) +from hyvideo.commons import maybe_fallback_attn_mode + +try: + from torch.nn.attention.flex_attention import flex_attention + + flex_attention = torch.compile(flex_attention, dynamic=False) + torch._dynamo.config.cache_size_limit = 192 + torch._dynamo.config.accumulated_cache_size_limit = 192 + flex_mask_cache = {} +except Exception: + logger.warning("Could not load Sliding Tile Attention of FlexAttn.") + +from hyvideo.models.transformers.modules.ssta_attention import ssta_3d_attention +from hyvideo.commons.infer_state import get_infer_state +from hyvideo.models.transformers.modules.ar_sparse_operation import run_sparse_attention +from hyvideo.models.transformers.modules.bi_sparse_operation_for_KV_cache import ( + run_sparse_prefill_attention, +) + + +@torch.compiler.disable +def attention( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + drop_rate: float = 0.0, + attn_mask: Optional[torch.Tensor] = None, + causal: bool = False, + attn_mode: str = "flash", +) -> torch.Tensor: + """ + Compute attention using flash_attn_no_pad or torch scaled_dot_product_attention. + + Args: + q: Query tensor of shape [B, L, H, D] + k: Key tensor of shape [B, L, H, D] + v: Value tensor of shape [B, L, H, D] + drop_rate: Dropout rate for attention weights. + attn_mask: Optional attention mask of shape [B, L]. + causal: Whether to apply causal masking. + attn_mode: Attention mode, either "flash" or "torch". Defaults to "flash". + + Returns: + Output tensor after attention of shape [B, L, H*D] + """ + attn_mode = maybe_fallback_attn_mode(attn_mode) + + if attn_mode == "torch": + # transpose q,k,v dim to fit scaled_dot_product_attention + query = q.transpose(1, 2) # B * H * L * D + key = k.transpose(1, 2) # B * H * L * D + value = v.transpose(1, 2) # B * H * L * D + + if attn_mask is not None: + if attn_mask.dtype != torch.bool and attn_mask.dtype in [ + torch.int64, + torch.int32, + ]: + assert ( + attn_mask.max() <= 1 and attn_mask.min() >= 0 + ), f"attention mask must be (0,1)." + attn_mask = attn_mask.to(torch.bool) + elif attn_mask.dtype != torch.bool: + attn_mask = attn_mask.to(query.dtype) + raise NotImplementedError( + f"Float attention mask is not implemented for torch attention." + ) + attn_mask1 = einops.rearrange(attn_mask, "b l -> b 1 l 1") + attn_mask2 = einops.rearrange(attn_mask1, "b 1 l 1 -> b 1 1 l") + attn_mask = attn_mask1 & attn_mask2 + + x = F.scaled_dot_product_attention( + query, + key, + value, + attn_mask=attn_mask, + dropout_p=drop_rate, + is_causal=causal, + ) + + # transpose back + x = x.transpose(1, 2) # B * L * H * D + b, s, h, d = x.shape + out = x.reshape(b, s, -1) + return out + else: + # flash mode (default) + qkv = torch.stack([q, k, v], dim=2) + if attn_mask is not None and attn_mask.dtype != torch.bool: + attn_mask = attn_mask.bool() + x = flash_attn_no_pad( + qkv, attn_mask, causal=causal, dropout_p=drop_rate, softmax_scale=None + ) + b, s, a, d = x.shape + out = x.reshape(b, s, -1) + return out + + +@torch.compiler.disable +def sequence_parallel_attention_txt( + q, + k, + v, + img_q_len, + img_kv_len, + attn_mode=None, + text_mask=None, + attn_param=None, + block_idx=None, + kv_cache=None, + cache_txt=False, +): + encoder_query = q + encoder_key = k + encoder_value = v + + parallel_dims = get_parallel_state() + enable_sp = parallel_dims.sp_enabled + + if enable_sp: + sp_group = parallel_dims.sp_group + sp_size = parallel_dims.sp + sp_rank = parallel_dims.sp_rank + + if enable_sp: + + def shrink_head(encoder_state, dim): + local_heads = encoder_state.shape[dim] // sp_size + return encoder_state.narrow(dim, sp_rank * local_heads, local_heads) + + encoder_query = shrink_head(encoder_query, dim=2) + encoder_key = shrink_head(encoder_key, dim=2) + encoder_value = shrink_head(encoder_value, dim=2) + + encoder_query = encoder_query.transpose(1, 2) + encoder_key = encoder_key.transpose(1, 2) + encoder_value = encoder_value.transpose(1, 2) + t_kv = {} + if cache_txt: + t_kv["k_txt"] = encoder_key + t_kv["v_txt"] = encoder_value + + infer_state = get_infer_state() + enable_sageattn = ( + infer_state.enable_sageattn and block_idx in infer_state.sage_blocks_range + ) + if enable_sageattn: + from sageattention import sageattn + + encoder_hidden_states = sageattn( + encoder_query, + encoder_key, + encoder_value, + tensor_layout="HND", + is_causal=False, + ) + else: + encoder_hidden_states = F.scaled_dot_product_attention( + encoder_query, encoder_key, encoder_value, dropout_p=0.0, is_causal=False + ) + + # transpose back + encoder_hidden_states = encoder_hidden_states.transpose(1, 2) # [B, S, H, D] + + if enable_sp: + encoder_hidden_states = all_gather( + encoder_hidden_states, dim=2, group=sp_group + ).contiguous() + encoder_hidden_states = encoder_hidden_states.to(q.dtype) + + b, s, a, d = encoder_hidden_states.shape + encoder_hidden_states = encoder_hidden_states.reshape(b, s, -1) + + return encoder_hidden_states, t_kv + + +@torch.compiler.disable +def sequence_parallel_attention_vision( + q, + k, + v, + block_idx=None, + kv_cache=None, + cache_vision=False, + attn_param=None, + anchor_states=None, + enable_sparse=False, + enable_prefill_sparse=False, + sparse_params=None, +): + enable_decode_sparse = enable_sparse + sparse_decode_start_chunk = 3 + sparse_tile_size = (4, 8, 4) + sparse_prefill_topk_ratio = 0.3 + sparse_decode_topk_ratio = 0.175 + sparse_recent_protect = 4 + if attn_param is not None: + enable_sparse = attn_param.get("enable_sparse", enable_sparse) + enable_prefill_sparse = attn_param.get( + "enable_prefill_sparse", enable_prefill_sparse + ) + enable_decode_sparse = attn_param.get("enable_decode_sparse", enable_sparse) + sparse_decode_start_chunk = attn_param.get( + "sparse_decode_start_chunk", sparse_decode_start_chunk + ) + sparse_tile_size = tuple(attn_param.get("sparse_tile_size", sparse_tile_size)) + sparse_prefill_topk_ratio = attn_param.get( + "sparse_prefill_topk_ratio", sparse_prefill_topk_ratio + ) + sparse_decode_topk_ratio = attn_param.get( + "sparse_decode_topk_ratio", sparse_decode_topk_ratio + ) + sparse_recent_protect = attn_param.get( + "sparse_recent_protect", sparse_recent_protect + ) + append_vision_cache = bool( + attn_param is not None and attn_param.get("append_vision_cache", False) + ) + + prefill_config = { + "tile_size": sparse_tile_size, + "topk_ratio": sparse_prefill_topk_ratio, + "sim_metric": "scaled_dot_product", + } + decode_config = { + "tile_size": sparse_tile_size, + "topk_ratio": sparse_decode_topk_ratio, + "recent_protect": sparse_recent_protect, + "sim_metric": "scaled_dot_product", + } + + infer_state = get_infer_state() + if cache_vision and hasattr(infer_state, "cached_k_all"): + infer_state.cached_k_all.clear() + infer_state.cached_v_all.clear() + infer_state.cached_k_pool.clear() + + assert kv_cache is not None + query, query_prope = q + key, key_prope = k + value, value_prope = v + + query = torch.cat([query, query_prope], dim=0) + key = torch.cat([key, key_prope], dim=0) + value = torch.cat([value, value_prope], dim=0) + + parallel_dims = get_parallel_state() + enable_sp = parallel_dims.sp_enabled + + if enable_sp: + sp_group = parallel_dims.sp_group + + if enable_sp: + # batch_size, seq_len, attn_heads, head_dim + query = all_to_all_4D(query, sp_group, scatter_dim=2, gather_dim=1) + key = all_to_all_4D(key, sp_group, scatter_dim=2, gather_dim=1) + value = all_to_all_4D(value, sp_group, scatter_dim=2, gather_dim=1) + + query = query.transpose(1, 2) + key = key.transpose(1, 2) + value = value.transpose(1, 2) + + vision_kv = {} + use_cached_vision = ( + kv_cache[block_idx]["k_vision"] is not None + and (not cache_vision or append_vision_cache) + ) + if use_cached_vision: + k_vis = torch.cat([kv_cache[block_idx]["k_vision"], key], dim=2) + v_vis = torch.cat([kv_cache[block_idx]["v_vision"], value], dim=2) + else: + k_vis, v_vis = key, value + if cache_vision: + vision_kv["k_vision"] = k_vis + vision_kv["v_vision"] = v_vis + + k_txt = kv_cache[block_idx]["k_txt"] + v_txt = kv_cache[block_idx]["v_txt"] + if query.shape[0] != k_txt.shape[0]: + repeat_factor = query.shape[0] // k_txt.shape[0] + if query.shape[0] % k_txt.shape[0]: + raise ValueError("Text KV batch cannot be expanded to AR vision batch") + # query is ordered [regular batch, ProPE batch]. Concatenation preserves + # that ordering; einops (B R) would interleave branches for batch > 1. + k_txt = torch.cat([k_txt] * repeat_factor, dim=0) + v_txt = torch.cat([v_txt] * repeat_factor, dim=0) + + attention_valid_mask = None + text_valid_mask = kv_cache[block_idx].get("text_valid_mask") + context_valid_mask = kv_cache[block_idx].get("context_valid_mask") + if text_valid_mask is not None or context_valid_mask is not None: + logical_batch = query.shape[0] // 2 + if query.shape[0] != 2 * logical_batch: + raise ValueError("AR regular/ProPE query batch is not even") + if text_valid_mask is None: + text_valid_mask = torch.ones( + logical_batch, k_txt.shape[2], dtype=torch.bool, device=query.device + ) + else: + text_valid_mask = text_valid_mask.to(device=query.device, dtype=torch.bool) + if tuple(text_valid_mask.shape) != (logical_batch, k_txt.shape[2]): + raise ValueError("Text valid mask does not match padded Text KV") + + current_valid_mask = torch.ones( + logical_batch, key.shape[2], dtype=torch.bool, device=query.device + ) + if use_cached_vision: + cached_tokens = kv_cache[block_idx]["k_vision"].shape[2] + if context_valid_mask is None: + context_valid_mask = torch.ones( + logical_batch, + cached_tokens, + dtype=torch.bool, + device=query.device, + ) + else: + context_valid_mask = context_valid_mask.to( + device=query.device, dtype=torch.bool + ) + if tuple(context_valid_mask.shape) != (logical_batch, cached_tokens): + raise ValueError("Context valid mask does not match padded Vision KV") + vision_valid_mask = torch.cat( + [context_valid_mask, current_valid_mask], dim=1 + ) + else: + vision_valid_mask = current_valid_mask + logical_valid_mask = torch.cat( + [text_valid_mask, vision_valid_mask], dim=1 + ) + attention_valid_mask = torch.cat( + [logical_valid_mask, logical_valid_mask], dim=0 + )[:, None, None, :] + + hidden_states = None + latent_h, latent_w = getattr(infer_state, "latent_shape", (30, 52)) + if cache_vision: + if enable_prefill_sparse and block_idx > 0 and attention_valid_mask is None: + hidden_states = run_sparse_prefill_attention( + query, + k_txt, + v_txt, + k_vis, + v_vis, + block_idx, + (latent_h, latent_w), + prefill_config, + ) + else: + current_chunk_index = getattr(infer_state, "current_chunk_index", 0) + if ( + enable_decode_sparse + and block_idx > 0 + and current_chunk_index >= sparse_decode_start_chunk + and attention_valid_mask is None + ): + hidden_states = run_sparse_attention( + query, + k_txt, + v_txt, + k_vis, + v_vis, + block_idx, + (latent_h, latent_w), + decode_config, + ) + + if hidden_states is None: + full_key = torch.cat([k_txt, k_vis], dim=2) + full_value = torch.cat([v_txt, v_vis], dim=2) + enable_sageattn = ( + infer_state.enable_sageattn and block_idx in infer_state.sage_blocks_range + ) + if enable_sageattn and attention_valid_mask is None: + from sageattention import sageattn + + hidden_states = sageattn( + query, full_key, full_value, tensor_layout="HND", is_causal=False + ) + else: + hidden_states = F.scaled_dot_product_attention( + query, + full_key, + full_value, + attn_mask=attention_valid_mask, + dropout_p=0.0, + is_causal=False, + ) + + # transpose back + hidden_states = hidden_states.transpose(1, 2) # [B, S, H, D] + + if enable_sp: + hidden_states = all_to_all_4D( + hidden_states, sp_group, scatter_dim=1, gather_dim=2 + ) + hidden_states = hidden_states.to(query.dtype) + + b, s, a, d = hidden_states.shape + hidden_states = hidden_states.reshape(b, s, -1) + hidden_states, hidden_states_prope = torch.chunk(hidden_states, chunks=2, dim=0) + + return hidden_states, hidden_states_prope, vision_kv + + +@torch.compiler.disable +def parallel_attention( + q, + k, + v, + img_q_len, + img_kv_len, + attn_mode=None, + text_mask=None, + attn_param=None, + block_idx=None, +): + return sequence_parallel_attention( + q, + k, + v, + img_q_len, + img_kv_len, + attn_mode, + text_mask, + attn_param=attn_param, + block_idx=block_idx, + ) + + +def sequence_parallel_attention( + q, + k, + v, + img_q_len, + img_kv_len, + attn_mode=None, + text_mask=None, + attn_param=None, + block_idx=None, +): + assert attn_mode is not None + query, encoder_query = q + key, encoder_key = k + value, encoder_value = v + + parallel_dims = get_parallel_state() + enable_sp = parallel_dims.sp_enabled + + if enable_sp: + sp_group = parallel_dims.sp_group + sp_size = parallel_dims.sp + sp_rank = parallel_dims.sp_rank + + if enable_sp: + # batch_size, seq_len, attn_heads, head_dim + query = all_to_all_4D(query, sp_group, scatter_dim=2, gather_dim=1) + key = all_to_all_4D(key, sp_group, scatter_dim=2, gather_dim=1) + value = all_to_all_4D(value, sp_group, scatter_dim=2, gather_dim=1) + + def shrink_head(encoder_state, dim): + local_heads = encoder_state.shape[dim] // sp_size + return encoder_state.narrow(dim, sp_rank * local_heads, local_heads) + + encoder_query = shrink_head(encoder_query, dim=2) + encoder_key = shrink_head(encoder_key, dim=2) + encoder_value = shrink_head(encoder_value, dim=2) + + sequence_length = query.size(1) + encoder_sequence_length = encoder_query.size(1) + + attn_mode = maybe_fallback_attn_mode(attn_mode, get_infer_state(), block_idx) + + if attn_mode == "sageattn": + from sageattention import sageattn + + query = torch.cat([query, encoder_query], dim=1) + key = torch.cat([key, encoder_key], dim=1) + value = torch.cat([value, encoder_value], dim=1) + hidden_states = sageattn( + query, key, value, tensor_layout="NHD", is_causal=False + ) + elif attn_mode == "torch": + query = torch.cat([query, encoder_query], dim=1) + key = torch.cat([key, encoder_key], dim=1) + value = torch.cat([value, encoder_value], dim=1) + if text_mask is not None: + attn_mask = F.pad(text_mask, (sequence_length, 0), value=True) + else: + attn_mask = None + + if attn_mask is not None: + if attn_mask.dtype != torch.bool and attn_mask.dtype in [ + torch.int64, + torch.int32, + ]: + assert ( + attn_mask.max() <= 1 and attn_mask.min() >= 0 + ), f"attention mask must be (0,1)." + attn_mask = attn_mask.to(torch.bool) + elif attn_mask.dtype != torch.bool: + attn_mask = attn_mask.to(query.dtype) + raise NotImplementedError( + f"Float attention mask is not implemented for torch attention." + ) + + # transpose q,k,v dim to fit scaled_dot_product_attention + query = query.transpose(1, 2) # B * Head_num * length * dim + key = key.transpose(1, 2) # B * Head_num * length * dim + value = value.transpose(1, 2) # B * Head_num * length * dim + if attn_mask is not None: + attn_mask1 = einops.rearrange(attn_mask, "b l -> b 1 l 1") + attn_mask2 = einops.rearrange(attn_mask1, "b 1 l 1 -> b 1 1 l") + attn_mask = attn_mask1 & attn_mask2 + hidden_states = F.scaled_dot_product_attention( + query, key, value, attn_mask=attn_mask, dropout_p=0.0, is_causal=False + ) + + # transpose back + hidden_states = hidden_states.transpose(1, 2) + + # add new attn for chunk-wise attn + elif attn_mode == "torch_causal": # now: we set text_mask = None + # attention: here we concat the encoder text sequence first, then apply causal attention + vision_seq_length = query.shape[1] + text_seq_length = encoder_query.shape[1] + total_seq_length = vision_seq_length + text_seq_length + + query = torch.cat([encoder_query, query], dim=1) + key = torch.cat([encoder_key, key], dim=1) + value = torch.cat([encoder_value, value], dim=1) + + # prepare causal mask for chunk-wise attention + latent_seq_length = attn_param["thw"][-1] * attn_param["thw"][-2] + chunk_seq_length = latent_seq_length * 4 + chunk_num = (vision_seq_length) // chunk_seq_length + causal_mask = torch.zeros( + (total_seq_length, total_seq_length), device=query.device + ) + causal_mask[:, :text_seq_length] = 1 # no attention for the rest + for i in range(chunk_num): + start_i = text_seq_length + i * chunk_seq_length + end_i = min(start_i + chunk_seq_length, total_seq_length) + for j in range(i + 1): + start_j = text_seq_length + j * chunk_seq_length + end_j = min(start_j + chunk_seq_length, total_seq_length) + # full attention within chunk i for j == i, causal for j < i + causal_mask[start_i:end_i, start_j:end_j] = 1 + + causal_mask = causal_mask.unsqueeze(0).unsqueeze(1) # 1, 1, S, S + causal_mask = causal_mask.expand(query.shape[0], 1, -1, -1) + causal_mask = causal_mask.to(torch.bool) # Force bool dtype + + query = query.transpose(1, 2) + key = key.transpose(1, 2) + value = value.transpose(1, 2) + + hidden_states = F.scaled_dot_product_attention( + query, key, value, attn_mask=causal_mask, dropout_p=0.0, is_causal=False + ) + + # transpose back + hidden_states = hidden_states.transpose(1, 2) # [B, S, H, D] + + # return back to the original order: [query, encoder_query] + hidden_states, encoder_hidden_states = ( + hidden_states[:, text_seq_length:], + hidden_states[:, :text_seq_length], + ) + hidden_states = torch.cat([hidden_states, encoder_hidden_states], dim=1) + + elif attn_mode == "flash2": + query = torch.cat([query, encoder_query], dim=1) + key = torch.cat([key, encoder_key], dim=1) + value = torch.cat([value, encoder_value], dim=1) + # B, S, 3, H, D + qkv = torch.stack([query, key, value], dim=2) + + attn_mask = F.pad(text_mask, (sequence_length, 0), value=True) + hidden_states = flash_attn_no_pad( + qkv, attn_mask, causal=False, dropout_p=0.0, softmax_scale=None + ) + + elif attn_mode == "flash3": + query = torch.cat([query, encoder_query], dim=1) + key = torch.cat([key, encoder_key], dim=1) + value = torch.cat([value, encoder_value], dim=1) + # B, S, 3, H, D + qkv = torch.stack([query, key, value], dim=2) + attn_mask = F.pad(text_mask, (sequence_length, 0), value=True) + hidden_states = flash_attn_no_pad_v3( + qkv, attn_mask, causal=False, dropout_p=0.0, softmax_scale=None + ) + + elif attn_mode == "flex-block-attn": + sparse_type = attn_param["attn_sparse_type"] # sta/block_attn/ssta + ssta_threshold = attn_param["ssta_threshold"] + ssta_lambda = attn_param["ssta_lambda"] + ssta_sampling_type = attn_param["ssta_sampling_type"] + ssta_adaptive_pool = attn_param["ssta_adaptive_pool"] + + attn_pad_type = attn_param["attn_pad_type"] # repeat/zero + attn_use_text_mask = attn_param["attn_use_text_mask"] + attn_mask_share_within_head = attn_param["attn_mask_share_within_head"] + + ssta_topk = attn_param["ssta_topk"] + thw = attn_param["thw"] + tile_size = attn_param["tile_size"] + win_size = attn_param["win_size"][0].copy() + + def get_image_tile(tile_size): + block_size = np.prod(tile_size) + if block_size == 384: + tile_size = (1, 16, 24) + elif block_size == 128: + tile_size = (1, 16, 8) + elif block_size == 64: + tile_size = (1, 8, 8) + elif block_size == 16: + tile_size = (1, 4, 4) + else: + raise ValueError( + f"Error tile_size {tile_size}, only support in [16, 64, 128, 384]" + ) + return tile_size + + if thw[0] == 1: + tile_size = get_image_tile(tile_size) + win_size = [1, 1, 1] + elif thw[0] <= 31: # 16fps: 5 * 16 / 4 + 1 = 21; 24fps: 5 * 24 / 4 + 1 = 31 + ssta_topk = ssta_topk // 2 + + # Concatenate and permute query, key, value to (B, H, S, D) + query = torch.cat([query, encoder_query], dim=1).permute(0, 2, 1, 3) + key = torch.cat([key, encoder_key], dim=1).permute(0, 2, 1, 3) + value = torch.cat([value, encoder_value], dim=1).permute(0, 2, 1, 3) + + assert ( + query.shape[-1] == 128 + ), "The last dimension of query, key and value must be 128 for flex-block-attn." + + hidden_states = ssta_3d_attention( + query, + key, + value, + thw, + topk=ssta_topk, + tile_thw=tile_size, + kernel_thw=win_size, + text_len=encoder_sequence_length, + sparse_type=sparse_type, + threshold=ssta_threshold, + lambda_=ssta_lambda, + pad_type=attn_pad_type, + text_mask=text_mask if attn_use_text_mask else None, + sampling_type=ssta_sampling_type, + adaptive_pool=ssta_adaptive_pool, + mask_share_within_head=attn_mask_share_within_head, + ) + hidden_states, sparse_ratio = hidden_states + hidden_states = hidden_states.permute(0, 2, 1, 3) + + else: + raise NotImplementedError(f"Unsupported attention mode: {attn_mode}.") + + if enable_sp: + hidden_states, encoder_hidden_states = hidden_states.split_with_sizes( + (sequence_length, encoder_sequence_length), dim=1 + ) + hidden_states = all_to_all_4D( + hidden_states, sp_group, scatter_dim=1, gather_dim=2 + ) + encoder_hidden_states = all_gather( + encoder_hidden_states, dim=2, group=sp_group + ).contiguous() + hidden_states = hidden_states.to(query.dtype) + encoder_hidden_states = encoder_hidden_states.to(query.dtype) + hidden_states = torch.cat([hidden_states, encoder_hidden_states], dim=1) + + b, s, a, d = hidden_states.shape + hidden_states = hidden_states.reshape(b, s, -1) + + return hidden_states diff --git a/hyvideo/models/transformers/modules/bi_sparse_operation_for_KV_cache.py b/hyvideo/models/transformers/modules/bi_sparse_operation_for_KV_cache.py new file mode 100644 index 0000000000000000000000000000000000000000..8c55c63383a252b9a7fe5b04417951d7f092f07d --- /dev/null +++ b/hyvideo/models/transformers/modules/bi_sparse_operation_for_KV_cache.py @@ -0,0 +1,395 @@ +# Copyright (c) 2026 Jiacheng Lu and contributors. +# Licensed under the MIT License. +# See the LICENSE file in the repository root for details. + +import torch +import torch.nn.functional as F +import math +from hyvideo.commons.infer_state import get_infer_state + +try: + import triton + import triton.language as tl + from .longcat_kernel import longcat_flash_attn + HAS_TRITON = True +except ImportError: + HAS_TRITON = False + print("[SparseAttn] Triton not found. Falling back to SDPA.") + + +# ============================================================================== +# Triton kernels. +# ============================================================================== +if HAS_TRITON: + # -------------------------------------------------------------------------- + # Kernel 1: Q Preparation (Fusion: Padding + Tiling + Mean Pooling) + # -------------------------------------------------------------------------- + @triton.jit + def _fused_q_prep_kernel( + Src_ptr, Dst_ptr, Mean_ptr, + s_b, s_h, s_l, s_d, d_b, d_h, d_n, d_blk, d_d, m_b, m_h, m_n, m_d, + NUM_HEADS, T, H_img, W_img, tt, th, tw, nh, nw, + BLOCK_SIZE: tl.constexpr, HEAD_DIM: tl.constexpr, BLOCK_SIZE_FLOAT: tl.constexpr + ): + blk_idx, pid_bh = tl.program_id(0), tl.program_id(1) + cur_head, cur_batch = pid_bh % NUM_HEADS, pid_bh // NUM_HEADS + + # Pointers + src_base = Src_ptr + cur_batch * s_b + cur_head * s_h + dst_base = Dst_ptr + cur_batch * d_b + cur_head * d_h + mean_base = Mean_ptr + cur_batch * m_b + cur_head * m_h + + # Grid Logic (Mapping 1D block index back to 3D coordinates) + cur_nw = blk_idx % nw + rem = blk_idx // nw + cur_nh, cur_nt = rem % nh, rem // nh + + base_t, base_h, base_w = cur_nt * tt, cur_nh * th, cur_nw * tw + + # Inner Loop + offs_flat = tl.arange(0, BLOCK_SIZE) + offs_tw = offs_flat % tw + rem_t = offs_flat // tw + offs_th, offs_tt = rem_t % th, rem_t // th + + coords_t = base_t + offs_tt + coords_h = base_h + offs_th + coords_w = base_w + offs_tw + + # Boundary Check (Padding Logic Fused Here) + mask = (coords_t < T) & (coords_h < H_img) & (coords_w < W_img) + idx_l = coords_t * (H_img * W_img) + coords_h * W_img + coords_w + offs_d = tl.arange(0, HEAD_DIM) + + # Load (Implicit Padding with other=0.0) + val = tl.load(src_base + idx_l[:, None] * s_l + offs_d[None, :] * s_d, mask=mask[:, None], other=0.0) + + # Store Tiled + dst_off = blk_idx * d_n + offs_flat[:, None] * d_blk + offs_d[None, :] * d_d + tl.store(dst_base + dst_off, val) + + # Mean Pooling (Fused) + mean_val = tl.sum(val.to(tl.float32), axis=0) / BLOCK_SIZE_FLOAT + tl.store(mean_base + blk_idx * m_n + offs_d * m_d, mean_val.to(val.dtype)) + + # -------------------------------------------------------------------------- + # Kernel 2: KV Preparation (Fusion: Padding + Tiling + Mean Pooling + Concat Offset) + # -------------------------------------------------------------------------- + @triton.jit + def _fused_kv_prep_kernel( + SK_ptr, SV_ptr, DK_ptr, DV_ptr, MK_ptr, + sk_b, sk_h, sk_l, sk_d, sv_b, sv_h, sv_l, sv_d, + dk_b, dk_h, dk_n, dk_blk, dk_d, dv_b, dv_h, dv_n, dv_blk, dv_d, + m_b, m_h, m_n, m_d, + NUM_HEADS, T, H_img, W_img, tt, th, tw, nh, nw, + dst_offset_blk, pool_offset_blk, + BLOCK_SIZE: tl.constexpr, HEAD_DIM: tl.constexpr, BLOCK_SIZE_FLOAT: tl.constexpr + ): + blk_idx, pid_bh = tl.program_id(0), tl.program_id(1) + cur_head, cur_batch = pid_bh % NUM_HEADS, pid_bh // NUM_HEADS + + # Base Pointers + sk_base, sv_base = SK_ptr + cur_batch * sk_b + cur_head * sk_h, SV_ptr + cur_batch * sv_b + cur_head * sv_h + dk_base, dv_base = DK_ptr + cur_batch * dk_b + cur_head * dk_h, DV_ptr + cur_batch * dv_b + cur_head * dv_h + mk_base = MK_ptr + cur_batch * m_b + cur_head * m_h + + # Grid Logic + cur_nw = blk_idx % nw + rem = blk_idx // nw + cur_nh, cur_nt = rem % nh, rem // nh + + offs_flat = tl.arange(0, BLOCK_SIZE) + offs_tw = offs_flat % tw + rem_t = offs_flat // tw + offs_th, offs_tt = rem_t % th, rem_t // th + + coords_t = cur_nt * tt + offs_tt + coords_h = cur_nh * th + offs_th + coords_w = cur_nw * tw + offs_tw + + mask = (coords_t < T) & (coords_h < H_img) & (coords_w < W_img) + idx_l = coords_t * (H_img * W_img) + coords_h * W_img + coords_w + offs_d = tl.arange(0, HEAD_DIM) + + # Load + val_k = tl.load(sk_base + idx_l[:, None] * sk_l + offs_d[None, :] * sk_d, mask=mask[:, None], other=0.0) + val_v = tl.load(sv_base + idx_l[:, None] * sv_l + offs_d[None, :] * sv_d, mask=mask[:, None], other=0.0) + + # Store Tiled (Notice dst_offset_blk for Text skipping) + dst_off = (blk_idx + dst_offset_blk) * dk_n + offs_flat[:, None] * dk_blk + offs_d[None, :] * dk_d + tl.store(dk_base + dst_off, val_k) + tl.store(dv_base + dst_off, val_v) + + # Store K Mean (Separate buffer, usually no offset needed relative to itself) + # But if mean buffer includes text, we need offset. Here we assume Mean buffer is VISUAL ONLY for Prefill logic usually. + # However, to be safe, we use pool_offset_blk if needed. + tl.store(mk_base + (blk_idx + pool_offset_blk) * m_n + offs_d * m_d, (tl.sum(val_k.to(tl.float32), axis=0) / BLOCK_SIZE_FLOAT).to(val_k.dtype)) + + # -------------------------------------------------------------------------- + # Kernel 3: Untile (Fusion: Scatter + Padding Removal) + # -------------------------------------------------------------------------- + @triton.jit + def _fused_untile_kernel( + Src_ptr, Dst_ptr, + s_b, s_h, s_n, s_blk, s_d, d_b, d_h, d_l, d_d, + NUM_HEADS, T, H_img, W_img, tt, th, tw, nh, nw, + BLOCK_SIZE: tl.constexpr, HEAD_DIM: tl.constexpr + ): + blk_idx, pid_bh = tl.program_id(0), tl.program_id(1) + cur_head, cur_batch = pid_bh % NUM_HEADS, pid_bh // NUM_HEADS + + src_base = Src_ptr + cur_batch * s_b + cur_head * s_h + dst_base = Dst_ptr + cur_batch * d_b + cur_head * d_h + + cur_nw = blk_idx % nw + rem = blk_idx // nw + cur_nh, cur_nt = rem % nh, rem // nh + + offs_flat = tl.arange(0, BLOCK_SIZE) + offs_tw = offs_flat % tw + rem_t = offs_flat // tw + offs_th, offs_tt = rem_t % th, rem_t // th + + coords_t = cur_nt * tt + offs_tt + coords_h = cur_nh * th + offs_th + coords_w = cur_nw * tw + offs_tw + + mask = (coords_t < T) & (coords_h < H_img) & (coords_w < W_img) + offs_d = tl.arange(0, HEAD_DIM) + + val = tl.load(src_base + blk_idx * s_n + offs_flat[:, None] * s_blk + offs_d[None, :] * s_d) + + idx_l = coords_t * (H_img * W_img) + coords_h * W_img + coords_w + tl.store(dst_base + idx_l[:, None] * d_l + offs_d[None, :] * d_d, val, mask=mask[:, None]) + + # -------------------------------------------------------------------------- + # Launchers (Helper functions) + # -------------------------------------------------------------------------- + def launch_q_prep(src, dst, mean, T, H, W, tt, th, tw, nh, nw): + B, NH, _, D = src.shape + sb, sh, sl, sd = src.stride() + db, dh, dn, dblk, dd = dst.stride() + mb, mh, mn, md = mean.stride() + num_blocks = ((T + tt - 1) // tt) * nh * nw + BLOCK_SIZE = tt * th * tw + _fused_q_prep_kernel[(num_blocks, B * NH)]( + src, dst, mean, + sb, sh, sl, sd, db, dh, dn, dblk, dd, mb, mh, mn, md, + NH, T, H, W, tt, th, tw, nh, nw, + BLOCK_SIZE=BLOCK_SIZE, HEAD_DIM=D, BLOCK_SIZE_FLOAT=float(BLOCK_SIZE), + num_warps=4, num_stages=2 + ) + + def launch_kv_prep(k_src, v_src, k_dst, v_dst, k_mean, dst_off, pool_off, T, H, W, tt, th, tw, nh, nw): + B, NH, _, D = k_src.shape + skb, skh, skl, skd = k_src.stride() + svb, svh, svl, svd = v_src.stride() + dkb, dkh, dkn, dkblk, dkd = k_dst.stride() + dvb, dvh, dvn, dvblk, dvd = v_dst.stride() + mb, mh, mn, md = k_mean.stride() + num_blocks = ((T + tt - 1) // tt) * nh * nw + BLOCK_SIZE = tt * th * tw + _fused_kv_prep_kernel[(num_blocks, B * NH)]( + k_src, v_src, k_dst, v_dst, k_mean, + skb, skh, skl, skd, svb, svh, svl, svd, + dkb, dkh, dkn, dkblk, dkd, dvb, dvh, dvn, dvblk, dvd, + mb, mh, mn, md, + NH, T, H, W, tt, th, tw, nh, nw, dst_off, pool_off, + BLOCK_SIZE=BLOCK_SIZE, HEAD_DIM=D, BLOCK_SIZE_FLOAT=float(BLOCK_SIZE), + num_warps=4, num_stages=2 + ) + + def launch_untile(src, dst, T, H, W, tt, th, tw, nh, nw): + B, NH, _, D = dst.shape + sb, sh, sn, sblk, sd = src.stride() + db, dh, dl, dd = dst.stride() + num_blocks = ((T + tt - 1) // tt) * nh * nw + BLOCK_SIZE = tt * th * tw + _fused_untile_kernel[(num_blocks, B * NH)]( + src, dst, + sb, sh, sn, sblk, sd, db, dh, dl, dd, + NH, T, H, W, tt, th, tw, nh, nw, + BLOCK_SIZE=BLOCK_SIZE, HEAD_DIM=D, + num_warps=4 + ) + +# ============================================================================== +# ============================================================================== +# Sparse index generation +# ============================================================================== +def generate_prefill_indices(q_in, k_in, text_len, grid_thw, params, device): + """Build block-sparse indices for the bidirectional prefill phase. + + Similar to the AR decode path, but adds a spatio-temporal neighbour bias + (Manhattan distance ≤ 1) so that blocks adjacent to the query block are + always retained. This preserves local structure during KV-cache + construction. + + Args: + q_in: Query pool ``(B, H, Nq, D)`` or tiled ``(B, H, Nq_blk, Bs, D)``. + k_in: Key pool ``(B, H, Nk, D)`` or tiled. + text_len: Text token count (offsets visual indices). + grid_thw: Grid ``(T_chunks, H_blks, W_blks)``. + params: Dict with ``tile_size``, ``topk_ratio``, ``sim_metric``. + device: Output device. + + Returns: + Tuple ``(final_indices, final_counts)``. + """ + if q_in.dim() == 5: + q_pool = q_in.mean(dim=-2) + BLOCK_SIZE = q_in.shape[-2] + else: + q_pool = q_in + tt, th, tw = params['tile_size'] + BLOCK_SIZE = tt * th * tw + + if k_in.dim() == 5: + k_pool = k_in.mean(dim=-2) + else: + k_pool = k_in + + HEAD_DIM = q_pool.shape[-1] + + metric = params.get('sim_metric', 'cosine') + if metric == 'cosine': + sim = torch.matmul(F.normalize(q_pool, dim=-1), F.normalize(k_pool, dim=-1).transpose(-2, -1)) + else: + sim = torch.matmul(q_pool, k_pool.transpose(-2, -1)) + if metric == 'scaled_dot_product': + sim *= (1.0 / math.sqrt(HEAD_DIM)) + + B, H, N_q, N_k = sim.shape + Nt, Nh, Nw = grid_thw + + score_for_ranking = sim.clone() + LARGE_BIAS = 1e6 + + idx_arr = torch.arange(N_k, device=device) + t_idx = idx_arr // (Nh * Nw) + rem = idx_arr % (Nh * Nw) + h_idx = rem // Nw + w_idx = rem % Nw + + t_q, t_k = t_idx.unsqueeze(1), t_idx.unsqueeze(0) + h_q, h_k = h_idx.unsqueeze(1), h_idx.unsqueeze(0) + w_q, w_k = w_idx.unsqueeze(1), w_idx.unsqueeze(0) + + dist = (t_q - t_k).abs() + (h_q - h_k).abs() + (w_q - w_k).abs() + is_neighbor = dist <= 1 + score_for_ranking = score_for_ranking + is_neighbor.to(score_for_ranking.dtype) * LARGE_BIAS + + target_k = int(N_k * params['topk_ratio']) + min_budget = 7 + k_budget = max(target_k, min_budget) + k_budget = min(k_budget, N_k) + + _, vis_indices = torch.topk(score_for_ranking, k=k_budget, dim=-1) + + vis_indices, _ = torch.sort(vis_indices, dim=-1) + num_txt_blk = (text_len + BLOCK_SIZE - 1) // BLOCK_SIZE + vis_indices = vis_indices + num_txt_blk + txt_indices = torch.arange(num_txt_blk, device=device).view(1, 1, 1, -1).expand(B, H, N_q, -1) + final_indices = torch.cat([txt_indices, vis_indices], dim=-1).int().contiguous() + final_counts = torch.full((B, H, N_q), final_indices.shape[-1], dtype=torch.int32, device=device) + + return final_indices, final_counts + +# ============================================================================== +# Prefill sparse attention entry point +# ============================================================================== +def run_sparse_prefill_attention(q, k_txt, v_txt, k_vis, v_vis, block_idx, latent_hw, sparse_params): + """Execute a single bidirectional (prefill) sparse-attention forward pass. + + The first three chunks always run dense (warm-up heuristic). After that, + Triton-fused tiling and sparse-index generation are applied; if Triton is + unavailable the call falls back to full SDPA. + + Args: + q: Query ``(B, H, Nq, D)``. + k_txt, v_txt: Text KV ``(B, H, N_txt, D)``. + k_vis, v_vis: Visual KV ``(B, H, N_vis, D)``. + block_idx: Current chunk index. + latent_hw: ``(H_lat, W_lat)`` of the latent grid. + sparse_params: Dict with ``tile_size``, ``topk_ratio``, + ``recent_protect``, ``sim_metric``. + + Returns: + Attention output ``(B, H, Nq, D)``. + """ + infer_state = get_infer_state() + current_global_chunk = getattr(infer_state, "current_chunk_index", 0) + + LATENT_H, LATENT_W = latent_hw + vis_len_q = q.shape[2] + + text_len = k_txt.shape[2] + vis_len_k = k_vis.shape[2] + + # Dense fallback: early warm-up chunks, missing Triton, or non-uniform + # latent shape (cannot tile cleanly). + need_dense = ( + current_global_chunk < 3 + or not HAS_TRITON + or (vis_len_q % (LATENT_H * LATENT_W) != 0) + ) + if need_dense: + return F.scaled_dot_product_attention( + q, + torch.cat([k_txt, k_vis], dim=2), + torch.cat([v_txt, v_vis], dim=2), + dropout_p=0.0, + is_causal=False + ) + + from .longcat_kernel import longcat_flash_attn + + tt, th, tw = sparse_params.get('tile_size', (4, 8, 4)) + BLOCK_SIZE = tt * th * tw + + nh = (LATENT_H + (th - LATENT_H % th) % th) // th + nw = (LATENT_W + (tw - LATENT_W % tw) % tw) // tw + + T_q = vis_len_q // (LATENT_H * LATENT_W) + T_k = vis_len_k // (LATENT_H * LATENT_W) + + nt_q = (T_q + (tt - T_q % tt) % tt) // tt + nt_k = (T_k + (tt - T_k % tt) % tt) // tt + + num_txt_blk = (text_len + (BLOCK_SIZE - text_len % BLOCK_SIZE) % BLOCK_SIZE) // BLOCK_SIZE + num_vis_blk_q = nt_q * nh * nw + num_vis_blk_k = nt_k * nh * nw + B, H, _, D = q.shape + q_tiled = torch.empty((B, H, num_vis_blk_q, BLOCK_SIZE, D), device=q.device, dtype=q.dtype) + q_pool_vis = torch.empty((B, H, num_vis_blk_q, D), device=q.device, dtype=q.dtype) + + k_all = torch.empty((B, H, num_txt_blk + num_vis_blk_k, BLOCK_SIZE, D), device=q.device, dtype=q.dtype) + v_all = torch.empty((B, H, num_txt_blk + num_vis_blk_k, BLOCK_SIZE, D), device=q.device, dtype=q.dtype) + k_pool_vis = torch.empty((B, H, num_vis_blk_k, D), device=q.device, dtype=q.dtype) + pad_txt = (BLOCK_SIZE - text_len % BLOCK_SIZE) % BLOCK_SIZE + k_txt_padded, v_txt_padded = k_txt, v_txt + if pad_txt > 0: + k_txt_padded = F.pad(k_txt, (0, 0, 0, pad_txt)) + v_txt_padded = F.pad(v_txt, (0, 0, 0, pad_txt)) + k_all[:, :, :num_txt_blk].copy_(k_txt_padded.view(B, H, -1, BLOCK_SIZE, D)) + v_all[:, :, :num_txt_blk].copy_(v_txt_padded.view(B, H, -1, BLOCK_SIZE, D)) + + launch_q_prep(q, q_tiled, q_pool_vis, T_q, LATENT_H, LATENT_W, tt, th, tw, nh, nw) + + launch_kv_prep(k_vis, v_vis, k_all, v_all, k_pool_vis, num_txt_blk, 0, + T_k, LATENT_H, LATENT_W, tt, th, tw, nh, nw) + + indices, counts = generate_prefill_indices( + q_pool_vis, k_pool_vis, text_len, (nt_k, nh, nw), + sparse_params, q.device + ) + + o_flat = longcat_flash_attn( + q_tiled.flatten(2, 3), k_all.flatten(2, 3), v_all.flatten(2, 3), + indices, counts, sm_scale=1.0/(D**0.5), logical_block_size=BLOCK_SIZE + ) + + o_out = torch.empty_like(q) + launch_untile(o_flat.view(B, H, num_vis_blk_q, BLOCK_SIZE, D), o_out, T_q, LATENT_H, LATENT_W, tt, th, tw, nh, nw) + + return o_out diff --git a/hyvideo/models/transformers/modules/embed_layers.py b/hyvideo/models/transformers/modules/embed_layers.py new file mode 100644 index 0000000000000000000000000000000000000000..92e3b7b11f00f615b9c76647c003318b6aff7726 --- /dev/null +++ b/hyvideo/models/transformers/modules/embed_layers.py @@ -0,0 +1,224 @@ +# Licensed under the TENCENT HUNYUAN COMMUNITY LICENSE AGREEMENT (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://github.com/Tencent-Hunyuan/HunyuanVideo-1.5/blob/main/LICENSE +# +# Unless and only to the extent required by applicable law, the Tencent Hunyuan works and any +# output and results therefrom are provided "AS IS" without any express or implied warranties of +# any kind including any warranties of title, merchantability, noninfringement, course of dealing, +# usage of trade, or fitness for a particular purpose. You are solely responsible for determining the +# appropriateness of using, reproducing, modifying, performing, displaying or distributing any of +# the Tencent Hunyuan works or outputs and assume any and all risks associated with your or a +# third party's use or distribution of any of the Tencent Hunyuan works or outputs and your exercise +# of rights and permissions under this agreement. +# See the License for the specific language governing permissions and limitations under the License. + +import math +import torch +import torch.nn as nn + +from hyvideo.commons import to_2tuple + + +class PatchEmbed(nn.Module): + """2D Image to Patch Embedding + + Image to Patch Embedding using Conv2d + + A convolution based approach to patchifying a 2D image w/ embedding projection. + + Based on the impl in https://github.com/google-research/vision_transformer + + Hacked together by / Copyright 2020 Ross Wightman + + Remove the _assert function in forward function to be compatible with multi-resolution images. + """ + + def __init__( + self, + patch_size=16, + in_chans=3, + embed_dim=768, + is_reshape_temporal_channels=False, + concat_condition=True, + norm_layer=None, + flatten=True, + bias=True, + dtype=None, + device=None, + ): + factory_kwargs = {"dtype": dtype, "device": device} + super().__init__() + patch_size = to_2tuple(patch_size) + self.patch_size = patch_size + self.flatten = flatten + + # Only support concat mode (multitask mask training) + orig_in_chans = in_chans + if concat_condition: + if is_reshape_temporal_channels: + in_chans = in_chans + in_chans // 2 + 1 + else: + in_chans = in_chans * 2 + 1 + + self.proj = nn.Conv3d( + in_chans, + embed_dim, + kernel_size=patch_size, + stride=patch_size, + bias=bias, + **factory_kwargs, + ) + + nn.init.xavier_uniform_( + self.proj.weight[:, :orig_in_chans].view( + self.proj.weight[:, :orig_in_chans].size(0), -1 + ) + ) + # Special initialization for concat mode + nn.init.zeros_( + self.proj.weight[:, orig_in_chans:].view( + self.proj.weight[:, orig_in_chans:].size(0), -1 + ) + ) + + if bias: + nn.init.zeros_(self.proj.bias) + + self.norm = norm_layer(embed_dim) if norm_layer else nn.Identity() + + def forward(self, x): + x = self.proj(x) + if self.flatten: + x = x.flatten(2).transpose(1, 2) # BCHW -> BNC + x = self.norm(x) + return x + + +class TextProjection(nn.Module): + """ + Projects text embeddings. Also handles dropout for classifier-free guidance. + + Adapted from https://github.com/PixArt-alpha/PixArt-alpha/blob/master/diffusion/model/nets/PixArt_blocks.py + """ + + def __init__(self, in_channels, hidden_size, act_layer, dtype=None, device=None): + factory_kwargs = {"dtype": dtype, "device": device} + super().__init__() + self.linear_1 = nn.Linear( + in_features=in_channels, + out_features=hidden_size, + bias=True, + **factory_kwargs, + ) + self.act_1 = act_layer() + self.linear_2 = nn.Linear( + in_features=hidden_size, + out_features=hidden_size, + bias=True, + **factory_kwargs, + ) + + def forward(self, caption): + hidden_states = self.linear_1(caption) + hidden_states = self.act_1(hidden_states) + hidden_states = self.linear_2(hidden_states) + return hidden_states + + +class VisionProjection(torch.nn.Module): + + def __init__(self, input_dim, output_dim): + super().__init__() + + self.proj = torch.nn.Sequential( + torch.nn.LayerNorm(input_dim), + torch.nn.Linear(input_dim, input_dim), + torch.nn.GELU(), + torch.nn.Linear(input_dim, output_dim), + torch.nn.LayerNorm(output_dim), + ) + + def forward(self, vision_embeds): + return self.proj(vision_embeds) + + +class ClipVisionProjection(nn.Module): + def __init__(self, in_channels, out_channels): + super().__init__() + self.up = nn.Linear(in_channels, out_channels * 3) + self.down = nn.Linear(out_channels * 3, out_channels) + torch.nn.init.zeros_(self.down.weight) + torch.nn.init.zeros_(self.down.bias) + + def forward(self, x): + projected_x = self.down(nn.functional.silu(self.up(x))) + return projected_x + + +def timestep_embedding(t, dim, max_period=10000): + """ + Create sinusoidal timestep embeddings. + + Args: + t (torch.Tensor): a 1-D Tensor of N indices, one per batch element. These may be fractional. + dim (int): the dimension of the output. + max_period (int): controls the minimum frequency of the embeddings. + + Returns: + embedding (torch.Tensor): An (N, D) Tensor of positional embeddings. + + .. ref_link: https://github.com/openai/glide-text2im/blob/main/glide_text2im/nn.py + """ + half = dim // 2 + freqs = torch.exp( + -math.log(max_period) + * torch.arange(start=0, end=half, dtype=torch.float32) + / half + ).to(device=t.device) + args = t[:, None].float() * freqs[None] + embedding = torch.cat([torch.cos(args), torch.sin(args)], dim=-1) + if dim % 2: + embedding = torch.cat([embedding, torch.zeros_like(embedding[:, :1])], dim=-1) + return embedding + + +class TimestepEmbedder(nn.Module): + """ + Embeds scalar timesteps into vector representations. + """ + + def __init__( + self, + hidden_size, + act_layer, + frequency_embedding_size=256, + max_period=10000, + out_size=None, + dtype=None, + device=None, + ): + factory_kwargs = {"dtype": dtype, "device": device} + super().__init__() + self.frequency_embedding_size = frequency_embedding_size + self.max_period = max_period + if out_size is None: + out_size = hidden_size + + self.mlp = nn.Sequential( + nn.Linear( + frequency_embedding_size, hidden_size, bias=True, **factory_kwargs + ), + act_layer(), + nn.Linear(hidden_size, out_size, bias=True, **factory_kwargs), + ) + nn.init.normal_(self.mlp[0].weight, std=0.02) + nn.init.normal_(self.mlp[2].weight, std=0.02) + + def forward(self, t): + t_freq = timestep_embedding( + t, self.frequency_embedding_size, self.max_period + ).type(self.mlp[0].weight.dtype) + t_emb = self.mlp(t_freq) + return t_emb diff --git a/hyvideo/models/transformers/modules/longcat_kernel.py b/hyvideo/models/transformers/modules/longcat_kernel.py new file mode 100644 index 0000000000000000000000000000000000000000..010c3acb81ac74ef3dfe6e42a5370f8f33e8527a --- /dev/null +++ b/hyvideo/models/transformers/modules/longcat_kernel.py @@ -0,0 +1,233 @@ +# Adapted from LongCat-Video. +# Original copyright (c) 2025 Meituan. +# Licensed under the MIT License. +# +# Modifications for Light Interaction: +# - adapted the sparse-attention kernel to HY-WorldPlay +# - integrated it with AR denoising and KV-cache recomputation paths +# +# Modifications copyright (c) 2026 Jiacheng Lu and contributors. + +import torch +import triton +import triton.language as tl +import os + +# Autotune configs for the aligned sparse-attention kernel. +# In this variant, BLOCK_N must match the logical sparse block size. +configs_fwd_bsa_align = [ + triton.Config({}, num_stages=3, num_warps=8), + # triton.Config({}, num_stages=3, num_warps=4), + # triton.Config({}, num_stages=4, num_warps=4), + # triton.Config({}, num_stages=4, num_warps=8), +] + +fwd_bsa_reevaluate_keys = ['Q_LEN', 'K_LEN', 'BLOCK_M', 'SPARSITY'] if os.environ.get('TRITON_REEVALUATE_KEY', '0') == '1' else [] + +@triton.autotune(list(configs_fwd_bsa_align), key=fwd_bsa_reevaluate_keys) +@triton.jit +def _attn_fwd_bsa_align( + Q, K, V, sm_scale, M, Out, + block_indices, block_indices_lens, + stride_qz, stride_qh, stride_qm, stride_qk, + stride_kz, stride_kh, stride_kn, stride_kk, + stride_vz, stride_vh, stride_vn, stride_vk, + stride_oz, stride_oh, stride_om, stride_ok, + stride_bz, stride_bh, stride_bm, stride_bs, + stride_lz, stride_lh, stride_lm, + H, + Q_LEN, K_LEN, + HEAD_DIM: tl.constexpr, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, + SPARSITY: tl.constexpr, + ): + + # Grid ID + start_m = tl.program_id(0) + off_hz = tl.program_id(1) + off_z = off_hz // H + off_h = off_hz % H + + # Offsets + q_offset = off_z.to(tl.int64) * stride_qz + off_h.to(tl.int64) * stride_qh + k_offset = off_z.to(tl.int64) * stride_kz + off_h.to(tl.int64) * stride_kh + v_offset = off_z.to(tl.int64) * stride_vz + off_h.to(tl.int64) * stride_vh + o_offset = off_z.to(tl.int64) * stride_oz + off_h.to(tl.int64) * stride_oh + b_offset = off_z.to(tl.int64) * stride_bz + off_h.to(tl.int64) * stride_bh + l_offset = off_z.to(tl.int64) * stride_lz + off_h.to(tl.int64) * stride_lh + + # Block Pointers + # Q: [BLOCK_M, HEAD_DIM] + Q_block_ptr = tl.make_block_ptr( + base=Q + q_offset, + shape=(Q_LEN, HEAD_DIM), + strides=(stride_qm, stride_qk), + offsets=(start_m * BLOCK_M, 0), + block_shape=(BLOCK_M, HEAD_DIM), + order=(1, 0), + ) + + V_block_ptr = tl.make_block_ptr( + base=V + v_offset, + shape=(K_LEN, HEAD_DIM), + strides=(stride_vn, stride_vk), + offsets=(0, 0), + block_shape=(BLOCK_N, HEAD_DIM), + order=(1, 0), + ) + KT_block_ptr = tl.make_block_ptr( + base=K + k_offset, + shape=(HEAD_DIM, K_LEN), + strides=(stride_kk, stride_kn), + offsets=(0, 0), + block_shape=(HEAD_DIM, BLOCK_N), + order=(0, 1), + ) + + O_block_ptr = tl.make_block_ptr( + base=Out + o_offset, + shape=(Q_LEN, HEAD_DIM), + strides=(stride_om, stride_ok), + offsets=(start_m * BLOCK_M, 0), + block_shape=(BLOCK_M, HEAD_DIM), + order=(1, 0), + ) + + # Indices pointers + block_indices += b_offset + start_m * stride_bm + block_indices_lens += l_offset + start_m * stride_lm + + # Accumulators + m_i = tl.zeros([BLOCK_M], dtype=tl.float32) - float("inf") + l_i = tl.zeros([BLOCK_M], dtype=tl.float32) + 1.0 + acc = tl.zeros([BLOCK_M, HEAD_DIM], dtype=tl.float32) + + # Pre-computation + qk_scale = sm_scale * 1.44269504 + + # Load Q (SRAM persistent) + q = tl.load(Q_block_ptr) + + # Load Loop Length + S = tl.load(block_indices_lens) + + # Aligned sparse-block loop. Each sparse index maps to one BLOCK_N-sized + # physical K/V block. + for i in range(S): + block_id = tl.load(block_indices + i * stride_bs).to(tl.int32) + + lo = block_id * BLOCK_N + + KT_curr = tl.advance(KT_block_ptr, (0, lo)) + V_curr = tl.advance(V_block_ptr, (lo, 0)) + + kT = tl.load(KT_curr) + qkT = tl.dot(q, kT) # [BLOCK_M, BLOCK_N] + + m_ij = tl.maximum(m_i, tl.max(qkT, 1) * qk_scale) + alpha = tl.math.exp2(m_i - m_ij) + acc = acc * alpha[:, None] + + qkT = qkT * qk_scale - m_ij[:, None] + p = tl.math.exp2(qkT) + + l_ij = tl.sum(p, 1) + l_i = l_i * alpha + l_ij + + m_i = m_ij + + v = tl.load(V_curr) + acc = tl.dot(p.to(v.dtype), v, acc) + + # ============================================================ + # Epilogue + # ============================================================ + m_i += tl.math.log2(l_i) + acc = acc / l_i[:, None] + + # Store M + offs_m = start_m * BLOCK_M + tl.arange(0, BLOCK_M) + m_ptrs = M + off_hz * Q_LEN + offs_m + tl.store(m_ptrs, m_i) + + # Store O + tl.store(O_block_ptr, acc.to(Out.type.element_ty)) + + +def longcat_flash_attn(q, k, v, block_indices, block_counts, sm_scale=None, logical_block_size=128): + """Triton-based block-sparse flash attention. + + Args: + q: Query ``(B, H, Q_len, D)`` (contiguous). + k: Key ``(B, H, K_len, D)`` (contiguous). + v: Value ``(B, H, K_len, D)`` (contiguous). + block_indices: Sparse block indices ``(B, H, Nq_blk, K_blk)`` int32. + block_counts: Number of valid blocks per query ``(B, H, Nq_blk)`` int32. + sm_scale: Scaling factor (default ``1/sqrt(D)``). + logical_block_size: Sparse block size used during index generation + (must evenly divide *K_len*). + + Returns: + Attention output ``(B, H, Q_len, D)``. + + Raises: + ValueError: If *K_len* is not divisible by *logical_block_size* + or if any block index is out of bounds. + """ + q = q.contiguous() + k = k.contiguous() + v = v.contiguous() + block_indices = block_indices.contiguous() + block_counts = block_counts.contiguous() + + B, H, Q_len, HEAD_DIM = q.shape + K_len = k.shape[2] + + if sm_scale is None: + sm_scale = 1.0 / (HEAD_DIM ** 0.5) + + BLOCK_N = logical_block_size + + if K_len % BLOCK_N != 0: + raise ValueError(f"K Length ({K_len}) must be divisible by logical_block_size ({BLOCK_N})") + if block_indices.numel() and block_indices.max() >= (K_len // BLOCK_N): + raise ValueError("Sparse indices out of bounds.") + + # 3. Buffer + o = torch.empty_like(q) + m_buffer = torch.empty((B, H, Q_len), device=q.device, dtype=torch.float32) + + # 4. Grid + BLOCK_M = 128 + grid = lambda META: ( + triton.cdiv(Q_len, BLOCK_M), + B * H + ) + + _attn_fwd_bsa_align[grid]( + Q=q, K=k, V=v, sm_scale=sm_scale, M=m_buffer, Out=o, + block_indices=block_indices, + block_indices_lens=block_counts, + + stride_qz=q.stride(0), stride_qh=q.stride(1), stride_qm=q.stride(2), stride_qk=q.stride(3), + stride_kz=k.stride(0), stride_kh=k.stride(1), stride_kn=k.stride(2), stride_kk=k.stride(3), + stride_vz=v.stride(0), stride_vh=v.stride(1), stride_vn=v.stride(2), stride_vk=v.stride(3), + stride_oz=o.stride(0), stride_oh=o.stride(1), stride_om=o.stride(2), stride_ok=o.stride(3), + + stride_bz=block_indices.stride(0), stride_bh=block_indices.stride(1), + stride_bm=block_indices.stride(2), stride_bs=block_indices.stride(3), + + stride_lz=block_counts.stride(0), stride_lh=block_counts.stride(1), + stride_lm=block_counts.stride(2), + + H=H, + Q_LEN=Q_len, + K_LEN=K_len, + HEAD_DIM=HEAD_DIM, + + BLOCK_M=BLOCK_M, + BLOCK_N=BLOCK_N, + SPARSITY=0 + ) + return o diff --git a/hyvideo/models/transformers/modules/mlp_layers.py b/hyvideo/models/transformers/modules/mlp_layers.py new file mode 100644 index 0000000000000000000000000000000000000000..0cabc8562e11bbdc6018b8659976bf6225c21558 --- /dev/null +++ b/hyvideo/models/transformers/modules/mlp_layers.py @@ -0,0 +1,151 @@ +# Licensed under the TENCENT HUNYUAN COMMUNITY LICENSE AGREEMENT (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://github.com/Tencent-Hunyuan/HunyuanVideo-1.5/blob/main/LICENSE +# +# Unless and only to the extent required by applicable law, the Tencent Hunyuan works and any +# output and results therefrom are provided "AS IS" without any express or implied warranties of +# any kind including any warranties of title, merchantability, noninfringement, course of dealing, +# usage of trade, or fitness for a particular purpose. You are solely responsible for determining the +# appropriateness of using, reproducing, modifying, performing, displaying or distributing any of +# the Tencent Hunyuan works or outputs and assume any and all risks associated with your or a +# third party's use or distribution of any of the Tencent Hunyuan works or outputs and your exercise +# of rights and permissions under this agreement. +# See the License for the specific language governing permissions and limitations under the License. + +# Modified from timm library: +# https://github.com/huggingface/pytorch-image-models/blob/648aaa41233ba83eb38faf5ba9d415d574823241/timm/layers/mlp.py#L13 + +from functools import partial + +import torch +import torch.nn as nn + +from hyvideo.commons import to_2tuple +from .modulate_layers import modulate + + +class MLP(nn.Module): + """MLP as used in Vision Transformer, MLP-Mixer and related networks""" + + def __init__( + self, + in_channels, + hidden_channels=None, + out_features=None, + act_layer=nn.GELU, + norm_layer=None, + bias=True, + drop=0.0, + use_conv=False, + device=None, + dtype=None, + ): + factory_kwargs = {"device": device, "dtype": dtype} + super().__init__() + out_features = out_features or in_channels + hidden_channels = hidden_channels or in_channels + bias = to_2tuple(bias) + drop_probs = to_2tuple(drop) + linear_layer = partial(nn.Conv2d, kernel_size=1) if use_conv else nn.Linear + + self.fc1 = linear_layer( + in_channels, hidden_channels, bias=bias[0], **factory_kwargs + ) + self.act = act_layer() + self.drop1 = nn.Dropout(drop_probs[0]) + self.norm = ( + norm_layer(hidden_channels, **factory_kwargs) + if norm_layer is not None + else nn.Identity() + ) + self.fc2 = linear_layer( + hidden_channels, out_features, bias=bias[1], **factory_kwargs + ) + self.drop2 = nn.Dropout(drop_probs[1]) + + def forward(self, x): + x = self.fc1(x) + x = self.act(x) + x = self.drop1(x) + x = self.norm(x) + x = self.fc2(x) + x = self.drop2(x) + return x + + +class LinearWarpforSingle(nn.Module): + def __init__(self, in_dim: int, out_dim: int, bias=False, device=None, dtype=None): + factory_kwargs = {"device": device, "dtype": dtype} + super().__init__() + self.fc = nn.Linear(in_dim, out_dim, bias=bias, **factory_kwargs) + + def forward(self, x, y): + input = torch.cat([x.contiguous(), y.contiguous()], dim=2).contiguous() + return self.fc(input) + + +# +class MLPEmbedder(nn.Module): + """copied from https://github.com/black-forest-labs/flux/blob/main/src/flux/modules/layers.py""" + + def __init__(self, in_dim: int, hidden_dim: int, device=None, dtype=None): + factory_kwargs = {"device": device, "dtype": dtype} + super().__init__() + self.in_layer = nn.Linear(in_dim, hidden_dim, bias=True, **factory_kwargs) + self.silu = nn.SiLU() + self.out_layer = nn.Linear(hidden_dim, hidden_dim, bias=True, **factory_kwargs) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.out_layer(self.silu(self.in_layer(x))) + + +class FinalLayer(nn.Module): + """The final layer of DiT.""" + + def __init__( + self, hidden_size, patch_size, out_channels, act_layer, device=None, dtype=None + ): + factory_kwargs = {"device": device, "dtype": dtype} + super().__init__() + + # Just use LayerNorm for the final layer + self.norm_final = nn.LayerNorm( + hidden_size, elementwise_affine=False, eps=1e-6, **factory_kwargs + ) + if isinstance(patch_size, int): + self.linear = nn.Linear( + hidden_size, + patch_size * patch_size * out_channels, + bias=True, + **factory_kwargs, + ) + else: + out_size = ( + patch_size[0] * patch_size[1] * patch_size[2] + if len(patch_size) == 3 + else patch_size[0] * patch_size[1] + ) * out_channels + self.linear = nn.Linear( + hidden_size, + out_size, + bias=True, + ) + nn.init.zeros_(self.linear.weight) + nn.init.zeros_(self.linear.bias) + + # Here we don't distinguish between the modulate types. Just use the simple one. + self.adaLN_modulation = nn.Sequential( + act_layer(), + nn.Linear(hidden_size, 2 * hidden_size, bias=True, **factory_kwargs), + ) + # Zero-initialize the modulation + nn.init.zeros_(self.adaLN_modulation[1].weight) + nn.init.zeros_(self.adaLN_modulation[1].bias) + + def forward(self, x, c): + shift, scale = self.adaLN_modulation(c).chunk(2, dim=1) + x = modulate(self.norm_final(x), shift=shift, scale=scale) + x = self.linear(x) + return x diff --git a/hyvideo/models/transformers/modules/modulate_layers.py b/hyvideo/models/transformers/modules/modulate_layers.py new file mode 100644 index 0000000000000000000000000000000000000000..019cbd9f2c3c9009936782e02ccba179a4ebd137 --- /dev/null +++ b/hyvideo/models/transformers/modules/modulate_layers.py @@ -0,0 +1,123 @@ +# Licensed under the TENCENT HUNYUAN COMMUNITY LICENSE AGREEMENT (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://github.com/Tencent-Hunyuan/HunyuanVideo-1.5/blob/main/LICENSE +# +# Unless and only to the extent required by applicable law, the Tencent Hunyuan works and any +# output and results therefrom are provided "AS IS" without any express or implied warranties of +# any kind including any warranties of title, merchantability, noninfringement, course of dealing, +# usage of trade, or fitness for a particular purpose. You are solely responsible for determining the +# appropriateness of using, reproducing, modifying, performing, displaying or distributing any of +# the Tencent Hunyuan works or outputs and assume any and all risks associated with your or a +# third party's use or distribution of any of the Tencent Hunyuan works or outputs and your exercise +# of rights and permissions under this agreement. +# See the License for the specific language governing permissions and limitations under the License. + +from typing import Callable + +import torch +import torch.nn as nn +from einops import rearrange + + +class ModulateDiT(nn.Module): + """Modulation layer for DiT.""" + + def __init__( + self, + hidden_size: int, + factor: int, + act_layer: Callable, + dtype=None, + device=None, + ): + factory_kwargs = {"dtype": dtype, "device": device} + super().__init__() + self.act = act_layer() + self.linear = nn.Linear( + hidden_size, factor * hidden_size, bias=True, **factory_kwargs + ) + # Zero-initialize the modulation + nn.init.zeros_(self.linear.weight) + nn.init.zeros_(self.linear.bias) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.linear(self.act(x)) + + +def modulate(x, shift=None, scale=None): + """modulate by shift and scale + + Args: + x (torch.Tensor): input tensor. + shift (torch.Tensor, optional): shift tensor. Defaults to None. + scale (torch.Tensor, optional): scale tensor. Defaults to None. + + Returns: + torch.Tensor: the output tensor after modulate. + """ + if scale is None and shift is None: + return x + elif shift is None: + scale = scale.unsqueeze(0) + scale = rearrange(scale, "B (N T) C -> (B N) T C", N=x.shape[0]) + latent_length = scale.shape[1] # latent length + token_length = x.shape[1] // latent_length + # operate on the hidden_states + scale = scale.repeat_interleave(token_length, dim=1).type_as(x) + return x * (1 + scale) + + elif scale is None: + shift = shift.unsqueeze(0) + shift = rearrange(shift, "B (N T) C -> (B N) T C", N=x.shape[0]) + latent_length = shift.shape[1] # latent length + token_length = x.shape[1] // latent_length + # operate on the hidden_states + shift = shift.repeat_interleave(token_length, dim=1).type_as(x) + return x + shift + + else: + shift = shift.unsqueeze(0) + scale = scale.unsqueeze(0) + shift = rearrange(shift, "B (N T) C -> (B N) T C", N=x.shape[0]) + scale = rearrange(scale, "B (N T) C -> (B N) T C", N=x.shape[0]) + + latent_length = shift.shape[1] # latent length + token_length = x.shape[1] // latent_length + + scale = scale.repeat_interleave(token_length, dim=1).type_as(x) + shift = shift.repeat_interleave(token_length, dim=1).type_as(x) + return x * (1 + scale) + shift + + +def apply_gate(x, gate=None, tanh=False): + """AI is creating summary for apply_gate + + Args: + x (torch.Tensor): input tensor. + gate (torch.Tensor, optional): gate tensor. Defaults to None. + tanh (bool, optional): whether to use tanh function. Defaults to False. + + Returns: + torch.Tensor: the output tensor after apply gate. + """ + if gate is None: + return x + gate = gate.unsqueeze(0) + gate = rearrange(gate, "B (N T) C -> (B N) T C", N=x.shape[0]) + latent_length = gate.shape[1] # latent length + token_length = x.shape[1] // latent_length + gate = gate.repeat_interleave(token_length, dim=1).type_as(x) + if tanh: + return x * gate.tanh() + else: + return x * gate + + +def ckpt_wrapper(module): + def ckpt_forward(*inputs): + outputs = module(*inputs) + return outputs + + return ckpt_forward diff --git a/hyvideo/models/transformers/modules/norm_layers.py b/hyvideo/models/transformers/modules/norm_layers.py new file mode 100644 index 0000000000000000000000000000000000000000..1d02aa8d220c225581e4028062784f0104ec8598 --- /dev/null +++ b/hyvideo/models/transformers/modules/norm_layers.py @@ -0,0 +1,97 @@ +# Licensed under the TENCENT HUNYUAN COMMUNITY LICENSE AGREEMENT (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://github.com/Tencent-Hunyuan/HunyuanVideo-1.5/blob/main/LICENSE +# +# Unless and only to the extent required by applicable law, the Tencent Hunyuan works and any +# output and results therefrom are provided "AS IS" without any express or implied warranties of +# any kind including any warranties of title, merchantability, noninfringement, course of dealing, +# usage of trade, or fitness for a particular purpose. You are solely responsible for determining the +# appropriateness of using, reproducing, modifying, performing, displaying or distributing any of +# the Tencent Hunyuan works or outputs and assume any and all risks associated with your or a +# third party's use or distribution of any of the Tencent Hunyuan works or outputs and your exercise +# of rights and permissions under this agreement. +# See the License for the specific language governing permissions and limitations under the License. + +import torch +import torch.nn as nn + + +class RMSNorm(nn.Module): + def __init__( + self, + dim: int, + elementwise_affine=True, + eps: float = 1e-6, + device=None, + dtype=None, + ): + """ + Initialize the RMSNorm normalization layer. + + Args: + dim (int): The dimension of the input tensor. + eps (float, optional): A small value added to the denominator for numerical stability. Default is 1e-6. + + Attributes: + eps (float): A small value added to the denominator for numerical stability. + weight (nn.Parameter): Learnable scaling parameter. + + """ + factory_kwargs = {"device": device, "dtype": dtype} + super().__init__() + self.eps = eps + if elementwise_affine: + self.weight = nn.Parameter(torch.ones(dim, **factory_kwargs)) + + def _norm(self, x): + """ + Apply the RMSNorm normalization to the input tensor. + + Args: + x (torch.Tensor): The input tensor. + + Returns: + torch.Tensor: The normalized tensor. + + """ + return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps) + + def reset_parameters(self): + if hasattr(self, "weight"): + self.weight.fill_(1) + + def forward(self, x): + """ + Forward pass through the RMSNorm layer. + + Args: + x (torch.Tensor): The input tensor. + + Returns: + torch.Tensor: The output tensor after applying RMSNorm. + + """ + output = self._norm(x.float()).type_as(x) + if hasattr(self, "weight"): + output = output * self.weight + return output + + +def get_norm_layer(norm_layer): + """ + Get the normalization layer. + + Args: + norm_layer (str): The type of normalization layer. + + Returns: + norm_layer (nn.Module): The normalization layer. + """ + if norm_layer == "layer": + return nn.LayerNorm + elif norm_layer == "rms": + return RMSNorm + else: + raise NotImplementedError(f"Norm layer {norm_layer} is not implemented") diff --git a/hyvideo/models/transformers/modules/posemb_layers.py b/hyvideo/models/transformers/modules/posemb_layers.py new file mode 100644 index 0000000000000000000000000000000000000000..5cec490b911b4e015621b2a9e71eb8bd5ceb8763 --- /dev/null +++ b/hyvideo/models/transformers/modules/posemb_layers.py @@ -0,0 +1,340 @@ +# Licensed under the TENCENT HUNYUAN COMMUNITY LICENSE AGREEMENT (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://github.com/Tencent-Hunyuan/HunyuanVideo-1.5/blob/main/LICENSE +# +# Unless and only to the extent required by applicable law, the Tencent Hunyuan works and any +# output and results therefrom are provided "AS IS" without any express or implied warranties of +# any kind including any warranties of title, merchantability, noninfringement, course of dealing, +# usage of trade, or fitness for a particular purpose. You are solely responsible for determining the +# appropriateness of using, reproducing, modifying, performing, displaying or distributing any of +# the Tencent Hunyuan works or outputs and assume any and all risks associated with your or a +# third party's use or distribution of any of the Tencent Hunyuan works or outputs and your exercise +# of rights and permissions under this agreement. +# See the License for the specific language governing permissions and limitations under the License. + +import torch +from typing import Union, Tuple, List + + +def _to_tuple(x, dim=2): + if isinstance(x, int): + return (x,) * dim + elif len(x) == dim: + return x + else: + raise ValueError(f"Expected length {dim} or int, but got {x}") + + +def get_meshgrid_nd(start, *args, dim=2): + """ + Get n-D meshgrid with start, stop and num. + + Args: + start (int or tuple): If len(args) == 0, start is num; If len(args) == 1, start is start, args[0] is stop, + step is 1; If len(args) == 2, start is start, args[0] is stop, args[1] is num. For n-dim, start/stop/num + should be int or n-tuple. If n-tuple is provided, the meshgrid will be stacked following the dim order in + n-tuples. + *args: See above. + dim (int): Dimension of the meshgrid. Defaults to 2. + + Returns: + grid (np.ndarray): [dim, ...] + """ + if len(args) == 0: + # start is grid_size + num = _to_tuple(start, dim=dim) + start = (0,) * dim + stop = num + elif len(args) == 1: + # start is start, args[0] is stop, step is 1 + start = _to_tuple(start, dim=dim) + stop = _to_tuple(args[0], dim=dim) + num = [stop[i] - start[i] for i in range(dim)] + elif len(args) == 2: + # start is start, args[0] is stop, args[1] is num + start = _to_tuple(start, dim=dim) # Left-Top eg: 12,0 + stop = _to_tuple(args[0], dim=dim) # Right-Bottom eg: 20,32 + num = _to_tuple(args[1], dim=dim) # Target Size eg: 32,124 + else: + raise ValueError(f"len(args) should be 0, 1 or 2, but got {len(args)}") + + # PyTorch implement of np.linspace(start[i], stop[i], num[i], endpoint=False) + axis_grid = [] + for i in range(dim): + a, b, n = start[i], stop[i], num[i] + g = torch.linspace(a, b, n + 1, dtype=torch.float32)[:n] + axis_grid.append(g) + grid = torch.meshgrid(*axis_grid, indexing="ij") # dim x [W, H, D] + grid = torch.stack(grid, dim=0) # [dim, W, H, D] + + return grid + + +################################################################################# +# Rotary Positional Embedding Functions # +################################################################################# +# https://github.com/meta-llama/llama/blob/be327c427cc5e89cc1d3ab3d3fec4484df771245/llama/model.py#L80 + + +def reshape_for_broadcast( + freqs_cis: Union[torch.Tensor, Tuple[torch.Tensor]], + x: torch.Tensor, + head_first=False, +): + """ + Reshape frequency tensor for broadcasting it with another tensor. + + This function reshapes the frequency tensor to have the same shape as the target tensor 'x' + for the purpose of broadcasting the frequency tensor during element-wise operations. + + Notes: + When using FlashMHAModified, head_first should be False. + When using Attention, head_first should be True. + + Args: + freqs_cis (Union[torch.Tensor, Tuple[torch.Tensor]]): Frequency tensor to be reshaped. + x (torch.Tensor): Target tensor for broadcasting compatibility. + head_first (bool): head dimension first (except batch dim) or not. + + Returns: + torch.Tensor: Reshaped frequency tensor. + + Raises: + AssertionError: If the frequency tensor doesn't match the expected shape. + AssertionError: If the target tensor 'x' doesn't have the expected number of dimensions. + """ + ndim = x.ndim + assert 0 <= 1 < ndim + + if isinstance(freqs_cis, tuple): + # freqs_cis: (cos, sin) in real space + if freqs_cis[0].ndim == 3: + if head_first: + assert freqs_cis[0].shape == ( + x.shape[0], + x.shape[-2], + x.shape[-1], + ), f"batched freqs_cis shape {freqs_cis[0].shape} does not match x shape {x.shape}" + return freqs_cis[0][:, None], freqs_cis[1][:, None] + assert freqs_cis[0].shape == ( + x.shape[0], + x.shape[1], + x.shape[-1], + ), f"batched freqs_cis shape {freqs_cis[0].shape} does not match x shape {x.shape}" + return freqs_cis[0][:, :, None], freqs_cis[1][:, :, None] + if head_first: + assert freqs_cis[0].shape == ( + x.shape[-2], + x.shape[-1], + ), f"freqs_cis shape {freqs_cis[0].shape} does not match x shape {x.shape}" + shape = [ + d if i == ndim - 2 or i == ndim - 1 else 1 + for i, d in enumerate(x.shape) + ] + else: + assert freqs_cis[0].shape == ( + x.shape[1], + x.shape[-1], + ), f"freqs_cis shape {freqs_cis[0].shape} does not match x shape {x.shape}" + shape = [d if i == 1 or i == ndim - 1 else 1 for i, d in enumerate(x.shape)] + return freqs_cis[0].view(*shape), freqs_cis[1].view(*shape) + else: + # freqs_cis: values in complex space + if head_first: + assert freqs_cis.shape == ( + x.shape[-2], + x.shape[-1], + ), f"freqs_cis shape {freqs_cis.shape} does not match x shape {x.shape}" + shape = [ + d if i == ndim - 2 or i == ndim - 1 else 1 + for i, d in enumerate(x.shape) + ] + else: + assert freqs_cis.shape == ( + x.shape[1], + x.shape[-1], + ), f"freqs_cis shape {freqs_cis.shape} does not match x shape {x.shape}" + shape = [d if i == 1 or i == ndim - 1 else 1 for i, d in enumerate(x.shape)] + return freqs_cis.view(*shape) + + +def rotate_half(x): + x_real, x_imag = ( + x.float().reshape(*x.shape[:-1], -1, 2).unbind(-1) + ) # [B, S, H, D//2] + return torch.stack([-x_imag, x_real], dim=-1).flatten(3) + + +def apply_rotary_emb( + xq: torch.Tensor, + xk: torch.Tensor, + freqs_cis: Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]], + head_first: bool = False, +) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Apply rotary embeddings to input tensors using the given frequency tensor. + + This function applies rotary embeddings to the given query 'xq' and key 'xk' tensors using the provided + frequency tensor 'freqs_cis'. The input tensors are reshaped as complex numbers, and the frequency tensor + is reshaped for broadcasting compatibility. The resulting tensors contain rotary embeddings and are + returned as real tensors. + + Args: + xq (torch.Tensor): Query tensor to apply rotary embeddings. [B, S, H, D] + xk (torch.Tensor): Key tensor to apply rotary embeddings. [B, S, H, D] + freqs_cis (torch.Tensor or tuple): Precomputed frequency tensor for complex exponential. + head_first (bool): head dimension first (except batch dim) or not. + + Returns: + Tuple[torch.Tensor, torch.Tensor]: Tuple of modified query tensor and key tensor with rotary embeddings. + + """ + xk_out = None + if isinstance(freqs_cis, tuple): + cos, sin = reshape_for_broadcast(freqs_cis, xq, head_first) # [S, D] + cos, sin = cos.to(xq.device), sin.to(xq.device) + # real * cos - imag * sin + # imag * cos + real * sin + xq_out = (xq.float() * cos + rotate_half(xq.float()) * sin).type_as(xq) + xk_out = (xk.float() * cos + rotate_half(xk.float()) * sin).type_as(xk) + else: + # view_as_complex will pack [..., D/2, 2](real) to [..., D/2](complex) + xq_ = torch.view_as_complex( + xq.float().reshape(*xq.shape[:-1], -1, 2) + ) # [B, S, H, D//2] + freqs_cis = reshape_for_broadcast(freqs_cis, xq_, head_first).to( + xq.device + ) # [S, D//2] --> [1, S, 1, D//2] + # (real, imag) * (cos, sin) = (real * cos - imag * sin, imag * cos + real * sin) + # view_as_real will expand [..., D/2](complex) to [..., D/2, 2](real) + xq_out = torch.view_as_real(xq_ * freqs_cis).flatten(3).type_as(xq) + xk_ = torch.view_as_complex( + xk.float().reshape(*xk.shape[:-1], -1, 2) + ) # [B, S, H, D//2] + xk_out = torch.view_as_real(xk_ * freqs_cis).flatten(3).type_as(xk) + + return xq_out, xk_out + + +def get_nd_rotary_pos_embed( + rope_dim_list, + start, + *args, + theta=10000.0, + use_real=False, + theta_rescale_factor: Union[float, List[float]] = 1.0, + interpolation_factor: Union[float, List[float]] = 1.0, +): + """ + This is a n-d version of precompute_freqs_cis, which is a RoPE for tokens with n-d structure. + + Args: + rope_dim_list (list of int): Dimension of each rope. len(rope_dim_list) should equal to n. + sum(rope_dim_list) should equal to head_dim of attention layer. + start (int | tuple of int | list of int): If len(args) == 0, start is num; If len(args) == 1, start is start, + args[0] is stop, step is 1; If len(args) == 2, start is start, args[0] is stop, args[1] is num. + *args: See above. + theta (float): Scaling factor for frequency computation. Defaults to 10000.0. + use_real (bool): If True, return real part and imaginary part separately. Otherwise, return complex numbers. + Some libraries such as TensorRT does not support complex64 data type. So it is useful to provide a real + part and an imaginary part separately. + theta_rescale_factor (float): Rescale factor for theta. Defaults to 1.0. + + Returns: + pos_embed (torch.Tensor): [HW, D/2] + """ + + grid = get_meshgrid_nd( + start, *args, dim=len(rope_dim_list) + ) # [3, W, H, D] / [2, W, H] + + if isinstance(theta_rescale_factor, int) or isinstance(theta_rescale_factor, float): + theta_rescale_factor = [theta_rescale_factor] * len(rope_dim_list) + elif isinstance(theta_rescale_factor, list) and len(theta_rescale_factor) == 1: + theta_rescale_factor = [theta_rescale_factor[0]] * len(rope_dim_list) + assert len(theta_rescale_factor) == len( + rope_dim_list + ), "len(theta_rescale_factor) should equal to len(rope_dim_list)" + + if isinstance(interpolation_factor, int) or isinstance(interpolation_factor, float): + interpolation_factor = [interpolation_factor] * len(rope_dim_list) + elif isinstance(interpolation_factor, list) and len(interpolation_factor) == 1: + interpolation_factor = [interpolation_factor[0]] * len(rope_dim_list) + assert len(interpolation_factor) == len( + rope_dim_list + ), "len(interpolation_factor) should equal to len(rope_dim_list)" + + # use 1/ndim of dimensions to encode grid_axis + embs = [] + for i in range(len(rope_dim_list)): + emb = get_1d_rotary_pos_embed( + rope_dim_list[i], + grid[i].reshape(-1), + theta, + use_real=use_real, + theta_rescale_factor=theta_rescale_factor[i], + interpolation_factor=interpolation_factor[i], + ) # 2 x [WHD, rope_dim_list[i]] + embs.append(emb) + + if use_real: + cos = torch.cat([emb[0] for emb in embs], dim=1) # (WHD, D/2) + sin = torch.cat([emb[1] for emb in embs], dim=1) # (WHD, D/2) + return cos, sin + else: + emb = torch.cat(embs, dim=1) # (WHD, D/2) + return emb + + +def get_1d_rotary_pos_embed( + dim: int, + pos: Union[torch.FloatTensor, int], + theta: float = 10000.0, + use_real: bool = False, + theta_rescale_factor: float = 1.0, + interpolation_factor: float = 1.0, +) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: + """ + Precompute the frequency tensor for complex exponential (cis) with given dimensions. + (Note: `cis` means `cos + i * sin`, where i is the imaginary unit.) + + This function calculates a frequency tensor with complex exponential using the given dimension 'dim' + and the end index 'end'. The 'theta' parameter scales the frequencies. + The returned tensor contains complex values in complex64 data type. + + Args: + dim (int): Dimension of the frequency tensor. + pos (int or torch.FloatTensor): Position indices for the frequency tensor. [S] or scalar + theta (float, optional): Scaling factor for frequency computation. Defaults to 10000.0. + use_real (bool, optional): If True, return real part and imaginary part separately. + Otherwise, return complex numbers. + theta_rescale_factor (float, optional): Rescale factor for theta. Defaults to 1.0. + + Returns: + freqs_cis: Precomputed frequency tensor with complex exponential. [S, D/2] + freqs_cos, freqs_sin: Precomputed frequency tensor with real and imaginary parts separately. [S, D] + """ + if isinstance(pos, int): + pos = torch.arange(pos).float() + + # proposed by reddit user bloc97, to rescale rotary embeddings to longer sequence length without fine-tuning + # has some connection to NTK literature + if theta_rescale_factor != 1.0: + theta *= theta_rescale_factor ** (dim / (dim - 2)) + + freqs = 1.0 / ( + theta ** (torch.arange(0, dim, 2)[: (dim // 2)].float() / dim) + ) # [D/2] + # assert interpolation_factor == 1.0, f"interpolation_factor: {interpolation_factor}" + freqs = torch.outer(pos * interpolation_factor, freqs) # [S, D/2] + if use_real: + freqs_cos = freqs.cos().repeat_interleave(2, dim=1) # [S, D] + freqs_sin = freqs.sin().repeat_interleave(2, dim=1) # [S, D] + return freqs_cos, freqs_sin + else: + freqs_cis = torch.polar( + torch.ones_like(freqs), freqs + ) # complex64 # [S, D/2] + return freqs_cis diff --git a/hyvideo/models/transformers/modules/ssta_attention.py b/hyvideo/models/transformers/modules/ssta_attention.py new file mode 100644 index 0000000000000000000000000000000000000000..fbc474d636957326a16f8e13b97e4d8eb34042e1 --- /dev/null +++ b/hyvideo/models/transformers/modules/ssta_attention.py @@ -0,0 +1,819 @@ +# Licensed under the TENCENT HUNYUAN COMMUNITY LICENSE AGREEMENT (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://github.com/Tencent-Hunyuan/HunyuanVideo-1.5/blob/main/LICENSE +# +# Unless and only to the extent required by applicable law, the Tencent Hunyuan works and any +# output and results therefrom are provided "AS IS" without any express or implied warranties of +# any kind including any warranties of title, merchantability, noninfringement, course of dealing, +# usage of trade, or fitness for a particular purpose. You are solely responsible for determining the +# appropriateness of using, reproducing, modifying, performing, displaying or distributing any of +# the Tencent Hunyuan works or outputs and assume any and all risks associated with your or a +# third party's use or distribution of any of the Tencent Hunyuan works or outputs and your exercise +# of rights and permissions under this agreement. +# See the License for the specific language governing permissions and limitations under the License. + +import math +import torch +import numpy as np +from einops import rearrange +from functools import lru_cache + + +def tile(x, canvas_thw, tile_thw, sp_size=1): + r"""Rearrange tensor into tiles for block-based attention. + + Args: + x: Input tensor with shape (b, head, s, d) where s = t * h * w + canvas_thw: Tuple of (t, h, w) representing temporal, height, width dimensions + tile_thw: Tuple of (tile_t, tile_h, tile_w) representing tile dimensions + sp_size: Spatial size parameter, defaults to 1 + + Returns: + Rearranged tensor organized by tiles + """ + b, h, s, d = x.shape + t, h, w = canvas_thw + assert t * h * w == s, f"t:{t} * h:{h} * w:{w} == s:{s}" + + tile_t_dim, tile_h_dim, tile_w_dim = tile_thw + n_t = int(t / tile_t_dim) + n_h = int(h / tile_h_dim) + n_w = int(w / tile_w_dim) + x = rearrange( + x, + "b head (sp t h w) d -> b head (t sp h w) d", + sp=sp_size, + t=t // sp_size, + h=h, + w=w, + ) + return rearrange( + x, + "b h (n_t ts_t n_h ts_h n_w ts_w) d -> b h (n_t n_h n_w ts_t ts_h ts_w) d", + n_t=n_t, + n_h=n_h, + n_w=n_w, + ts_t=tile_t_dim, + ts_h=tile_h_dim, + ts_w=tile_w_dim, + ) + + +def untile(x, canvas_thw, tile_thw, sp_size=1): + r"""Reverse the tiling operation to restore original tensor layout. + + Args: + x: Tiled tensor + canvas_thw: Tuple of (t, h, w) representing temporal, height, width dimensions + tile_thw: Tuple of (tile_t, tile_h, tile_w) representing tile dimensions + sp_size: Spatial size parameter, defaults to 1 + + Returns: + Restored tensor with original layout + """ + t, h, w = canvas_thw + + tile_t_dim, tile_h_dim, tile_w_dim = tile_thw + n_t = int(t / tile_t_dim) + n_h = int(h / tile_h_dim) + n_w = int(w / tile_w_dim) + + x = rearrange( + x, + "b h (n_t n_h n_w ts_t ts_h ts_w) d -> b h (n_t ts_t n_h ts_h n_w ts_w) d", + n_t=n_t, + n_h=n_h, + n_w=n_w, + ts_t=tile_t_dim, + ts_h=tile_h_dim, + ts_w=tile_w_dim, + ) + return rearrange( + x, + "b head (t sp h w) d -> b head (sp t h w) d", + sp=sp_size, + t=t // sp_size, + h=h, + w=w, + ) + + +def get_tile_t_h_w(tile_id, tile_thw_dim): + """Extract temporal, height, and width indices from a flattened tile ID.""" + tile_t_dim, tile_h_dim, tile_w_dim = tile_thw_dim + tile_t = tile_id // (tile_h_dim * tile_w_dim) + tile_h = (tile_id % (tile_h_dim * tile_w_dim)) // tile_w_dim + tile_w = tile_id % tile_w_dim + + return tile_t, tile_h, tile_w + + +def importance_sampling(q, k, topk, threshold=0.0, lambda_=0.9, adaptive_pool=None): + r"""Select top-k blocks based on importance scores considering both similarity and redundancy. + + Args: + q: Query tensor with shape (B, H, S, D) + k: Key tensor with shape (B, H, K, D) + topk: Number of top blocks to select + threshold: Threshold parameter (not implemented) + lambda_: Weight factor balancing similarity and redundancy + adaptive_pool: Adaptive pooling parameter (unused) + + Returns: + top_block_indices: Indices of selected blocks with shape (B, H, S, topk) + """ + if threshold > 0.0: + raise NotImplementedError("importance_sampling with threshold not implemented") + + q = q / q.norm(dim=-1, keepdim=True) + k = k / k.norm(dim=-1, keepdim=True) + gate_similarity = torch.einsum("bhsd,bhkd->bhsk", q, k) + gate_similarity = (gate_similarity + 1.0) / 2.0 + gate_unique = torch.einsum("bhsd,bhkd->bhsk", k, k) + gate_unique = (gate_unique + 1.0) / 2.0 + + B, H, K_num, D = k.shape + diag_indices = torch.arange(K_num, device=k.device) + gate_unique[:, :, diag_indices, diag_indices] = torch.nan + + mean_redundancy = torch.nanmean(gate_unique, dim=-2, keepdim=True) + + importance_scores = lambda_ * gate_similarity - (1 - lambda_) * mean_redundancy + + topk = min(topk, importance_scores.size(-1)) + _, top_block_indices = importance_scores.topk(k=topk, dim=-1, sorted=False) + return top_block_indices + + +def similarity_sampling( + q, k, topk, threshold=0.0, block_num=None, adaptive_pool=None, temperature=0.01 +): + r"""Select top-k blocks based on similarity scores between query and key averages. + + Args: + q: Query tensor with shape (B, H, S, D) + k: Key tensor with shape (B, H, K, D) + topk: Number of top blocks to select + threshold: Cumulative score threshold for dynamic topk selection + block_num: Total number of blocks (unused) + adaptive_pool: Adaptive pooling parameter (unused) + temperature: Temperature scaling for softmax + + Returns: + top_block_indices: Indices of selected blocks with shape (B, H, S, topk) + """ + if threshold > 0.0: + gate = torch.einsum("bhsd,bhkd->bhsk", q, k) + gate = gate / temperature + gate_ = torch.softmax(gate, dim=-1) + sorted_gate, sorted_indices = torch.sort(gate_, dim=-1, descending=True) + cum_scores = torch.cumsum(sorted_gate, dim=-1) + above_threshold = cum_scores >= threshold + has_any_above = above_threshold.any(dim=-1, keepdim=True) + # Find the first position exceeding threshold, add 1 to get the number of blocks to select + dynamic_topk = above_threshold.int().argmax(dim=-1, keepdim=True) + 1 + dynamic_topk = torch.where( + has_any_above, dynamic_topk, torch.full_like(dynamic_topk, topk) + ) + # Generate fixed-shape indices and pad with top1 indices for parts not meeting threshold + # Limit minimum k value + dynamic_topk = torch.clamp(dynamic_topk, min=8, max=topk) + indices = torch.arange(gate.size(-1), device=gate.device).expand(gate.size()) + mask = (indices < dynamic_topk).int() + # Use top1 to pad to topk to ensure same size + top_block_indices = ( + torch.gather(sorted_indices, -1, indices) * mask + + (1 - mask) * sorted_indices[..., 0:1] + ) + else: + gate = torch.einsum("bhsd,bhkd->bhsk", q, k) + topk = min(topk, gate.size(-1)) + _, top_block_indices = gate.topk(k=topk, dim=-1, sorted=False) + return top_block_indices + + +def create_moba_3d_mask( + q, + k, + canvas_thw, + topk, + tile_thw, + kernel_thw, + text_block_num=0, + add_text_mask=False, + threshold=0.0, + lambda_=None, + mask_share_within_head=True, + q_block_avg_pool=True, + adaptive_pool=None, + sampling_type=None, +): + r"""Create MOBA (Mixture of Block Attention) 3D mask for sparse attention. + + Args: + q: Query tensor + k: Key tensor + canvas_thw: Canvas dimensions (t, h, w) + topk: Number of top blocks to attend to + tile_thw: Tile dimensions + kernel_thw: Kernel dimensions + text_block_num: Number of text blocks + add_text_mask: Whether to add text mask + threshold: Threshold for similarity sampling + lambda_: Weight factor for importance sampling + mask_share_within_head: Whether to share mask across heads + q_block_avg_pool: Whether to apply average pooling to query blocks + adaptive_pool: Adaptive pooling size + sampling_type: Type of sampling ("similarity" or "importance") + + Returns: + moba_3d_mask: 3D attention mask with shape (num_heads, block_num, block_num) + """ + seq_len = q.size(2) + block_size = np.prod(tile_thw) + block_num = int(seq_len / block_size) + + image_shape = canvas_thw + block_shape = tile_thw + batch_size, num_heads, seq_len, head_dim = k.shape + num_blocks_t = np.ceil(image_shape[0] / block_shape[0]).astype(int) + num_blocks_h = np.ceil(image_shape[1] / block_shape[1]).astype(int) + num_blocks_w = np.ceil(image_shape[2] / block_shape[2]).astype(int) + + def get_block_avg_feat( + x: torch.Tensor, adaptive_pool=None, pooling_type="avg" + ) -> torch.Tensor: + x_block_means = x.view( + batch_size, + num_heads, + num_blocks_t, + num_blocks_h, + num_blocks_w, + block_shape[0], + block_shape[1], + block_shape[2], + head_dim, + ) + if adaptive_pool is not None: + x_block_means = x_block_means.view( + -1, block_shape[0], block_shape[1], block_shape[2], head_dim + ) + x_block_means = x_block_means.permute(0, 4, 1, 2, 3) + if pooling_type == "avg": + global_avg = torch.nn.functional.adaptive_avg_pool3d( + x_block_means, adaptive_pool + ) + global_avg = global_avg.permute(0, 2, 3, 4, 1) + global_avg = global_avg.reshape( + batch_size, + num_heads, + -1, + head_dim * adaptive_pool[0] * adaptive_pool[1] * adaptive_pool[2], + ) + x_block_means = global_avg + elif pooling_type == "max": + max_pool = torch.nn.functional.adaptive_max_pool3d( + x_block_means, adaptive_pool + ) + max_pool = max_pool.permute(0, 2, 3, 4, 1) + max_pool = max_pool.reshape( + batch_size, + num_heads, + -1, + head_dim * adaptive_pool[0] * adaptive_pool[1] * adaptive_pool[2], + ) + x_block_means = max_pool + elif pooling_type == "mix": + global_avg = torch.nn.functional.adaptive_avg_pool3d( + x_block_means, (1, 1, 1) + ) + global_avg = global_avg / (global_avg.norm(dim=1, keepdim=True) + 1e-8) + global_avg = global_avg.permute(0, 2, 3, 4, 1) + global_avg = global_avg.reshape(batch_size, num_heads, -1, head_dim) + + max_pool = torch.nn.functional.adaptive_max_pool3d( + x_block_means, adaptive_pool + ) + max_pool = max_pool / (max_pool.norm(dim=1, keepdim=True) + 1e-8) + max_pool = max_pool.permute(0, 2, 3, 4, 1) + max_pool = max_pool.reshape( + batch_size, + num_heads, + -1, + head_dim * adaptive_pool[0] * adaptive_pool[1] * adaptive_pool[2], + ) + x_block_means = torch.cat([max_pool, global_avg], dim=-1) + else: + raise ValueError(f"pooling_type={pooling_type} is not Supported") + else: + x_block_means = x_block_means.mean(dim=(-2, -3, -4)).view( + batch_size, num_heads, -1, head_dim + ) + return x_block_means + + if (sampling_type == "similarity" and threshold > 0.0) and adaptive_pool is None: + adaptive_pool = (2, 2, 2) + + k_block_means = get_block_avg_feat(k, adaptive_pool) + + if q_block_avg_pool: + q = get_block_avg_feat(q, adaptive_pool) + q = q.type(torch.float32) + k_block_means = k_block_means.type( + torch.float32 + ) # float logit for better gate logit perception + + if mask_share_within_head: + q = q.mean(dim=1, keepdim=True) + k_block_means = k_block_means.mean(dim=1, keepdim=True) + + if sampling_type == "similarity": + top_block_indices = similarity_sampling( + q, + k_block_means, + topk, + threshold, + block_num=block_num, + adaptive_pool=adaptive_pool, + ) + elif sampling_type == "importance": + top_block_indices = importance_sampling( + q, + k_block_means, + topk, + threshold, + lambda_=lambda_, + adaptive_pool=adaptive_pool, + ) + else: + raise NotImplementedError(f"sampling_type={sampling_type} is not Supported") + + q = q.type_as(k) + # Keep head dimension to dynamically build mask + assert top_block_indices.size(0) == 1, "top_block_indices batch size must be 1" + top_block_indices = top_block_indices.squeeze( + 0 + ) # Remove batch dimension [h, s_block, topk] + + # Create 3D mask template (heads, block_num, block_num) + gate_idx_mask = torch.zeros( + (top_block_indices.size(0), block_num, block_num), + dtype=torch.bool, + device=q.device, + ) + + # Iterate over each attention head using dim=0 + for head_idx in range(top_block_indices.size(0)): + # Build index matrix for each head + head_mask = torch.zeros( + (block_num, block_num), dtype=torch.bool, device=q.device + ) + # Fill indices for current head + head_mask.scatter_(dim=-1, index=top_block_indices[head_idx], value=True) + gate_idx_mask[head_idx] = head_mask + + if text_block_num > 0: + pad_block_num = block_num + text_block_num + moba_3d_mask = torch.full( + (gate_idx_mask.size(0), pad_block_num, pad_block_num), + False, + dtype=torch.bool, + device=gate_idx_mask.device, + ) + moba_3d_mask[:, :block_num, :block_num] = gate_idx_mask + if add_text_mask: + moba_3d_mask[:, :, -text_block_num:] = True + moba_3d_mask[:, -text_block_num:, :] = True + else: + moba_3d_mask = gate_idx_mask + + return moba_3d_mask + + +@lru_cache(maxsize=4096) +def create_sta_3d_mask_optimize(canvas_thw, tile_thw, kernel_thw): + r"""Create optimized STA (Spatio-Temporal Attention) 3D mask using vectorized operations. + + Args: + canvas_thw: String representation of canvas dimensions "t_h_w" + tile_thw: String representation of tile dimensions "t_h_w" + kernel_thw: String representation of kernel dimensions "t_h_w" + + Returns: + block_mask: Boolean mask tensor with shape (block_num, block_num) + """ + canvas_thw = tuple(map(int, canvas_thw.split("_"))) + seq_len = np.prod(canvas_thw) + tile_thw = tuple(map(int, tile_thw.split("_"))) + kernel_thw = tuple(map(int, kernel_thw.split("_"))) + + kernel_t, kernel_h, kernel_w = kernel_thw + + block_size = np.prod(tile_thw) + block_num = int(seq_len / block_size) + + block_mask = np.full((block_num, block_num), False, dtype=bool) + tile_thw_num = ( + canvas_thw[0] // tile_thw[0], + canvas_thw[1] // tile_thw[1], + canvas_thw[2] // tile_thw[2], + ) + + i_indices = np.arange(block_num) + j_indices = np.arange(block_num) + + i_grid, j_grid = np.meshgrid(i_indices, j_indices, indexing="ij") + + q_t_tile = i_grid // (tile_thw_num[1] * tile_thw_num[2]) + q_h_tile = (i_grid % (tile_thw_num[1] * tile_thw_num[2])) // tile_thw_num[2] + q_w_tile = i_grid % tile_thw_num[2] + + kv_t_tile = j_grid // (tile_thw_num[1] * tile_thw_num[2]) + kv_h_tile = (j_grid % (tile_thw_num[1] * tile_thw_num[2])) // tile_thw_num[2] + kv_w_tile = j_grid % tile_thw_num[2] + + kernel_center_t = np.clip( + q_t_tile, kernel_t // 2, (tile_thw_num[0] - 1) - kernel_t // 2 + ) + kernel_center_h = np.clip( + q_h_tile, kernel_h // 2, (tile_thw_num[1] - 1) - kernel_h // 2 + ) + kernel_center_w = np.clip( + q_w_tile, kernel_w // 2, (tile_thw_num[2] - 1) - kernel_w // 2 + ) + + time_mask = np.abs(kernel_center_t - kv_t_tile) <= kernel_t // 2 + hori_mask = np.abs(kernel_center_h - kv_h_tile) <= kernel_h // 2 + vert_mask = np.abs(kernel_center_w - kv_w_tile) <= kernel_w // 2 + + block_mask = time_mask & hori_mask & vert_mask + + block_mask = torch.tensor(block_mask, dtype=torch.bool) + return block_mask + + +@torch.no_grad() +def create_sta_3d_mask(canvas_thw, tile_thw, kernel_thw, text_block_num=0): + r"""Create STA (Spatio-Temporal Attention) 3D mask. + + Args: + canvas_thw: Canvas dimensions (t, h, w) + tile_thw: Tile dimensions + kernel_thw: Kernel dimensions + text_block_num: Number of text blocks to pad + + Returns: + sta_mask: Boolean mask tensor with optional text block padding + """ + block_mask = create_sta_3d_mask_optimize( + "_".join([str(x) for x in canvas_thw]), + "_".join([str(x) for x in tile_thw]), + "_".join([str(x) for x in kernel_thw]), + ) + sta_mask = None + block_num = block_mask.size(0) + if text_block_num > 0: + pad_block_num = block_num + text_block_num + sta_mask = torch.full( + (pad_block_num, pad_block_num), + False, + dtype=torch.bool, + ) + sta_mask[:block_num, :block_num] = block_mask + sta_mask[:, -text_block_num:] = True + sta_mask[-text_block_num:, :] = True + else: + sta_mask = block_mask + return sta_mask + + +@torch.no_grad() +def create_ssta_3d_mask( + q, + k, + canvas_thw, + topk, + tile_thw, + kernel_thw, + text_block_num=0, + threshold=0.0, + lambda_=None, + text_mask=None, + mask_share_within_head=True, + adaptive_pool=None, + sampling_type=None, +): + r"""Create SSTA (Sparse Spatio-Temporal Attention) 3D mask combining STA and MOBA masks. + + Args: + q: Query tensor + k: Key tensor + canvas_thw: Canvas dimensions (t, h, w) + topk: Number of top blocks to attend to + tile_thw: Tile dimensions + kernel_thw: Kernel dimensions + text_block_num: Number of text blocks + threshold: Threshold for similarity sampling + lambda_: Weight factor for importance sampling + text_mask: Optional text mask tensor + mask_share_within_head: Whether to share mask across heads + adaptive_pool: Adaptive pooling size + sampling_type: Type of sampling ("similarity" or "importance") + + Returns: + ssta_3d_mask: Combined 3D attention mask + """ + sta_3d_mask = create_sta_3d_mask(canvas_thw, tile_thw, kernel_thw, text_block_num) + sta_3d_mask = sta_3d_mask.to(q.device) + + moba_3d_mask = create_moba_3d_mask( + q, + k, + canvas_thw, + topk, + tile_thw, + kernel_thw, + text_block_num, + threshold=threshold, + lambda_=lambda_, + mask_share_within_head=mask_share_within_head, + adaptive_pool=adaptive_pool, + sampling_type=sampling_type, + ) + ssta_3d_mask = torch.logical_or(sta_3d_mask.unsqueeze(0), moba_3d_mask) + assert len(ssta_3d_mask.size()) == 3, "ssta_3d_mask should be 3D" + + # set text block mask + if text_mask is not None: + block_size = np.prod(tile_thw) + seq_len = q.size(2) + block_num = int(seq_len / block_size) + text_mask_index = torch.ceil(text_mask.sum() / block_size).long() + text_mask_index = text_mask_index.clamp(min=1).item() + assert ( + ssta_3d_mask.shape[-1] == ssta_3d_mask.shape[-2] + ), "ssta_3d_mask should be square" + + pad_start_index = block_num + text_mask_index + ssta_3d_mask[:, pad_start_index:, :] = False + ssta_3d_mask[:, :, pad_start_index:] = False + eye_mask = torch.eye( + ssta_3d_mask.shape[1] - pad_start_index, + dtype=torch.bool, + device=ssta_3d_mask.device, + ).unsqueeze(0) + ssta_3d_mask[:, pad_start_index:, pad_start_index:] = ( + ssta_3d_mask[:, pad_start_index:, pad_start_index:] | eye_mask + ) + return ssta_3d_mask + + +def ssta_3d_attention( + all_q, + all_k, + all_v, + canvas_thw, + topk=1, + tile_thw=(6, 8, 8), + kernel_thw=(1, 1, 1), + text_len=0, + sparse_type="ssta", + threshold=0.0, + lambda_=None, + pad_type="zero", + text_mask=None, + mask_share_within_head=True, + sampling_type=None, + adaptive_pool=None, +): + r"""Sparse Spatio-Temporal Attention (SSTA) 3D attention mechanism. + + Args: + all_q: Query tensor with shape (B, H, S, D) + all_k: Key tensor with shape (B, H, S, D) + all_v: Value tensor with shape (B, H, S, D) + canvas_thw: Canvas dimensions (t, h, w) + topk: Number of top blocks to attend to + tile_thw: Tile dimensions + kernel_thw: Kernel dimensions + text_len: Length of text sequence + sparse_type: Type of sparse attention ('sta', 'block_attn', or 'ssta') + threshold: Threshold for similarity sampling + lambda_: Weight factor for importance sampling + pad_type: Padding type ("zero" or "repeat") + text_mask: Optional text mask tensor + mask_share_within_head: Whether to share mask across heads + sampling_type: Type of sampling ("similarity" or "importance") + adaptive_pool: Adaptive pooling size + + Returns: + tuple: (output tensor, sparse_ratio) + - output: Attention output with shape (B, H, S, D) + - sparse_ratio: Ratio of non-zero attention weights + """ + try: + from flex_block_attn import flex_block_attn_func + except ImportError as e: + raise Exception( + "Could not load flex-block-attn. Please install the flex-block-attn package. " + "You can install it via 'pip install flex-block-attn'." + ) from e + assert pad_type in ["zero", "repeat"] + assert sampling_type in ["similarity", "importance"] + assert ( + lambda_ is not None and sampling_type == "importance" + ) or sampling_type == "similarity" + + if text_len > 0: + image_q = all_q[:, :, :-text_len, :] + image_k = all_k[:, :, :-text_len, :] + image_v = all_v[:, :, :-text_len, :] + + text_q = all_q[:, :, -text_len:, :] + text_k = all_k[:, :, -text_len:, :] + text_v = all_v[:, :, -text_len:, :] + else: + image_q = all_q + image_k = all_k + image_v = all_v + + b, hd, s, d = image_q.shape + t, h, w = canvas_thw + assert t * h * w == s, f"t:{t} * h:{h} * w:{w} != s:{s}" + tile_t, tile_h, tile_w = tile_thw + block_size = np.prod(tile_thw) + + need_pad = False + if t % tile_t != 0 or h % tile_h != 0 or w % tile_w != 0: + need_pad = True + pad_image_q = image_q.reshape(b, hd, t, h, w, d) + pad_image_k = image_k.reshape(b, hd, t, h, w, d) + pad_image_v = image_v.reshape(b, hd, t, h, w, d) + + pad_t = 0 if t % tile_t == 0 else tile_t - t % tile_t + if pad_t > 0: + t = t + pad_t + repeat_q = pad_image_q[:, :, -1:, :, :, :].expand(-1, -1, pad_t, -1, -1, -1) + repeat_k = pad_image_k[:, :, -1:, :, :, :].expand(-1, -1, pad_t, -1, -1, -1) + repeat_v = pad_image_v[:, :, -1:, :, :, :].expand(-1, -1, pad_t, -1, -1, -1) + if pad_type == "zero": + repeat_q = torch.zeros_like(repeat_q) + repeat_k = torch.zeros_like(repeat_k) + repeat_v = torch.zeros_like(repeat_v) + pad_image_q = torch.cat([pad_image_q, repeat_q], dim=2) + pad_image_k = torch.cat([pad_image_k, repeat_k], dim=2) + pad_image_v = torch.cat([pad_image_v, repeat_v], dim=2) + + pad_h = 0 if h % tile_h == 0 else tile_h - h % tile_h + if pad_h > 0: + h = h + pad_h + repeat_q = pad_image_q[:, :, :, -1:, :, :].expand(-1, -1, -1, pad_h, -1, -1) + repeat_k = pad_image_k[:, :, :, -1:, :, :].expand(-1, -1, -1, pad_h, -1, -1) + repeat_v = pad_image_v[:, :, :, -1:, :, :].expand(-1, -1, -1, pad_h, -1, -1) + if pad_type == "zero": + repeat_q = torch.zeros_like(repeat_q) + repeat_k = torch.zeros_like(repeat_k) + repeat_v = torch.zeros_like(repeat_v) + pad_image_q = torch.cat([pad_image_q, repeat_q], dim=3) + pad_image_k = torch.cat([pad_image_k, repeat_k], dim=3) + pad_image_v = torch.cat([pad_image_v, repeat_v], dim=3) + + pad_w = 0 if w % tile_w == 0 else tile_w - w % tile_w + if pad_w > 0: + w = w + pad_w + repeat_q = pad_image_q[:, :, :, :, -1:, :].expand(-1, -1, -1, -1, pad_w, -1) + repeat_k = pad_image_k[:, :, :, :, -1:, :].expand(-1, -1, -1, -1, pad_w, -1) + repeat_v = pad_image_v[:, :, :, :, -1:, :].expand(-1, -1, -1, -1, pad_w, -1) + if pad_type == "zero": + repeat_q = torch.zeros_like(repeat_q) + repeat_k = torch.zeros_like(repeat_k) + repeat_v = torch.zeros_like(repeat_v) + pad_image_q = torch.cat([pad_image_q, repeat_q], dim=4) + pad_image_k = torch.cat([pad_image_k, repeat_k], dim=4) + pad_image_v = torch.cat([pad_image_v, repeat_v], dim=4) + + image_q = pad_image_q.reshape(b, hd, -1, d) + image_k = pad_image_k.reshape(b, hd, -1, d) + image_v = pad_image_v.reshape(b, hd, -1, d) + + canvas_thw = (t, h, w) + + need_pad_text = False + text_block_num = math.ceil(text_len / block_size) + text_target_size = text_block_num * block_size + if text_len % block_size > 0: + need_pad_text = True + text_pad_size = text_target_size - text_len + + pad_text_q = text_q[:, :, -1, :].unsqueeze(2).expand(-1, -1, text_pad_size, -1) + pad_text_k = text_k[:, :, -1, :].unsqueeze(2).expand(-1, -1, text_pad_size, -1) + pad_text_v = text_v[:, :, -1, :].unsqueeze(2).expand(-1, -1, text_pad_size, -1) + + text_q = torch.cat([text_q, pad_text_q], dim=2) + text_k = torch.cat([text_k, pad_text_k], dim=2) + text_v = torch.cat([text_v, pad_text_v], dim=2) + + image_q = tile(image_q, canvas_thw, tile_thw) + image_k = tile(image_k, canvas_thw, tile_thw) + image_v = tile(image_v, canvas_thw, tile_thw) + + if text_len > 0: + q = torch.cat([image_q, text_q], dim=2) + k = torch.cat([image_k, text_k], dim=2) + v = torch.cat([image_v, text_v], dim=2) + else: + q = image_q + k = image_k + v = image_v + + if sparse_type == "sta": + assert text_mask is None, "text_mask do not support in sta sparse_type" + block_mask = create_sta_3d_mask( + canvas_thw, tile_thw, kernel_thw, text_block_num + ).to(q.device) + o = flex_block_attn_func(q, k, v, block_size, block_size, block_mask) + elif sparse_type == "block_attn": + image_q_list = torch.split(image_q, 1, dim=0) + image_k_list = torch.split(image_k, 1, dim=0) + + mask_list = [] + for i in range(b): + block_mask = create_moba_3d_mask( + image_q_list[i], + image_k_list[i], + canvas_thw=canvas_thw, + topk=topk, + tile_thw=tile_thw, + kernel_thw=kernel_thw, + text_block_num=text_block_num, + add_text_mask=True, + lambda_=lambda_, + threshold=threshold, + mask_share_within_head=mask_share_within_head, + adaptive_pool=adaptive_pool, + sampling_type=sampling_type, + ) + mask_list.append(block_mask) + block_mask = torch.stack(mask_list, dim=0) + o = flex_block_attn_func(q, k, v, block_size, block_size, block_mask) + + elif sparse_type == "ssta": + image_q_list = torch.split(image_q, 1, dim=0) + image_k_list = torch.split(image_k, 1, dim=0) + + mask_list = [] + for i in range(b): + block_mask = create_ssta_3d_mask( + image_q_list[i], + image_k_list[i], + canvas_thw=canvas_thw, + tile_thw=tile_thw, + kernel_thw=kernel_thw, + text_block_num=text_block_num, + topk=topk, + threshold=threshold, + lambda_=lambda_, + text_mask=text_mask[i] if text_mask is not None else None, + mask_share_within_head=mask_share_within_head, + adaptive_pool=adaptive_pool, + sampling_type=sampling_type, + ) + mask_list.append(block_mask) + block_mask = torch.stack(mask_list, dim=0) + if mask_share_within_head: + block_mask = block_mask.unsqueeze(1) # [b, 1, s_block, s_block] + o = flex_block_attn_func(q, k, v, block_size, block_size, block_mask) + else: + raise Exception(f"unsupported sparse_type:{sparse_type}") + sparse_ratio = block_mask.float().mean().cpu().item() + + if text_len > 0: + image_o = o[:, :, :-text_target_size, :] + if need_pad_text: + text_o = o[:, :, -text_target_size:-text_pad_size, :] + else: + text_o = o[:, :, -text_target_size:, :] + else: + image_o = o + + image_o = untile(image_o, canvas_thw, tile_thw) + + if need_pad: + # Remove padding from output + unpad_image_o = image_o.reshape(b, hd, t, h, w, d) + if pad_t > 0: + unpad_image_o = unpad_image_o[:, :, :-pad_t, :, :, :] + if pad_h > 0: + unpad_image_o = unpad_image_o[:, :, :, :-pad_h, :, :] + if pad_w > 0: + unpad_image_o = unpad_image_o[:, :, :, :, :-pad_w, :] + image_o = unpad_image_o.reshape(b, hd, -1, d) + + if text_len > 0: + o = torch.cat([image_o, text_o], dim=2) + else: + o = image_o + + return o, sparse_ratio diff --git a/hyvideo/models/transformers/modules/token_refiner.py b/hyvideo/models/transformers/modules/token_refiner.py new file mode 100644 index 0000000000000000000000000000000000000000..77e352e76171c302cd1ee9570986971eb03c0171 --- /dev/null +++ b/hyvideo/models/transformers/modules/token_refiner.py @@ -0,0 +1,299 @@ +# Licensed under the TENCENT HUNYUAN COMMUNITY LICENSE AGREEMENT (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://github.com/Tencent-Hunyuan/HunyuanVideo-1.5/blob/main/LICENSE +# +# Unless and only to the extent required by applicable law, the Tencent Hunyuan works and any +# output and results therefrom are provided "AS IS" without any express or implied warranties of +# any kind including any warranties of title, merchantability, noninfringement, course of dealing, +# usage of trade, or fitness for a particular purpose. You are solely responsible for determining the +# appropriateness of using, reproducing, modifying, performing, displaying or distributing any of +# the Tencent Hunyuan works or outputs and assume any and all risks associated with your or a +# third party's use or distribution of any of the Tencent Hunyuan works or outputs and your exercise +# of rights and permissions under this agreement. +# See the License for the specific language governing permissions and limitations under the License. + +from typing import Optional + +import torch +import torch.nn as nn +from einops import rearrange + +from .activation_layers import get_activation_layer +from .embed_layers import TextProjection, TimestepEmbedder +from .mlp_layers import MLP +from .modulate_layers import apply_gate +from .norm_layers import get_norm_layer +from .attention import attention + + +class IndividualTokenRefinerBlock(nn.Module): + """ + A single block for token refinement with self-attention and MLP. + + Args: + hidden_size: Hidden dimension size. + heads_num: Number of attention heads. + mlp_width_ratio: Expansion ratio for MLP hidden size. + mlp_drop_rate: Dropout rate for MLP. + act_type: Activation function type. + qk_norm: Whether to use QK normalization. + qk_norm_type: Type of QK normalization. + qkv_bias: Whether to use bias in QKV projections. + dtype: Optional torch dtype. + device: Optional torch device. + """ + + def __init__( + self, + hidden_size: int, + heads_num: int, + mlp_width_ratio: float = 4.0, + mlp_drop_rate: float = 0.0, + act_type: str = "silu", + qk_norm: bool = False, + qk_norm_type: str = "layer", + qkv_bias: bool = True, + dtype: Optional[torch.dtype] = None, + device: Optional[torch.device] = None, + ): + factory_kwargs = {"device": device, "dtype": dtype} + super().__init__() + self.heads_num = heads_num + head_dim = hidden_size // heads_num + mlp_hidden_dim = int(hidden_size * mlp_width_ratio) + + self.norm1 = nn.LayerNorm( + hidden_size, elementwise_affine=True, eps=1e-6, **factory_kwargs + ) + self.self_attn_qkv = nn.Linear( + hidden_size, hidden_size * 3, bias=qkv_bias, **factory_kwargs + ) + qk_norm_layer = get_norm_layer(qk_norm_type) + self.self_attn_q_norm = ( + qk_norm_layer(head_dim, elementwise_affine=True, eps=1e-6, **factory_kwargs) + if qk_norm + else nn.Identity() + ) + self.self_attn_k_norm = ( + qk_norm_layer(head_dim, elementwise_affine=True, eps=1e-6, **factory_kwargs) + if qk_norm + else nn.Identity() + ) + self.self_attn_proj = nn.Linear( + hidden_size, hidden_size, bias=qkv_bias, **factory_kwargs + ) + + self.norm2 = nn.LayerNorm( + hidden_size, elementwise_affine=True, eps=1e-6, **factory_kwargs + ) + act_layer = get_activation_layer(act_type) + self.mlp = MLP( + in_channels=hidden_size, + hidden_channels=mlp_hidden_dim, + act_layer=act_layer, + drop=mlp_drop_rate, + **factory_kwargs, + ) + + self.adaLN_modulation = nn.Sequential( + act_layer(), + nn.Linear(hidden_size, 2 * hidden_size, bias=True, **factory_kwargs), + ) + # Zero-initialize the modulation + nn.init.zeros_(self.adaLN_modulation[1].weight) + nn.init.zeros_(self.adaLN_modulation[1].bias) + + def forward( + self, + x: torch.Tensor, + c: torch.Tensor, # timestep_aware_representations + context_aware_representations + attn_mask: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + """ + Forward pass for IndividualTokenRefinerBlock. + + Args: + x: Input tensor of shape [B, L, C]. + c: Conditioning tensor of shape [B, C]. + attn_mask: Optional attention mask of shape [B, L]. + + Returns: + Refined tensor of shape [B, L, C]. + """ + gate_msa, gate_mlp = self.adaLN_modulation(c).chunk(2, dim=1) + norm_x = self.norm1(x) + qkv = self.self_attn_qkv(norm_x) + q, k, v = rearrange(qkv, "B L (K H D) -> K B L H D", K=3, H=self.heads_num) + q = self.self_attn_q_norm(q).to(v) + k = self.self_attn_k_norm(k).to(v) + attn = attention(q, k, v, attn_mask=attn_mask) + x = x + apply_gate(self.self_attn_proj(attn), gate_msa) + x = x + apply_gate(self.mlp(self.norm2(x)), gate_mlp) + return x + + +class IndividualTokenRefiner(nn.Module): + """ + Stacks multiple IndividualTokenRefinerBlock modules. + + Args: + hidden_size: Hidden dimension size. + heads_num: Number of attention heads. + depth: Number of blocks. + mlp_width_ratio: Expansion ratio for MLP hidden size. + mlp_drop_rate: Dropout rate for MLP. + act_type: Activation function type. + qk_norm: Whether to use QK normalization. + qk_norm_type: Type of QK normalization. + qkv_bias: Whether to use bias in QKV projections. + dtype: Optional torch dtype. + device: Optional torch device. + """ + + def __init__( + self, + hidden_size: int, + heads_num: int, + depth: int, + mlp_width_ratio: float = 4.0, + mlp_drop_rate: float = 0.0, + act_type: str = "silu", + qk_norm: bool = False, + qk_norm_type: str = "layer", + qkv_bias: bool = True, + dtype: Optional[torch.dtype] = None, + device: Optional[torch.device] = None, + ): + factory_kwargs = {"device": device, "dtype": dtype} + super().__init__() + self.blocks = nn.ModuleList( + [ + IndividualTokenRefinerBlock( + hidden_size=hidden_size, + heads_num=heads_num, + mlp_width_ratio=mlp_width_ratio, + mlp_drop_rate=mlp_drop_rate, + act_type=act_type, + qk_norm=qk_norm, + qk_norm_type=qk_norm_type, + qkv_bias=qkv_bias, + **factory_kwargs, + ) + for _ in range(depth) + ] + ) + + def forward( + self, + x: torch.Tensor, + c: torch.LongTensor, + mask: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + """ + Forward pass for IndividualTokenRefiner. + + Args: + x: Input tensor of shape [B, L, C]. + c: Conditioning tensor of shape [B, C]. + mask: Optional mask tensor of shape [B, L]. + + Returns: + Refined tensor of shape [B, L, C]. + """ + if mask is not None: + mask = mask.clone().bool() + mask[:, 0] = True # Prevent attention weights from becoming NaN + for block in self.blocks: + x = block(x, c, mask) + return x + + +class SingleTokenRefiner(nn.Module): + """ + Single token refiner block for LLM text embedding refinement. + + Args: + in_channels: Input feature dimension. + hidden_size: Hidden dimension size. + heads_num: Number of attention heads. + depth: Number of blocks. + mlp_width_ratio: Expansion ratio for MLP hidden size. + mlp_drop_rate: Dropout rate for MLP. + act_type: Activation function type. + qk_norm: Whether to use QK normalization. + qk_norm_type: Type of QK normalization. + qkv_bias: Whether to use bias in QKV projections. + dtype: Optional torch dtype. + device: Optional torch device. + """ + + def __init__( + self, + in_channels: int, + hidden_size: int, + heads_num: int, + depth: int, + mlp_width_ratio: float = 4.0, + mlp_drop_rate: float = 0.0, + act_type: str = "silu", + qk_norm: bool = False, + qk_norm_type: str = "layer", + qkv_bias: bool = True, + dtype: Optional[torch.dtype] = None, + device: Optional[torch.device] = None, + ): + factory_kwargs = {"device": device, "dtype": dtype} + super().__init__() + self.input_embedder = nn.Linear( + in_channels, hidden_size, bias=True, **factory_kwargs + ) + act_layer = get_activation_layer(act_type) + self.t_embedder = TimestepEmbedder(hidden_size, act_layer, **factory_kwargs) + self.c_embedder = TextProjection( + in_channels, hidden_size, act_layer, **factory_kwargs + ) + self.individual_token_refiner = IndividualTokenRefiner( + hidden_size=hidden_size, + heads_num=heads_num, + depth=depth, + mlp_width_ratio=mlp_width_ratio, + mlp_drop_rate=mlp_drop_rate, + act_type=act_type, + qk_norm=qk_norm, + qk_norm_type=qk_norm_type, + qkv_bias=qkv_bias, + **factory_kwargs, + ) + + def forward( + self, + x: torch.Tensor, + t: torch.LongTensor, + mask: Optional[torch.LongTensor] = None, + ) -> torch.Tensor: + """ + Forward pass for SingleTokenRefiner. + + Args: + x: Input tensor of shape [B, L, in_channels]. + t: Timestep tensor of shape [B]. + mask: Optional mask tensor of shape [B, L]. + + Returns: + Refined tensor of shape [B, L, hidden_size]. + """ + timestep_aware_representations = self.t_embedder(t) + if mask is None: + context_aware_representations = x.mean(dim=1) + else: + mask_float = mask.float().unsqueeze(-1) # [B, L, 1] + context_aware_representations = (x * mask_float).sum( + dim=1 + ) / mask_float.sum(dim=1) + context_aware_representations = self.c_embedder(context_aware_representations) + c = timestep_aware_representations + context_aware_representations + x = self.input_embedder(x) + x = self.individual_token_refiner(x, c, mask) + return x diff --git a/hyvideo/models/transformers/modules/upsample.py b/hyvideo/models/transformers/modules/upsample.py new file mode 100644 index 0000000000000000000000000000000000000000..fe28ad78203ead9578e4e3b2477a5129018e94c6 --- /dev/null +++ b/hyvideo/models/transformers/modules/upsample.py @@ -0,0 +1,175 @@ +# Licensed under the TENCENT HUNYUAN COMMUNITY LICENSE AGREEMENT (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://github.com/Tencent-Hunyuan/HunyuanVideo-1.5/blob/main/LICENSE +# +# Unless and only to the extent required by applicable law, the Tencent Hunyuan works and any +# output and results therefrom are provided "AS IS" without any express or implied warranties of +# any kind including any warranties of title, merchantability, noninfringement, course of dealing, +# usage of trade, or fitness for a particular purpose. You are solely responsible for determining the +# appropriateness of using, reproducing, modifying, performing, displaying or distributing any of +# the Tencent Hunyuan works or outputs and assume any and all risks associated with your or a +# third party's use or distribution of any of the Tencent Hunyuan works or outputs and your exercise +# of rights and permissions under this agreement. +# See the License for the specific language governing permissions and limitations under the License. + +from collections.abc import Sequence +from dataclasses import dataclass +from enum import Enum + +import torch +import torch.nn as nn +import torch.nn.functional as F +from einops import rearrange +from torch import Tensor +from diffusers.models import ModelMixin +from diffusers.configuration_utils import ConfigMixin, register_to_config + +from hyvideo.models.autoencoders.hunyuanvideo_15_vae_w_cache import ( + CausalConv3d, + ResnetBlock, + RMS_norm, + forward_with_checkpointing, + swish, +) + + +class UpsamplerType(Enum): + LEARNED = "learned" + FIXED = "fixed" + NONE = "none" + LEARNED_FIXED = "learned_fixed" + + +@dataclass +class UpsamplerConfig: + load_from: str + enable: bool = False + hidden_channels: int = 128 + num_blocks: int = 16 + model_type: UpsamplerType = UpsamplerType.NONE + version: str = "720p" + + +class SRResidualCausalBlock3D(nn.Module): + def __init__(self, channels: int): + super().__init__() + self.block = nn.Sequential( + CausalConv3d(channels, channels, kernel_size=3), + nn.SiLU(inplace=True), + CausalConv3d(channels, channels, kernel_size=3), + nn.SiLU(inplace=True), + CausalConv3d(channels, channels, kernel_size=3), + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return x + self.block(x) + + +class SRTo720pUpsampler(ModelMixin, ConfigMixin): + + @register_to_config + def __init__( + self, + in_channels: int, + out_channels: int, + hidden_channels: int | None = None, + num_blocks: int = 6, + global_residual: bool = False, + ): + super().__init__() + if hidden_channels is None: + hidden_channels = 64 + self.in_conv = CausalConv3d(in_channels, hidden_channels, kernel_size=3) + self.blocks = nn.ModuleList( + [SRResidualCausalBlock3D(hidden_channels) for _ in range(num_blocks)] + ) + self.out_conv = CausalConv3d(hidden_channels, out_channels, kernel_size=3) + self.global_residual = bool(global_residual) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + residual = x + y = self.in_conv(x) + for blk in self.blocks: + y = blk(y) + y = self.out_conv(y) + if self.global_residual and (y.shape == residual.shape): + y = y + residual + return y + + +class SRTo1080pUpsampler(ModelMixin, ConfigMixin): + + @register_to_config + def __init__( + self, + z_channels: int, + out_channels: int, + block_out_channels: tuple[int, ...], + num_res_blocks: int = 2, + is_residual: bool = False, + ): + super().__init__() + self.num_res_blocks = num_res_blocks + self.block_out_channels = block_out_channels + self.z_channels = z_channels + + block_in = block_out_channels[0] + self.conv_in = CausalConv3d(z_channels, block_in, kernel_size=3) + + self.up = nn.ModuleList() + for i_level, ch in enumerate(block_out_channels): + block = nn.ModuleList() + block_out = ch + for _ in range(self.num_res_blocks + 1): + block.append(ResnetBlock(in_channels=block_in, out_channels=block_out)) + block_in = block_out + up = nn.Module() + up.block = block + + self.up.append(up) + + self.norm_out = RMS_norm(block_in, images=False) + self.conv_out = CausalConv3d(block_in, out_channels, kernel_size=3) + + self.gradient_checkpointing = False + self.is_residual = is_residual + + def forward(self, z: Tensor, target_shape: Sequence[int] = None) -> Tensor: + """ + Args: + z: (B, C, T, H, W) + target_shape: (H, W) + """ + use_checkpointing = bool(self.training and self.gradient_checkpointing) + if target_shape is not None and z.shape[-2:] != target_shape: + bsz = z.shape[0] + z = rearrange(z, "b c f h w -> (b f) c h w") + z = F.interpolate( + z, size=target_shape, mode="bilinear", align_corners=False + ) + z = rearrange(z, "(b f) c h w -> b c f h w", b=bsz) + + # z to block_in + repeats = self.block_out_channels[0] // (self.z_channels) + h = self.conv_in(z) + z.repeat_interleave(repeats=repeats, dim=1) + + # upsampling + for i_level in range(len(self.block_out_channels)): + for i_block in range(self.num_res_blocks + 1): + h = forward_with_checkpointing( + self.up[i_level].block[i_block], + h, + use_checkpointing=use_checkpointing, + ) + if hasattr(self.up[i_level], "upsample"): + h = forward_with_checkpointing( + self.up[i_level].upsample, h, use_checkpointing=use_checkpointing + ) + + # end + h = self.norm_out(h) + h = swish(h) + h = self.conv_out(h) + return h diff --git a/hyvideo/models/transformers/worldplay_1_5_transformer.py b/hyvideo/models/transformers/worldplay_1_5_transformer.py new file mode 100644 index 0000000000000000000000000000000000000000..658ffa53fe1c0937a6fd73403f708e460ab08c24 --- /dev/null +++ b/hyvideo/models/transformers/worldplay_1_5_transformer.py @@ -0,0 +1,2239 @@ +# Licensed under the TENCENT HUNYUAN COMMUNITY LICENSE AGREEMENT (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://github.com/Tencent-Hunyuan/HunyuanVideo-1.5/blob/main/LICENSE +# +# Unless and only to the extent required by applicable law, the Tencent Hunyuan works and any +# output and results therefrom are provided "AS IS" without any express or implied warranties of +# any kind including any warranties of title, merchantability, noninfringement, course of dealing, +# usage of trade, or fitness for a particular purpose. You are solely responsible for determining the +# appropriateness of using, reproducing, modifying, performing, displaying or distributing any of +# the Tencent Hunyuan works or outputs and assume any and all risks associated with your or a +# third party's use or distribution of any of the Tencent Hunyuan works or outputs and your exercise +# of rights and permissions under this agreement. +# See the License for the specific language governing permissions and limitations under the License. + +from typing import Any, Tuple, Optional, Union, Dict + +import torch +import torch.nn as nn +from einops import rearrange, repeat +from loguru import logger + +from diffusers.models import ModelMixin +from diffusers.configuration_utils import ConfigMixin, register_to_config + +from .modules.activation_layers import get_activation_layer +from .modules.norm_layers import get_norm_layer +from .modules.embed_layers import ( + TimestepEmbedder, + PatchEmbed, + TextProjection, + VisionProjection, +) +from .modules.attention import ( + parallel_attention, + sequence_parallel_attention_txt, + sequence_parallel_attention_vision, +) +from .modules.posemb_layers import apply_rotary_emb, get_nd_rotary_pos_embed +from .modules.mlp_layers import MLP, MLPEmbedder, FinalLayer, LinearWarpforSingle +from .modules.modulate_layers import ModulateDiT, modulate, apply_gate +from .modules.token_refiner import SingleTokenRefiner + +from hyvideo.utils.communications import all_gather +from hyvideo.utils.infer_utils import torch_compile_wrapper +from hyvideo.models.text_encoders.byT5 import ByT5Mapper +from hyvideo.commons.parallel_states import get_parallel_state +from hyvideo.commons.infer_state import get_infer_state + +from hyvideo.prope.camera_rope import prope_qkv, _prepare_apply_fns_all_dim + + +def is_blocks(n: str, m) -> bool: + print("is_blocks", n, flush=True) + is_valid = ("double_blocks" in n and str.isdigit(n.split(".")[-1])) or ( + "single_blocks" in n and str.isdigit(n.split(".")[-1]) + ) + return is_valid + + +class MMDoubleStreamBlock(nn.Module): + + def __init__( + self, + hidden_size: int, + heads_num: int, + mlp_width_ratio: float, + mlp_act_type: str = "gelu_tanh", + attn_mode: str = None, + qk_norm: bool = True, + qk_norm_type: str = "rms", + qkv_bias: bool = False, + dtype: Optional[torch.dtype] = None, + device: Optional[torch.device] = None, + ): + factory_kwargs = {"device": device, "dtype": dtype} + super().__init__() + + self.deterministic = False + self.heads_num = heads_num + self.attn_mode = attn_mode + + self.hidden_size = hidden_size + self.qkv_bias = qkv_bias + self.factory_kwargs = factory_kwargs + + head_dim = hidden_size // heads_num + mlp_hidden_dim = int(hidden_size * mlp_width_ratio) + + self.img_mod = ModulateDiT( + hidden_size, + factor=6, + act_layer=get_activation_layer("silu"), + **factory_kwargs, + ) + self.img_norm1 = nn.LayerNorm( + hidden_size, elementwise_affine=False, eps=1e-6, **factory_kwargs + ) + self.img_attn_q = nn.Linear( + hidden_size, hidden_size, bias=qkv_bias, **factory_kwargs + ) + self.img_attn_k = nn.Linear( + hidden_size, hidden_size, bias=qkv_bias, **factory_kwargs + ) + self.img_attn_v = nn.Linear( + hidden_size, hidden_size, bias=qkv_bias, **factory_kwargs + ) + + qk_norm_layer = get_norm_layer(qk_norm_type) + self.img_attn_q_norm = ( + qk_norm_layer(head_dim, elementwise_affine=True, eps=1e-6, **factory_kwargs) + if qk_norm + else nn.Identity() + ) + self.img_attn_k_norm = ( + qk_norm_layer(head_dim, elementwise_affine=True, eps=1e-6, **factory_kwargs) + if qk_norm + else nn.Identity() + ) + self.img_attn_proj = nn.Linear( + hidden_size, hidden_size, bias=qkv_bias, **factory_kwargs + ) + + self.img_norm2 = nn.LayerNorm( + hidden_size, elementwise_affine=False, eps=1e-6, **factory_kwargs + ) + self.img_mlp = MLP( + hidden_size, + mlp_hidden_dim, + act_layer=get_activation_layer(mlp_act_type), + bias=True, + **factory_kwargs, + ) + + self.txt_mod = ModulateDiT( + hidden_size, + factor=6, + act_layer=get_activation_layer("silu"), + **factory_kwargs, + ) + self.txt_norm1 = nn.LayerNorm( + hidden_size, elementwise_affine=False, eps=1e-6, **factory_kwargs + ) + + self.txt_attn_q = nn.Linear( + hidden_size, hidden_size, bias=qkv_bias, **factory_kwargs + ) + self.txt_attn_k = nn.Linear( + hidden_size, hidden_size, bias=qkv_bias, **factory_kwargs + ) + self.txt_attn_v = nn.Linear( + hidden_size, hidden_size, bias=qkv_bias, **factory_kwargs + ) + + self.txt_attn_q_norm = ( + qk_norm_layer(head_dim, elementwise_affine=True, eps=1e-6, **factory_kwargs) + if qk_norm + else nn.Identity() + ) + self.txt_attn_k_norm = ( + qk_norm_layer(head_dim, elementwise_affine=True, eps=1e-6, **factory_kwargs) + if qk_norm + else nn.Identity() + ) + self.txt_attn_proj = nn.Linear( + hidden_size, hidden_size, bias=qkv_bias, **factory_kwargs + ) + self.txt_norm2 = nn.LayerNorm( + hidden_size, elementwise_affine=False, eps=1e-6, **factory_kwargs + ) + self.txt_mlp = MLP( + hidden_size, + mlp_hidden_dim, + act_layer=get_activation_layer(mlp_act_type), + bias=True, + **factory_kwargs, + ) + + self.hybrid_seq_parallel_attn = None + + def enable_deterministic(self): + self.deterministic = True + + def disable_deterministic(self): + self.deterministic = False + + # modulating module for txt embedding + def modulate_txt(self, vec_txt, txt): + # Generate modulation parameters (shift, scale, gate) for text embeddings using adaptive norm + ( + txt_mod1_shift, + txt_mod1_scale, + txt_mod1_gate, + txt_mod2_shift, + txt_mod2_scale, + txt_mod2_gate, + ) = self.txt_mod(vec_txt).chunk(6, dim=-1) + + txt_modulated = self.txt_norm1(txt) + txt_modulated = modulate( + txt_modulated, shift=txt_mod1_shift, scale=txt_mod1_scale + ) + txt_q = self.txt_attn_q(txt_modulated) + txt_k = self.txt_attn_k(txt_modulated) + txt_v = self.txt_attn_v(txt_modulated) + txt_q = rearrange(txt_q, "B L (H D) -> B L H D", H=self.heads_num) + txt_k = rearrange(txt_k, "B L (H D) -> B L H D", H=self.heads_num) + txt_v = rearrange(txt_v, "B L (H D) -> B L H D", H=self.heads_num) + txt_q = self.txt_attn_q_norm(txt_q).to(txt_v) + txt_k = self.txt_attn_k_norm(txt_k).to(txt_v) + return ( + txt_q, + txt_k, + txt_v, + txt_mod1_gate, + txt_mod2_shift, + txt_mod2_scale, + txt_mod2_gate, + ) + + # modulating module for img embedding + def modulate_img(self, vec, img): + ( + img_mod1_shift, + img_mod1_scale, + img_mod1_gate, + img_mod2_shift, + img_mod2_scale, + img_mod2_gate, + ) = self.img_mod(vec).chunk(6, dim=-1) + + img_modulated = self.img_norm1(img) + img_modulated = modulate( + img_modulated, shift=img_mod1_shift, scale=img_mod1_scale + ) + + img_q = self.img_attn_q(img_modulated) + img_k = self.img_attn_k(img_modulated) + img_v = self.img_attn_v(img_modulated) + img_q = rearrange(img_q, "B L (H D) -> B L H D", H=self.heads_num) + img_k = rearrange(img_k, "B L (H D) -> B L H D", H=self.heads_num) + img_v = rearrange(img_v, "B L (H D) -> B L H D", H=self.heads_num) + img_q = self.img_attn_q_norm(img_q).to(img_v) + img_k = self.img_attn_k_norm(img_k).to(img_v) + return ( + img_q, + img_k, + img_v, + img_mod1_gate, + img_mod2_shift, + img_mod2_scale, + img_mod2_gate, + ) + + @torch_compile_wrapper() + def forward_txt( + self, + txt: torch.Tensor, + vec_txt: torch.Tensor, + text_mask=None, + attn_param=None, + is_flash=False, + block_idx=None, + kv_cache: Optional[dict] = None, + cache_txt: bool = False, + ) -> Tuple[torch.Tensor]: + ( + txt_q, + txt_k, + txt_v, + txt_mod1_gate, + txt_mod2_shift, + txt_mod2_scale, + txt_mod2_gate, + ) = self.modulate_txt(vec_txt, txt) + + attn_mode = "torch_causal" # for ar model, the default mode is torch_causal + txt_attn, t_kv = sequence_parallel_attention_txt( + (txt_q), + (txt_k), + (txt_v), + img_q_len=txt_q.shape[1], + img_kv_len=txt_k.shape[1], + text_mask=text_mask, + attn_mode=attn_mode, + attn_param=attn_param, + block_idx=block_idx, + kv_cache=kv_cache, + cache_txt=cache_txt, + ) + + txt = txt + apply_gate(self.txt_attn_proj(txt_attn), gate=txt_mod1_gate) + txt = txt + apply_gate( + self.txt_mlp( + modulate( + self.txt_norm2(txt), shift=txt_mod2_shift, scale=txt_mod2_scale + ) + ), + gate=txt_mod2_gate, + ) + return txt, t_kv + + @torch_compile_wrapper() + def forward_vision( + self, + img: torch.Tensor, + vec: torch.Tensor, + freqs_cis: tuple = None, + attn_param=None, + block_idx=None, + viewmats: Optional[torch.Tensor] = None, + Ks: Optional[torch.Tensor] = None, + kv_cache: Optional[dict] = None, + cache_vision: bool = False, + ) -> Tuple[torch.Tensor, torch.Tensor]: + ( + img_q, + img_k, + img_v, + img_mod1_gate, + img_mod2_shift, + img_mod2_scale, + img_mod2_gate, + ) = self.modulate_img(vec, img) + + # Add camera pose conditioning through ProPE (Projective Positional Encoding) + # delete rope components (original attn included) + # Apply ProPE transformation to Q, K, V using camera view matrices and intrinsics + img_q_prope, img_k_prope, img_v_prope, apply_fn_o = prope_qkv( + img_q.permute(0, 2, 1, 3), + img_k.permute(0, 2, 1, 3), + img_v.permute(0, 2, 1, 3), + viewmats=viewmats, + Ks=Ks, + ) # [batch, num_heads, seqlen, head_dim] + img_q_prope = img_q_prope.permute( + 0, 2, 1, 3 + ) # [batch, seqlen, num_heads, head_dim] + img_k_prope = img_k_prope.permute( + 0, 2, 1, 3 + ) # [batch, seqlen, num_heads, head_dim] + img_v_prope = img_v_prope.permute( + 0, 2, 1, 3 + ) # [batch, seqlen, num_heads, head_dim] + + if freqs_cis is not None: + img_qq, img_kk = apply_rotary_emb(img_q, img_k, freqs_cis, head_first=False) + assert ( + img_qq.shape == img_q.shape and img_kk.shape == img_k.shape + ), f"img_kk: {img_qq.shape}, img_q: {img_q.shape}, img_kk: {img_kk.shape}, img_k: {img_k.shape}" + img_q, img_k = img_qq, img_kk + + readonly_after_layer = int( + attn_param.get("context_readonly_after_layer", 0) + if attn_param is not None + else 0 + ) + readonly_keep_last_frames = int( + attn_param.get("context_readonly_keep_last_frames", 0) + if attn_param is not None + else 0 + ) + readonly_prefix_tokens = 0 + if ( + cache_vision + and readonly_after_layer > 0 + and block_idx + 1 > readonly_after_layer + and readonly_keep_last_frames > 0 + ): + temporal_frames, spatial_h, spatial_w = attn_param["thw"] + if temporal_frames > readonly_keep_last_frames: + readonly_prefix_tokens = ( + temporal_frames - readonly_keep_last_frames + ) * spatial_h * spatial_w + img_q = img_q[:, readonly_prefix_tokens:] + img_q_prope = img_q_prope[:, readonly_prefix_tokens:] + _, _, apply_fn_o = _prepare_apply_fns_all_dim( + head_dim=img_q_prope.shape[-1], + viewmats=viewmats[:, readonly_prefix_tokens:], + Ks=Ks[:, readonly_prefix_tokens:], + patches_x=None, + patches_y=None, + image_width=None, + image_height=None, + ) + + img_attn, img_attn_prope, vision_kv = sequence_parallel_attention_vision( + (img_q, img_q_prope), + (img_k, img_k_prope), + (img_v, img_v_prope), + block_idx=block_idx, + kv_cache=kv_cache, + cache_vision=cache_vision, + attn_param=attn_param, + ) + + img_attn_prope = rearrange( + img_attn_prope, "B L (H D) -> B H L D", H=self.heads_num + ) + img_attn_prope = apply_fn_o( + img_attn_prope + ) # [batch, num_heads, seqlen, head_dim] + img_attn_prope = rearrange(img_attn_prope, "B H L D -> B L (H D)") + + if readonly_prefix_tokens: + batch_size, sequence_length, _ = img.shape + + def dynamic_modulation(values): + return values.reshape(batch_size, sequence_length, -1)[ + :, readonly_prefix_tokens: + ].reshape(-1, values.shape[-1]) + + readonly_img = img[:, :readonly_prefix_tokens] + img = img[:, readonly_prefix_tokens:] + img_mod1_gate = dynamic_modulation(img_mod1_gate) + img_mod2_shift = dynamic_modulation(img_mod2_shift) + img_mod2_scale = dynamic_modulation(img_mod2_scale) + img_mod2_gate = dynamic_modulation(img_mod2_gate) + + img = img + apply_gate( + self.img_attn_proj(img_attn) + self.img_attn_prope_proj(img_attn_prope), + gate=img_mod1_gate, + ) + img = img + apply_gate( + self.img_mlp( + modulate( + self.img_norm2(img), shift=img_mod2_shift, scale=img_mod2_scale + ) + ), + gate=img_mod2_gate, + ) + + if readonly_prefix_tokens: + img = torch.cat([readonly_img, img], dim=1) + + return img, vision_kv + + @torch_compile_wrapper() + def forward_bi( + self, + img: torch.Tensor, + txt: torch.Tensor, + vec_txt: torch.Tensor, + vec: torch.Tensor, + freqs_cis: tuple = None, + text_mask=None, + attn_param=None, + is_flash=False, + block_idx=None, + viewmats: Optional[torch.Tensor] = None, + Ks: Optional[torch.Tensor] = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + ( + img_q, + img_k, + img_v, + img_mod1_gate, + img_mod2_shift, + img_mod2_scale, + img_mod2_gate, + ) = self.modulate_img(vec, img) + ( + txt_q, + txt_k, + txt_v, + txt_mod1_gate, + txt_mod2_shift, + txt_mod2_scale, + txt_mod2_gate, + ) = self.modulate_txt(vec_txt, txt) + + # add camera pose through prope + img_q_prope, img_k_prope, img_v_prope, apply_fn_o = prope_qkv( + img_q.permute(0, 2, 1, 3), + img_k.permute(0, 2, 1, 3), + img_v.permute(0, 2, 1, 3), + viewmats=viewmats, + Ks=Ks, + ) # [batch, num_heads, seqlen, head_dim] + img_q_prope = img_q_prope.permute( + 0, 2, 1, 3 + ) # [batch, seqlen, num_heads, head_dim] + img_k_prope = img_k_prope.permute( + 0, 2, 1, 3 + ) # [batch, seqlen, num_heads, head_dim] + img_v_prope = img_v_prope.permute( + 0, 2, 1, 3 + ) # [batch, seqlen, num_heads, head_dim] + + if freqs_cis is not None: + img_qq, img_kk = apply_rotary_emb(img_q, img_k, freqs_cis, head_first=False) + assert ( + img_qq.shape == img_q.shape and img_kk.shape == img_k.shape + ), f"img_kk: {img_qq.shape}, img_q: {img_q.shape}, img_kk: {img_kk.shape}, img_k: {img_k.shape}" + img_q, img_k = img_qq, img_kk + + attn_mode = "flash" if is_flash else self.attn_mode + attn = parallel_attention( + (img_q, txt_q), + (img_k, txt_k), + (img_v, txt_v), + img_q_len=img_q.shape[1], + img_kv_len=img_k.shape[1], + text_mask=text_mask, + attn_mode=attn_mode, + attn_param=attn_param, + block_idx=block_idx, + ) + img_attn, txt_attn = ( + attn[:, : img_q.shape[1]].contiguous(), + attn[:, img_q.shape[1] :].contiguous(), + ) + + attn_prope = parallel_attention( + (img_q_prope, txt_q), + (img_k_prope, txt_k), + (img_v_prope, txt_v), + img_q_len=img_q.shape[1], + img_kv_len=img_k.shape[1], + text_mask=text_mask, + attn_mode=attn_mode, + attn_param=attn_param, + block_idx=block_idx, + ) + img_attn_prope = attn_prope[:, : img_q_prope.shape[1]].contiguous() + img_attn_prope = rearrange( + img_attn_prope, "B L (H D) -> B H L D", H=self.heads_num + ) + img_attn_prope = apply_fn_o( + img_attn_prope + ) # [batch, num_heads, seqlen, head_dim] + img_attn_prope = rearrange(img_attn_prope, "B H L D -> B L (H D)") + + img = img + apply_gate( + self.img_attn_proj(img_attn) + self.img_attn_prope_proj(img_attn_prope), + gate=img_mod1_gate, + ) + img = img + apply_gate( + self.img_mlp( + modulate( + self.img_norm2(img), shift=img_mod2_shift, scale=img_mod2_scale + ) + ), + gate=img_mod2_gate, + ) + + txt = txt + apply_gate(self.txt_attn_proj(txt_attn), gate=txt_mod1_gate) + txt = txt + apply_gate( + self.txt_mlp( + modulate( + self.txt_norm2(txt), shift=txt_mod2_shift, scale=txt_mod2_scale + ) + ), + gate=txt_mod2_gate, + ) + return img, txt + + @torch_compile_wrapper() + def forward_sr( + self, + img: torch.Tensor, + txt: torch.Tensor, + vec_txt: torch.Tensor, + vec: torch.Tensor, + freqs_cis: tuple = None, + text_mask=None, + attn_param=None, + is_flash=False, + block_idx=None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + ( + img_q, + img_k, + img_v, + img_mod1_gate, + img_mod2_shift, + img_mod2_scale, + img_mod2_gate, + ) = self.modulate_img(vec, img) + ( + txt_q, + txt_k, + txt_v, + txt_mod1_gate, + txt_mod2_shift, + txt_mod2_scale, + txt_mod2_gate, + ) = self.modulate_txt(vec_txt, txt) + + if freqs_cis is not None: + img_qq, img_kk = apply_rotary_emb(img_q, img_k, freqs_cis, head_first=False) + assert ( + img_qq.shape == img_q.shape and img_kk.shape == img_k.shape + ), f"img_kk: {img_qq.shape}, img_q: {img_q.shape}, img_kk: {img_kk.shape}, img_k: {img_k.shape}" + img_q, img_k = img_qq, img_kk + + attn_mode = "flash" if is_flash else self.attn_mode + attn = parallel_attention( + (img_q, txt_q), + (img_k, txt_k), + (img_v, txt_v), + img_q_len=img_q.shape[1], + img_kv_len=img_k.shape[1], + text_mask=text_mask, + attn_mode=attn_mode, + attn_param=attn_param, + block_idx=block_idx, + ) + + img_attn, txt_attn = ( + attn[:, : img_q.shape[1]].contiguous(), + attn[:, img_q.shape[1] :].contiguous(), + ) + + img = img + apply_gate(self.img_attn_proj(img_attn), gate=img_mod1_gate) + img = img + apply_gate( + self.img_mlp( + modulate( + self.img_norm2(img), shift=img_mod2_shift, scale=img_mod2_scale + ) + ), + gate=img_mod2_gate, + ) + + txt = txt + apply_gate(self.txt_attn_proj(txt_attn), gate=txt_mod1_gate) + txt = txt + apply_gate( + self.txt_mlp( + modulate( + self.txt_norm2(txt), shift=txt_mod2_shift, scale=txt_mod2_scale + ) + ), + gate=txt_mod2_gate, + ) + + return img, txt + + @torch_compile_wrapper() + def forward( + self, + bi_inference=True, + ar_txt_inference=False, + ar_vision_inference=False, + **kwargs, + ): + if bi_inference: + return self.forward_bi(**kwargs) + elif ar_txt_inference: + return self.forward_txt(**kwargs) + elif ar_vision_inference: + return self.forward_vision(**kwargs) + else: + return self.forward_sr(**kwargs) + + +class MMSingleStreamBlock(nn.Module): + + def __init__( + self, + hidden_size: int, + heads_num: int, + mlp_width_ratio: float = 4.0, + mlp_act_type: str = "gelu_tanh", + attn_mode: str = None, + qk_norm: bool = True, + qk_norm_type: str = "rms", + qk_scale: float = None, + dtype: Optional[torch.dtype] = None, + device: Optional[torch.device] = None, + ): + factory_kwargs = {"device": device, "dtype": dtype} + super().__init__() + + self.deterministic = False + self.attn_mode = attn_mode + + self.hidden_size = hidden_size + self.heads_num = heads_num + head_dim = hidden_size // heads_num + mlp_hidden_dim = int(hidden_size * mlp_width_ratio) + self.mlp_hidden_dim = mlp_hidden_dim + self.scale = qk_scale or head_dim**-0.5 + + self.linear1_q = nn.Linear(hidden_size, hidden_size, **factory_kwargs) + self.linear1_k = nn.Linear(hidden_size, hidden_size, **factory_kwargs) + self.linear1_v = nn.Linear(hidden_size, hidden_size, **factory_kwargs) + self.linear1_mlp = nn.Linear(hidden_size, mlp_hidden_dim, **factory_kwargs) + self.linear2 = LinearWarpforSingle( + hidden_size + mlp_hidden_dim, hidden_size, bias=True, **factory_kwargs + ) + self.mlp_act = get_activation_layer(mlp_act_type)() + + qk_norm_layer = get_norm_layer(qk_norm_type) + self.q_norm = ( + qk_norm_layer(head_dim, elementwise_affine=True, eps=1e-6, **factory_kwargs) + if qk_norm + else nn.Identity() + ) + self.k_norm = ( + qk_norm_layer(head_dim, elementwise_affine=True, eps=1e-6, **factory_kwargs) + if qk_norm + else nn.Identity() + ) + + self.pre_norm = nn.LayerNorm( + hidden_size, elementwise_affine=False, eps=1e-6, **factory_kwargs + ) + self.modulation = ModulateDiT( + hidden_size, + factor=3, + act_layer=get_activation_layer("silu"), + **factory_kwargs, + ) + self.hybrid_seq_parallel_attn = None + + def enable_deterministic(self): + self.deterministic = True + + def disable_deterministic(self): + self.deterministic = False + + def forward( + self, + x: torch.Tensor, + vec_txt: torch.Tensor, + vec: torch.Tensor, + txt_len: int, + freqs_cis: Tuple[torch.Tensor, torch.Tensor] = None, + text_mask=None, + attn_param=None, + is_flash=False, + ) -> torch.Tensor: + """Forward pass for the single stream block.""" + mod_shift, mod_scale, mod_gate = self.modulation(vec).chunk(3, dim=-1) + txt_mod_shift, txt_mod_scale, txt_mod_gate = self.modulation(vec_txt).chunk( + 3, dim=-1 + ) + + x_pre_norm = self.pre_norm(x) + img_token, txt_token = x_pre_norm[:, :-txt_len, :], x_pre_norm[:, -txt_len:, :] + + img_mod = modulate(img_token, shift=mod_shift, scale=mod_scale) + txt_mod = modulate(txt_token, shift=txt_mod_shift, scale=txt_mod_scale) + x_mod = torch.cat([img_mod, txt_mod], dim=1) + + q = self.linear1_q(x_mod) + k = self.linear1_k(x_mod) + v = self.linear1_v(x_mod) + + q = rearrange(q, "B L (H D) -> B L H D", H=self.heads_num) + k = rearrange(k, "B L (H D) -> B L H D", H=self.heads_num) + v = rearrange(v, "B L (H D) -> B L H D", H=self.heads_num) + + mlp = self.linear1_mlp(x_mod) + + # Apply QK-Norm if needed. + q = self.q_norm(q).to(v) + k = self.k_norm(k).to(v) + + img_q, txt_q = q[:, :-txt_len, :, :], q[:, -txt_len:, :, :] + img_k, txt_k = k[:, :-txt_len, :, :], k[:, -txt_len:, :, :] + img_v, txt_v = v[:, :-txt_len, :, :], v[:, -txt_len:, :, :] + img_qq, img_kk = apply_rotary_emb(img_q, img_k, freqs_cis, head_first=False) + assert ( + img_qq.shape == img_q.shape and img_kk.shape == img_k.shape + ), f"img_kk: {img_qq.shape}, img_q: {img_q.shape}, img_kk: {img_kk.shape}, img_k: {img_k.shape}" + img_q, img_k = img_qq, img_kk + + if is_flash: + attn_mode = "flash" + else: + attn_mode = self.attn_mode + attn = parallel_attention( + (img_q, txt_q), + (img_k, txt_k), + (img_v, txt_v), + img_q_len=img_q.shape[1], + img_kv_len=img_k.shape[1], + text_mask=text_mask, + attn_mode=attn_mode, + attn_param=attn_param, + ) + output = self.linear2(attn, self.mlp_act(mlp)) + img_output, txt_output = output[:, :-txt_len, :], output[:, -txt_len:, :] + + img_output = apply_gate(img_output, gate=mod_gate) + txt_output = apply_gate(txt_output, gate=txt_mod_gate) + gate_output = torch.cat([img_output, txt_output], dim=1) + + return x + gate_output + + +class HunyuanVideo_1_5_DiffusionTransformer(ModelMixin, ConfigMixin): + """ + HunyuanVideo Transformer backbone. + + Args: + patch_size (list): The size of the patch. + in_channels (int): The number of input channels. + out_channels (int): The number of output channels. + hidden_size (int): The hidden size of the transformer backbone. + heads_num (int): The number of attention heads. + mlp_width_ratio (float): Width ratio for the transformer MLPs. + mlp_act_type (str): Activation type for the transformer MLPs. + mm_double_blocks_depth (int): Number of double-stream transformer blocks. + mm_single_blocks_depth (int): Number of single-stream transformer blocks. + rope_dim_list (list): Rotary embedding dim for t, h, w. + qkv_bias (bool): Use bias in qkv projection. + qk_norm (bool): Whether to use qk norm. + qk_norm_type (str): Type of qk norm. + guidance_embed (bool): Use guidance embedding for distillation. + text_projection (str): Text input projection. Default is "single_refiner". + use_attention_mask (bool): If to use attention mask. + text_states_dim (int): Text encoder output dim. + text_states_dim_2 (int): Secondary text encoder output dim. + text_pool_type (str): Type for text pooling. + rope_theta (int): Rotary embedding theta parameter. + attn_mode (str): Attention mode identifier. + attn_param (dict): Attention parameter dictionary. + glyph_byT5_v2 (bool): Use ByT5 glyph module. + vision_projection (str): Vision condition embedding mode. + vision_states_dim (int): Vision encoder states input dim. + is_reshape_temporal_channels (bool): For video VAE adaptation. + use_cond_type_embedding (bool): Use condition type embedding. + """ + + @register_to_config + def __init__( + self, + patch_size: list = [1, 2, 2], + in_channels: int = 4, + concat_condition: bool = True, + out_channels: int = None, + hidden_size: int = 3072, + heads_num: int = 24, + mlp_width_ratio: float = 4.0, + mlp_act_type: str = "gelu_tanh", + mm_double_blocks_depth: int = 20, + mm_single_blocks_depth: int = 40, + rope_dim_list: list = [16, 56, 56], + qkv_bias: bool = True, + qk_norm: bool = True, + qk_norm_type: str = "rms", + guidance_embed: bool = False, + use_meanflow: bool = False, + text_projection: str = "single_refiner", + use_attention_mask: bool = True, + text_states_dim: int = 4096, + text_states_dim_2: int = 768, + text_pool_type: str = None, + rope_theta: int = 256, + attn_mode: str = "flash", + attn_param: dict = None, + glyph_byT5_v2: bool = False, + vision_projection: str = "none", + vision_states_dim: int = 1280, + is_reshape_temporal_channels: bool = False, + use_cond_type_embedding: bool = False, + ideal_resolution: str = None, + ideal_task: str = None, + ): + super().__init__() + factory_kwargs = {} + + self.patch_size = patch_size + self.in_channels = in_channels + self.out_channels = in_channels if out_channels is None else out_channels + self.unpatchify_channels = self.out_channels + self.guidance_embed = guidance_embed + self.rope_dim_list = rope_dim_list + self.rope_theta = rope_theta + # Text projection. Default to linear projection. + # Alternative: TokenRefiner. See more details (LI-DiT): http://arxiv.org/abs/2406.11831 + self.use_attention_mask = use_attention_mask + self.text_projection = text_projection + self.attn_mode = attn_mode + self.text_pool_type = text_pool_type + self.text_states_dim = text_states_dim + self.text_states_dim_2 = text_states_dim_2 + self.vision_states_dim = vision_states_dim + + self.glyph_byT5_v2 = glyph_byT5_v2 + if self.glyph_byT5_v2: + self.byt5_in = ByT5Mapper( + in_dim=1472, + out_dim=2048, + hidden_dim=2048, + out_dim1=hidden_size, + use_residual=False, + ) + + if hidden_size % heads_num != 0: + raise ValueError( + f"Hidden size {hidden_size} must be divisible by heads_num {heads_num}" + ) + pe_dim = hidden_size // heads_num + if sum(rope_dim_list) != pe_dim: + raise ValueError( + f"Got {rope_dim_list} but expected positional dim {pe_dim}" + ) + self.hidden_size = hidden_size + self.heads_num = heads_num + + self.img_in = PatchEmbed( + self.patch_size, + self.in_channels, + self.hidden_size, + is_reshape_temporal_channels=is_reshape_temporal_channels, + concat_condition=concat_condition, + **factory_kwargs, + ) + + # Vision projection + if vision_projection == "linear": + self.vision_in = VisionProjection( + input_dim=self.vision_states_dim, output_dim=self.hidden_size + ) + else: + self.vision_in = None + + # Text projection + if self.text_projection == "linear": + self.txt_in = TextProjection( + text_states_dim, + self.hidden_size, + get_activation_layer("silu"), + **factory_kwargs, + ) + elif self.text_projection == "single_refiner": + self.txt_in = SingleTokenRefiner( + text_states_dim, + hidden_size, + heads_num, + depth=2, + **factory_kwargs, + ) + else: + raise NotImplementedError( + f"Unsupported text_projection: {self.text_projection}" + ) + + # time modulation + self.time_in = TimestepEmbedder( + self.hidden_size, get_activation_layer("silu"), **factory_kwargs + ) + self.vector_in = ( + MLPEmbedder( + self.config.text_states_dim_2, self.hidden_size, **factory_kwargs + ) + if self.text_pool_type is not None + else None + ) + self.guidance_in = ( + TimestepEmbedder( + self.hidden_size, get_activation_layer("silu"), **factory_kwargs + ) + if guidance_embed + else None + ) + + self.time_r_in = ( + TimestepEmbedder( + self.hidden_size, get_activation_layer("silu"), **factory_kwargs + ) + if use_meanflow + else None + ) + + self.double_blocks = nn.ModuleList( + [ + MMDoubleStreamBlock( + self.hidden_size, + self.heads_num, + mlp_width_ratio=mlp_width_ratio, + mlp_act_type=mlp_act_type, + attn_mode=attn_mode, + qk_norm=qk_norm, + qk_norm_type=qk_norm_type, + qkv_bias=qkv_bias, + **factory_kwargs, + ) + for _ in range(mm_double_blocks_depth) + ] + ) + + self.single_blocks = nn.ModuleList( + [ + MMSingleStreamBlock( + self.hidden_size, + self.heads_num, + mlp_width_ratio=mlp_width_ratio, + mlp_act_type=mlp_act_type, + attn_mode=attn_mode, + qk_norm=qk_norm, + qk_norm_type=qk_norm_type, + **factory_kwargs, + ) + for _ in range(mm_single_blocks_depth) + ] + ) + + self.final_layer = FinalLayer( + self.hidden_size, + self.patch_size, + self.out_channels, + get_activation_layer("silu"), + **factory_kwargs, + ) + + # STA + if attn_param is None: + self.attn_param = { + # STA + "win_size": [[3, 3, 3]], + "win_type": "fixed", + "win_ratio": 10, + "tile_size": [6, 8, 8], + # SSTA + "ssta_topk": 64, + "ssta_threshold": 0.0, + "ssta_lambda": 0.7, + "ssta_sampling_type": "importance", + "ssta_adaptive_pool": None, + # flex-block-attn: + "attn_sparse_type": "ssta", + "attn_pad_type": "zero", + "attn_use_text_mask": 1, + "attn_mask_share_within_head": 0, + } + else: + self.attn_param = attn_param + + if attn_mode == "flex-block-attn": + self.register_to_config(attn_param=self.attn_param) + + if use_cond_type_embedding: + self.cond_type_embedding = nn.Embedding(3, self.hidden_size) + self.cond_type_embedding.weight.data.fill_(0) + assert ( + self.glyph_byT5_v2 + ), "text type embedding is only used when glyph_byT5_v2 is True" + assert ( + vision_projection is not None + ), "text type embedding is only used when vision_projection is not None" + # 0: text_encoder feature + # 1: byt5 feature + # 2: vision_encoder feature + else: + self.cond_type_embedding = None + + self.gradient_checkpointing = False + + def load_hunyuan_state_dict(self, model_path): + load_key = "module" + bare_model = "unknown" + + if model_path.endswith(".safetensors"): + from safetensors.torch import load_file + + state_dict = load_file(model_path, device="cpu") + else: + state_dict = torch.load(model_path, map_location="cpu", weights_only=True) + + if bare_model == "unknown" and ("ema" in state_dict or "module" in state_dict): + bare_model = False + if bare_model is False: + if load_key in state_dict: + state_dict = state_dict[load_key] + else: + raise KeyError( + f"Missing key: `{load_key}` in the checkpoint: {model_path}. The keys in the checkpoint " + f"are: {list(state_dict.keys())}." + ) + + result = self.load_state_dict(state_dict, strict=False) + + if result.missing_keys: + logger.info("[load.py] Missing keys when loading state_dict:") + for key in result.missing_keys: + logger.info(f"[load.py] Missing key: {key}") + if result.unexpected_keys: + logger.info("[load.py] Unexpected keys when loading state_dict:") + for key in result.unexpected_keys: + logger.info(f"[load.py] Unexpected key: {key}") + if result.missing_keys or result.unexpected_keys: + raise ValueError( + f"Missing: {result.missing_keys}, Unexpected: {result.unexpected_keys}" + ) + + return result + + def enable_deterministic(self): + for block in self.double_blocks: + block.enable_deterministic() + for block in self.single_blocks: + block.enable_deterministic() + + def disable_deterministic(self): + for block in self.double_blocks: + block.disable_deterministic() + for block in self.single_blocks: + block.disable_deterministic() + + def get_rotary_pos_embed(self, rope_sizes): + target_ndim = 3 + head_dim = self.hidden_size // self.heads_num + rope_dim_list = self.rope_dim_list + if rope_dim_list is None: + rope_dim_list = [head_dim // target_ndim for _ in range(target_ndim)] + assert ( + sum(rope_dim_list) == head_dim + ), "sum(rope_dim_list) should equal to head_dim of attention layer" + freqs_cos, freqs_sin = get_nd_rotary_pos_embed( + rope_dim_list, + rope_sizes, + theta=self.rope_theta, + use_real=True, + theta_rescale_factor=1, + ) + return freqs_cos, freqs_sin + + def _get_vision_rotary_pos_embed( + self, + rope_temporal_size: int, + th: int, + tw: int, + tt: int, + start_rope_start_idx: int = 0, + rope_frame_indices: Optional[Union[torch.Tensor, list, tuple]] = None, + ): + rope_temporal_size = rope_temporal_size // self.patch_size[0] + freqs_cos, freqs_sin = self.get_rotary_pos_embed( + (rope_temporal_size, th, tw) + ) + per_latent_size = th * tw + + if rope_frame_indices is not None: + if torch.is_tensor(rope_frame_indices): + rope_frame_indices = rope_frame_indices.to( + device=freqs_cos.device, dtype=torch.long + ).reshape(-1) + else: + rope_frame_indices = torch.tensor( + rope_frame_indices, + device=freqs_cos.device, + dtype=torch.long, + ).reshape(-1) + + assert rope_frame_indices.numel() == tt, ( + f"rope_frame_indices length {rope_frame_indices.numel()} " + f"must match latent token temporal size {tt}" + ) + if rope_frame_indices.numel() > 0: + min_slot = int(rope_frame_indices.min().item()) + max_slot = int(rope_frame_indices.max().item()) + assert min_slot >= 0, f"Negative RoPE slot: {min_slot}" + assert max_slot < rope_temporal_size, ( + f"RoPE slot {max_slot} exceeds temporal size " + f"{rope_temporal_size}" + ) + + spatial_offsets = torch.arange( + per_latent_size, + device=freqs_cos.device, + dtype=torch.long, + ) + token_indices = ( + rope_frame_indices[:, None] * per_latent_size + + spatial_offsets[None, :] + ).reshape(-1) + freqs_cos = freqs_cos.index_select(0, token_indices) + freqs_sin = freqs_sin.index_select(0, token_indices) + else: + assert start_rope_start_idx + tt <= rope_temporal_size, ( + f"Continuous RoPE slice [{start_rope_start_idx}, " + f"{start_rope_start_idx + tt}) exceeds temporal size " + f"{rope_temporal_size}" + ) + start_index = start_rope_start_idx * per_latent_size + end_index = (start_rope_start_idx + tt) * per_latent_size + freqs_cos = freqs_cos[start_index:end_index, ...] + freqs_sin = freqs_sin[start_index:end_index, ...] + + expected_tokens = tt * per_latent_size + assert freqs_cos.shape[0] == expected_tokens, ( + f"freqs_cos token count {freqs_cos.shape[0]} != {expected_tokens}" + ) + assert freqs_sin.shape[0] == expected_tokens, ( + f"freqs_sin token count {freqs_sin.shape[0]} != {expected_tokens}" + ) + return freqs_cos, freqs_sin + + def reorder_txt_token( + self, byt5_txt, txt, byt5_text_mask, text_mask, zero_feat=False, is_reorder=True + ): + if is_reorder: + reorder_txt = [] + reorder_mask = [] + for i in range(text_mask.shape[0]): + byt5_text_mask_i = byt5_text_mask[i].bool() + text_mask_i = text_mask[i].bool() + + byt5_txt_i = byt5_txt[i] + txt_i = txt[i] + if zero_feat: + # When using block mask with approximate computation, set pad to zero to reduce error + pad_byt5 = torch.zeros_like(byt5_txt_i[~byt5_text_mask_i]) + pad_text = torch.zeros_like(txt_i[~text_mask_i]) + reorder_txt_i = torch.cat( + [ + byt5_txt_i[byt5_text_mask_i], + txt_i[text_mask_i], + pad_byt5, + pad_text, + ], + dim=0, + ) + else: + reorder_txt_i = torch.cat( + [ + byt5_txt_i[byt5_text_mask_i], + txt_i[text_mask_i], + byt5_txt_i[~byt5_text_mask_i], + txt_i[~text_mask_i], + ], + dim=0, + ) + reorder_mask_i = torch.cat( + [ + byt5_text_mask_i[byt5_text_mask_i], + text_mask_i[text_mask_i], + byt5_text_mask_i[~byt5_text_mask_i], + text_mask_i[~text_mask_i], + ], + dim=0, + ) + + reorder_txt.append(reorder_txt_i) + reorder_mask.append(reorder_mask_i) + + reorder_txt = torch.stack(reorder_txt) + reorder_mask = torch.stack(reorder_mask).to(dtype=torch.int64) + else: + reorder_txt = torch.concat([byt5_txt, txt], dim=1) + reorder_mask = torch.concat([byt5_text_mask, text_mask], dim=1).to( + dtype=torch.int64 + ) + + return reorder_txt, reorder_mask + + def add_action_parameters(self): + # Initialize the embedding module for action conditioning (using Silu activation). + # This method adds action control capabilities to the pre-trained transformer + self.action_in = TimestepEmbedder( + self.hidden_size, get_activation_layer("silu") + ) + + # Zero-initialize the weights of the final layer (mlp[2]) of the embedder. + # This ensures the initial output is zero, preventing the action conditioning + # from affecting the model's behavior at the start of training. + nn.init.zeros_(self.action_in.mlp[2].weight) + + # Zero-initialize the bias term if it exists. + if self.action_in.mlp[2].bias is not None: + nn.init.zeros_(self.action_in.mlp[2].bias) + + # Iterate through each transformer double-block to inject action-specific projections. + for block in self.double_blocks: + # Create a new linear projection layer for image attention properties. + block.img_attn_prope_proj = nn.Linear( + block.hidden_size, + block.hidden_size, + bias=block.qkv_bias, + **block.factory_kwargs, + ) + + # Zero-initialize the weights of this new projection layer. + # Again, this ensures the new layer acts as an identity function initially. + nn.init.zeros_(block.img_attn_prope_proj.weight) + + # Zero-initialize the bias term for the projection layer. + if block.img_attn_prope_proj.bias is not None: + nn.init.zeros_(block.img_attn_prope_proj.bias) + + def get_text_and_mask( + self, + encoder_attention_mask, + text_states, + timestep_txt, + extra_kwargs, + vision_states, + mask_type, + ): + text_mask = encoder_attention_mask + txt = text_states + bs = txt.shape[0] + + # Prepare modulation vectors + vec_txt = self.time_in(timestep_txt) + + # Embed text tokens + if self.text_projection == "linear": + txt = self.txt_in(txt) + elif self.text_projection == "single_refiner": + txt = self.txt_in( + txt, timestep_txt, text_mask if self.use_attention_mask else None + ) + else: + raise NotImplementedError( + f"Unsupported text_projection: {self.text_projection}" + ) + if self.cond_type_embedding is not None: + cond_emb = self.cond_type_embedding( + torch.zeros_like( + txt[:, :, 0], device=text_mask.device, dtype=torch.long + ) + ) + txt = txt + cond_emb + + if self.glyph_byT5_v2: + byt5_text_states = extra_kwargs["byt5_text_states"] + byt5_text_mask = extra_kwargs["byt5_text_mask"] + byt5_txt = self.byt5_in(byt5_text_states) + if self.cond_type_embedding is not None: + cond_emb = self.cond_type_embedding( + torch.ones_like( + byt5_txt[:, :, 0], device=byt5_txt.device, dtype=torch.long + ) + ) + byt5_txt = byt5_txt + cond_emb + txt, text_mask = self.reorder_txt_token( + byt5_txt, txt, byt5_text_mask, text_mask, zero_feat=True + ) + + if self.vision_in is not None and vision_states is not None: + extra_encoder_hidden_states = self.vision_in(vision_states) + # If t2v, set extra_attention_mask to 0 to avoid attention to semantic tokens + if mask_type == "t2v" and torch.all(vision_states == 0): + extra_attention_mask = torch.zeros( + (bs, extra_encoder_hidden_states.shape[1]), + dtype=text_mask.dtype, + device=text_mask.device, + ) + # Set vision tokens to zero to mitigate potential block mask error in SSTA + extra_encoder_hidden_states = extra_encoder_hidden_states * 0.0 + else: + extra_attention_mask = torch.ones( + (bs, extra_encoder_hidden_states.shape[1]), + dtype=text_mask.dtype, + device=text_mask.device, + ) + # Ensure valid tokens precede padding tokens + if self.cond_type_embedding is not None: + cond_emb = self.cond_type_embedding( + 2 + * torch.ones_like( + extra_encoder_hidden_states[:, :, 0], + dtype=torch.long, + device=extra_encoder_hidden_states.device, + ) + ) + extra_encoder_hidden_states = extra_encoder_hidden_states + cond_emb + + txt, text_mask = self.reorder_txt_token( + extra_encoder_hidden_states, txt, extra_attention_mask, text_mask + ) + return txt, text_mask, vec_txt + + # txt embedding kv cache + def forward_txt( + self, + timestep_txt: torch.Tensor, + text_states: torch.Tensor, + encoder_attention_mask: torch.Tensor, + vision_states: torch.Tensor = None, + mask_type="t2v", + extra_kwargs=None, + kv_cache: Optional[dict] = None, + cache_txt: Optional[bool] = False, + ) -> Union[torch.Tensor, Dict[str, torch.Tensor]]: + # Forward pass for text embeddings only, optionally caching K/V for autoregressive generation + if cache_txt: + _kv_cache_new = [] + transformer_num_layers = len(self.double_blocks) + for _ in range(transformer_num_layers): + _kv_cache_new.append( + {"k_vision": None, "v_vision": None, "k_txt": None, "v_txt": None} + ) + + txt, text_mask, vec_txt = self.get_text_and_mask( + encoder_attention_mask, + text_states, + timestep_txt, + extra_kwargs, + vision_states, + mask_type, + ) + + # mask the txt tokens in advance for efficiency + txt = txt[text_mask.bool().to(txt.device)].unsqueeze(0) + + # Pass through double-stream blocks + for index, block in enumerate(self.double_blocks): + txt, t_kv = block( + bi_inference=False, + ar_txt_inference=True, + ar_vision_inference=False, + txt=txt, + vec_txt=vec_txt, + text_mask=None, # we have masked txt tokens already, set None here + attn_param=None, + is_flash=False, + block_idx=index, + kv_cache=kv_cache, + cache_txt=cache_txt, + ) + + if cache_txt: + _kv_cache_new[index]["k_txt"] = t_kv["k_txt"] + _kv_cache_new[index]["v_txt"] = t_kv["v_txt"] + + if cache_txt: + return _kv_cache_new + + # using kv cache to calculate vision embedding + def forward_vision( + self, + hidden_states: torch.Tensor, + timestep: torch.LongTensor, + timestep_r=None, + freqs_cos: Optional[torch.Tensor] = None, + freqs_sin: Optional[torch.Tensor] = None, + return_dict: bool = False, + mask_type="t2v", + extra_kwargs=None, + action: Optional[torch.Tensor] = None, + viewmats: Optional[torch.Tensor] = None, + Ks: Optional[torch.Tensor] = None, + kv_cache: Optional[dict] = None, + cache_vision: bool = False, + rope_temporal_size=4, + start_rope_start_idx=0, + rope_frame_indices: Optional[Union[torch.Tensor, list, tuple]] = None, + context_frame_indices: Optional[Union[torch.Tensor, list, tuple]] = None, + context_query_viewmats: Optional[torch.Tensor] = None, + context_query_Ks: Optional[torch.Tensor] = None, + return_final_hidden: bool = False, + ) -> Union[torch.Tensor, Dict[str, torch.Tensor]]: + infer_state = get_infer_state() + current_step = getattr(infer_state, "current_step_index", 0) + use_fov_cache = getattr(infer_state, "use_fov_cache", False) + + if cache_vision and self.attn_param.get("context_reuse_vision_cache", False): + return kv_cache + + enable_denoise_cache = self.attn_param.get("enable_denoise_cache", False) + denoise_cache_steps = tuple(self.attn_param.get("denoise_cache_steps", (1, 2))) + if ( + enable_denoise_cache + and not cache_vision + and use_fov_cache + and current_step in denoise_cache_steps + and getattr(self, "_zoh_cached_noise", None) is not None + ): + return (self._zoh_cached_noise, None) + + if cache_vision: + _kv_cache_new = [] + transformer_num_layers = len(self.double_blocks) + for i in range(transformer_num_layers): + _kv_cache_new.append( + { + "k_vision": None, + "v_vision": None, + "k_txt": kv_cache[i]["k_txt"], + "v_txt": kv_cache[i]["v_txt"], + } + ) + + img = x = hidden_states + t = timestep + bs, _, ot, oh, ow = x.shape + tt, th, tw = ( + ot // self.patch_size[0], + oh // self.patch_size[1], + ow // self.patch_size[2], + ) + self.attn_param["thw"] = [tt, th, tw] + infer_state = get_infer_state() + infer_state.latent_shape = (th, tw) + if freqs_cos is None and freqs_sin is None: + freqs_cos, freqs_sin = self._get_vision_rotary_pos_embed( + rope_temporal_size=rope_temporal_size, + th=th, + tw=tw, + tt=tt, + start_rope_start_idx=start_rope_start_idx, + rope_frame_indices=rope_frame_indices, + ) + + img = self.img_in(img) + + action = action.reshape(-1) + t = t.reshape(-1) + + # Prepare modulation vectors + vec = self.time_in(t) + vec = vec + self.action_in(action) # inject discrete action + + frame_viewmats = viewmats + frame_Ks = Ks + + # boardcast to match the sequence length of video latents + vec = repeat(vec, "(B T) C->B (T H W) C", B=img.shape[0], H=th, W=tw) + viewmats = repeat(viewmats, "B T M N->B (T H W) M N", H=th, W=tw) + Ks = repeat(Ks, "B T M N->B (T H W) M N", H=th, W=tw) + + spatial_keep_ratio = float( + self.attn_param.get("context_spatial_keep_ratio", 1.0) + ) + spatial_full_last_frames = int( + self.attn_param.get("context_spatial_full_last_frames", 4) + ) + spatial_pool = bool(self.attn_param.get("context_spatial_pool", False)) + spatial_adaptive_visibility = bool( + self.attn_param.get("context_spatial_adaptive_visibility", False) + ) + if ( + cache_vision + and spatial_pool + and not self.attn_param.get("context_spatial_force_full", False) + and 0.0 < spatial_keep_ratio < 1.0 + and tt > spatial_full_last_frames + ): + pool_factor = int(round(1.0 / spatial_keep_ratio)) + if ( + pool_factor <= 1 + or tw % pool_factor != 0 + or abs(spatial_keep_ratio - (1.0 / pool_factor)) > 1e-6 + ): + raise ValueError( + "Spatial context pooling requires keep_ratio=1/N and width divisible by N" + ) + tokens_per_frame = th * tw + compressed_start_frame = max( + 0, + int( + self.attn_param.get( + "context_spatial_compress_start_frame", 0 + ) + ), + ) + compressed_end_frame = max( + compressed_start_frame, tt - spatial_full_last_frames + ) + compressed_frame_count = compressed_end_frame - compressed_start_frame + pooled_width = tw // pool_factor + hidden_dim = img.shape[-1] + prefix_end = compressed_start_frame * tokens_per_frame + compressed_end = compressed_end_frame * tokens_per_frame + prefix_img = img[:, :prefix_end] + compressed_img = img[:, prefix_end:compressed_end] + compressed_img = compressed_img.reshape( + bs, + compressed_frame_count, + th, + pooled_width, + pool_factor, + hidden_dim, + ).mean(dim=4) + compressed_img = compressed_img.reshape(bs, -1, hidden_dim) + suffix_img = img[:, compressed_end:] + img = torch.cat([prefix_img, compressed_img, suffix_img], dim=1) + + rows = torch.arange(th, device=img.device)[:, None] + cols = torch.arange(pooled_width, device=img.device)[None, :] + representative_spatial_indices = ( + rows * tw + cols * pool_factor + (pool_factor // 2) + ).flatten() + token_indices = [ + torch.arange(0, prefix_end, device=img.device), + *[ + representative_spatial_indices + frame_idx * tokens_per_frame + for frame_idx in range( + compressed_start_frame, compressed_end_frame + ) + ], + ] + token_indices.append( + torch.arange(compressed_end, tt * tokens_per_frame, device=img.device) + ) + token_indices = torch.cat(token_indices) + vec = vec.index_select(1, token_indices) + viewmats = viewmats.index_select(1, token_indices) + Ks = Ks.index_select(1, token_indices) + rope_token_indices = token_indices.to(freqs_cos.device) + freqs_cos = freqs_cos.index_select(0, rope_token_indices) + freqs_sin = freqs_sin.index_select(0, rope_token_indices) + self.attn_param["context_spatial_token_count"] = int( + token_indices.numel() + ) + assert img.shape[1] == freqs_cos.shape[0] == freqs_sin.shape[0] + elif ( + cache_vision + and not self.attn_param.get("context_spatial_force_full", False) + and ( + 0.0 < spatial_keep_ratio < 1.0 + or spatial_adaptive_visibility + ) + and tt > spatial_full_last_frames + ): + tokens_per_frame = th * tw + compressed_start_frame = max( + 0, + int( + self.attn_param.get( + "context_spatial_compress_start_frame", 0 + ) + ), + ) + compressed_end_frame = max( + compressed_start_frame, tt - spatial_full_last_frames + ) + keep_per_memory_frame = max( + 1, + min( + tokens_per_frame, + int(round(tokens_per_frame * spatial_keep_ratio)), + ), + ) + token_indices = [] + if compressed_start_frame > 0: + token_indices.append( + torch.arange( + 0, + compressed_start_frame * tokens_per_frame, + device=img.device, + ) + ) + for frame_idx in range(compressed_start_frame, compressed_end_frame): + frame_keep_count = keep_per_memory_frame + sampled_spatial_indices = None + if self.attn_param.get("context_spatial_center", False): + kept_width = max(1, min(tw, int(round(tw * spatial_keep_ratio)))) + start_col = (tw - kept_width) // 2 + center_rows = torch.arange(th, device=img.device)[:, None] + center_cols = torch.arange( + start_col, start_col + kept_width, device=img.device + )[None, :] + sampled_spatial_indices = ( + center_rows * tw + center_cols + ).flatten() + if ( + sampled_spatial_indices is None + and self.attn_param.get("context_spatial_projected", False) + and context_query_viewmats is not None + and context_query_Ks is not None + ): + if bs != 1: + raise ValueError( + "Projected context token selection currently requires batch size 1" + ) + geometry_dtype = torch.float32 + rows = ( + torch.arange(th, device=img.device, dtype=geometry_dtype) + + 0.5 + ) / th + cols = ( + torch.arange(tw, device=img.device, dtype=geometry_dtype) + + 0.5 + ) / tw + grid_y, grid_x = torch.meshgrid(rows, cols, indexing="ij") + pixels = torch.stack( + [grid_x.flatten(), grid_y.flatten(), torch.ones(tokens_per_frame, device=img.device, dtype=geometry_dtype)], + dim=-1, + ) + history_K_inv = torch.linalg.inv( + frame_Ks[0, frame_idx].to(geometry_dtype) + ) + rays = pixels @ history_K_inv.T + points_camera = rays * 4.0 + points_camera_h = torch.cat( + [ + points_camera, + torch.ones(tokens_per_frame, 1, device=img.device, dtype=geometry_dtype), + ], + dim=-1, + ) + history_c2w = torch.linalg.inv( + frame_viewmats[0, frame_idx].to(geometry_dtype) + ) + points_world = points_camera_h @ history_c2w.T + visible = torch.zeros(tokens_per_frame, device=img.device, dtype=torch.bool) + for query_idx in range(context_query_viewmats.shape[1]): + points_query = points_world @ context_query_viewmats[ + 0, query_idx + ].to(geometry_dtype).T + projected = points_query[:, :3] @ context_query_Ks[ + 0, query_idx + ].to(geometry_dtype).T + depth = projected[:, 2] + projected_x = projected[:, 0] / depth.clamp_min(1e-6) + projected_y = projected[:, 1] / depth.clamp_min(1e-6) + visible |= ( + (depth > 0) + & (projected_x >= 0) + & (projected_x <= 1) + & (projected_y >= 0) + & (projected_y <= 1) + ) + visible_indices = torch.nonzero(visible, as_tuple=False).flatten() + if spatial_adaptive_visibility: + min_keep_ratio = float( + self.attn_param.get( + "context_spatial_min_keep_ratio", 0.1 + ) + ) + frame_keep_count = max( + 1, + min( + tokens_per_frame, + max( + int(round(tokens_per_frame * min_keep_ratio)), + int(visible_indices.numel()), + ), + ), + ) + if visible_indices.numel() >= frame_keep_count: + positions = torch.linspace( + 0, + visible_indices.numel() - 1, + frame_keep_count, + device=img.device, + ).round().to(torch.long) + sampled_spatial_indices = visible_indices.index_select(0, positions) + else: + missing_count = frame_keep_count - visible_indices.numel() + background_indices = torch.nonzero(~visible, as_tuple=False).flatten() + if missing_count > 0: + positions = torch.linspace( + 0, + background_indices.numel() - 1, + missing_count, + device=img.device, + ).round().to(torch.long) + sampled_spatial_indices = torch.cat( + [visible_indices, background_indices.index_select(0, positions)] + ).sort().values + else: + sampled_spatial_indices = visible_indices + if sampled_spatial_indices is None: + sampled_spatial_indices = torch.linspace( + 0, + tokens_per_frame - 1, + frame_keep_count, + device=img.device, + ).round().to(torch.long) + saliency_fraction = float( + self.attn_param.get("context_spatial_saliency_fraction", 0.0) + ) + saliency_count = min( + frame_keep_count, + max(0, int(round(frame_keep_count * saliency_fraction))), + ) + if saliency_count > 0: + frame_start = frame_idx * tokens_per_frame + frame_tokens = img[ + 0, frame_start : frame_start + tokens_per_frame + ] + saliency_scores = frame_tokens.float().pow(2).mean(dim=-1) + salient_indices = torch.topk( + saliency_scores, saliency_count, sorted=False + ).indices + fill_count = frame_keep_count - saliency_count + base_mask = torch.zeros( + tokens_per_frame, device=img.device, dtype=torch.bool + ) + base_mask[sampled_spatial_indices] = True + base_mask[salient_indices] = False + base_candidates = torch.nonzero( + base_mask, as_tuple=False + ).flatten() + if base_candidates.numel() > fill_count: + fill_positions = torch.linspace( + 0, + base_candidates.numel() - 1, + fill_count, + device=img.device, + ).round().to(torch.long) + base_candidates = base_candidates.index_select( + 0, fill_positions + ) + sampled_spatial_indices = torch.cat( + [salient_indices, base_candidates] + ).sort().values + if ( + self.attn_param.get("context_spatial_merge", False) + and sampled_spatial_indices.numel() < tokens_per_frame + ): + spatial_rows = torch.arange( + th, device=img.device, dtype=torch.float32 + )[:, None].expand(th, tw) + spatial_cols = torch.arange( + tw, device=img.device, dtype=torch.float32 + )[None, :].expand(th, tw) + all_coordinates = torch.stack( + [spatial_rows.flatten(), spatial_cols.flatten()], dim=-1 + ) + selected_coordinates = all_coordinates.index_select( + 0, sampled_spatial_indices + ) + merge_assignments = torch.cdist( + all_coordinates, selected_coordinates + ).argmin(dim=1) + merge_counts = torch.bincount( + merge_assignments, + minlength=sampled_spatial_indices.numel(), + ).clamp_min(1).to(img.dtype) + frame_start = frame_idx * tokens_per_frame + frame_tokens = img[ + :, frame_start : frame_start + tokens_per_frame + ] + merged_tokens = [] + for batch_idx in range(frame_tokens.shape[0]): + merged = torch.zeros( + sampled_spatial_indices.numel(), + frame_tokens.shape[-1], + device=img.device, + dtype=img.dtype, + ) + merged.index_add_( + 0, merge_assignments, frame_tokens[batch_idx] + ) + merged_tokens.append(merged / merge_counts[:, None]) + merged_tokens = torch.stack(merged_tokens, dim=0) + img[ + :, frame_start + sampled_spatial_indices + ] = merged_tokens + token_indices.append( + sampled_spatial_indices + frame_idx * tokens_per_frame + ) + recent_start = compressed_end_frame * tokens_per_frame + token_indices.append( + torch.arange(recent_start, tt * tokens_per_frame, device=img.device) + ) + token_indices = torch.cat(token_indices) + + img = img.index_select(1, token_indices) + vec = vec.index_select(1, token_indices) + viewmats = viewmats.index_select(1, token_indices) + Ks = Ks.index_select(1, token_indices) + rope_token_indices = token_indices.to(freqs_cos.device) + freqs_cos = freqs_cos.index_select(0, rope_token_indices) + freqs_sin = freqs_sin.index_select(0, rope_token_indices) + self.attn_param["context_spatial_token_count"] = int( + token_indices.numel() + ) + assert img.shape[1] == freqs_cos.shape[0] == freqs_sin.shape[0] + + # Sequence parallel: split spatial tokens across multiple GPUs for memory efficiency + parallel_dims = get_parallel_state() + sp_enabled = parallel_dims.sp_enabled + if sp_enabled: + sp_size = parallel_dims.sp + sp_rank = parallel_dims.sp_rank + # Verify sequence length is sufficient for parallelization + if img.shape[1] % sp_size != 0: + n_token = img.shape[1] + assert n_token > (n_token // sp_size + 1) * ( + sp_size - 1 + ), f"Too short context length for SP {sp_size}" + img = torch.chunk(img, sp_size, dim=1)[sp_rank] + freqs_cos = torch.chunk(freqs_cos, sp_size, dim=0)[sp_rank] + freqs_sin = torch.chunk(freqs_sin, sp_size, dim=0)[sp_rank] + + vec = torch.chunk(vec, sp_size, dim=1)[sp_rank] + vec = rearrange(vec, "B S C->(B S) C") + viewmats = torch.chunk(viewmats, sp_size, dim=1)[sp_rank] + Ks = torch.chunk(Ks, sp_size, dim=1)[sp_rank] + else: + vec = rearrange(vec, "B S C->(B S) C") + + freqs_cis = (freqs_cos, freqs_sin) if freqs_cos is not None else None + + # Pass through double-stream blocks + for index, block in enumerate(self.double_blocks): + compile_state = get_infer_state() + if ( + cache_vision + and compile_state is not None + and self.attn_param.get("context_compile_prefill", False) + ): + compile_state.enable_torch_compile = bool( + index + >= int( + self.attn_param.get( + "context_compile_prefill_start_layer", 0 + ) + ) + and index + < int( + self.attn_param.get( + "context_compile_prefill_end_layer", + len(self.double_blocks), + ) + ) + ) + self.attn_param["layer-name"] = f"double_block_{index + 1}" + + img, vision_kv = block( + bi_inference=False, + ar_txt_inference=False, + ar_vision_inference=True, + img=img, + vec=vec, + freqs_cis=freqs_cis, + attn_param=self.attn_param, + block_idx=index, + viewmats=viewmats, + Ks=Ks, + kv_cache=kv_cache, + cache_vision=cache_vision, + ) + if cache_vision: + _kv_cache_new[index]["k_vision"] = vision_kv["k_vision"] + _kv_cache_new[index]["v_vision"] = vision_kv["v_vision"] + + prune_after_layer = int( + self.attn_param.get("context_prune_after_layer", 0) + ) + keep_last_frames = int( + self.attn_param.get("context_prune_keep_last_frames", 0) + ) + keep_frame_indices = self.attn_param.get( + "context_prune_keep_frame_indices", None + ) + expected_frame_count = int( + self.attn_param.get("context_prune_expected_frame_count", 0) + ) + has_explicit_keep = bool(keep_frame_indices) + use_explicit_keep = bool( + has_explicit_keep + and (expected_frame_count <= 0 or tt == expected_frame_count) + and all(0 <= int(frame_idx) < tt for frame_idx in keep_frame_indices) + ) + if ( + cache_vision + and prune_after_layer > 0 + and index + 1 == prune_after_layer + and ( + use_explicit_keep + or ( + not has_explicit_keep + and keep_last_frames > 0 + and tt > keep_last_frames + ) + ) + ): + if sp_enabled: + raise ValueError( + "Depth-adaptive context pruning currently requires single-GPU inference" + ) + tokens_per_frame = th * tw + if use_explicit_keep: + kept_frames = sorted(set(int(idx) for idx in keep_frame_indices)) + token_indices = torch.cat( + [ + torch.arange( + frame_idx * tokens_per_frame, + (frame_idx + 1) * tokens_per_frame, + device=img.device, + ) + for frame_idx in kept_frames + ] + ) + new_frame_count = len(kept_frames) + else: + first_kept_frame = tt - keep_last_frames + token_indices = torch.arange( + first_kept_frame * tokens_per_frame, + tt * tokens_per_frame, + device=img.device, + ) + new_frame_count = keep_last_frames + img = img.index_select(1, token_indices) + batch_size = img.shape[0] + vec = vec.reshape(batch_size, -1, vec.shape[-1]) + vec = vec.index_select(1, token_indices).reshape(-1, vec.shape[-1]) + viewmats = viewmats.index_select(1, token_indices) + Ks = Ks.index_select(1, token_indices) + rope_token_indices = token_indices.to(freqs_cos.device) + freqs_cos = freqs_cos.index_select(0, rope_token_indices) + freqs_sin = freqs_sin.index_select(0, rope_token_indices) + freqs_cis = (freqs_cos, freqs_sin) + tt = new_frame_count + self.attn_param["thw"] = [tt, th, tw] + + if cache_vision: + return _kv_cache_new + + # Final Layer + final_hidden = img if return_final_hidden else None + frame_condition = None + if return_final_hidden: + if sp_enabled: + raise ValueError( + "Returning final hidden features currently requires single-GPU inference" + ) + frame_condition = vec.reshape(bs, tt, th * tw, -1)[:, :, 0] + img = self.final_layer(img, vec) + if sp_enabled: + img = all_gather(img, dim=1) + img = self.unpatchify(img, tt, th, tw) + assert return_dict is False, "return_dict is not supported." + features_list = None + if not cache_vision and current_step == 0: + self._zoh_cached_noise = img.clone().detach() + if return_final_hidden: + return (img, features_list, final_hidden, frame_condition) + return (img, features_list) + + def forward_bi( + self, + hidden_states: torch.Tensor, + timestep: torch.LongTensor, + timestep_txt: torch.Tensor, + text_states: torch.Tensor, + text_states_2: torch.Tensor, + encoder_attention_mask: torch.Tensor, + timestep_r=None, + vision_states: torch.Tensor = None, + output_features=False, + output_features_stride=8, + attention_kwargs: Optional[Dict[str, Any]] = None, + freqs_cos: Optional[torch.Tensor] = None, + freqs_sin: Optional[torch.Tensor] = None, + return_dict: bool = False, + guidance=None, + mask_type="t2v", + extra_kwargs=None, + action: Optional[torch.Tensor] = None, + viewmats: Optional[torch.Tensor] = None, + Ks: Optional[torch.Tensor] = None, + rope_temporal_size: Optional[int] = None, + start_rope_start_idx: int = 0, + rope_frame_indices: Optional[Union[torch.Tensor, list, tuple]] = None, + ) -> Union[torch.Tensor, Dict[str, torch.Tensor]]: + if guidance is None: + guidance = torch.tensor( + [6016.0], device=hidden_states.device, dtype=torch.bfloat16 + ) + + img = x = hidden_states + t = timestep + bs, _, ot, oh, ow = x.shape + tt, th, tw = ( + ot // self.patch_size[0], + oh // self.patch_size[1], + ow // self.patch_size[2], + ) + self.attn_param["thw"] = [tt, th, tw] + infer_state = get_infer_state() + infer_state.latent_shape = (th, tw) + if freqs_cos is None and freqs_sin is None: + if rope_temporal_size is None: + rope_temporal_size = ot + freqs_cos, freqs_sin = self._get_vision_rotary_pos_embed( + rope_temporal_size=rope_temporal_size, + th=th, + tw=tw, + tt=tt, + start_rope_start_idx=start_rope_start_idx, + rope_frame_indices=rope_frame_indices, + ) + + img = self.img_in(img) + vec = self.time_in(t) + + if text_states_2 is not None: + vec_2 = self.vector_in(text_states_2) + vec = vec + vec_2 + + if self.guidance_embed: + if guidance is None: + raise ValueError( + "Didn't get guidance strength for guidance distilled model." + ) + vec = vec + self.guidance_in(guidance) + + if timestep_r is not None: + vec = vec + self.time_r_in(timestep_r) + + if action is not None: + vec = vec + self.action_in(action) # 添加离散的action + + # boardcast to match the sequence length of video latents + vec = repeat(vec, "(B T) C->B (T H W) C", B=img.shape[0], H=th, W=tw) + if viewmats is not None: + viewmats = repeat(viewmats, "B T M N->B (T H W) M N", H=th, W=tw) + Ks = repeat(Ks, "B T M N->B (T H W) M N", H=th, W=tw) + + # sequence parallel + parallel_dims = get_parallel_state() + sp_enabled = parallel_dims.sp_enabled + if sp_enabled: + sp_size = parallel_dims.sp + sp_rank = parallel_dims.sp_rank + if img.shape[1] % sp_size != 0: + n_token = img.shape[1] + assert n_token > (n_token // sp_size + 1) * ( + sp_size - 1 + ), f"Too short context length for SP {sp_size}" + img = torch.chunk(img, sp_size, dim=1)[sp_rank] + freqs_cos = torch.chunk(freqs_cos, sp_size, dim=0)[sp_rank] + freqs_sin = torch.chunk(freqs_sin, sp_size, dim=0)[sp_rank] + + vec = torch.chunk(vec, sp_size, dim=1)[sp_rank] + vec = rearrange(vec, "B S C->(B S) C") + if viewmats is not None: + viewmats = torch.chunk(viewmats, sp_size, dim=1)[sp_rank] + Ks = torch.chunk(Ks, sp_size, dim=1)[sp_rank] + else: + vec = rearrange(vec, "B S C->(B S) C") + + txt, text_mask, vec_txt = self.get_text_and_mask( + encoder_attention_mask, + text_states, + timestep_txt, + extra_kwargs, + vision_states, + mask_type, + ) + + freqs_cis = (freqs_cos, freqs_sin) if freqs_cos is not None else None + + # Pass through double-stream blocks + for index, block in enumerate(self.double_blocks): + force_full_attn = ( + self.attn_mode in ["flex-block-attn"] + and self.attn_param["win_type"] == "hybrid" + and self.attn_param["win_ratio"] > 0 + and ( + (index + 1) % self.attn_param["win_ratio"] == 0 + or (index + 1) == len(self.double_blocks) + ) + ) + self.attn_param["layer-name"] = f"double_block_{index + 1}" + if viewmats is not None: + img, txt = block( + bi_inference=True, + ar_txt_inference=False, + ar_vision_inference=False, + img=img, + txt=txt, + vec_txt=vec_txt, + vec=vec, + freqs_cis=freqs_cis, + text_mask=text_mask, # we have masked txt tokens already, set None here + attn_param=self.attn_param, + is_flash=force_full_attn, + block_idx=index, + viewmats=viewmats, + Ks=Ks, + ) + else: + img, txt = block( + bi_inference=False, + ar_txt_inference=False, + ar_vision_inference=False, + img=img, + txt=txt, + vec_txt=vec_txt, + vec=vec, + freqs_cis=freqs_cis, + text_mask=text_mask, # we have masked txt tokens already, set None here + attn_param=self.attn_param, + is_flash=force_full_attn, + block_idx=index, + ) + + # Final Layer + img = self.final_layer(img, vec) + if sp_enabled: + img = all_gather(img, dim=1) + img = self.unpatchify(img, tt, th, tw) + assert return_dict is False, "return_dict is not supported." + features_list = None + return (img, features_list) + + def forward( + self, + bi_inference=True, + ar_txt_inference=False, + ar_vision_inference=False, + **kwargs, + ): + if bi_inference: + return self.forward_bi(**kwargs) + elif ar_txt_inference: + return self.forward_txt(**kwargs) + elif ar_vision_inference: + return self.forward_vision(**kwargs) + else: + raise NotImplementedError + + def unpatchify(self, x, t, h, w): + """ + Unpatchify a tensorized input back to frame format. + + Args: + x (Tensor): Input tensor of shape (N, T, patch_size**2 * C) + t (int): Number of time steps + h (int): Height in patch units + w (int): Width in patch units + + Returns: + Tensor: Output tensor of shape (N, C, t * pt, h * ph, w * pw) + """ + c = self.unpatchify_channels + pt, ph, pw = self.patch_size + assert t * h * w == x.shape[1] + x = x.reshape(shape=(x.shape[0], t, h, w, c, pt, ph, pw)) + x = torch.einsum("nthwcopq->nctohpwq", x) + imgs = x.reshape(shape=(x.shape[0], c, t * pt, h * ph, w * pw)) + return imgs + + def set_attn_mode(self, attn_mode: str): + # Set the attention mode for the current object + self.attn_mode = attn_mode + + # Iterate through each block in the double_blocks list + for block in self.double_blocks: + # Set the attention mode for each block in double_blocks + block.attn_mode = attn_mode + + # Iterate through each block in the single_blocks list + for block in self.single_blocks: + # Set the attention mode for each block in single_blocks + block.attn_mode = attn_mode diff --git a/hyvideo/models/vision_encoder/__init__.py b/hyvideo/models/vision_encoder/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8d847fa2814ab28881049707b55930c8f52695df --- /dev/null +++ b/hyvideo/models/vision_encoder/__init__.py @@ -0,0 +1,240 @@ +# Licensed under the TENCENT HUNYUAN COMMUNITY LICENSE AGREEMENT (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://github.com/Tencent-Hunyuan/HunyuanVideo-1.5/blob/main/LICENSE +# +# Unless and only to the extent required by applicable law, the Tencent Hunyuan works and any +# output and results therefrom are provided "AS IS" without any express or implied warranties of +# any kind including any warranties of title, merchantability, noninfringement, course of dealing, +# usage of trade, or fitness for a particular purpose. You are solely responsible for determining the +# appropriateness of using, reproducing, modifying, performing, displaying or distributing any of +# the Tencent Hunyuan works or outputs and assume any and all risks associated with your or a +# third party's use or distribution of any of the Tencent Hunyuan works or outputs and your exercise +# of rights and permissions under this agreement. +# See the License for the specific language governing permissions and limitations under the License. + +from dataclasses import dataclass +from typing import Optional, Tuple +import torch +import torch.nn as nn +from transformers import SiglipImageProcessor, SiglipVisionModel +from transformers.utils import ModelOutput +import numpy as np +from hyvideo.commons import PRECISION_TO_TYPE + +VISION_ENCODER_PATH = {} + + +def use_default(value, default): + return value if value is not None else default + + +def load_vision_encoder( + vision_encoder_type, + vision_encoder_precision=None, + vision_encoder_path=None, + logger=None, + device=None, +): + if vision_encoder_path is None: + vision_encoder_path = VISION_ENCODER_PATH[vision_encoder_type] + + if vision_encoder_type == "siglip": + vision_encoder = SiglipVisionModel.from_pretrained( + vision_encoder_path, subfolder="image_encoder" + ) + else: + raise ValueError(f"Unsupported vision encoder type: {vision_encoder_type}") + + # from_pretrained will ensure that the model is in eval mode. + if vision_encoder_precision is not None: + vision_encoder = vision_encoder.to( + dtype=PRECISION_TO_TYPE[vision_encoder_precision] + ) + + vision_encoder.requires_grad_(False) + + if device is not None: + vision_encoder = vision_encoder.to(device) + + return vision_encoder, vision_encoder_path + + +def load_image_processor(processor_type, processor_path=None, logger=None): + if processor_path is None: + processor_path = VISION_ENCODER_PATH[processor_type] + + if processor_type == "siglip": + processor = SiglipImageProcessor.from_pretrained( + processor_path, subfolder="feature_extractor" + ) + else: + raise ValueError(f"Unsupported processor type: {processor_type}") + + return processor, processor_path + + +@dataclass +class VisionEncoderModelOutput(ModelOutput): + """ + Base class for vision encoder model's outputs. + + Args: + last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): + Sequence of hidden-states at the output of the last layer of the model. + pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`, *optional*): + Last layer hidden-state of the first token of the sequence (classification token) + after further processing through the layers used for the auxiliary pretraining task. + hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed): + Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. + Hidden-states of the model at the output of each layer plus the optional initial embedding outputs. + """ + + last_hidden_state: torch.FloatTensor = None + pooler_output: Optional[torch.FloatTensor] = None + hidden_states: Optional[Tuple[torch.FloatTensor, ...]] = None + + +class VisionEncoder(nn.Module): + def __init__( + self, + vision_encoder_type: str, + vision_encoder_precision: Optional[str] = None, + vision_encoder_path: Optional[str] = None, + processor_type: Optional[str] = None, + processor_path: Optional[str] = None, + output_key: Optional[str] = None, + logger=None, + device=None, + ): + super().__init__() + self.vision_encoder_type = vision_encoder_type + self.precision = vision_encoder_precision + self.model_path = vision_encoder_path + self.processor_type = ( + processor_type if processor_type is not None else vision_encoder_type + ) + self.processor_path = ( + processor_path if processor_path is not None else vision_encoder_path + ) + self.logger = logger + + if "siglip" in vision_encoder_type: + self.output_key = output_key or "last_hidden_state" + else: + raise ValueError(f"Unsupported vision encoder type: {vision_encoder_type}") + + self.model, self.model_path = load_vision_encoder( + vision_encoder_type=self.vision_encoder_type, + vision_encoder_precision=self.precision, + vision_encoder_path=self.model_path, + logger=self.logger, + device=device, + ) + self.dtype = self.model.dtype + self.device = self.model.device + + self.processor, self.processor_path = load_image_processor( + processor_type=self.processor_type, + processor_path=self.processor_path, + logger=self.logger, + ) + + def __repr__(self): + return f"{self.vision_encoder_type} ({self.precision} - {self.model_path})" + + def encode_latents_to_images(self, latents, vae, reorg_token=False): + """ + Convert latents to images using VAE decoder. + + Args: + latents: Input latents tensor + vae: VAE model for decoding + reorg_token: Whether to reorg the token + Returns: + images: Decoded images as numpy array + """ + # Handle both 4D and 5D latents (for video, take first frame) + first_image_latents = ( + latents[:, :, 0, ...] if len(latents.shape) == 5 else latents + ) + first_image_latents = 1 / vae.config.scaling_factor * first_image_latents + first_image = vae.decode( + first_image_latents.unsqueeze(2).to(vae.dtype), return_dict=False + )[0].cpu() + first_image = first_image[:, :, 0, :, :] + first_image = (first_image / 2 + 0.5).clamp(0, 1) + first_image = (first_image * 255.0).clamp(0, 255.0) + first_image = first_image.to(torch.uint8).numpy() + first_image = first_image.transpose(0, 2, 3, 1) + + assert isinstance(first_image, np.ndarray) + assert first_image.ndim == 4 and first_image.shape[3] == 3 + assert first_image.dtype == np.uint8 + + return first_image + + def encode_images(self, images): + """ + Encode images using the vision encoder. + + Args: + images: Input images (numpy array or preprocessed tensor) + + Returns: + VisionEncoderModelOutput with encoded features + """ + if isinstance(images, np.ndarray): + # Preprocess images if they're numpy arrays + preprocessed = self.processor.preprocess( + images=images, return_tensors="pt" + ).to(device=self.model.device, dtype=self.model.dtype) + else: + # Assume already preprocessed + preprocessed = images + + outputs = self.model(**preprocessed) + + return VisionEncoderModelOutput( + last_hidden_state=outputs.last_hidden_state, + pooler_output=( + outputs.pooler_output if hasattr(outputs, "pooler_output") else None + ), + hidden_states=( + outputs.hidden_states if hasattr(outputs, "hidden_states") else None + ), + ) + + def encode_latents(self, latents, vae, reorg_token=False): + """ + Encode latents by first converting to images, then encoding. + This is the main function that replaces sigclip_vision_encode. + + Args: + latents: Input latent tensors + vae: VAE model for decoding latents to images + + Returns: + Encoded image features + """ + # Convert latents to images + images = self.encode_latents_to_images(latents, vae, reorg_token) + + # Encode images + outputs = self.encode_images(images) + + return outputs.last_hidden_state + + def forward(self, images): + """ + Forward pass for direct image encoding. + + Args: + images: Input images + + Returns: + VisionEncoderModelOutput with encoded features + """ + return self.encode_images(images) diff --git a/hyvideo/pipelines/hunyuan_video_sr_pipeline.py b/hyvideo/pipelines/hunyuan_video_sr_pipeline.py new file mode 100644 index 0000000000000000000000000000000000000000..aec3f0230c47128b4ae333df9a8e5216b3d71c58 --- /dev/null +++ b/hyvideo/pipelines/hunyuan_video_sr_pipeline.py @@ -0,0 +1,550 @@ +# Licensed under the TENCENT HUNYUAN COMMUNITY LICENSE AGREEMENT (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://github.com/Tencent-Hunyuan/HunyuanVideo-1.5/blob/main/LICENSE +# +# Unless and only to the extent required by applicable law, the Tencent Hunyuan works and any +# output and results therefrom are provided "AS IS" without any express or implied warranties of +# any kind including any warranties of title, merchantability, noninfringement, course of dealing, +# usage of trade, or fitness for a particular purpose. You are solely responsible for determining the +# appropriateness of using, reproducing, modifying, performing, displaying or distributing any of +# the Tencent Hunyuan works or outputs and assume any and all risks associated with your or a +# third party's use or distribution of any of the Tencent Hunyuan works or outputs and your exercise +# of rights and permissions under this agreement. +# See the License for the specific language governing permissions and limitations under the License. + +from dataclasses import dataclass +from typing import List, Optional, Union, Dict, Any + +import numpy as np +import torch +from einops import rearrange +from PIL import Image +from torch.nn import functional as F + +from diffusers.models import AutoencoderKL +from diffusers.schedulers import KarrasDiffusionSchedulers +from diffusers.utils import BaseOutput + +from hyvideo.commons.parallel_states import get_parallel_state +from hyvideo.commons import auto_offload_model, get_rank +from hyvideo.models.text_encoders import TextEncoder +from hyvideo.models.transformers.worldplay_1_5_transformer import ( + HunyuanVideo_1_5_DiffusionTransformer, +) +from hyvideo.models.transformers.modules.upsample import SRTo720pUpsampler +from hyvideo.utils.data_utils import generate_crop_size_list + +from .worldplay_video_pipeline import HunyuanVideo_1_5_Pipeline +from .pipeline_utils import rescale_noise_cfg, retrieve_timesteps + + +def expand_dims(tensor: torch.Tensor, ndim: int): + shape = tensor.shape + (1,) * (ndim - tensor.ndim) + return tensor.reshape(shape) + + +class BucketMap: + """Maps low-resolution bucket sizes to corresponding high-resolution bucket sizes.""" + + def __init__(self, lr_base_size, hr_base_size, lr_patch_size, hr_patch_size): + self.lr_buckets = generate_crop_size_list( + base_size=lr_base_size, patch_size=lr_patch_size + ) + self.hr_buckets = generate_crop_size_list( + base_size=hr_base_size, patch_size=hr_patch_size + ) + + self.lr_aspect_ratios = np.array( + [float(w) / float(h) for w, h in self.lr_buckets] + ) + self.hr_aspect_ratios = np.array( + [float(w) / float(h) for w, h in self.hr_buckets] + ) + + self.hr_bucket_map = {} + for i, (lr_w, lr_h) in enumerate(self.lr_buckets): + lr_ratio = self.lr_aspect_ratios[i] + closest_hr_ratio_id = np.abs(self.hr_aspect_ratios - lr_ratio).argmin() + self.hr_bucket_map[(lr_w, lr_h)] = self.hr_buckets[closest_hr_ratio_id] + + def __call__(self, lr_bucket): + """ + Args: + lr_bucket (tuple): Low-resolution bucket size as (width, height). + + Returns: + tuple: High-resolution bucket size as (width, height). + """ + if lr_bucket not in self.hr_bucket_map: + raise ValueError(f"LR bucket {lr_bucket} not found in bucket map") + return self.hr_bucket_map[lr_bucket] + + +SizeMap = { + "480p": 640, + "720p": 960, + "1080p": 1440, +} + + +@dataclass +class HunyuanVideo_1_5_SR_PipelineOutput(BaseOutput): + videos: Union[torch.Tensor, np.ndarray] + + +class HunyuanVideo_1_5_SR_Pipeline(HunyuanVideo_1_5_Pipeline): + + def __init__( + self, + vae: AutoencoderKL, + text_encoder: TextEncoder, + transformer: HunyuanVideo_1_5_DiffusionTransformer, + scheduler: KarrasDiffusionSchedulers, + upsampler: SRTo720pUpsampler, + flow_shift: float = 6.0, + guidance_scale: float = 6.0, + num_inference_steps: int = 30, + embedded_guidance_scale: Optional[float] = None, + base_resolution: str = "480p", + text_encoder_2: Optional[TextEncoder] = None, + progress_bar_config: Dict[str, Any] = None, + vision_num_semantic_tokens=729, + vision_states_dim=1152, + glyph_byT5_v2=True, + byt5_model=None, + byt5_tokenizer=None, + byt5_max_length=256, + prompt_format=None, + execution_device=None, + vision_encoder=None, + enable_offloading=False, + ): + super().__init__( + vae=vae, + text_encoder=text_encoder, + transformer=transformer, + scheduler=scheduler, + text_encoder_2=text_encoder_2, + progress_bar_config=progress_bar_config, + vision_num_semantic_tokens=vision_num_semantic_tokens, + vision_states_dim=vision_states_dim, + glyph_byT5_v2=glyph_byT5_v2, + byt5_model=byt5_model, + byt5_tokenizer=byt5_tokenizer, + byt5_max_length=byt5_max_length, + prompt_format=prompt_format, + execution_device=execution_device, + vision_encoder=vision_encoder, + enable_offloading=enable_offloading, + ) + assert upsampler is not None + assert base_resolution is not None + self.register_modules(upsampler=upsampler) + self.register_to_config( + base_resolution=base_resolution, + flow_shift=flow_shift, + guidance_scale=guidance_scale, + embedded_guidance_scale=embedded_guidance_scale, + num_inference_steps=num_inference_steps, + ) + + def add_noise_to_lq(self, lq_latents, strength=0.7): + noise = torch.randn_like(lq_latents) + timestep = torch.tensor([1000.0], device=self.execution_device) * strength + t = expand_dims(timestep, lq_latents.ndim) + return (1 - t / 1000.0) * lq_latents + (t / 1000.0) * noise + + def _prepare_lq_cond_latents(self, lq_latents): + """ + Prepare conditional latents and mask for multitask training. + + Args: + lq_latents: Low-resolution latent tensor. + + Returns: + torch.Tensor: Low-resolution conditional latent tensor. + """ + b, _, f, h, w = lq_latents.shape + mask_ones = torch.ones(b, 1, f, h, w).to(lq_latents.device) + cond_latents = torch.concat([lq_latents, mask_ones], dim=1) + + return cond_latents + + @torch.no_grad() + def __call__( + self, + prompt: Union[str, List[str]], + video_length: int, + num_inference_steps: Optional[int] = None, + guidance_scale: Optional[float] = None, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_videos_per_prompt: Optional[int] = 1, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + seed: Optional[int] = None, + embedded_guidance_scale: Optional[float] = None, + reference_image=None, + lq_latents=None, + output_type: Optional[str] = "pt", + return_dict: bool = True, + **kwargs, + ): + r""" + Runs the super-resolution (SR) pipeline for video generation. + + Args: + prompt (`str` or `List[str]`): + Text prompt(s) that describe the desired video content. + video_length (`int`): + Number of frames in the video to generate. + num_inference_steps (`int`, *optional*, defaults to value in config): + Denoising steps during SR. Higher values may improve visual quality at the cost of slower inference. + guidance_scale (`float`, *optional*, defaults to value in config): + How closely to follow the prompt. `guidance_scale > 1` enables classifier-free guidance. + negative_prompt (`str` or `List[str]`, *optional*): + Prompt(s) of what should not appear in the generated video. + num_videos_per_prompt (`int`, *optional*, defaults to 1): + Number of videos to generate per prompt. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + PyTorch random generator(s) for deterministic and reproducible results. + seed (`int`, *optional*): + If specified, used to construct a generator for reproducibility. + embedded_guidance_scale (`float`, *optional*): + Additional guidance scale for enhanced control, if model supports it. + reference_image (PIL.Image or `str`, *optional*): + Reference image for image-to-video (i2v) tasks. Can be a PIL image or local file path. + Set to `None` for text-to-video (t2v) mode. + lq_latents (`torch.Tensor`, *optional*): + Low-quality (LQ) video latents to use as input for SR upsampling step. + Should have shape (B, C, F, H, W). + output_type (`str`, *optional*, defaults to "pt"): + Output format, either `"pt"` (PyTorch tensor) or `"np"` (NumPy array). + return_dict (`bool`, *optional*, defaults to True): + Whether to return a [`HunyuanVideo_1_5_SR_PipelineOutput`] or a tuple. + **kwargs: + Additional keyword arguments. + """ + target_resolution = self.ideal_resolution + + if guidance_scale is None: + guidance_scale = self.config.guidance_scale + if embedded_guidance_scale is None: + embedded_guidance_scale = self.config.embedded_guidance_scale + if num_inference_steps is None: + num_inference_steps = self.config.num_inference_steps + + if reference_image is not None: + task_type = "i2v" + if isinstance(reference_image, str): + reference_image = Image.open(reference_image).convert("RGB") + elif not isinstance(reference_image, Image.Image): + raise ValueError( + "reference_image must be a PIL Image or path to image file" + ) + semantic_images_np = np.array(reference_image) + else: + task_type = "t2v" + semantic_images_np = None + + self.scheduler = self._create_scheduler(self.config.flow_shift) + + if get_parallel_state().sp_enabled: + assert seed is not None + if generator is None and seed is not None: + generator = torch.Generator(device=self.execution_device).manual_seed(seed) + + sr_stride = 16 + base_size = SizeMap[self.config.base_resolution] + sr_size = SizeMap[self.ideal_resolution] + bucket_map = BucketMap( + lr_base_size=base_size, + hr_base_size=sr_size, + lr_patch_size=16, + hr_patch_size=sr_stride, + ) + lr_video_height, lr_video_width = [x * 16 for x in lq_latents.shape[-2:]] + width, height = bucket_map((lr_video_width, lr_video_height)) + + latent_target_length, latent_height, latent_width = self.get_latent_size( + video_length, height, width + ) + n_tokens = latent_target_length * latent_height * latent_width + + self._guidance_scale = guidance_scale + self._guidance_rescale = kwargs.get("guidance_rescale", 0.0) + self._clip_skip = kwargs.get("clip_skip", None) + + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = 1 + device = self.execution_device + + if get_rank() == 0: + print( + "\n" + f"{'=' * 60}\n" + f"🎬 HunyuanVideo SR Generation Task\n" + f"{'-' * 60}\n" + f"Prompt: {prompt}\n" + f"Video Length: {video_length}\n" + f"Reference Image: {reference_image}\n" + f"Guidance Scale: {guidance_scale}\n" + f"Guidance Embedded Scale: {embedded_guidance_scale}\n" + f"Shift: {self.config.flow_shift}\n" + f"Seed: {seed}\n" + f"Video Resolution: {width} x {height}\n" + f"Transformer dtype: {self.transformer.dtype}\n" + f"Sampling Steps: {num_inference_steps}\n" + f"Use Meanflow: {self.use_meanflow}\n" + f"{'=' * 60}\n" + ) + + with auto_offload_model( + self.text_encoder, self.execution_device, enabled=self.enable_offloading + ): + ( + prompt_embeds, + negative_prompt_embeds, + prompt_mask, + negative_prompt_mask, + ) = self.encode_prompt( + prompt, + device, + num_videos_per_prompt, + self.do_classifier_free_guidance, + negative_prompt, + clip_skip=self.clip_skip, + data_type="video", + ) + + extra_kwargs = {} + if self.config.glyph_byT5_v2: + with auto_offload_model( + self.byt5_model, self.execution_device, enabled=self.enable_offloading + ): + extra_kwargs = self._prepare_byt5_embeddings(prompt, device) + + if self.do_classifier_free_guidance: + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds]) + if prompt_mask is not None: + prompt_mask = torch.cat([negative_prompt_mask, prompt_mask]) + + extra_set_timesteps_kwargs = self.prepare_extra_func_kwargs( + self.scheduler.set_timesteps, {"n_tokens": n_tokens} + ) + + timesteps, num_inference_steps = retrieve_timesteps( + self.scheduler, + num_inference_steps, + device, + **extra_set_timesteps_kwargs, + ) + + latents = self.prepare_latents( + batch_size * num_videos_per_prompt, + 32, + latent_height, + latent_width, + latent_target_length, + self.target_dtype, + device, + generator, + ) + + with auto_offload_model( + self.vae, self.execution_device, enabled=self.enable_offloading + ): + image_cond = self.get_image_condition_latents( + task_type, reference_image, height, width + ) + + tgt_shape = latents.shape[-2:] # (h w) + bsz = lq_latents.shape[0] + lq_latents = rearrange(lq_latents, "b c f h w -> (b f) c h w") + lq_latents = F.interpolate( + lq_latents, size=tgt_shape, mode="bilinear", align_corners=False + ) + lq_latents = rearrange(lq_latents, "(b f) c h w -> b c f h w", b=bsz) + with auto_offload_model( + self.upsampler, self.execution_device, enabled=self.enable_offloading + ): + lq_latents = self.upsampler( + lq_latents.to(dtype=torch.float32, device=self.execution_device) + ) + lq_latents = lq_latents.to(dtype=latents.dtype) + + noise_scale = 0.7 + lq_latents = self.add_noise_to_lq(lq_latents, noise_scale) + + multitask_mask = self.get_task_mask(task_type, latent_target_length) + cond_latents = self._prepare_cond_latents( + task_type, image_cond, latents, multitask_mask + ) + lq_cond_latents = self._prepare_lq_cond_latents(lq_latents) + + condition = torch.concat([cond_latents, lq_cond_latents], dim=1) + + c = lq_latents.shape[1] + zero_lq_condition = condition.clone() + zero_lq_condition[:, c + 1 : 2 * c + 1] = torch.zeros_like(lq_latents) + zero_lq_condition[:, 2 * c + 1] = 0 + + with auto_offload_model( + self.vision_encoder, self.execution_device, enabled=self.enable_offloading + ): + vision_states = self._prepare_vision_states( + semantic_images_np, target_resolution, latents, device + ) + + extra_step_kwargs = self.prepare_extra_func_kwargs( + self.scheduler.step, + {"generator": generator, "eta": kwargs.get("eta", 0.0)}, + ) + + self._num_timesteps = len(timesteps) + num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order + + with ( + self.progress_bar(total=num_inference_steps) as progress_bar, + auto_offload_model( + self.transformer, self.execution_device, enabled=self.enable_offloading + ), + ): + for i, t in enumerate(timesteps): + if t < 1000 * noise_scale: + condition = zero_lq_condition + + latents_concat = torch.concat([latents, condition], dim=1) + if self.do_classifier_free_guidance: + latent_model_input = torch.cat([latents_concat] * 2) + else: + latent_model_input = latents_concat + + latent_model_input = self.scheduler.scale_model_input( + latent_model_input, t + ) + + t_expand = t.repeat( + latent_model_input.shape[0] * latent_model_input.shape[2] + ) + t_txt_expand = t.repeat(latent_model_input.shape[0]) + if not self.use_meanflow: + timesteps_r = None + else: + if i == len(timesteps) - 1: + timesteps_r = torch.tensor([0.0], device=self.execution_device) + else: + timesteps_r = timesteps[i + 1] + timesteps_r = timesteps_r.repeat(latent_model_input.shape[0]) + + guidance_expand = ( + torch.tensor( + [embedded_guidance_scale] * latent_model_input.shape[0], + dtype=torch.float32, + device=device, + ).to(self.target_dtype) + * 1000.0 + if embedded_guidance_scale is not None + else None + ) + + with torch.autocast( + device_type="cuda", + dtype=self.target_dtype, + enabled=self.autocast_enabled, + ): + output = self.transformer( + bi_inference=True, + ar_txt_inference=False, + ar_vision_inference=False, + hidden_states=latent_model_input, + timestep=t_expand, + timestep_txt=t_txt_expand, + text_states=prompt_embeds, + text_states_2=None, + encoder_attention_mask=prompt_mask, + timestep_r=timesteps_r, + vision_states=vision_states, + mask_type=task_type, + guidance=guidance_expand, + return_dict=False, + extra_kwargs=extra_kwargs, + ) + noise_pred = output[0] + + if self.do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + self.guidance_scale * ( + noise_pred_text - noise_pred_uncond + ) + + if self.guidance_rescale > 0.0 and self.do_classifier_free_guidance: + noise_pred = rescale_noise_cfg( + noise_pred, + noise_pred_text, + guidance_rescale=self.guidance_rescale, + ) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step( + noise_pred, t, latents, **extra_step_kwargs, return_dict=False + )[0] + + # Update progress bar + if i == len(timesteps) - 1 or ( + (i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0 + ): + if progress_bar is not None: + progress_bar.update() + + if output_type == "latent": + video_frames = latents + else: + if len(latents.shape) == 4: + latents = latents.unsqueeze(2) + elif len(latents.shape) != 5: + raise ValueError( + f"Only support latents with shape (b, c, h, w) or (b, c, f, h, w), but got {latents.shape}." + ) + + if ( + hasattr(self.vae.config, "shift_factor") + and self.vae.config.shift_factor + ): + latents = ( + latents / self.vae.config.scaling_factor + + self.vae.config.shift_factor + ) + else: + latents = latents / self.vae.config.scaling_factor + + if hasattr(self.vae, "enable_tile_parallelism"): + self.vae.enable_tile_parallelism() + with ( + torch.autocast( + device_type="cuda", + dtype=self.vae_dtype, + enabled=self.vae_autocast_enabled, + ), + auto_offload_model( + self.vae, self.execution_device, enabled=self.enable_offloading + ), + ): + video_frames = self.vae.decode( + latents, return_dict=False, generator=generator + )[0] + + if video_frames is not None: + video_frames = (video_frames / 2 + 0.5).clamp(0, 1).cpu().float() + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + return video_frames + + return HunyuanVideo_1_5_SR_PipelineOutput(videos=video_frames) diff --git a/hyvideo/pipelines/pipeline_utils.py b/hyvideo/pipelines/pipeline_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..6e3cf5b4ca18727af89a6f81f4a5abbb96e27e6d --- /dev/null +++ b/hyvideo/pipelines/pipeline_utils.py @@ -0,0 +1,102 @@ +# Licensed under the TENCENT HUNYUAN COMMUNITY LICENSE AGREEMENT (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://github.com/Tencent-Hunyuan/HunyuanVideo-1.5/blob/main/LICENSE +# +# Unless and only to the extent required by applicable law, the Tencent Hunyuan works and any +# output and results therefrom are provided "AS IS" without any express or implied warranties of +# any kind including any warranties of title, merchantability, noninfringement, course of dealing, +# usage of trade, or fitness for a particular purpose. You are solely responsible for determining the +# appropriateness of using, reproducing, modifying, performing, displaying or distributing any of +# the Tencent Hunyuan works or outputs and assume any and all risks associated with your or a +# third party's use or distribution of any of the Tencent Hunyuan works or outputs and your exercise +# of rights and permissions under this agreement. +# See the License for the specific language governing permissions and limitations under the License. + +from typing import Optional, Union, List +import inspect +import torch + + +def retrieve_timesteps( + scheduler, + num_inference_steps: Optional[int] = None, + device: Optional[Union[str, torch.device]] = None, + timesteps: Optional[List[int]] = None, + sigmas: Optional[List[float]] = None, + **kwargs, +): + """ + Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles + custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`. + + Args: + scheduler (`SchedulerMixin`): + The scheduler to get timesteps from. + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. If used, `timesteps` + must be `None`. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + timesteps (`List[int]`, *optional*): + Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, + `num_inference_steps` and `sigmas` must be `None`. + sigmas (`List[float]`, *optional*): + Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, + `num_inference_steps` and `timesteps` must be `None`. + + Returns: + `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + second element is the number of inference steps. + """ + if timesteps is not None and sigmas is not None: + raise ValueError( + "Only one of `timesteps` or `sigmas` can be passed. Please choose one to set custom values" + ) + if timesteps is not None: + accepts_timesteps = "timesteps" in set( + inspect.signature(scheduler.set_timesteps).parameters.keys() + ) + if not accepts_timesteps: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" timestep schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + elif sigmas is not None: + accept_sigmas = "sigmas" in set( + inspect.signature(scheduler.set_timesteps).parameters.keys() + ) + if not accept_sigmas: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" sigmas schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + else: + scheduler.set_timesteps(num_inference_steps, device=device, **kwargs) + timesteps = scheduler.timesteps + return timesteps, num_inference_steps + + +def rescale_noise_cfg(noise_cfg, noise_pred_text, guidance_rescale=0.0): + """ + Rescale `noise_cfg` according to `guidance_rescale`. Based on findings of [Common Diffusion Noise Schedules and + Sample Steps are Flawed](https://arxiv.org/pdf/2305.08891.pdf). See Section 3.4 + """ + std_text = noise_pred_text.std( + dim=list(range(1, noise_pred_text.ndim)), keepdim=True + ) + std_cfg = noise_cfg.std(dim=list(range(1, noise_cfg.ndim)), keepdim=True) + # rescale the results from guidance (fixes overexposure) + noise_pred_rescaled = noise_cfg * (std_text / std_cfg) + # mix with the original results from guidance by factor guidance_rescale to avoid "plain looking" images + noise_cfg = ( + guidance_rescale * noise_pred_rescaled + (1 - guidance_rescale) * noise_cfg + ) + return noise_cfg diff --git a/hyvideo/pipelines/worldplay_video_pipeline.py b/hyvideo/pipelines/worldplay_video_pipeline.py new file mode 100644 index 0000000000000000000000000000000000000000..e5c12616b6809e5341bfb81d624629b1d9c39b02 --- /dev/null +++ b/hyvideo/pipelines/worldplay_video_pipeline.py @@ -0,0 +1,2704 @@ +# Licensed under the TENCENT HUNYUAN COMMUNITY LICENSE AGREEMENT (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://github.com/Tencent-Hunyuan/HunyuanVideo-1.5/blob/main/LICENSE +# +# Unless and only to the extent required by applicable law, the Tencent Hunyuan works and any +# output and results therefrom are provided "AS IS" without any express or implied warranties of +# any kind including any warranties of title, merchantability, noninfringement, course of dealing, +# usage of trade, or fitness for a particular purpose. You are solely responsible for determining the +# appropriateness of using, reproducing, modifying, performing, displaying or distributing any of +# the Tencent Hunyuan works or outputs and assume any and all risks associated with your or a +# third party's use or distribution of any of the Tencent Hunyuan works or outputs and your exercise +# of rights and permissions under this agreement. +# See the License for the specific language governing permissions and limitations under the License. + + +import inspect +import os +import random +import re +import time +from contextlib import contextmanager +from dataclasses import dataclass +from typing import Any, Dict, List, Optional, Union + +import loguru + +import numpy as np +import torch +from einops import rearrange +from PIL import Image +import torchvision.transforms as transforms +from torch import distributed as dist +from einops import repeat + +from diffusers.configuration_utils import FrozenDict +from diffusers.image_processor import VaeImageProcessor +from diffusers.models import AutoencoderKL +from diffusers.pipelines.pipeline_utils import DiffusionPipeline +from diffusers.schedulers import KarrasDiffusionSchedulers +from diffusers.utils import BaseOutput, deprecate, logging +from diffusers.utils.torch_utils import randn_tensor + +from hyvideo.commons import ( + PIPELINE_CONFIGS, + SR_PIPELINE_CONFIGS, + TRANSFORMER_VERSION_TO_SR_VERSION, + auto_offload_model, + get_gpu_memory, + get_rank, + is_flash3_available, + is_sparse_attn_supported, +) +from hyvideo.commons.parallel_states import get_parallel_state + +from hyvideo.commons.infer_state import get_infer_state + +from hyvideo.models.autoencoders import hunyuanvideo_15_vae_w_cache +from hyvideo.models.text_encoders import PROMPT_TEMPLATE, TextEncoder +from hyvideo.models.text_encoders.byT5 import load_glyph_byT5_v2 +from hyvideo.models.text_encoders.byT5.format_prompt import MultilingualPromptFormat + +from hyvideo.models.transformers.worldplay_1_5_transformer import ( + HunyuanVideo_1_5_DiffusionTransformer, +) +from hyvideo.models.transformers.modules.upsample import ( + SRTo720pUpsampler, + SRTo1080pUpsampler, +) +from hyvideo.models.vision_encoder import VisionEncoder + +from hyvideo.schedulers.scheduling_flow_match_discrete import FlowMatchDiscreteScheduler + +from hyvideo.utils.data_utils import ( + generate_crop_size_list, + get_closest_ratio, + resize_and_center_crop, +) +from hyvideo.utils.multitask_utils import ( + merge_tensor_by_mask, +) +from hyvideo.utils.retrieval_context import ( + generate_points_in_sphere, + select_aligned_memory_frames, +) + + +from .pipeline_utils import retrieve_timesteps, rescale_noise_cfg + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +@dataclass +class HunyuanVideoPipelineOutput(BaseOutput): + videos: Union[torch.Tensor, np.ndarray] + sr_videos: Optional[Union[torch.Tensor, np.ndarray]] = None + + +class HunyuanVideo_1_5_Pipeline(DiffusionPipeline): + + model_cpu_offload_seq = "text_encoder->text_encoder_2->byt5_model->transformer->vae" + _optional_components = ["text_encoder_2"] + + def __init__( + self, + vae: AutoencoderKL, + text_encoder: TextEncoder, + transformer: HunyuanVideo_1_5_DiffusionTransformer, + scheduler: KarrasDiffusionSchedulers, + text_encoder_2: Optional[TextEncoder] = None, + flow_shift: float = 7.0, + guidance_scale: float = 6.0, + embedded_guidance_scale: Optional[float] = None, + progress_bar_config: Dict[str, Any] = None, + vision_num_semantic_tokens=729, + vision_states_dim=1152, + glyph_byT5_v2=True, + byt5_model=None, + byt5_tokenizer=None, + byt5_max_length=256, + prompt_format=None, + execution_device=None, + vision_encoder=None, + enable_offloading=False, + ): + super().__init__() + + self.register_to_config( + glyph_byT5_v2=glyph_byT5_v2, + byt5_max_length=byt5_max_length, + vision_num_semantic_tokens=vision_num_semantic_tokens, + vision_states_dim=vision_states_dim, + flow_shift=flow_shift, + guidance_scale=guidance_scale, + embedded_guidance_scale=embedded_guidance_scale, + ) + + if progress_bar_config is None: + progress_bar_config = {} + if not hasattr(self, "_progress_bar_config"): + self._progress_bar_config = {} + self._progress_bar_config.update(progress_bar_config) + + if glyph_byT5_v2: + self.byt5_max_length = byt5_max_length + self.register_modules( + vae=vae, + text_encoder=text_encoder, + transformer=transformer, + scheduler=scheduler, + text_encoder_2=text_encoder_2, + byt5_model=byt5_model, + byt5_tokenizer=byt5_tokenizer, + ) + self.prompt_format = prompt_format + else: + self.register_modules( + vae=vae, + text_encoder=text_encoder, + transformer=transformer, + scheduler=scheduler, + text_encoder_2=text_encoder_2, + ) + self.byt5_model = None + self.byt5_tokenizer = None + + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor) + self.text_len = text_encoder.max_length + self.target_dtype = torch.bfloat16 + self.vae_dtype = torch.float16 + self.autocast_enabled = True + self.vae_autocast_enabled = True + self.enable_offloading = enable_offloading + self.execution_device = torch.device(execution_device) + + if vision_encoder: + self.register_modules(vision_encoder=vision_encoder) + else: + self.vision_encoder = None + + # Default i2v target size configurations + self.target_size_config = { + "360p": {"bucket_hw_base_size": 480, "bucket_hw_bucket_stride": 16}, + "480p": {"bucket_hw_base_size": 640, "bucket_hw_bucket_stride": 16}, + "720p": {"bucket_hw_base_size": 960, "bucket_hw_bucket_stride": 16}, + "1080p": {"bucket_hw_base_size": 1440, "bucket_hw_bucket_stride": 16}, + } + self._timing_enabled = False + self._timing_records = [] + + def _profile_timing_enabled(self): + return os.environ.get("HY_PROFILE_TIMING", "0").lower() in { + "1", + "true", + "yes", + "on", + } + + def _profile_sync(self): + if torch.cuda.is_available(): + torch.cuda.synchronize(self.execution_device) + + def _record_timing(self, name, elapsed_s, **metadata): + if not self._timing_enabled or get_rank() != 0: + return + record = {"name": name, "elapsed_s": elapsed_s} + record.update(metadata) + self._timing_records.append(record) + + @contextmanager + def _timed(self, name, **metadata): + if not self._timing_enabled: + yield + return + self._profile_sync() + start = time.perf_counter() + try: + yield + finally: + self._profile_sync() + self._record_timing(name, time.perf_counter() - start, **metadata) + + @classmethod + def _create_scheduler(cls, flow_shift): + scheduler = FlowMatchDiscreteScheduler( + shift=flow_shift, + reverse=True, + solver="euler", + ) + return scheduler + + @classmethod + def _load_byt5(cls, cached_folder, glyph_byT5_v2, byt5_max_length, device): + if not glyph_byT5_v2: + byt5_kwargs = None + prompt_format = None + return byt5_kwargs, prompt_format + try: + load_from = os.path.join(cached_folder, "text_encoder") + glyph_root = os.path.join(load_from, "Glyph-SDXL-v2") + if not os.path.exists(glyph_root): + raise RuntimeError( + f"Glyph checkpoint not found from '{glyph_root}'. \n" + "Please download from https://modelscope.cn/models/AI-ModelScope/Glyph-SDXL-v2/files.\n\n" + "- Required files:\n" + " Glyph-SDXL-v2\n" + " ├── assets\n" + " │ ├── color_idx.json\n" + " │ └── multilingual_10-lang_idx.json\n" + " └── checkpoints\n" + " └── byt5_model.pt\n" + ) + + byT5_google_path = os.path.join(load_from, "byt5-small") + if not os.path.exists(byT5_google_path): + loguru.logger.warning( + f"ByT5 google path not found from: {byT5_google_path}. " + f"Try downloading from https://huggingface.co/google/byt5-small." + ) + byT5_google_path = "google/byt5-small" + + multilingual_prompt_format_color_path = os.path.join( + glyph_root, "assets/color_idx.json" + ) + multilingual_prompt_format_font_path = os.path.join( + glyph_root, "assets/multilingual_10-lang_idx.json" + ) + + byt5_args = dict( + byT5_google_path=byT5_google_path, + byT5_ckpt_path=os.path.join(glyph_root, "checkpoints/byt5_model.pt"), + multilingual_prompt_format_color_path=multilingual_prompt_format_color_path, + multilingual_prompt_format_font_path=multilingual_prompt_format_font_path, + byt5_max_length=byt5_max_length, + ) + + byt5_kwargs = load_glyph_byT5_v2(byt5_args, device=device) + prompt_format = MultilingualPromptFormat( + font_path=multilingual_prompt_format_font_path, + color_path=multilingual_prompt_format_color_path, + ) + return byt5_kwargs, prompt_format + except Exception as e: + raise RuntimeError("Error loading byT5 glyph processor") from e + + def encode_prompt( + self, + prompt, + device, + num_videos_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.Tensor] = None, + attention_mask: Optional[torch.Tensor] = None, + negative_prompt_embeds: Optional[torch.Tensor] = None, + negative_attention_mask: Optional[torch.Tensor] = None, + clip_skip: Optional[int] = None, + text_encoder: Optional[TextEncoder] = None, + data_type: Optional[str] = "image", + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + device: (`torch.device`): + torch device + num_videos_per_prompt (`int`): + number of videos that should be generated per prompt + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the video generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + prompt_embeds (`torch.Tensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + attention_mask (`torch.Tensor`, *optional*): + negative_prompt_embeds (`torch.Tensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + negative_attention_mask (`torch.Tensor`, *optional*): + Attention mask for negative prompt embeddings. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + text_encoder (TextEncoder, *optional*): + Text encoder to use. If None, uses the pipeline's default text encoder. + data_type (`str`, *optional*): + Type of data being encoded. Defaults to "image". + """ + if text_encoder is None: + text_encoder = self.text_encoder + + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if prompt_embeds is None: + text_inputs = text_encoder.text2tokens( + prompt, data_type=data_type, max_length=self.text_len + ) + if clip_skip is None: + prompt_outputs = text_encoder.encode( + text_inputs, data_type=data_type, device=device + ) + prompt_embeds = prompt_outputs.hidden_state + else: + prompt_outputs = text_encoder.encode( + text_inputs, + output_hidden_states=True, + data_type=data_type, + device=device, + ) + # Access the `hidden_states` first, that contains a tuple of + # all the hidden states from the encoder layers. Then index into + # the tuple to access the hidden states from the desired layer. + prompt_embeds = prompt_outputs.hidden_states_list[-(clip_skip + 1)] + # We also need to apply the final LayerNorm here to not mess with the + # representations. The `last_hidden_states` that we typically use for + # obtaining the final prompt representations passes through the LayerNorm + # layer. + prompt_embeds = text_encoder.model.text_model.final_layer_norm( + prompt_embeds + ) + + attention_mask = prompt_outputs.attention_mask + if attention_mask is not None: + attention_mask = attention_mask.to(device) + bs_embed, seq_len = attention_mask.shape + attention_mask = attention_mask.repeat(1, num_videos_per_prompt) + attention_mask = attention_mask.view( + bs_embed * num_videos_per_prompt, seq_len + ) + + if text_encoder is not None: + prompt_embeds_dtype = text_encoder.dtype + elif self.transformer is not None: + prompt_embeds_dtype = self.transformer.dtype + else: + prompt_embeds_dtype = prompt_embeds.dtype + + prompt_embeds = prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + if prompt_embeds.ndim == 2: + bs_embed, _ = prompt_embeds.shape + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_videos_per_prompt) + prompt_embeds = prompt_embeds.view(bs_embed * num_videos_per_prompt, -1) + else: + bs_embed, seq_len, _ = prompt_embeds.shape + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_videos_per_prompt, 1) + prompt_embeds = prompt_embeds.view( + bs_embed * num_videos_per_prompt, seq_len, -1 + ) + + # get unconditional embeddings for classifier free guidance + if do_classifier_free_guidance and negative_prompt_embeds is None: + uncond_tokens: List[str] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + + uncond_input = text_encoder.text2tokens( + uncond_tokens, data_type=data_type, max_length=self.text_len + ) + + negative_prompt_outputs = text_encoder.encode( + uncond_input, data_type=data_type, is_uncond=True + ) + negative_prompt_embeds = negative_prompt_outputs.hidden_state + + negative_attention_mask = negative_prompt_outputs.attention_mask + if negative_attention_mask is not None: + negative_attention_mask = negative_attention_mask.to(device) + _, seq_len = negative_attention_mask.shape + negative_attention_mask = negative_attention_mask.repeat( + 1, num_videos_per_prompt + ) + negative_attention_mask = negative_attention_mask.view( + batch_size * num_videos_per_prompt, seq_len + ) + + if do_classifier_free_guidance: + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + + negative_prompt_embeds = negative_prompt_embeds.to( + dtype=prompt_embeds_dtype, device=device + ) + + if negative_prompt_embeds.ndim == 2: + negative_prompt_embeds = negative_prompt_embeds.repeat( + 1, num_videos_per_prompt + ) + negative_prompt_embeds = negative_prompt_embeds.view( + batch_size * num_videos_per_prompt, -1 + ) + else: + negative_prompt_embeds = negative_prompt_embeds.repeat( + 1, num_videos_per_prompt, 1 + ) + negative_prompt_embeds = negative_prompt_embeds.view( + batch_size * num_videos_per_prompt, seq_len, -1 + ) + + return ( + prompt_embeds, + negative_prompt_embeds, + attention_mask, + negative_attention_mask, + ) + + def prepare_extra_func_kwargs(self, func, kwargs): + """ + Prepare extra keyword arguments for scheduler functions. + + Filters kwargs to only include parameters that the function accepts. + This is useful since not all schedulers have the same signature. + """ + extra_step_kwargs = {} + + for k, v in kwargs.items(): + accepts = k in set(inspect.signature(func).parameters.keys()) + if accepts: + extra_step_kwargs[k] = v + return extra_step_kwargs + + def prepare_latents( + self, + batch_size, + num_channels_latents, + latent_height, + latent_width, + video_length, + dtype, + device, + generator, + latents=None, + ): + """ + Prepare latents for video generation. + + Args: + batch_size: Batch size for generation. + num_channels_latents: Number of channels in latent space. + latent_height: Height of latent tensors. + latent_width: Width of latent tensors. + video_length: Number of frames in the video. + dtype: Data type for latents. + device: Target device for latents. + generator: Random number generator. + latents: Pre-computed latents. If None, random latents are generated. + + Returns: + torch.Tensor: Prepared latents tensor. + """ + shape = ( + batch_size, + num_channels_latents, + video_length, + latent_height, + latent_width, + ) + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + if latents is None: + latents = randn_tensor( + shape, generator=generator, device=device, dtype=dtype + ) + else: + latents = latents.to(device) + + # Check existence to make it compatible with FlowMatchEulerDiscreteScheduler + if hasattr(self.scheduler, "init_noise_sigma"): + # scale the initial noise by the standard deviation required by the scheduler + latents = latents * self.scheduler.init_noise_sigma + return latents + + # Copied from diffusers.pipelines.latent_consistency_models. + # pipeline_latent_consistency_text2img.LatentConsistencyModelPipeline.get_guidance_scale_embedding + def get_guidance_scale_embedding( + self, + w: torch.Tensor, + embedding_dim: int = 512, + dtype: torch.dtype = torch.float32, + ) -> torch.Tensor: + """ + See https://github.com/google-research/vdm/blob/dc27b98a554f65cdc654b800da5aa1846545d41b/model_vdm.py#L298 + + Args: + w (`torch.Tensor`): + Generate embedding vectors with a specified guidance scale to subsequently enrich timestep embeddings. + embedding_dim (`int`, *optional*, defaults to 512): + Dimension of the embeddings to generate. + dtype (`torch.dtype`, *optional*, defaults to `torch.float32`): + Data type of the generated embeddings. + + Returns: + `torch.Tensor`: Embedding vectors with shape `(len(w), embedding_dim)`. + """ + assert len(w.shape) == 1 + w = w * 1000.0 + + half_dim = embedding_dim // 2 + emb = torch.log(torch.tensor(10000.0)) / (half_dim - 1) + emb = torch.exp(torch.arange(half_dim, dtype=dtype) * -emb) + emb = w.to(dtype)[:, None] * emb[None, :] + emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=1) + if embedding_dim % 2 == 1: # zero pad + emb = torch.nn.functional.pad(emb, (0, 1)) + assert emb.shape == (w.shape[0], embedding_dim) + return emb + + @property + def guidance_scale(self): + return self._guidance_scale + + @property + def guidance_rescale(self): + return self._guidance_rescale + + @property + def clip_skip(self): + return self._clip_skip + + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + @property + def do_classifier_free_guidance(self): + return self._guidance_scale is not None and self._guidance_scale > 1 + + @property + def cross_attention_kwargs(self): + return self._cross_attention_kwargs + + @property + def num_timesteps(self): + return self._num_timesteps + + @property + def interrupt(self): + return self._interrupt + + @staticmethod + def get_byt5_text_tokens(byt5_tokenizer, byt5_max_length, text_prompt): + """ + Tokenize text prompt for byT5 model. + + Args: + byt5_tokenizer: The byT5 tokenizer. + byt5_max_length: Maximum sequence length for tokenization. + text_prompt: Text prompt string to tokenize. + + Returns: + tuple[torch.Tensor, torch.Tensor]: + - input_ids: Tokenized input IDs. + - attention_mask: Attention mask tensor. + """ + byt5_text_inputs = byt5_tokenizer( + text_prompt, + padding="max_length", + max_length=byt5_max_length, + truncation=True, + add_special_tokens=True, + return_tensors="pt", + ) + + return byt5_text_inputs.input_ids, byt5_text_inputs.attention_mask + + def _extract_glyph_texts(self, prompt): + """ + Extract glyph texts from prompt using regex pattern. + + Args: + prompt: Input prompt string containing quoted text. + + Returns: + List[str]: List of extracted glyph texts (deduplicated if multiple). + """ + pattern = r"\"(.*?)\"|“(.*?)”" + matches = re.findall(pattern, prompt) + result = [match[0] or match[1] for match in matches] + result = list(dict.fromkeys(result)) if len(result) > 1 else result + return result + + def _process_single_byt5_prompt(self, prompt_text, device): + """ + Process a single prompt for byT5 encoding. + + Args: + prompt_text: The prompt text to process. + device: Target device for tensors. + + Returns: + tuple[torch.Tensor, torch.Tensor]: + - byt5_embeddings: Encoded embeddings tensor. + - byt5_mask: Attention mask tensor. + """ + byt5_embeddings = torch.zeros((1, self.byt5_max_length, 1472), device=device) + byt5_mask = torch.zeros( + (1, self.byt5_max_length), device=device, dtype=torch.int64 + ) + + glyph_texts = self._extract_glyph_texts(prompt_text) + + if len(glyph_texts) > 0: + text_styles = [ + {"color": None, "font-family": None} for _ in range(len(glyph_texts)) + ] + formatted_text = self.prompt_format.format_prompt(glyph_texts, text_styles) + + text_ids, text_mask = self.get_byt5_text_tokens( + self.byt5_tokenizer, self.byt5_max_length, formatted_text + ) + text_ids = text_ids.to(device=device) + text_mask = text_mask.to(device=device) + + byt5_outputs = self.byt5_model(text_ids, attention_mask=text_mask.float()) + byt5_embeddings = byt5_outputs[0] + byt5_mask = text_mask + + return byt5_embeddings, byt5_mask + + def _prepare_byt5_embeddings(self, prompts, device): + """ + Prepare byT5 embeddings for both positive and negative prompts. + + Args: + prompts: List of prompt strings or single prompt string. + device: Target device for tensors. + + Returns: + dict: Dictionary containing: + - "byt5_text_states": Combined embeddings tensor. + - "byt5_text_mask": Combined attention mask tensor. + Returns empty dict if glyph_byT5_v2 is disabled. + """ + if not self.config.glyph_byT5_v2: + return {} + + if isinstance(prompts, str): + prompt_list = [prompts] + elif isinstance(prompts, list): + prompt_list = prompts + else: + raise ValueError("prompts must be str or list of str") + + positive_embeddings = [] + positive_masks = [] + negative_embeddings = [] + negative_masks = [] + + for prompt in prompt_list: + pos_emb, pos_mask = self._process_single_byt5_prompt(prompt, device) + positive_embeddings.append(pos_emb) + positive_masks.append(pos_mask) + + if self.do_classifier_free_guidance: + neg_emb, neg_mask = self._process_single_byt5_prompt("", device) + negative_embeddings.append(neg_emb) + negative_masks.append(neg_mask) + + byt5_positive = torch.cat(positive_embeddings, dim=0) + byt5_positive_mask = torch.cat(positive_masks, dim=0) + + if self.do_classifier_free_guidance: + byt5_negative = torch.cat(negative_embeddings, dim=0) + byt5_negative_mask = torch.cat(negative_masks, dim=0) + + byt5_embeddings = torch.cat([byt5_negative, byt5_positive], dim=0) + byt5_masks = torch.cat([byt5_negative_mask, byt5_positive_mask], dim=0) + else: + byt5_embeddings = byt5_positive + byt5_masks = byt5_positive_mask + + return {"byt5_text_states": byt5_embeddings, "byt5_text_mask": byt5_masks} + + def extract_image_features(self, reference_image): + """ + Extract features from a reference image using VisionEncoder. + + Args: + reference_image: numpy array of shape (H, W, 3) with dtype uint8. + + Returns: + VisionEncoderModelOutput: Encoded image features. + """ + assert isinstance(reference_image, np.ndarray) + assert reference_image.ndim == 3 and reference_image.shape[2] == 3 + assert reference_image.dtype == np.uint8 + + image_encoder_output = self.vision_encoder.encode_images(reference_image) + + return image_encoder_output + + def _prepare_vision_states( + self, reference_image, target_resolution, latents, device + ): + """ + Prepare vision states for multitask training. + + Args: + reference_image: Reference image for i2v tasks (None for t2v tasks). + target_resolution: Target size for i2v tasks. + latents: Latent tensors. + device: Target device. + + Returns: + torch.Tensor or None: Vision states tensor or None if vision encoder is unavailable. + """ + if reference_image is None: + vision_states = torch.zeros( + latents.shape[0], + self.config.vision_num_semantic_tokens, + self.config.vision_states_dim, + ).to(latents.device) + else: + reference_image = ( + np.array(reference_image) + if isinstance(reference_image, Image.Image) + else reference_image + ) + if len(reference_image.shape) == 4: + reference_image = reference_image[0] + + height, width = self.get_closest_resolution_given_reference_image( + reference_image, target_resolution + ) + + # Encode reference image to vision states + if self.vision_encoder is not None: + input_image_np = resize_and_center_crop( + reference_image, target_width=width, target_height=height + ) + vision_states = self.vision_encoder.encode_images(input_image_np) + vision_states = vision_states.last_hidden_state.to( + device=device, dtype=self.target_dtype + ) + else: + vision_states = None + + # Repeat image features for batch size if needed (for classifier-free guidance) + if self.do_classifier_free_guidance and vision_states is not None: + vision_states = vision_states.repeat(2, 1, 1) + + return vision_states + + def _prepare_cond_latents(self, task_type, cond_latents, latents, multitask_mask): + """ + Prepare conditional latents and mask for multitask training. + + Args: + task_type: Type of task ("i2v" or "t2v"). + cond_latents: Conditional latents tensor. + latents: Main latents tensor. + multitask_mask: Multitask mask tensor. + + Returns: + tuple[torch.Tensor, torch.Tensor]: + - latents_concat: Concatenated conditional latents. + - mask_concat: Concatenated mask tensor. + """ + latents_concat = None + mask_concat = None + + if cond_latents is not None and task_type == "i2v": + latents_concat = cond_latents.repeat(1, 1, latents.shape[2], 1, 1) + latents_concat[:, :, 1:, :, :] = 0.0 + else: + latents_concat = torch.zeros( + latents.shape[0], + latents.shape[1], + latents.shape[2], + latents.shape[3], + latents.shape[4], + ).to(latents.device) + + mask_zeros = torch.zeros( + latents.shape[0], 1, latents.shape[2], latents.shape[3], latents.shape[4] + ) + mask_ones = torch.ones( + latents.shape[0], 1, latents.shape[2], latents.shape[3], latents.shape[4] + ) + mask_concat = merge_tensor_by_mask( + mask_zeros.cpu(), mask_ones.cpu(), mask=multitask_mask.cpu(), dim=2 + ).to(device=latents.device) + + cond_latents = torch.concat([latents_concat, mask_concat], dim=1) + + return cond_latents + + def get_task_mask(self, task_type, latent_target_length): + if task_type == "t2v": + mask = torch.zeros(latent_target_length) + elif task_type == "i2v": + mask = torch.zeros(latent_target_length) + mask[0] = 1.0 + else: + raise ValueError(f"{task_type} is not supported !") + return mask + + def get_closest_resolution_given_reference_image( + self, reference_image, target_resolution + ): + """ + Get closest supported resolution for a reference image. + + Args: + reference_image: PIL Image or numpy array. + target_resolution: Target resolution string (e.g., "720p", "1080p"). + + Returns: + tuple[int, int]: (height, width) of closest supported resolution. + """ + assert reference_image is not None + + if isinstance(reference_image, Image.Image): + origin_size = reference_image.size + elif isinstance(reference_image, np.ndarray): + H, W, C = reference_image.shape + origin_size = (W, H) + else: + raise ValueError( + f"Unsupported reference_image type: {type(reference_image)}. " + f"Must be PIL Image or numpy array" + ) + + return self.get_closest_resolution_given_original_size( + origin_size, target_resolution + ) + + def get_closest_resolution_given_original_size(self, origin_size, target_size): + """ + Get closest supported resolution for given original size and target resolution. + + Args: + origin_size: Tuple of (width, height) of original image. + target_size: Target resolution string (e.g., "720p", "1080p"). + + Returns: + tuple[int, int]: (height, width) of closest supported resolution. + """ + bucket_hw_base_size = self.target_size_config[target_size][ + "bucket_hw_base_size" + ] + bucket_hw_bucket_stride = self.target_size_config[target_size][ + "bucket_hw_bucket_stride" + ] + + assert bucket_hw_base_size in [ + 128, + 256, + 480, + 512, + 640, + 720, + 960, + 1440, + ], f"bucket_hw_base_size must be in [128, 256, 480, 512, 640, 720, 960, 1440], but got {bucket_hw_base_size}" + + crop_size_list = generate_crop_size_list( + bucket_hw_base_size, bucket_hw_bucket_stride + ) + aspect_ratios = np.array( + [round(float(h) / float(w), 5) for h, w in crop_size_list] + ) + closest_size, closest_ratio = get_closest_ratio( + origin_size[1], origin_size[0], aspect_ratios, crop_size_list + ) + + height = closest_size[0] + width = closest_size[1] + + return height, width + + def get_image_condition_latents(self, task_type, reference_image, height, width): + + if task_type == "t2v": + cond_latents = None + + elif task_type == "i2v": + origin_size = reference_image.size + + target_height, target_width = height, width + original_width, original_height = origin_size + + scale_factor = max( + target_width / original_width, target_height / original_height + ) + resize_width = int(round(original_width * scale_factor)) + resize_height = int(round(original_height * scale_factor)) + + ref_image_transform = transforms.Compose( + [ + transforms.Resize( + (resize_height, resize_width), + interpolation=transforms.InterpolationMode.LANCZOS, + ), + transforms.CenterCrop((target_height, target_width)), + transforms.ToTensor(), + transforms.Normalize([0.5], [0.5]), + ] + ) + + ref_images_pixel_values = ref_image_transform(reference_image) + ref_images_pixel_values = ( + ref_images_pixel_values.unsqueeze(0) + .unsqueeze(2) + .to(self.execution_device) + ) + + with torch.autocast(device_type="cuda", dtype=torch.float16, enabled=True): + cond_latents = self.vae.encode( + ref_images_pixel_values + ).latent_dist.mode() + cond_latents.mul_(self.vae.config.scaling_factor) + + else: + raise ValueError( + f"Unsupported task_type: {task_type}. Must be 't2v' or 'i2v'" + ) + + return cond_latents + + @property + def vae_spatial_compression_ratio(self): + if hasattr(self.vae.config, "ffactor_spatial"): + return self.vae.config.ffactor_spatial + else: + return 16 + + @property + def vae_temporal_compression_ratio(self): + if hasattr(self.vae.config, "ffactor_temporal"): + return self.vae.config.ffactor_temporal + else: + return 4 + + def get_latent_size(self, video_length, height, width): + spatial_compression_ratio = self.vae_spatial_compression_ratio + temporal_compression_ratio = self.vae_temporal_compression_ratio + video_length = (video_length - 1) // temporal_compression_ratio + 1 + height, width = ( + height // spatial_compression_ratio, + width // spatial_compression_ratio, + ) + + assert height > 0 and width > 0 and video_length > 0, ( + f"Invalid dimensions - height: {height}, width: {width}, " + f"video_length: {video_length}" + ) + + return video_length, height, width + + def init_kv_cache(self): + self._kv_cache = [] + self._kv_cache_neg = [] + transformer_num_layers = len(self.transformer.double_blocks) + for i in range(transformer_num_layers): + self._kv_cache.append( + {"k_vision": None, "v_vision": None, "k_txt": None, "v_txt": None} + ) + self._kv_cache_neg.append( + {"k_vision": None, "v_vision": None, "k_txt": None, "v_txt": None} + ) + + def ar_rollout( + self, + latents, + timesteps, + prompt_embeds, + prompt_mask, + vision_states, + cond_latents, + task_type, + extra_kwargs, + viewmats, + Ks, + action, + device, + transformer_resident=False, + predictor_mode="full", + predictor_runtime=None, + ): + # When transformer_resident is True, keep the transformer on GPU for the + # entire AR rollout instead of loading/offloading it per chunk. This does + # not increase peak VRAM (other large modules are already offloaded before + # ar_rollout is called) but removes per-chunk transfer overhead. + if transformer_resident and self.enable_offloading: + with auto_offload_model( + self.transformer, self.execution_device, enabled=True + ): + orig = self.enable_offloading + self.enable_offloading = False + try: + return self._ar_rollout_inner( + latents, timesteps, prompt_embeds, prompt_mask, + vision_states, cond_latents, task_type, extra_kwargs, + viewmats, Ks, action, device, + predictor_mode, predictor_runtime, + ) + finally: + self.enable_offloading = orig + else: + return self._ar_rollout_inner( + latents, timesteps, prompt_embeds, prompt_mask, + vision_states, cond_latents, task_type, extra_kwargs, + viewmats, Ks, action, device, + predictor_mode, predictor_runtime, + ) + + def _ar_rollout_inner( + self, + latents, + timesteps, + prompt_embeds, + prompt_mask, + vision_states, + cond_latents, + task_type, + extra_kwargs, + viewmats, + Ks, + action, + device, + predictor_mode="full", + predictor_runtime=None, + ): + predictor_model = None + if predictor_mode not in {"full", "reuse", "predictor"}: + raise ValueError(f"Unsupported predictor_mode: {predictor_mode!r}") + if predictor_mode != "full" and self.do_classifier_free_guidance: + raise ValueError("Predictor/reuse rollout requires few-step guidance_scale=1") + if predictor_mode == "predictor": + required = {"model"} + missing = required.difference(predictor_runtime or {}) + if missing: + raise ValueError(f"Missing predictor runtime values: {sorted(missing)}") + predictor_model = predictor_runtime["model"] + if not getattr(predictor_model, "uses_context_kv", False): + legacy_required = {"current_txt", "cached_txt", "vec_txt"} + missing = legacy_required.difference(predictor_runtime) + if missing: + raise ValueError( + f"Missing legacy predictor runtime values: {sorted(missing)}" + ) + confidence_head = predictor_runtime.get("confidence_head") + confidence_threshold = predictor_runtime.get("confidence_threshold") + confidence_threshold_by_step = predictor_runtime.get( + "confidence_threshold_by_step" + ) + confidence_target = predictor_runtime.get("confidence_target", "c1") + threshold_modes = sum( + value is not None + for value in (confidence_threshold, confidence_threshold_by_step) + ) + if confidence_head is None and threshold_modes: + raise ValueError( + "confidence thresholds require confidence_head" + ) + if confidence_head is not None and threshold_modes != 1: + raise ValueError( + "confidence_head requires exactly one global or per-step threshold" + ) + else: + confidence_head = None + confidence_threshold = None + confidence_threshold_by_step = None + confidence_target = "c1" + requires_previous_chunk_hidden = bool( + predictor_model is not None + and getattr(predictor_model, "requires_previous_chunk_hidden", False) + ) + force_first_chunk_full = bool( + (predictor_runtime or {}).get("force_first_chunk_full", False) + ) + previous_condition_mode = ( + getattr(predictor_model, "previous_condition_mode", "none") + if predictor_model is not None + else "none" + ) + if predictor_mode == "full": + replace_steps = () + else: + replace_steps = tuple( + sorted(set((predictor_runtime or {}).get("replace_steps", (1, 3)))) + ) + invalid_replace_steps = [ + step + for step in replace_steps + if not isinstance(step, int) + or isinstance(step, bool) + or step <= 0 + or step >= self.num_inference_steps + ] + if invalid_replace_steps: + raise ValueError( + "Predictor/reuse replace_steps must be integer denoising steps in " + f"[1, {self.num_inference_steps - 1}], got {replace_steps}" + ) + if not replace_steps: + raise ValueError("Predictor/reuse replace_steps must not be empty") + condition_hidden_steps = set() + if previous_condition_mode in {"same_step", "both", "both_delta_gated"}: + condition_hidden_steps.update(replace_steps) + if previous_condition_mode in {"previous_step", "both", "both_delta_gated"}: + condition_hidden_steps.update(step - 1 for step in replace_steps) + previous_chunk_hidden_by_step = {} + + with self._timed("ar_init_kv_cache"): + self.init_kv_cache() + positive_idx = 1 if self.do_classifier_free_guidance else 0 + stabilization_level = 15 + # text, siglip, byt5 embedding cache + with self._timed("ar_text_kv_cache"): + with ( + torch.autocast( + device_type="cuda", + dtype=self.target_dtype, + enabled=self.autocast_enabled, + ), + auto_offload_model( + self.transformer, + self.execution_device, + enabled=self.enable_offloading, + ), + ): + extra_kwargs_pos = { + "byt5_text_states": extra_kwargs["byt5_text_states"][ + positive_idx, None, ... + ], + "byt5_text_mask": extra_kwargs["byt5_text_mask"][ + positive_idx, None, ... + ], + } + t_expand_txt = torch.tensor([0]).to(device).to(latents.dtype) + self._kv_cache = self.transformer( + bi_inference=False, + ar_txt_inference=True, + ar_vision_inference=False, + timestep_txt=t_expand_txt, + text_states=prompt_embeds[positive_idx, None, ...], + encoder_attention_mask=prompt_mask[positive_idx, None, ...], + vision_states=vision_states[positive_idx, None, ...], + mask_type=task_type, + extra_kwargs=extra_kwargs_pos, + kv_cache=self._kv_cache, + cache_txt=True, + ) + if self.do_classifier_free_guidance: + extra_kwargs_neg = { + "byt5_text_states": extra_kwargs["byt5_text_states"][0, None, ...], + "byt5_text_mask": extra_kwargs["byt5_text_mask"][0, None, ...], + } + t_expand_txt = torch.tensor([0]).to(device).to(latents.dtype) + self._kv_cache_neg = self.transformer( + bi_inference=False, + ar_txt_inference=True, + ar_vision_inference=False, + timestep_txt=t_expand_txt, + text_states=prompt_embeds[0, None, ...], + encoder_attention_mask=prompt_mask[0, None, ...], + vision_states=vision_states[0, None, ...], + mask_type=task_type, + extra_kwargs=extra_kwargs_neg, + kv_cache=self._kv_cache_neg, + cache_txt=True, + ) + + selected_frame_indices = [] + + for chunk_i in range(self.chunk_num): + current_chunk_hidden_by_step = {} + if self._timing_enabled: + self._profile_sync() + chunk_start = time.perf_counter() + else: + chunk_start = None + if chunk_i > 0: + current_frame_idx = ( + chunk_i * self.chunk_latent_frames + ) # the current frame index to generate + + with self._timed("ar_history_select", chunk=chunk_i): + selected_frame_indices = [] + for chunk_start_idx in range( + current_frame_idx, current_frame_idx + self.chunk_latent_frames, 4 + ): + selected_history_frame_id = select_aligned_memory_frames( + viewmats[0].cpu().detach().numpy(), + chunk_start_idx, + memory_frames=20, + temporal_context_size=12, + pred_latent_size=4, + points_local=self.points_local, + device=device, + ) + selected_frame_indices += selected_history_frame_id + selected_frame_indices = sorted(list(set(selected_frame_indices))) + to_remove = list( + range( + current_frame_idx, + current_frame_idx + self.chunk_latent_frames, + ) + ) + selected_frame_indices = [ + x for x in selected_frame_indices if x not in to_remove + ] + + with self._timed( + "ar_history_prepare", chunk=chunk_i, selected_frames=len(selected_frame_indices) + ): + context_latents = latents[:, :, selected_frame_indices] + context_cond_latents_input = cond_latents[:, :, selected_frame_indices] + context_latents_input = torch.concat( + [context_latents, context_cond_latents_input], dim=1 + ) + + context_viewmats = viewmats[:, selected_frame_indices].to(device) + context_Ks = Ks[:, selected_frame_indices].to(device) + context_action = action[:, selected_frame_indices].to(device) + + context_timestep = torch.full( + (len(selected_frame_indices),), + stabilization_level - 1, + device=device, + dtype=timesteps.dtype, + ) + # compute kv cache + with self._timed( + "ar_history_kv_cache", + chunk=chunk_i, + selected_frames=len(selected_frame_indices), + ): + with ( + torch.autocast( + device_type="cuda", + dtype=self.target_dtype, + enabled=self.autocast_enabled, + ), + auto_offload_model( + self.transformer, + self.execution_device, + enabled=self.enable_offloading, + ), + ): + self._kv_cache = self.transformer( + bi_inference=False, + ar_txt_inference=False, + ar_vision_inference=True, + hidden_states=context_latents_input, + timestep=context_timestep, + timestep_r=None, + mask_type=task_type, + return_dict=False, + viewmats=context_viewmats.to(self.target_dtype), + Ks=context_Ks.to(self.target_dtype), + action=context_action.to(self.target_dtype), + kv_cache=self._kv_cache, + cache_vision=True, + rope_temporal_size=context_latents_input.shape[2], + start_rope_start_idx=0, + context_frame_indices=selected_frame_indices, + ) + if self.do_classifier_free_guidance: + self._kv_cache_neg = self.transformer( + bi_inference=False, + ar_txt_inference=False, + ar_vision_inference=True, + hidden_states=context_latents_input, + timestep=context_timestep, + timestep_r=None, + mask_type=task_type, + return_dict=False, + viewmats=context_viewmats.to(self.target_dtype), + Ks=context_Ks.to(self.target_dtype), + action=context_action.to(self.target_dtype), + kv_cache=self._kv_cache_neg, + cache_vision=True, + rope_temporal_size=context_latents_input.shape[2], + start_rope_start_idx=0, + context_frame_indices=selected_frame_indices, + ) + + with self._timed("ar_chunk_scheduler_timesteps", chunk=chunk_i): + self.scheduler.set_timesteps(self.num_inference_steps, device=device) + + start_idx = chunk_i * self.chunk_latent_frames + end_idx = chunk_i * self.chunk_latent_frames + self.chunk_latent_frames + + with ( + self.progress_bar(total=self.num_inference_steps) as progress_bar, + auto_offload_model( + self.transformer, + self.execution_device, + enabled=self.enable_offloading, + ), + ): + anchor_hidden = None + for i, t in enumerate(timesteps): + timestep_input = torch.full( + (self.chunk_latent_frames,), + t, + device=device, + dtype=timesteps.dtype, + ) + latent_model_input = latents[:, :, start_idx:end_idx] + cond_latents_input = cond_latents[:, :, start_idx:end_idx] + + viewmats_input = viewmats[:, start_idx:end_idx].to(device) + Ks_input = Ks[:, start_idx:end_idx].to(device) + action_input = action[:, start_idx:end_idx].to(device) + + latents_concat = torch.concat( + [latent_model_input, cond_latents_input], dim=1 + ) + latents_concat = self.scheduler.scale_model_input(latents_concat, t) + + use_approximate_step = predictor_mode != "full" and i in replace_steps + if ( + requires_previous_chunk_hidden or force_first_chunk_full + ) and chunk_i == 0: + use_approximate_step = False + step_method = predictor_mode if use_approximate_step else "full" + timing_name = { + "full": "ar_step_transformer", + "reuse": "ar_step_reuse", + "predictor": "ar_step_predictor", + }[step_method] + with self._timed(timing_name, chunk=chunk_i, step=i): + with torch.autocast( + device_type="cuda", + dtype=self.target_dtype, + enabled=self.autocast_enabled, + ): + if step_method == "full": + return_anchor = ( + predictor_mode != "full" and (i + 1) in replace_steps + ) + return_previous_hidden = i in condition_hidden_steps + transformer_output = self.transformer( + bi_inference=False, + ar_txt_inference=False, + ar_vision_inference=True, + hidden_states=latents_concat, + timestep=timestep_input, + timestep_r=None, + mask_type=task_type, + return_dict=False, + viewmats=viewmats_input.to(self.target_dtype), + Ks=Ks_input.to(self.target_dtype), + action=action_input.to(self.target_dtype), + kv_cache=self._kv_cache, + cache_vision=False, + rope_temporal_size=latents_concat.shape[2] + + len(selected_frame_indices), + start_rope_start_idx=len(selected_frame_indices), + return_final_hidden=( + return_anchor or return_previous_hidden + ), + ) + noise_pred = transformer_output[0] + if return_anchor: + anchor_hidden = transformer_output[2] + if return_previous_hidden: + current_chunk_hidden_by_step[i] = ( + transformer_output[2].detach() + ) + else: + if anchor_hidden is None: + raise RuntimeError( + f"Missing Full-DiT anchor before denoising step {i}" + ) + frame_condition = self.transformer.time_in( + timestep_input.reshape(-1) + ) + self.transformer.action_in( + action_input.to(self.target_dtype).reshape(-1) + ) + frame_condition = frame_condition.reshape( + 1, self.chunk_latent_frames, -1 + ) + if step_method == "reuse": + spatial_tokens = ( + latents_concat.shape[-2] * latents_concat.shape[-1] + ) + condition_tokens = frame_condition.repeat_interleave( + spatial_tokens, dim=1 + ).reshape(-1, frame_condition.shape[-1]) + noise_tokens = self.transformer.final_layer( + anchor_hidden, condition_tokens + ) + noise_pred = self.transformer.unpatchify( + noise_tokens, + self.chunk_latent_frames, + latents_concat.shape[-2], + latents_concat.shape[-1], + ) + else: + predictor = predictor_runtime["model"] + predictor_anchor_hidden = anchor_hidden + common_predictor_inputs = { + "target_model_input": latents_concat, + "anchor_hidden": anchor_hidden, + "target_frame_condition": frame_condition, + "target_viewmats": viewmats_input.to(self.target_dtype), + "target_Ks": Ks_input.to(self.target_dtype), + "rope_temporal_size": latents_concat.shape[2] + + len(selected_frame_indices), + "start_rope_start_idx": len(selected_frame_indices), + "anchor_distance": ( + timesteps[i] - timesteps[i - 1] + ).abs().reshape(1), + } + if getattr(predictor, "uses_context_kv", False): + source_blocks = predictor.predictor_config.source_block_ids + previous_chunk_hidden = None + previous_step_chunk_hidden = None + if previous_condition_mode in {"same_step", "both", "both_delta_gated"}: + if i not in previous_chunk_hidden_by_step: + raise RuntimeError( + "Missing same-step previous-chunk hidden for " + f"chunk={chunk_i}, denoise_step={i}" + ) + previous_chunk_hidden = ( + previous_chunk_hidden_by_step[i] + ) + if previous_condition_mode in {"previous_step", "both", "both_delta_gated"}: + previous_step = i - 1 + if previous_step not in previous_chunk_hidden_by_step: + raise RuntimeError( + "Missing previous-step previous-chunk hidden for " + f"chunk={chunk_i}, denoise_step={i}, " + f"previous_step={previous_step}" + ) + previous_step_chunk_hidden = ( + previous_chunk_hidden_by_step[previous_step] + ) + predictor_output = predictor( + **common_predictor_inputs, + context_kv={ + block_id: self._kv_cache[block_id] + for block_id in source_blocks + }, + previous_chunk_hidden=previous_chunk_hidden, + previous_step_chunk_hidden=( + previous_step_chunk_hidden + ), + ) + else: + predictor_output = predictor( + **common_predictor_inputs, + current_txt=predictor_runtime["current_txt"], + cached_txt=predictor_runtime["cached_txt"], + vec_txt=predictor_runtime["vec_txt"], + ) + fallback_to_full = False + confidence_score = None + if confidence_head is not None: + step_confidence_threshold = ( + confidence_threshold_by_step[i] + if confidence_threshold_by_step is not None + else confidence_threshold + ) + pred_hidden_float = predictor_output[ + "pred_hidden" + ].float() + pooled_hidden = torch.nn.functional.layer_norm( + pred_hidden_float, + (pred_hidden_float.shape[-1],), + ).mean(dim=1) + log_residual = ( + ( + pred_hidden_float + - predictor_anchor_hidden.float() + ) + .abs() + .mean((1, 2)) + + 1e-8 + ).log() + confidence_score_tensor = confidence_head( + pooled_hidden, + torch.full( + (pooled_hidden.shape[0],), + i, + device=pooled_hidden.device, + dtype=torch.long, + ), + log_residual, + ) + confidence_score = float( + confidence_score_tensor.mean() + ) + fallback_to_full = bool( + confidence_score + > step_confidence_threshold + ) + decisions = predictor_runtime.get( + "confidence_decisions" + ) + if decisions is not None: + decision = { + "chunk_id": int(chunk_i), + "denoise_step": int(i), + "timestep": float(t), + "confidence_target": confidence_target, + "log_confidence": confidence_score, + "confidence": float( + confidence_score_tensor.exp().mean() + ), + "threshold_log_confidence": float( + step_confidence_threshold + ), + "fallback_to_full": fallback_to_full, + } + if confidence_target == "c1": + decision.update( + log_c1=confidence_score, + c1=decision["confidence"], + threshold_log_c1=float( + step_confidence_threshold + ), + ) + decisions.append(decision) + if fallback_to_full: + return_anchor = (i + 1) in replace_steps + return_previous_hidden = ( + i in condition_hidden_steps + ) + with self._timed( + "ar_step_confidence_fallback_transformer", + chunk=chunk_i, + step=i, + ): + transformer_output = self.transformer( + bi_inference=False, + ar_txt_inference=False, + ar_vision_inference=True, + hidden_states=latents_concat, + timestep=timestep_input, + timestep_r=None, + mask_type=task_type, + return_dict=False, + viewmats=viewmats_input.to( + self.target_dtype + ), + Ks=Ks_input.to(self.target_dtype), + action=action_input.to( + self.target_dtype + ), + kv_cache=self._kv_cache, + cache_vision=False, + rope_temporal_size=( + latents_concat.shape[2] + + len(selected_frame_indices) + ), + start_rope_start_idx=len( + selected_frame_indices + ), + return_final_hidden=( + return_anchor + or return_previous_hidden + ), + ) + noise_pred = transformer_output[0] + if return_anchor: + anchor_hidden = transformer_output[2] + if return_previous_hidden: + current_chunk_hidden_by_step[i] = ( + transformer_output[2].detach() + ) + else: + noise_pred = predictor_output["pred_velocity"] + if i in condition_hidden_steps: + current_chunk_hidden_by_step[i] = ( + predictor_output["pred_hidden"].detach() + ) + # Consecutive Predictor replacements form a chain: + # step N+1 consumes step N's predicted final hidden. + if (i + 1) in replace_steps: + anchor_hidden = predictor_output["pred_hidden"] + gate_stats_collector = predictor_runtime.get( + "gate_stats" + ) + if ( + gate_stats_collector is not None + and "gate_stats" in predictor_output + ): + predictor_diagnostics = { + name: float(value) + for name, value in predictor_output[ + "gate_stats" + ].items() + } + predictor_diagnostics.update( + { + name: float(value) + for name, value in predictor_output.get( + "atc_stats", {} + ).items() + } + ) + gate_stats_collector.append( + { + "chunk_id": int(chunk_i), + "denoise_step": int(i), + "timestep": float(t), + **predictor_diagnostics, + } + ) + if self.do_classifier_free_guidance: + noise_pred_uncond = self.transformer( + bi_inference=False, + ar_txt_inference=False, + ar_vision_inference=True, + hidden_states=latents_concat, + timestep=timestep_input, + timestep_r=None, + mask_type=task_type, + return_dict=False, + viewmats=viewmats_input.to(self.target_dtype), + Ks=Ks_input.to(self.target_dtype), + action=action_input.to(self.target_dtype), + kv_cache=self._kv_cache_neg, + cache_vision=False, + rope_temporal_size=latents_concat.shape[2] + + len(selected_frame_indices), + start_rope_start_idx=len(selected_frame_indices), + )[0] + + if self.do_classifier_free_guidance: + noise_pred = noise_pred_uncond + self.guidance_scale * ( + noise_pred - noise_pred_uncond + ) + + with self._timed("ar_step_scheduler", chunk=chunk_i, step=i): + latent_model_input = self.scheduler.step( + noise_pred, t, latent_model_input, return_dict=False + )[0] + latents[:, :, start_idx:end_idx] = latent_model_input[ + :, :, -self.chunk_latent_frames : + ] + + if i == len(timesteps) - 1 or ( + (i + 1) > self.num_warmup_steps + and (i + 1) % self.scheduler.order == 0 + ): + if progress_bar is not None: + progress_bar.update() + if requires_previous_chunk_hidden: + missing_hidden_steps = condition_hidden_steps.difference( + current_chunk_hidden_by_step + ) + if missing_hidden_steps: + raise RuntimeError( + "Current chunk did not produce previous-hidden states for " + f"steps {sorted(missing_hidden_steps)}" + ) + previous_chunk_hidden_by_step = current_chunk_hidden_by_step + if chunk_start is not None: + self._profile_sync() + self._record_timing( + "ar_chunk_total", + time.perf_counter() - chunk_start, + chunk=chunk_i, + selected_frames=len(selected_frame_indices), + ) + return latents + + def bi_rollout( + self, + latents, + timesteps, + prompt_embeds, + prompt_mask, + vision_states, + cond_latents, + task_type, + extra_kwargs, + viewmats, + Ks, + action, + device, + ): + stabilization_level = 15 + for chunk_i in range(self.chunk_num): + if chunk_i > 0: + # the current frame index to generate (in latent space): 4, 8, 12, 16, 20 + current_frame_idx = chunk_i * self.chunk_latent_frames + + selected_frame_indices = [] + for chunk_start_idx in range( + current_frame_idx, current_frame_idx + self.chunk_latent_frames, 4 + ): + selected_history_frame_id = select_aligned_memory_frames( + viewmats[0].cpu().detach().numpy(), + chunk_start_idx, + memory_frames=20, + temporal_context_size=12, + pred_latent_size=4, + points_local=self.points_local, + device=device, + ) + selected_frame_indices = ( + selected_frame_indices + selected_history_frame_id + ) + selected_frame_indices = sorted(list(set(selected_frame_indices))) + to_remove = list( + range( + current_frame_idx, current_frame_idx + self.chunk_latent_frames + ) + ) + selected_frame_indices = [ + x for x in selected_frame_indices if x not in to_remove + ] + + context_latents = latents[:, :, selected_frame_indices] + context_w2c = viewmats[:, selected_frame_indices] + context_Ks = Ks[:, selected_frame_indices] + context_action = action[:, selected_frame_indices] + + self.scheduler.set_timesteps(self.num_inference_steps, device=device) + + start_idx = chunk_i * self.chunk_latent_frames + end_idx = chunk_i * self.chunk_latent_frames + self.chunk_latent_frames + + with ( + self.progress_bar(total=self.num_inference_steps) as progress_bar, + auto_offload_model( + self.transformer, + self.execution_device, + enabled=self.enable_offloading, + ), + ): + for i, t in enumerate(timesteps): + if chunk_i == 0: + timestep_input = torch.full( + (self.chunk_latent_frames,), + t, + device=device, + dtype=timesteps.dtype, + ) + latent_model_input = latents[:, :, : self.chunk_latent_frames] + cond_latents_input = cond_latents[ + :, :, : self.chunk_latent_frames + ] + else: + t_now = torch.full( + (self.chunk_latent_frames,), + t, + device=device, + dtype=timesteps.dtype, + ) + t_ctx = torch.full( + (len(selected_frame_indices),), + stabilization_level - 1, + device=device, + dtype=timesteps.dtype, + ) + timestep_input = torch.cat([t_ctx, t_now], dim=0) + + latents_model_now = latents[:, :, start_idx:end_idx] + latent_model_input = torch.cat( + [context_latents, latents_model_now], dim=2 + ) + cond_latents_input = cond_latents[ + :, :, : latent_model_input.shape[2] + ] + + viewmats_input = viewmats[:, start_idx:end_idx] + Ks_input = Ks[:, start_idx:end_idx] + action_input = action[:, start_idx:end_idx] + + if chunk_i > 0: + viewmats_input = torch.cat([context_w2c, viewmats_input], dim=1) + Ks_input = torch.cat([context_Ks, Ks_input], dim=1) + action_input = torch.cat([context_action, action_input], dim=1) + + latents_concat = torch.concat( + [latent_model_input, cond_latents_input], dim=1 + ) + if self.do_classifier_free_guidance: + latents_concat = torch.cat([latents_concat] * 2) + latents_concat = self.scheduler.scale_model_input(latents_concat, t) + + batch_size = latents_concat.shape[0] + t_expand_txt = t.repeat(batch_size) + t_expand = timestep_input.repeat(batch_size) + viewmats_input = repeat( + viewmats_input, "B L H W -> (B R) L H W", R=batch_size + ).to(device) + Ks_input = repeat( + Ks_input, "B L H W -> (B R) L H W", R=batch_size + ).to(device) + action_input = ( + repeat(action_input, "B L -> (B R) L", R=batch_size) + .reshape(-1) + .to(device) + ) + + with torch.autocast( + device_type="cuda", + dtype=self.target_dtype, + enabled=self.autocast_enabled, + ): + output = self.transformer( + bi_inference=True, + ar_txt_inference=False, + ar_vision_inference=False, + hidden_states=latents_concat, + timestep=t_expand, + timestep_txt=t_expand_txt, + text_states=prompt_embeds, + text_states_2=None, + encoder_attention_mask=prompt_mask, + timestep_r=None, + vision_states=vision_states, + mask_type=task_type, + guidance=None, + return_dict=False, + extra_kwargs=extra_kwargs, + viewmats=viewmats_input.to(self.target_dtype), + Ks=Ks_input.to(self.target_dtype), + action=action_input.to(self.target_dtype), + ) + noise_pred = output[0] + + if self.do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + self.guidance_scale * ( + noise_pred_text - noise_pred_uncond + ) + + if self.do_classifier_free_guidance and self.guidance_rescale > 0.0: + noise_pred = rescale_noise_cfg( + noise_pred, + noise_pred_text, + guidance_rescale=self.guidance_rescale, + ) + + latent_model_input = self.scheduler.step( + noise_pred, t, latent_model_input, return_dict=False + )[0] + latents[:, :, start_idx:end_idx] = latent_model_input[ + :, :, -self.chunk_latent_frames : + ] + + # Update progress bar + if i == len(timesteps) - 1 or ( + (i + 1) > self.num_warmup_steps + and (i + 1) % self.scheduler.order == 0 + ): + if progress_bar is not None: + progress_bar.update() + return latents + + @torch.no_grad() + def __call__( + self, + prompt: Union[str, List[str]], + aspect_ratio: str, + video_length: int, + prompt_rewrite: bool = True, + num_inference_steps: int = 50, + guidance_scale: Optional[float] = None, + enable_sr: bool = True, + sr_num_inference_steps: Optional[int] = None, + negative_prompt: Optional[Union[str, List[str]]] = None, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + seed: Optional[int] = None, + flow_shift: Optional[float] = None, + embedded_guidance_scale: Optional[float] = None, + reference_image=None, # For i2v tasks: PIL Image or path to image file + output_type: Optional[str] = "pt", + return_dict: bool = True, + return_pre_sr_video: bool = False, + viewmats: Optional[torch.Tensor] = None, + Ks: Optional[torch.Tensor] = None, + action: Optional[torch.Tensor] = None, + few_step: bool = False, + chunk_latent_frames: int = 4, + model_type: str = "ar", + user_height: Optional[int] = None, + user_width: Optional[int] = None, + **kwargs, + ): + r""" + Generates a video (or videos) based on text (and optionally image) conditions. + + Args: + prompt (`str` or `List[str]`): + Text prompt(s) to guide video generation. + aspect_ratio (`str`): + Output video aspect ratio as a string formatted like "720:1280" or "16:9". + Required for text-to-video tasks. + video_length (`int`): + Number of frames in the generated video. + num_inference_steps (`int`, *optional*, defaults to 50): + Number of denoising steps during generation. + Larger values may improve video quality at the expense of slower inference. + guidance_scale (`float`, *optional*, defaults to value in config): + Scale to encourage the model to better follow the prompt. + `guidance_scale > 1` enables classifier-free guidance. + enable_sr (`bool`, *optional*, defaults to True): + Whether to apply super-resolution to the generated video. + sr_num_inference_steps (`int`, *optional*, defaults to 30): + Number of inference steps in the super-resolution module (if enabled). + negative_prompt (`str` or `List[str]`, *optional*): + Negative prompt(s) that describe what should NOT be shown in the generated video. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + PyTorch random generator(s) for deterministic results. + seed (`int`, *optional*): + If specified, used to create the generator for reproducible sampling. + flow_shift (`float`, *optional*): + Flow shift parameter for the scheduler. Overrides the default pipeline configuration if provided. + embedded_guidance_scale (`float`, *optional*): + Additional control guidance scale, if supported. + reference_image (PIL.Image or `str`, *optional*): + Reference image for image-to-video (i2v) tasks. + Can be a PIL image or a path to an image file. Set to `None` for text-to-video (t2v) generation. + output_type (`str`, *optional*, defaults to "pt"): + Output format of the returned video(s). + Accepted values: `"pt"` for torch.Tensor or `"np"` for numpy.ndarray. + return_dict (`bool`, *optional*, defaults to True): + Whether to return a [`HunyuanVideoPipelineOutput`] or a tuple. + **kwargs: + Additional keyword arguments. + + Returns: + HunyuanVideoPipelineOutput or `tuple`: + If `return_dict` is True, returns a [`HunyuanVideoPipelineOutput`] with fields: + - `videos`: Generated video(s) as a tensor or numpy array. + - `sr_videos`: Super-resolved video(s) if `enable_sr` is True, else None. + Otherwise, returns a tuple containing the outputs as above. + + Example: + ```python + pipe = HunyuanVideoPipeline.from_pretrained("your_model_dir") + # Text-to-video + video = pipe( + prompt="A dog surfing on the beach", + aspect_ratio="9:16", + video_length=32 + ).videos + # Image-to-video + video = pipe( + prompt="Make this image move", + reference_image="img.jpg", + aspect_ratio="16:9", + video_length=24).videos + ``` + """ + self._timing_enabled = self._profile_timing_enabled() + self._timing_records = [] + if self._timing_enabled: + self._profile_sync() + pipeline_start = time.perf_counter() + else: + pipeline_start = None + + num_videos_per_prompt = 1 + target_resolution = self.ideal_resolution + + if guidance_scale is None: + guidance_scale = self.config.guidance_scale + if embedded_guidance_scale is None: + embedded_guidance_scale = self.config.embedded_guidance_scale + if flow_shift is None: + flow_shift = self.config.flow_shift + + # enable few step inference + if few_step: + guidance_scale = 1.0 + + if embedded_guidance_scale is not None: + assert not self.do_classifier_free_guidance + assert self.transformer.config.guidance_embed + else: + assert not self.transformer.config.guidance_embed + + user_reference_image = reference_image + user_prompt = prompt + + if reference_image is not None: + task_type = "i2v" + if isinstance(reference_image, str): + reference_image = Image.open(reference_image).convert("RGB") + elif not isinstance(reference_image, Image.Image): + raise ValueError( + "reference_image must be a PIL Image or path to image file" + ) + semantic_images_np = np.array(reference_image) + + else: + task_type = "t2v" + semantic_images_np = None + + if prompt_rewrite: + from hyvideo.utils.rewrite.rewrite_utils import run_prompt_rewrite + + if not dist.is_initialized() or get_parallel_state().sp_rank == 0: + try: + prompt = run_prompt_rewrite(user_prompt, reference_image, task_type) + except Exception as e: + loguru.logger.warning(f"Failed to rewrite prompt: {e}") + prompt = user_prompt + + if dist.is_initialized() and get_parallel_state().sp_enabled: + obj_list = [prompt] + # not use group_src to support old PyTorch + group_src_rank = dist.get_global_rank(get_parallel_state().sp_group, 0) + dist.broadcast_object_list( + obj_list, src=group_src_rank, group=get_parallel_state().sp_group + ) + prompt = obj_list[0] + + if self.ideal_task is not None and self.ideal_task != task_type: + raise ValueError( + f"The loaded pipeline is trained for '{self.ideal_task}' task, " + f"but received input for '{task_type}' task. " + "Please load a pipeline trained for the correct task, " + "or check and update your arguments accordingly." + ) + + if flow_shift is None: + self.scheduler = self._create_scheduler(self.config.flow_shift) + else: + self.scheduler = self._create_scheduler(flow_shift) + + if get_parallel_state().sp_enabled: + assert seed is not None + + if generator is None and seed is not None: + generator = torch.Generator(device=self.execution_device).manual_seed(seed) + + if reference_image is not None: + if ( + self.ideal_resolution is not None + and target_resolution != self.ideal_resolution + ): + raise ValueError( + f"The loaded pipeline is trained for {self.ideal_resolution} resolution, " + f"but received input for {target_resolution} resolution. " + ) + height, width = self.get_closest_resolution_given_reference_image( + reference_image, target_resolution + ) + else: + if self.ideal_resolution is not None: + if ":" not in aspect_ratio: + raise ValueError("aspect_ratio must be separated by a colon") + width, height = aspect_ratio.split(":") + # check if width and height are integers + if ( + not width.isdigit() + or not height.isdigit() + or int(width) <= 0 + or int(height) <= 0 + ): + raise ValueError( + "w and h must be positive and separated by a colon in aspect_ratio" + ) + width = int(width) + height = int(height) + height, width = self.get_closest_resolution_given_original_size( + (width, height), self.ideal_resolution + ) + + height = user_height if user_height is not None else height + width = user_width if user_width is not None else width + + latent_target_length, latent_height, latent_width = self.get_latent_size( + video_length, height, width + ) + n_tokens = latent_target_length * latent_height * latent_width + multitask_mask = self.get_task_mask(task_type, latent_target_length) + + self._guidance_scale = guidance_scale + self._guidance_rescale = kwargs.get("guidance_rescale", 0.0) + self._clip_skip = kwargs.get("clip_skip", None) + + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = 1 + device = self.execution_device + + if get_rank() == 0: + print( + "\n" + f"{'=' * 60}\n" + f"🎬 HunyuanVideo Generation Task\n" + f"{'-' * 60}\n" + f"User Prompt: {user_prompt}\n" + f"Rewritten Prompt: {prompt if prompt_rewrite else ''}\n" + f"Aspect Ratio: {aspect_ratio}\n" + f"Video Length: {video_length}\n" + f"Reference Image: {reference_image}\n" + f"Guidance Scale: {guidance_scale}\n" + f"Guidance Embedded Scale: {embedded_guidance_scale}\n" + f"Shift: {flow_shift}\n" + f"Seed: {seed}\n" + f"Video Resolution: {width} x {height}\n" + f"Attn mode: {self.transformer.config.attn_mode}\n" + f"Transformer dtype: {self.transformer.dtype}\n" + f"Sampling Steps: {num_inference_steps}\n" + f"Use Meanflow: {self.use_meanflow}\n" + f"{'=' * 60}" + "\n" + ) + + with self._timed("text_encoder_1"): + with auto_offload_model( + self.text_encoder, self.execution_device, enabled=self.enable_offloading + ): + ( + prompt_embeds, + negative_prompt_embeds, + prompt_mask, + negative_prompt_mask, + ) = self.encode_prompt( + prompt, + device, + num_videos_per_prompt, + self.do_classifier_free_guidance, + negative_prompt, + clip_skip=self.clip_skip, + data_type="video", + ) + + # Encode prompts with second encoder if available + if self.text_encoder_2 is not None: + with self._timed("text_encoder_2"): + with auto_offload_model( + self.text_encoder_2, + self.execution_device, + enabled=self.enable_offloading, + ): + ( + prompt_embeds_2, + negative_prompt_embeds_2, + prompt_mask_2, + negative_prompt_mask_2, + ) = self.encode_prompt( + prompt, + device, + num_videos_per_prompt, + self.do_classifier_free_guidance, + negative_prompt, + clip_skip=self.clip_skip, + text_encoder=self.text_encoder_2, + data_type="video", + ) + else: + prompt_embeds_2 = None + negative_prompt_embeds_2 = None + prompt_mask_2 = None + negative_prompt_mask_2 = None + + extra_kwargs = {} + if self.config.glyph_byT5_v2: + with self._timed("byt5_encoder"): + with auto_offload_model( + self.byt5_model, self.execution_device, enabled=self.enable_offloading + ): + extra_kwargs = self._prepare_byt5_embeddings(prompt, device) + + if self.do_classifier_free_guidance: + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds]) + if prompt_mask is not None: + prompt_mask = torch.cat([negative_prompt_mask, prompt_mask]) + if prompt_embeds_2 is not None: + prompt_embeds_2 = torch.cat([negative_prompt_embeds_2, prompt_embeds_2]) + if prompt_mask_2 is not None: + prompt_mask_2 = torch.cat([negative_prompt_mask_2, prompt_mask_2]) + + with self._timed("scheduler_timesteps"): + extra_set_timesteps_kwargs = self.prepare_extra_func_kwargs( + self.scheduler.set_timesteps, {"n_tokens": n_tokens} + ) + + timesteps, num_inference_steps = retrieve_timesteps( + self.scheduler, + num_inference_steps, + device, + **extra_set_timesteps_kwargs, + ) + + num_channels_latents = self.transformer.config.in_channels + with self._timed("prepare_latents"): + latents = self.prepare_latents( + batch_size * num_videos_per_prompt, + num_channels_latents, + latent_height, + latent_width, + latent_target_length, + self.target_dtype, + device, + generator, + ) + + with self._timed("vae_encode_image_condition"): + with auto_offload_model( + self.vae, self.execution_device, enabled=self.enable_offloading + ): + image_cond = self.get_image_condition_latents( + task_type, reference_image, height, width + ) + + with self._timed("prepare_cond_latents"): + cond_latents = self._prepare_cond_latents( + task_type, image_cond, latents, multitask_mask + ) + with self._timed("vision_encoder"): + with auto_offload_model( + self.vision_encoder, self.execution_device, enabled=self.enable_offloading + ): + vision_states = self._prepare_vision_states( + semantic_images_np, target_resolution, latents, device + ) + + self.num_warmup_steps = ( + len(timesteps) - num_inference_steps * self.scheduler.order + ) + self._num_timesteps = len(timesteps) + + latent_frames = latents.shape[2] + with self._timed("ar_context_setup"): + self.points_local = generate_points_in_sphere(50000, 8.0).to(device) + self.chunk_num = latent_frames // chunk_latent_frames + self.chunk_latent_frames = chunk_latent_frames + self.num_inference_steps = num_inference_steps + + if model_type == "ar": + with self._timed( + "ar_rollout_total", + chunk_num=self.chunk_num, + chunk_latent_frames=self.chunk_latent_frames, + ): + latents = self.ar_rollout( + latents=latents, + timesteps=timesteps, + prompt_embeds=prompt_embeds, + prompt_mask=prompt_mask, + vision_states=vision_states, + cond_latents=cond_latents, + task_type=task_type, + extra_kwargs=extra_kwargs, + viewmats=viewmats, + Ks=Ks, + action=action, + device=device, + transformer_resident=kwargs.get("transformer_resident_ar_rollout", False), + predictor_mode=kwargs.get("predictor_mode", "full"), + predictor_runtime=kwargs.get("predictor_runtime"), + ) + elif model_type == "bi": + with self._timed("bi_rollout_total", chunk_num=self.chunk_num): + latents = self.bi_rollout( + latents=latents, + timesteps=timesteps, + prompt_embeds=prompt_embeds, + prompt_mask=prompt_mask, + vision_states=vision_states, + cond_latents=cond_latents, + task_type=task_type, + extra_kwargs=extra_kwargs, + viewmats=viewmats, + Ks=Ks, + action=action, + device=device, + ) + + if enable_sr: + assert hasattr(self, "sr_pipeline") + sr_out = self.sr_pipeline( + prompt=prompt, + num_inference_steps=sr_num_inference_steps, + video_length=video_length, + negative_prompt="", + num_videos_per_prompt=num_videos_per_prompt, + seed=seed, + output_type=output_type, + lq_latents=latents, + reference_image=user_reference_image, + ) + + if output_type == "latent": + video_frames = latents + else: + if len(latents.shape) == 4: + latents = latents.unsqueeze(2) + elif len(latents.shape) != 5: + raise ValueError( + f"Only support latents with shape (b, c, h, w) or (b, c, f, h, w), but got {latents.shape}." + ) + + if ( + hasattr(self.vae.config, "shift_factor") + and self.vae.config.shift_factor + ): + latents = ( + latents / self.vae.config.scaling_factor + + self.vae.config.shift_factor + ) + else: + latents = latents / self.vae.config.scaling_factor + + if get_infer_state() and get_infer_state().use_vae_parallel: + self.vae.enable_spatial_tiling() + self.vae.enable_tile_parallelism() + + if return_pre_sr_video or not enable_sr: + with self._timed("vae_decode"): + with ( + torch.autocast( + device_type="cuda", + dtype=self.vae_dtype, + enabled=self.vae_autocast_enabled, + ), + auto_offload_model( + self.vae, self.execution_device, enabled=self.enable_offloading + ), + ): + video_frames = self.vae.decode( + latents, return_dict=False, generator=generator + )[0] + + if video_frames is not None: + with self._timed("video_postprocess_to_cpu"): + video_frames = (video_frames / 2 + 0.5).clamp(0, 1).cpu().float() + + else: + video_frames = sr_out.videos + + # Offload all models + self.maybe_free_model_hooks() + if enable_sr: + sr_video_frames = sr_out.videos + + if pipeline_start is not None: + self._profile_sync() + self._record_timing("pipeline_total", time.perf_counter() - pipeline_start) + + if not return_dict: + ret = video_frames + if enable_sr: + ret = (video_frames, sr_video_frames) + return ret + + if enable_sr: + return HunyuanVideoPipelineOutput( + videos=video_frames, sr_videos=sr_video_frames + ) + else: + return HunyuanVideoPipelineOutput(videos=video_frames) + + @property + def ideal_resolution(self): + return self.transformer.config.ideal_resolution + + @property + def ideal_task(self): + return self.transformer.config.ideal_task + + @property + def use_meanflow(self): + return self.transformer.config.use_meanflow + + @classmethod + def load_sr_transformer_upsampler( + cls, cached_folder, sr_version, transformer_dtype=torch.bfloat16, device=None + ): + sr_path = os.path.join(cached_folder, "transformer", sr_version) + transformer = HunyuanVideo_1_5_DiffusionTransformer.from_pretrained( + sr_path, torch_dtype=transformer_dtype + ).to(device) + upsampler_cls = ( + SRTo720pUpsampler if "720p" in sr_version else SRTo1080pUpsampler + ) + upsampler = upsampler_cls.from_pretrained( + os.path.join(cached_folder, "upsampler", sr_version) + ).to(device) + return transformer, upsampler + + def create_sr_pipeline( + self, cached_folder, sr_version, transformer_dtype=torch.bfloat16, device=None + ): + from .hunyuan_video_sr_pipeline import HunyuanVideo_1_5_SR_Pipeline + + transformer, upsampler = self.load_sr_transformer_upsampler( + cached_folder, + sr_version, + transformer_dtype=transformer_dtype, + device=device, + ) + + return HunyuanVideo_1_5_SR_Pipeline( + vae=self.vae, + transformer=transformer, + text_encoder=self.text_encoder, + scheduler=self.scheduler, + upsampler=upsampler, + text_encoder_2=self.text_encoder_2, + progress_bar_config=None, + glyph_byT5_v2=self.config.glyph_byT5_v2, + byt5_model=self.byt5_model, + byt5_tokenizer=self.byt5_tokenizer, + byt5_max_length=self.byt5_max_length, + prompt_format=self.prompt_format, + execution_device="cuda", + vision_encoder=self.vision_encoder, + enable_offloading=self.enable_offloading, + **SR_PIPELINE_CONFIGS[sr_version], + ) + + @classmethod + def create_pipeline( + cls, + pretrained_model_name_or_path, + transformer_version, + create_sr_pipeline=False, + force_sparse_attn=False, + transformer_dtype=torch.bfloat16, + enable_offloading=None, + enable_group_offloading=None, + overlap_group_offloading=True, + device=None, + action_ckpt=None, + **kwargs, + ): + # use snapshot download here to get it working from from_pretrained + if not os.path.isdir(pretrained_model_name_or_path): + if pretrained_model_name_or_path.count("/") > 1: + raise ValueError( + f'The provided pretrained_model_name_or_path "{pretrained_model_name_or_path}"' + " is neither a valid local path nor a valid repo id. Please check the parameter." + ) + cached_folder = cls.download( + pretrained_model_name_or_path, + **kwargs, + ) + else: + cached_folder = pretrained_model_name_or_path + + if enable_offloading: + # Assuming the user does not have sufficient GPU memory, we initialize the models on CPU + device = torch.device("cpu") + else: + if device is None: + device = torch.device("cuda") + + if enable_group_offloading: + # Assuming the user does not have sufficient GPU memory, we initialize the models on CPU + transformer_init_device = torch.device("cpu") + else: + transformer_init_device = device + + supported_transformer_version = os.listdir( + os.path.join(cached_folder, "transformer") + ) + if transformer_version not in supported_transformer_version: + raise ValueError( + f"Could not find {transformer_version} in {cached_folder}." + f"Only {supported_transformer_version} are available." + ) + + vae_inference_config = cls.get_vae_inference_config() + transformer = HunyuanVideo_1_5_DiffusionTransformer.from_pretrained( + os.path.join(cached_folder, "transformer", transformer_version), + torch_dtype=transformer_dtype, + low_cpu_mem_usage=True, + ) + + transformer.add_action_parameters() + if action_ckpt is not None: + from safetensors.torch import load_file + + safetensor_path = action_ckpt + state_dict = load_file(safetensor_path) + transformer.load_state_dict(state_dict, strict=True) + print("HY-World 1.5 loading from: ", action_ckpt) + + transformer = transformer.to(transformer_dtype).to(transformer_init_device) + + infer_state = get_infer_state() + if infer_state.use_fp8_gemm: + from angelslim.compressor.diffusion import DynamicDiTQuantizer + + quant_type = infer_state.quant_type + include_patterns = infer_state.include_patterns + quantizer = DynamicDiTQuantizer( + quant_type=quant_type, include_patterns=include_patterns + ) + quantizer.convert_linear(transformer) + + vae = hunyuanvideo_15_vae_w_cache.AutoencoderKLConv3D.from_pretrained( + os.path.join(cached_folder, "vae"), + torch_dtype=vae_inference_config["dtype"], + ).to(device) + scheduler = FlowMatchDiscreteScheduler.from_pretrained( + os.path.join(cached_folder, "scheduler") + ) + + if force_sparse_attn: + if not is_sparse_attn_supported(): + raise RuntimeError( + f"Current GPU is {torch.cuda.get_device_properties(0).name}, " + f"which does not support sparse attention." + ) + if transformer.config.attn_mode != "flex-block-attn": + loguru.logger.warning( + f"The transformer loaded ({transformer_version}) is not trained with sparse attention." + f"Forcing to use sparse attention may lead to artifacts in the generated video." + f"To enable sparse attention, we recommend loading `{transformer_version}_distilled_sparse`." + ) + transformer.set_attn_mode("flex-block-attn") + + byt5_kwargs, prompt_format = cls._load_byt5( + cached_folder, True, 256, device=device + ) + text_encoder, text_encoder_2 = cls._load_text_encoders( + cached_folder, device=device + ) + vision_encoder = cls._load_vision_encoder(cached_folder, device=device) + + group_offloading_kwargs = { + "onload_device": torch.device("cuda"), + "num_blocks_per_group": 4, + } + if overlap_group_offloading: + # Using streams is only supported for num_blocks_per_group=1 + group_offloading_kwargs["num_blocks_per_group"] = 1 + group_offloading_kwargs["use_stream"] = True + group_offloading_kwargs["record_stream"] = True + + if enable_group_offloading: + assert enable_offloading + transformer.enable_group_offload(**group_offloading_kwargs) + + pipeline = cls( + vae=vae, + text_encoder=text_encoder, + transformer=transformer, + scheduler=scheduler, + text_encoder_2=text_encoder_2, + progress_bar_config=None, + byt5_model=byt5_kwargs["byt5_model"], + byt5_tokenizer=byt5_kwargs["byt5_tokenizer"], + byt5_max_length=byt5_kwargs["byt5_max_length"], + prompt_format=prompt_format, + execution_device="cuda", + vision_encoder=vision_encoder, + enable_offloading=enable_offloading, + **PIPELINE_CONFIGS[transformer_version], + ) + + if create_sr_pipeline: + sr_version = TRANSFORMER_VERSION_TO_SR_VERSION[transformer_version] + sr_pipeline = pipeline.create_sr_pipeline( + cached_folder, + sr_version, + transformer_dtype=transformer_dtype, + device=device, + ) + pipeline.sr_pipeline = sr_pipeline + if enable_group_offloading: + sr_pipeline.transformer.enable_group_offload(**group_offloading_kwargs) + + return pipeline + + @staticmethod + def get_vae_inference_config(memory_limitation=None): + if memory_limitation is None: + memory_limitation = get_gpu_memory() + GB = 1024 * 1024 * 1024 + if memory_limitation < 23 * GB: + sample_size = 160 + tile_overlap_factor = 0.2 + dtype = torch.float16 + else: + sample_size = 256 + tile_overlap_factor = 0.25 + dtype = torch.float32 + return { + "sample_size": sample_size, + "tile_overlap_factor": tile_overlap_factor, + "dtype": dtype, + } + + @classmethod + def _load_text_encoders(cls, pretrained_model_path, device): + text_encoder_path = f"{pretrained_model_path}/text_encoder/llm" + if not os.path.exists(text_encoder_path): + msg = f"{text_encoder_path} not found. Please refer to checkpoints-download.md." + loguru.logger.error(msg) + raise FileNotFoundError(msg) + text_encoder = TextEncoder( + text_encoder_type="llm", + tokenizer_type="llm", + text_encoder_path=text_encoder_path, + max_length=1000, + text_encoder_precision="fp16", + prompt_template=PROMPT_TEMPLATE["li-dit-encode-image-json"], + prompt_template_video=PROMPT_TEMPLATE["li-dit-encode-video-json"], + hidden_state_skip_layer=2, + apply_final_norm=False, + reproduce=False, + logger=loguru.logger, + device=device, + ) + text_encoder_2 = None + + return text_encoder, text_encoder_2 + + @classmethod + def _load_vision_encoder(cls, pretrained_model_name_or_path, device): + vision_encoder_path = f"{pretrained_model_name_or_path}/vision_encoder/siglip" + if not os.path.exists(vision_encoder_path): + msg = f"{vision_encoder_path} not found. Please refer to checkpoints-download.md." + loguru.logger.error(msg) + raise FileNotFoundError(msg) + vision_encoder = VisionEncoder( + vision_encoder_type="siglip", + vision_encoder_precision="fp16", + vision_encoder_path=vision_encoder_path, + processor_type=None, + processor_path=None, + output_key=None, + logger=logger, + device=device, + ) + return vision_encoder diff --git a/hyvideo/prope/camera_rope.py b/hyvideo/prope/camera_rope.py new file mode 100644 index 0000000000000000000000000000000000000000..68b7b6ad4c92ae4b5d1628de1121be37001e1f66 --- /dev/null +++ b/hyvideo/prope/camera_rope.py @@ -0,0 +1,259 @@ +# MIT License +# +# Copyright (c) Authors of +# "PRoPE: Projective Positional Encoding for Multiview Transformers" +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# How to use PRoPE attention for self-attention: +# +# 1. Easiest way (fast): +# attn = PropeDotProductAttention(...) +# o = attn(q, k, v, viewmats, Ks) +# +# 2. More flexible way (fast): +# attn = PropeDotProductAttention(...) +# attn._precompute_and_cache_apply_fns(viewmats, Ks) +# q = attn._apply_to_q(q) +# k = attn._apply_to_kv(k) +# v = attn._apply_to_kv(v) +# o = F.scaled_dot_product_attention(q, k, v, **kwargs) +# o = attn._apply_to_o(o) +# +# 3. The most flexible way (but slower because repeated computation of RoPE coefficients): +# o = prope_dot_product_attention(q, k, v, ...) +# +# How to use PRoPE attention for cross-attention: +# +# attn_src = PropeDotProductAttention(...) +# attn_tgt = PropeDotProductAttention(...) +# attn_src._precompute_and_cache_apply_fns(viewmats_src, Ks_src) +# attn_tgt._precompute_and_cache_apply_fns(viewmats_tgt, Ks_tgt) +# q_src = attn_src._apply_to_q(q_src) +# k_tgt = attn_tgt._apply_to_kv(k_tgt) +# v_tgt = attn_tgt._apply_to_kv(v_tgt) +# o_src = F.scaled_dot_product_attention(q_src, k_tgt, v_tgt, **kwargs) +# o_src = attn_src._apply_to_o(o_src) + +from functools import partial +from typing import Callable, Optional, Tuple, List + +import torch +import torch.nn.functional as F + + +def prope_qkv( + q: torch.Tensor, # (batch, num_heads, seqlen, head_dim) + k: torch.Tensor, # (batch, num_heads, seqlen, head_dim) + v: torch.Tensor, # (batch, num_heads, seqlen, head_dim) + *, + viewmats: torch.Tensor, # (batch, cameras, 4, 4) + Ks: Optional[torch.Tensor], # (batch, cameras, 3, 3) + patches_x: int = None, # How many patches wide is each image? + patches_y: int = None, # How many patches tall is each image? + image_width: int = None, # Width of the image. Used to normalize intrinsics. + image_height: int = None, # Height of the image. Used to normalize intrinsics. + coeffs_x: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + coeffs_y: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + mask: Optional[torch.Tensor] = None, + kv_cache=None, + is_cache: bool = False, + **kwargs, +) -> torch.Tensor: + """Similar to torch.nn.functional.scaled_dot_product_attention, but applies PRoPE-style + positional encoding. + + Currently, we assume that the sequence length is equal to: + + cameras * patches_x * patches_y + + And token ordering allows the `(seqlen,)` axis to be reshaped into + `(cameras, patches_x, patches_y)`. + """ + # We're going to assume self-attention: all inputs are the same shape. + (batch, num_heads, seqlen, head_dim) = q.shape + cameras = viewmats.shape[1] + assert q.shape == k.shape == v.shape + assert viewmats.shape == (batch, cameras, 4, 4) + assert Ks is None or Ks.shape == (batch, cameras, 3, 3) + # assert seqlen == cameras * patches_x * patches_y + + apply_fn_q, apply_fn_kv, apply_fn_o = _prepare_apply_fns_all_dim( + head_dim=head_dim, + viewmats=viewmats, + Ks=Ks, + patches_x=patches_x, + patches_y=patches_y, + image_width=image_width, + image_height=image_height, + coeffs_x=coeffs_x, + coeffs_y=coeffs_y, + ) + + query = apply_fn_q(q) + key = apply_fn_kv(k) + value = apply_fn_kv(v) + + return query, key, value, apply_fn_o + + +def _prepare_apply_fns_all_dim( + head_dim: int, # Q/K/V will have this last dimension + viewmats: torch.Tensor, # (batch, cameras, 4, 4) + Ks: Optional[torch.Tensor], # (batch, cameras, 3, 3) + patches_x: int, # How many patches wide is each image? + patches_y: int, # How many patches tall is each image? + image_width: int, # Width of the image. Used to normalize intrinsics. + image_height: int, # Height of the image. Used to normalize intrinsics. + coeffs_x: Optional[torch.Tensor] = None, + coeffs_y: Optional[torch.Tensor] = None, +) -> Tuple[ + Callable[[torch.Tensor], torch.Tensor], + Callable[[torch.Tensor], torch.Tensor], + Callable[[torch.Tensor], torch.Tensor], +]: + """Prepare transforms for PRoPE-style positional encoding.""" + device = viewmats.device + (batch, cameras, _, _) = viewmats.shape + + # Normalize camera intrinsics. + if Ks is not None: + Ks_norm = torch.zeros_like(Ks) + Ks_norm[..., 0, 0] = Ks[..., 0, 0] + Ks_norm[..., 1, 1] = Ks[..., 1, 1] + Ks_norm[..., 0, 2] = 0 + Ks_norm[..., 1, 2] = 0 + Ks_norm[..., 2, 2] = 1.0 + Ks_norm = Ks_norm.to(dtype=Ks.dtype) + del Ks + + # Compute the camera projection matrices we use in PRoPE. + # - K is an `image<-camera` transform. + # - viewmats is a `camera<-world` transform. + # - P = lift(K) @ viewmats is an `image<-world` transform. + P = torch.einsum("...ij,...jk->...ik", _lift_K(Ks_norm), viewmats) + P_T = P.transpose(-1, -2).to(dtype=viewmats.dtype) + P_inv = torch.einsum( + "...ij,...jk->...ik", + _invert_SE3(viewmats), + _lift_K(_invert_K(Ks_norm)), + ).to(dtype=viewmats.dtype) + + else: + # GTA formula. P is `camera<-world` transform. + P = viewmats + P_T = P.transpose(-1, -2) + P_inv = _invert_SE3(viewmats) + + assert P.shape == P_inv.shape == (batch, cameras, 4, 4) + + # Block-diagonal transforms to the inputs and outputs of the attention operator. + assert head_dim % 4 == 0 + transforms_q = [ + (partial(_apply_tiled_projmat, matrix=P_T), head_dim), + ] + transforms_kv = [ + (partial(_apply_tiled_projmat, matrix=P_inv), head_dim), + ] + transforms_o = [ + (partial(_apply_tiled_projmat, matrix=P), head_dim), + ] + + apply_fn_q = partial(_apply_block_diagonal, func_size_pairs=transforms_q) + apply_fn_kv = partial(_apply_block_diagonal, func_size_pairs=transforms_kv) + apply_fn_o = partial(_apply_block_diagonal, func_size_pairs=transforms_o) + return apply_fn_q, apply_fn_kv, apply_fn_o + + +def _apply_tiled_projmat( + feats: torch.Tensor, # (batch, num_heads, seqlen, feat_dim) + matrix: torch.Tensor, # (batch, cameras, D, D) +) -> torch.Tensor: + """Apply projection matrix to features.""" + # - seqlen => (cameras, patches_x * patches_y) + # - feat_dim => (feat_dim // 4, 4) + (batch, num_heads, seqlen, feat_dim) = feats.shape + cameras = matrix.shape[1] + assert seqlen >= cameras and seqlen % cameras == 0 + D = matrix.shape[-1] + assert matrix.shape == (batch, cameras, D, D) + assert feat_dim % D == 0 + return torch.einsum( + "bcij,bncpkj->bncpki", + matrix, + feats.reshape((batch, num_heads, cameras, -1, feat_dim // D, D)), + ).reshape(feats.shape) + + +def _apply_block_diagonal( + feats: torch.Tensor, # (..., dim) + func_size_pairs: List[Tuple[Callable[[torch.Tensor], torch.Tensor], int]], +) -> torch.Tensor: + """Apply a block-diagonal function to an input array. + + Each function is specified as a tuple with form: + + ((Tensor) -> Tensor, int) + + Where the integer is the size of the input to the function. + """ + funcs, block_sizes = zip(*func_size_pairs) + assert feats.shape[-1] == sum(block_sizes) + x_blocks = torch.split(feats, block_sizes, dim=-1) + out = torch.cat( + [f(x_block) for f, x_block in zip(funcs, x_blocks)], + dim=-1, + ) + assert out.shape == feats.shape, "Input/output shapes should match." + return out + + +def _invert_SE3(transforms: torch.Tensor) -> torch.Tensor: + """Invert a 4x4 SE(3) matrix.""" + assert transforms.shape[-2:] == (4, 4) + Rinv = transforms[..., :3, :3].transpose(-1, -2) + out = torch.zeros_like(transforms) + out[..., :3, :3] = Rinv + out[..., :3, 3] = -torch.einsum("...ij,...j->...i", Rinv, transforms[..., :3, 3]) + out[..., 3, 3] = 1.0 + out = out.to(dtype=transforms.dtype) + return out + + +def _lift_K(Ks: torch.Tensor) -> torch.Tensor: + """Lift 3x3 matrices to homogeneous 4x4 matrices.""" + assert Ks.shape[-2:] == (3, 3) + out = torch.zeros(Ks.shape[:-2] + (4, 4), device=Ks.device) + out[..., :3, :3] = Ks + out[..., 3, 3] = 1.0 + out = out.to(dtype=Ks.dtype) + return out + + +def _invert_K(Ks: torch.Tensor) -> torch.Tensor: + """Invert 3x3 intrinsics matrices. Assumes no skew.""" + assert Ks.shape[-2:] == (3, 3) + out = torch.zeros_like(Ks) + out[..., 0, 0] = 1.0 / Ks[..., 0, 0] + out[..., 1, 1] = 1.0 / Ks[..., 1, 1] + out[..., 0, 2] = -Ks[..., 0, 2] / Ks[..., 0, 0] + out[..., 1, 2] = -Ks[..., 1, 2] / Ks[..., 1, 1] + out[..., 2, 2] = 1.0 + out = out.to(dtype=Ks.dtype) + return out diff --git a/hyvideo/schedulers/scheduling_flow_match_discrete.py b/hyvideo/schedulers/scheduling_flow_match_discrete.py new file mode 100644 index 0000000000000000000000000000000000000000..37f3ccd56740d81dc9e4226a7e4c52a3fb30c5db --- /dev/null +++ b/hyvideo/schedulers/scheduling_flow_match_discrete.py @@ -0,0 +1,309 @@ +# Licensed under the TENCENT HUNYUAN COMMUNITY LICENSE AGREEMENT (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://github.com/Tencent-Hunyuan/HunyuanVideo-1.5/blob/main/LICENSE +# +# Unless and only to the extent required by applicable law, the Tencent Hunyuan works and any +# output and results therefrom are provided "AS IS" without any express or implied warranties of +# any kind including any warranties of title, merchantability, noninfringement, course of dealing, +# usage of trade, or fitness for a particular purpose. You are solely responsible for determining the +# appropriateness of using, reproducing, modifying, performing, displaying or distributing any of +# the Tencent Hunyuan works or outputs and assume any and all risks associated with your or a +# third party's use or distribution of any of the Tencent Hunyuan works or outputs and your exercise +# of rights and permissions under this agreement. +# See the License for the specific language governing permissions and limitations under the License. +# +# ============================================================================== +# Modified from diffusers +# ============================================================================== +# Copyright 2024 Stability AI, Katherine Crowson and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math +from dataclasses import dataclass +from typing import Optional, Tuple, Union + +import numpy as np +import torch + +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.utils import BaseOutput, logging +from diffusers.schedulers.scheduling_utils import SchedulerMixin + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +@dataclass +class FlowMatchDiscreteSchedulerOutput(BaseOutput): + """ + Output class for the scheduler's `step` function output. + + Args: + prev_sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` for images): + Computed sample `(x_{t-1})` of previous timestep. `prev_sample` should be used as next model input in the + denoising loop. + """ + + prev_sample: torch.FloatTensor + + +class FlowMatchDiscreteScheduler(SchedulerMixin, ConfigMixin): + """ + Euler scheduler. + + This model inherits from [`SchedulerMixin`] and [`ConfigMixin`]. Check the superclass documentation for the generic + methods the library implements for all schedulers such as loading and saving. + + Args: + num_train_timesteps (`int`, defaults to 1000): + The number of diffusion steps to train the model. + timestep_spacing (`str`, defaults to `"linspace"`): + The way the timesteps should be scaled. Refer to Table 2 of the [Common Diffusion Noise Schedules and + Sample Steps are Flawed](https://huggingface.co/papers/2305.08891) for more information. + shift (`float`, defaults to 1.0): + The shift value for the timestep schedule. + reverse (`bool`, defaults to `True`): + Whether to reverse the timestep schedule. + """ + + _compatibles = [] + order = 1 + + @register_to_config + def __init__( + self, + num_train_timesteps: int = 1000, + shift: float = 1.0, + reverse: bool = True, + solver: str = "euler", + use_flux_shift: bool = False, + flux_base_shift: float = 0.5, + flux_max_shift: float = 1.15, + n_tokens: Optional[int] = None, + flux_base_token=256.0, + flux_max_token=4096.0, + flux_shift_factor=1.0, + ): + sigmas = torch.linspace(1, 0, num_train_timesteps + 1) + + if not reverse: + sigmas = sigmas.flip(0) + + self.sigmas = sigmas + # the value fed to model + self.timesteps = (sigmas[:-1] * num_train_timesteps).to(dtype=torch.float32) + + self._step_index = None + self._begin_index = None + + self.supported_solver = ["euler"] + if solver not in self.supported_solver: + raise ValueError( + f"Solver {solver} not supported. Supported solvers: {self.supported_solver}" + ) + + @property + def step_index(self): + """ + The index counter for current timestep. It will increase 1 after each scheduler step. + """ + return self._step_index + + @property + def begin_index(self): + """ + The index for the first timestep. It should be set from pipeline with `set_begin_index` method. + """ + return self._begin_index + + # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.set_begin_index + def set_begin_index(self, begin_index: int = 0): + """ + Sets the begin index for the scheduler. This function should be run from pipeline before the inference. + + Args: + begin_index (`int`): + The begin index for the scheduler. + """ + self._begin_index = begin_index + + def _sigma_to_t(self, sigma): + return sigma * self.config.num_train_timesteps + + def set_timesteps( + self, + num_inference_steps: int, + device: Union[str, torch.device] = None, + n_tokens: int = None, + ): + """ + Sets the discrete timesteps used for the diffusion chain (to be run before inference). + + Args: + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + n_tokens (`int`, *optional*): + Number of tokens in the input sequence. + """ + self.num_inference_steps = num_inference_steps + + sigmas = torch.linspace(1, 0, num_inference_steps + 1) + + # Apply timestep shift + if self.config.use_flux_shift: + assert isinstance( + n_tokens, int + ), "n_tokens should be provided for flux shift" + mu = self.get_lin_function( + x1=self.config.flux_base_token, + x2=self.config.flux_max_token, + y1=self.config.flux_base_shift * self.config.flux_shift_factor, + y2=self.config.flux_max_shift * self.config.flux_shift_factor, + )(n_tokens) + sigmas = self.flux_time_shift(mu, 1.0, sigmas) + elif self.config.shift != 1.0: + sigmas = self.sd3_time_shift(sigmas) + + if not self.config.reverse: + sigmas = 1 - sigmas + + self.sigmas = sigmas + self.timesteps = (sigmas[:-1] * self.config.num_train_timesteps).to( + dtype=torch.float32, device=device + ) + + # Reset step index + self._step_index = None + + def index_for_timestep(self, timestep, schedule_timesteps=None): + if schedule_timesteps is None: + schedule_timesteps = self.timesteps + + indices = (schedule_timesteps == timestep).nonzero() + + # The sigma index that is taken for the **very** first `step` + # is always the second index (or the last index if there is only 1) + # This way we can ensure we don't accidentally skip a sigma in + # case we start in the middle of the denoising schedule (e.g. for image-to-image) + pos = 1 if len(indices) > 1 else 0 + + return indices[pos].item() + + def _init_step_index(self, timestep): + if self.begin_index is None: + if isinstance(timestep, torch.Tensor): + timestep = timestep.to(self.timesteps.device) + self._step_index = self.index_for_timestep(timestep) + else: + self._step_index = self._begin_index + + def scale_model_input( + self, sample: torch.Tensor, timestep: Optional[int] = None + ) -> torch.Tensor: + return sample + + @staticmethod + def get_lin_function( + x1: float = 256, y1: float = 0.5, x2: float = 4096, y2: float = 1.15 + ): + m = (y2 - y1) / (x2 - x1) + b = y1 - m * x1 + return lambda x: m * x + b + + @staticmethod + def flux_time_shift(mu: float, sigma: float, t: torch.Tensor): + return math.exp(mu) / (math.exp(mu) + (1 / t - 1) ** sigma) + + def sd3_time_shift(self, t: torch.Tensor): + return (self.config.shift * t) / (1 + (self.config.shift - 1) * t) + + def step( + self, + model_output: torch.FloatTensor, + timestep: Union[float, torch.FloatTensor], + sample: torch.FloatTensor, + generator: Optional[torch.Generator] = None, + n_tokens: Optional[int] = None, + return_dict: bool = True, + ) -> Union[FlowMatchDiscreteSchedulerOutput, Tuple]: + """ + Predict the sample from the previous timestep by reversing the SDE. This function propagates the diffusion + process from the learned model outputs (most often the predicted noise). + + Args: + model_output (`torch.FloatTensor`): + The direct output from learned diffusion model. + timestep (`float`): + The current discrete timestep in the diffusion chain. + sample (`torch.FloatTensor`): + A current instance of a sample created by the diffusion process. + generator (`torch.Generator`, *optional*): + A random number generator. + n_tokens (`int`, *optional*): + Number of tokens in the input sequence. + return_dict (`bool`): + Whether or not to return a [`~schedulers.scheduling_euler_discrete.EulerDiscreteSchedulerOutput`] or + tuple. + + Returns: + [`~schedulers.scheduling_euler_discrete.EulerDiscreteSchedulerOutput`] or `tuple`: + If return_dict is `True`, [`~schedulers.scheduling_euler_discrete.EulerDiscreteSchedulerOutput`] is + returned, otherwise a tuple is returned where the first element is the sample tensor. + """ + + if ( + isinstance(timestep, int) + or isinstance(timestep, torch.IntTensor) + or isinstance(timestep, torch.LongTensor) + ): + raise ValueError( + ( + "Passing integer indices (e.g. from `enumerate(timesteps)`) as timesteps to" + " `EulerDiscreteScheduler.step()` is not supported. Make sure to pass" + " one of the `scheduler.timesteps` as a timestep." + ), + ) + + if self.step_index is None: + self._init_step_index(timestep) + + # Upcast to avoid precision issues when computing prev_sample + sample = sample.to(torch.float32) + + dt = self.sigmas[self.step_index + 1] - self.sigmas[self.step_index] + + if self.config.solver == "euler": + prev_sample = sample + model_output.float() * dt + else: + raise ValueError( + f"Solver {self.config.solver} not supported. Supported solvers: {self.supported_solver}" + ) + + # Cast sample back to model compatible dtype + # prev_sample = prev_sample.to(model_output.dtype) + + # upon completion increase step index by one + self._step_index += 1 + + if not return_dict: + return (prev_sample,) + + return FlowMatchDiscreteSchedulerOutput(prev_sample=prev_sample) + + def __len__(self): + return self.config.num_train_timesteps diff --git a/hyvideo/utils/communications.py b/hyvideo/utils/communications.py new file mode 100644 index 0000000000000000000000000000000000000000..ac0fa42302ef3c7e79ba1ad5498494c8bd00a716 --- /dev/null +++ b/hyvideo/utils/communications.py @@ -0,0 +1,309 @@ +# Licensed under the TENCENT HUNYUAN COMMUNITY LICENSE AGREEMENT (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://github.com/Tencent-Hunyuan/HunyuanVideo-1.5/blob/main/LICENSE +# +# Unless and only to the extent required by applicable law, the Tencent Hunyuan works and any +# output and results therefrom are provided "AS IS" without any express or implied warranties of +# any kind including any warranties of title, merchantability, noninfringement, course of dealing, +# usage of trade, or fitness for a particular purpose. You are solely responsible for determining the +# appropriateness of using, reproducing, modifying, performing, displaying or distributing any of +# the Tencent Hunyuan works or outputs and assume any and all risks associated with your or a +# third party's use or distribution of any of the Tencent Hunyuan works or outputs and your exercise +# of rights and permissions under this agreement. +# See the License for the specific language governing permissions and limitations under the License. + +from typing import Any, Tuple +import torch +import torch.distributed as dist + +from torch.nn import functional as F + + +def broadcast(input_: torch.Tensor, group: dist.ProcessGroup): + src = dist.get_global_rank(group, 0) + dist.broadcast(input_, src=src, group=group) + + +def _all_to_all_4D( + input: torch.tensor, scatter_idx: int = 2, gather_idx: int = 1, group=None +) -> torch.tensor: + """ + all-to-all for QKV + + Args: + input (torch.tensor): a tensor sharded along dim scatter dim + scatter_idx (int): default 1 + gather_idx (int): default 2 + group : torch process group + + Returns: + torch.tensor: resharded tensor (bs, seqlen/P, hc, hs) + """ + assert ( + input.dim() == 4 + ), f"input must be 4D tensor, got {input.dim()} and shape {input.shape}" + + seq_world_size = dist.get_world_size(group) + + if scatter_idx == 2 and gather_idx == 1: + + seq_lens = [None] * seq_world_size + dist.all_gather_object(seq_lens, input.shape[1], group) + # uneven + if seq_lens[-1] != seq_lens[0]: + assert seq_lens[0] > seq_lens[-1] + gap = seq_lens[0] - seq_lens[-1] + if dist.get_group_rank(group, dist.get_rank()) == seq_world_size - 1: + assert input.shape[1] == seq_lens[-1] + input = F.pad(input, (0, 0, 0, 0, 0, gap)) + else: + gap = 0 + + # input (torch.tensor): a tensor sharded along dim 1 (bs, seqlen/P, hc, hs) output: (bs, seqlen, hc/P, hs) + bs, shard_seqlen, hc, hs = input.shape + seqlen = shard_seqlen * seq_world_size + assert ( + hc % seq_world_size == 0 + ), f"Invalid size: {hc}, which should be divisible by {seq_world_size}" + shard_hc = hc // seq_world_size + + # transpose groups of heads with the seq-len parallel dimension, so that we can scatter them! + # (bs, seqlen/P, hc, hs) -reshape-> (bs, seq_len/P, P, hc/P, hs) -transpose(0,2)-> (P, seq_len/P, bs, hc/P, hs) + input_t = ( + input.reshape(bs, shard_seqlen, seq_world_size, shard_hc, hs) + .transpose(0, 2) + .contiguous() + ) + + output = torch.empty_like(input_t) + # https://pytorch.org/docs/stable/distributed.html#torch.distributed.all_to_all_single + # (P, seq_len/P, bs, hc/P, hs) scatter seqlen -all2all-> (P, seq_len/P, bs, hc/P, hs) scatter head + if seq_world_size > 1: + dist.all_to_all_single(output, input_t, group=group) + else: + output = input_t + # if scattering the seq-dim, transpose the heads back to the original dimension + output = output.reshape(seqlen, bs, shard_hc, hs) + + # (seq_len, bs, hc/P, hs) -reshape-> (bs, seq_len, hc/P, hs) + output = output.transpose(0, 1).contiguous().reshape(bs, seqlen, shard_hc, hs) + if gap > 0: + output = output[:, :-gap] + + return output + + elif scatter_idx == 1 and gather_idx == 2: + # input (torch.tensor): a tensor sharded along dim 1 (bs, seqlen, hc/P, hs) output: (bs, seqlen/P, hc, hs) + bs, seqlen, shard_hc, hs = input.shape + + hc = shard_hc * seq_world_size + if seqlen % seq_world_size != 0: + new_seqlen = (seqlen // seq_world_size + 1) * seq_world_size + gap = new_seqlen - seqlen + input = F.pad(input, (0, 0, 0, 0, 0, gap)) + bs, seqlen, shard_hc, hs = input.shape + else: + gap = 0 + + assert seqlen % seq_world_size == 0 + + shard_seqlen = seqlen // seq_world_size + seq_world_size = dist.get_world_size(group) + + # transpose groups of heads with the seq-len parallel dimension, so that we can scatter them! + # (bs, seqlen, hc/P, hs) -reshape-> (bs, P, seq_len/P, hc/P, hs) -transpose(0, 3)-> + # (hc/P, P, seqlen/P, bs, hs) -transpose(0, 1) -> (P, hc/P, seqlen/P, bs, hs) + input_t = ( + input.reshape(bs, seq_world_size, shard_seqlen, shard_hc, hs) + .transpose(0, 3) + .transpose(0, 1) + .contiguous() + .reshape(seq_world_size, shard_hc, shard_seqlen, bs, hs) + ) + + output = torch.empty_like(input_t) + # https://pytorch.org/docs/stable/distributed.html#torch.distributed.all_to_all_single + # (P, bs x hc/P, seqlen/P, hs) scatter seqlen -all2all-> (P, bs x seq_len/P, hc/P, hs) scatter head + if seq_world_size > 1: + dist.all_to_all_single(output, input_t, group=group) + else: + output = input_t + + # if scattering the seq-dim, transpose the heads back to the original dimension + output = output.reshape(hc, shard_seqlen, bs, hs) + + # (hc, seqlen/N, bs, hs) -tranpose(0,2)-> (bs, seqlen/N, hc, hs) + output = output.transpose(0, 2).contiguous().reshape(bs, shard_seqlen, hc, hs) + + if ( + gap > 0 + and dist.get_group_rank(group, dist.get_rank()) == seq_world_size - 1 + ): + output = output[:, :-gap] + + return output + else: + raise RuntimeError("scatter_idx must be 1 or 2 and gather_idx must be 1 or 2") + + +class SeqAllToAll4D(torch.autograd.Function): + @staticmethod + def forward( + ctx: Any, + group: dist.ProcessGroup, + input: torch.Tensor, + scatter_idx: int, + gather_idx: int, + ) -> torch.Tensor: + ctx.group = group + ctx.scatter_idx = scatter_idx + ctx.gather_idx = gather_idx + + return _all_to_all_4D(input, scatter_idx, gather_idx, group=group) + + @staticmethod + def backward( + ctx: Any, *grad_output: torch.Tensor + ) -> Tuple[None, torch.Tensor, None, None]: + return ( + None, + SeqAllToAll4D.apply( + ctx.group, *grad_output, ctx.gather_idx, ctx.scatter_idx + ), + None, + None, + ) + + +def all_to_all_4D( + input_: torch.Tensor, + group: dist.ProcessGroup, + scatter_dim: int = 2, + gather_dim: int = 1, +): + return SeqAllToAll4D.apply(group, input_, scatter_dim, gather_dim) + + +def _all_to_all( + input_: torch.Tensor, + world_size: int, + group: dist.ProcessGroup, + scatter_dim: int, + gather_dim: int, +): + input_list = [ + t.contiguous() for t in torch.tensor_split(input_, world_size, scatter_dim) + ] + output_list = [torch.empty_like(input_list[0]) for _ in range(world_size)] + dist.all_to_all(output_list, input_list, group=group) + return torch.cat(output_list, dim=gather_dim).contiguous() + + +class _AllToAll(torch.autograd.Function): + """All-to-all communication. + + Args: + input_: input matrix + process_group: communication group + scatter_dim: scatter dimension + gather_dim: gather dimension + """ + + @staticmethod + def forward(ctx, input_, process_group, scatter_dim, gather_dim): + ctx.process_group = process_group + ctx.scatter_dim = scatter_dim + ctx.gather_dim = gather_dim + ctx.world_size = dist.get_world_size(process_group) + output = _all_to_all( + input_, ctx.world_size, process_group, scatter_dim, gather_dim + ) + return output + + @staticmethod + def backward(ctx, grad_output): + grad_output = _all_to_all( + grad_output, + ctx.world_size, + ctx.process_group, + ctx.gather_dim, + ctx.scatter_dim, + ) + return ( + grad_output, + None, + None, + None, + ) + + +def all_to_all( + input_: torch.Tensor, + group: dist.ProcessGroup, + scatter_dim: int = 2, + gather_dim: int = 1, +): + return _AllToAll.apply(input_, group, scatter_dim, gather_dim) + + +class _AllGather(torch.autograd.Function): + """All-gather communication with autograd support. + + Args: + input_: input tensor + dim: dimension along which to concatenate + """ + + @staticmethod + def forward(ctx, input_, dim, group): + ctx.dim = dim + ctx.group = group + world_size = dist.get_world_size(group) + input_size = list(input_.size()) + + sizes = [None] * world_size + dist.all_gather_object(sizes, input_.shape, group) + + ctx.input_size = input_size[dim] + + tensor_list = [ + torch.empty(sizes[i], dtype=input_.dtype, device=input_.device) + for i in range(world_size) + ] + input_ = input_.contiguous() + dist.all_gather(tensor_list, input_, group=group) + + output = torch.cat(tensor_list, dim=dim) + return output + + @staticmethod + def backward(ctx, grad_output): + group = ctx.group + world_size = dist.get_world_size(group) + global_rank = dist.get_rank() + rank = dist.get_group_rank(group, global_rank) + dim = ctx.dim + input_size = ctx.input_size + + sizes = [None] * world_size + dist.all_gather_object(sizes, input_size, group=group) + + grad_input_list = torch.split(grad_output, sizes, dim=dim) + grad_input = grad_input_list[rank] + + return grad_input, None, None + + +def all_gather(input_: torch.Tensor, dim: int = 1, group=None): + """Performs an all-gather operation on the input tensor along the specified dimension. + + Args: + input_ (torch.Tensor): Input tensor of shape [B, H, S, D]. + dim (int, optional): Dimension along which to concatenate. Defaults to 1. + + Returns: + torch.Tensor: Output tensor after all-gather operation, concatenated along 'dim'. + """ + return _AllGather.apply(input_, dim, group) diff --git a/hyvideo/utils/data_utils.py b/hyvideo/utils/data_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..0d5ab23c33e134d83b83b8c624b80df10df5f73b --- /dev/null +++ b/hyvideo/utils/data_utils.py @@ -0,0 +1,74 @@ +# Licensed under the TENCENT HUNYUAN COMMUNITY LICENSE AGREEMENT (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://github.com/Tencent-Hunyuan/HunyuanVideo-1.5/blob/main/LICENSE +# +# Unless and only to the extent required by applicable law, the Tencent Hunyuan works and any +# output and results therefrom are provided "AS IS" without any express or implied warranties of +# any kind including any warranties of title, merchantability, noninfringement, course of dealing, +# usage of trade, or fitness for a particular purpose. You are solely responsible for determining the +# appropriateness of using, reproducing, modifying, performing, displaying or distributing any of +# the Tencent Hunyuan works or outputs and assume any and all risks associated with your or a +# third party's use or distribution of any of the Tencent Hunyuan works or outputs and your exercise +# of rights and permissions under this agreement. +# See the License for the specific language governing permissions and limitations under the License. + +import numpy as np +from PIL import Image + + +def resize_and_center_crop(image, target_width, target_height): + if target_height == image.shape[0] and target_width == image.shape[1]: + return image + + pil_image = Image.fromarray(image) + original_width, original_height = pil_image.size + scale_factor = max(target_width / original_width, target_height / original_height) + resized_width = int(round(original_width * scale_factor)) + resized_height = int(round(original_height * scale_factor)) + resized_image = pil_image.resize((resized_width, resized_height), Image.LANCZOS) + left = (resized_width - target_width) / 2 + top = (resized_height - target_height) / 2 + right = (resized_width + target_width) / 2 + bottom = (resized_height + target_height) / 2 + cropped_image = resized_image.crop((left, top, right, bottom)) + return np.array(cropped_image) + + +def get_closest_ratio(height: float, width: float, ratios: list, buckets: list): + """ + Get the closest ratio in the buckets. + + Args: + height (float): video height + width (float): video width + ratios (list): video aspect ratio + buckets (list): buckets generated by `generate_crop_size_list` + + Returns: + the closest size in the buckets and the corresponding ratio + """ + aspect_ratio = float(height) / float(width) + + ratios_array = np.array(ratios) + closest_ratio_id = np.abs(ratios_array - aspect_ratio).argmin() + closest_size = buckets[closest_ratio_id] + closest_ratio = ratios_array[closest_ratio_id] + + return closest_size, closest_ratio + + +def generate_crop_size_list(base_size=256, patch_size=16, max_ratio=4.0): + num_patches = round((base_size / patch_size) ** 2) + assert max_ratio >= 1.0 + crop_size_list = [] + wp, hp = num_patches, 1 + while wp > 0: + if max(wp, hp) / min(wp, hp) <= max_ratio: + crop_size_list.append((wp * patch_size, hp * patch_size)) + if (hp + 1) * wp <= num_patches: + hp += 1 + else: + wp -= 1 + return crop_size_list diff --git a/hyvideo/utils/flash_attn_no_pad.py b/hyvideo/utils/flash_attn_no_pad.py new file mode 100644 index 0000000000000000000000000000000000000000..1a81c8799d4e09256f1f1ddb1a4b8438c9b43f6e --- /dev/null +++ b/hyvideo/utils/flash_attn_no_pad.py @@ -0,0 +1,111 @@ +# Licensed under the TENCENT HUNYUAN COMMUNITY LICENSE AGREEMENT (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://github.com/Tencent-Hunyuan/HunyuanVideo-1.5/blob/main/LICENSE +# +# Unless and only to the extent required by applicable law, the Tencent Hunyuan works and any +# output and results therefrom are provided "AS IS" without any express or implied warranties of +# any kind including any warranties of title, merchantability, noninfringement, course of dealing, +# usage of trade, or fitness for a particular purpose. You are solely responsible for determining the +# appropriateness of using, reproducing, modifying, performing, displaying or distributing any of +# the Tencent Hunyuan works or outputs and assume any and all risks associated with your or a +# third party's use or distribution of any of the Tencent Hunyuan works or outputs and your exercise +# of rights and permissions under this agreement. +# See the License for the specific language governing permissions and limitations under the License. + +from einops import rearrange + + +def flash_attn_no_pad( + qkv, + key_padding_mask, + causal=False, + dropout_p=0.0, + softmax_scale=None, + deterministic=False, +): + from flash_attn import flash_attn_varlen_qkvpacked_func + from flash_attn.bert_padding import pad_input, unpad_input + + batch_size = qkv.shape[0] + seqlen = qkv.shape[1] + nheads = qkv.shape[-2] + x = rearrange(qkv, "b s three h d -> b s (three h d)") + x_unpad, indices, cu_seqlens, max_s, used_seqlens_in_batch = unpad_input( + x, key_padding_mask + ) + + x_unpad = rearrange(x_unpad, "nnz (three h d) -> nnz three h d", three=3, h=nheads) + output_unpad = flash_attn_varlen_qkvpacked_func( + x_unpad, + cu_seqlens, + max_s, + dropout_p, + softmax_scale=softmax_scale, + causal=causal, + deterministic=deterministic, + ) + output = rearrange( + pad_input( + rearrange(output_unpad, "nnz h d -> nnz (h d)"), indices, batch_size, seqlen + ), + "b s (h d) -> b s h d", + h=nheads, + ) + return output + + +def flash_attn_no_pad_v3( + qkv, + key_padding_mask, + causal=False, + dropout_p=0.0, + softmax_scale=None, + deterministic=False, +): + from flash_attn import flash_attn_varlen_qkvpacked_func + from flash_attn.bert_padding import pad_input, unpad_input + from flash_attn_interface import flash_attn_varlen_func as flash_attn_varlen_func_v3 + + if flash_attn_varlen_func_v3 is None: + raise ImportError("FlashAttention V3 backend not available") + + batch_size, seqlen, _, nheads, head_dim = qkv.shape + query, key, value = qkv.unbind(dim=2) + + query_unpad, indices, cu_seqlens_q, max_seqlen_q, _ = unpad_input( + rearrange(query, "b s h d -> b s (h d)"), key_padding_mask + ) + key_unpad, _, cu_seqlens_k, _, _ = unpad_input( + rearrange(key, "b s h d -> b s (h d)"), key_padding_mask + ) + value_unpad, _, _, _, _ = unpad_input( + rearrange(value, "b s h d -> b s (h d)"), key_padding_mask + ) + + query_unpad = rearrange(query_unpad, "nnz (h d) -> nnz h d", h=nheads) + key_unpad = rearrange(key_unpad, "nnz (h d) -> nnz h d", h=nheads) + value_unpad = rearrange(value_unpad, "nnz (h d) -> nnz h d", h=nheads) + + output_unpad = flash_attn_varlen_func_v3( + query_unpad, + key_unpad, + value_unpad, + cu_seqlens_q, + cu_seqlens_k, + max_seqlen_q, + max_seqlen_q, + softmax_scale=softmax_scale, + causal=causal, + deterministic=deterministic, + ) + + output = rearrange( + pad_input( + rearrange(output_unpad, "nnz h d -> nnz (h d)"), indices, batch_size, seqlen + ), + "b s (h d) -> b s h d", + h=nheads, + ) + return output diff --git a/hyvideo/utils/infer_utils.py b/hyvideo/utils/infer_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..f05c563e27f0e0d2eadde14819e0eb7892a912da --- /dev/null +++ b/hyvideo/utils/infer_utils.py @@ -0,0 +1,35 @@ +# Licensed under the TENCENT HUNYUAN COMMUNITY LICENSE AGREEMENT (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://github.com/Tencent-Hunyuan/HunyuanVideo-1.5/blob/main/LICENSE +# +# Unless and only to the extent required by applicable law, the Tencent Hunyuan works and any +# output and results therefrom are provided "AS IS" without any express or implied warranties of +# any kind including any warranties of title, merchantability, noninfringement, course of dealing, +# usage of trade, or fitness for a particular purpose. You are solely responsible for determining the +# appropriateness of using, reproducing, modifying, performing, displaying or distributing any of +# the Tencent Hunyuan works or outputs and assume any and all risks associated with your or a +# third party's use or distribution of any of the Tencent Hunyuan works or outputs and your exercise +# of rights and permissions under this agreement. +# See the License for the specific language governing permissions and limitations under the License. + +import torch + +from hyvideo.commons.infer_state import get_infer_state + + +def torch_compile_wrapper(): + """返回一个装饰器，延迟决定是否使用torch.compile""" + + def decorator(func): + def wrapper(*args, **kwargs): + if get_infer_state() and get_infer_state().enable_torch_compile: + compiled_func = torch.compile(func) + return compiled_func(*args, **kwargs) + else: + return func(*args, **kwargs) + + return wrapper + + return decorator diff --git a/hyvideo/utils/multitask_utils.py b/hyvideo/utils/multitask_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..eeff5814c06d5f1106dbc306841430e8bada8b5a --- /dev/null +++ b/hyvideo/utils/multitask_utils.py @@ -0,0 +1,56 @@ +# Licensed under the TENCENT HUNYUAN COMMUNITY LICENSE AGREEMENT (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://github.com/Tencent-Hunyuan/HunyuanVideo-1.5/blob/main/LICENSE +# +# Unless and only to the extent required by applicable law, the Tencent Hunyuan works and any +# output and results therefrom are provided "AS IS" without any express or implied warranties of +# any kind including any warranties of title, merchantability, noninfringement, course of dealing, +# usage of trade, or fitness for a particular purpose. You are solely responsible for determining the +# appropriateness of using, reproducing, modifying, performing, displaying or distributing any of +# the Tencent Hunyuan works or outputs and assume any and all risks associated with your or a +# third party's use or distribution of any of the Tencent Hunyuan works or outputs and your exercise +# of rights and permissions under this agreement. +# See the License for the specific language governing permissions and limitations under the License. + +import torch +from typing import List +import numpy as np +from PIL import Image + + +def numpy_to_pil(images: np.ndarray) -> List[Image.Image]: + """ + Convert a numpy image or a batch of images to a PIL image. + + Args: + images (np.ndarray): The image array to convert to PIL format. + + Returns: + List[Image.Image]: A list of PIL images. + """ + if images.ndim == 3: + images = images[None, ...] + images = (images * 255).round().astype("uint8") + if images.shape[-1] == 1: + # special case for grayscale (single channel) images + pil_images = [Image.fromarray(image.squeeze(), mode="L") for image in images] + else: + pil_images = [Image.fromarray(image) for image in images] + + return pil_images + + +def merge_tensor_by_mask(tensor_1, tensor_2, mask, dim): + assert tensor_1.shape == tensor_2.shape + # Mask is a 0/1 vector. Choose tensor_2 when the value is 1; otherwise, tensor_1 + masked_indices = torch.nonzero(mask).squeeze(1) + tmp = tensor_1.clone() + if dim == 0: + tmp[masked_indices] = tensor_2[masked_indices] + elif dim == 1: + tmp[:, masked_indices] = tensor_2[:, masked_indices] + elif dim == 2: + tmp[:, :, masked_indices] = tensor_2[:, :, masked_indices] + return tmp diff --git a/hyvideo/utils/retrieval_context.py b/hyvideo/utils/retrieval_context.py new file mode 100644 index 0000000000000000000000000000000000000000..112102794b8ec6a645db7610648570d3d15aa8f6 --- /dev/null +++ b/hyvideo/utils/retrieval_context.py @@ -0,0 +1,314 @@ +# Licensed under the TENCENT HUNYUAN COMMUNITY LICENSE AGREEMENT (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://github.com/Tencent-Hunyuan/HunyuanVideo-1.5/blob/main/LICENSE +# +# Unless and only to the extent required by applicable law, the Tencent Hunyuan works and any +# output and results therefrom are provided "AS IS" without any express or implied warranties of +# any kind including any warranties of title, merchantability, noninfringement, course of dealing, +# usage of trade, or fitness for a particular purpose. You are solely responsible for determining the +# appropriateness of using, reproducing, modifying, performing, displaying or distributing any of +# the Tencent Hunyuan works or outputs and assume any and all risks associated with your or a +# third party's use or distribution of any of the Tencent Hunyuan works or outputs and your exercise +# of rights and permissions under this agreement. +# See the License for the specific language governing permissions and limitations under the License. + +import torch +import numpy as np +from typing import List, Tuple, Dict +import math + + +def generate_points_in_sphere(n_points: int, radius: float) -> torch.Tensor: + """ + Uniformly sample points within a sphere of a specified radius. + + :param n_points: The number of points to generate. + :param radius: The radius of the sphere. + :return: A tensor of shape (n_points, 3), representing the (x, y, z) coordinates of the points. + """ + samples_r = torch.rand(n_points) + samples_phi = torch.rand(n_points) + samples_u = torch.rand(n_points) + + r = radius * torch.pow(samples_r, 1 / 3) + phi = 2 * math.pi * samples_phi + theta = torch.acos(1 - 2 * samples_u) + + # transfer the coordinates from spherical to cartesian + x = r * torch.sin(theta) * torch.cos(phi) + y = r * torch.sin(theta) * torch.sin(phi) + z = r * torch.cos(theta) + + points = torch.stack((x, y, z), dim=1) + return points + + +def rotation_matrix_to_angles(R: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Estimate the Pitch and Yaw angles from a 3x3 rotation matrix R in the camera coordinate system. + + Assumed Camera Coordinate System: X=Right, Y=Up, Z=Backward + (or NeRF style: X=Right, Y=Down, Z=Forward). + Here we adopt the common Computer Vision convention: Z-axis is Forward. + + Note: The angle calculations here are directly based on the conventions of your `is_inside_fov_3d_hv` function: + - Yaw/Azimuth angle is in the XZ plane (atan2(x, z)). + - Pitch/Elevation angle is relative to the horizontal plane (atan2(y, sqrt(x^2 + z^2))). + + For the third column R[:, 2] of the W2C matrix R (the direction of the World Z-axis in the Camera frame), + this typically corresponds to the direction the camera is looking + (the representation of the world Z-axis in the camera frame). + + To simplify and match your `is_inside_fov` logic, we directly use the camera's Z-axis vector: + Camera Z-axis direction in World Frame (Forward Vector): fwd = R_w2c_inv @ [0, 0, 1] + More simply, the Z-axis vector of the C2W matrix is the camera's forward vector in the world frame. + C2W = W2C_inv + """ + + R_c2w = R.T + fwd = R_c2w[:, 2] + + x = fwd[0] + y = fwd[1] + z = fwd[2] + + # compute yaw and pitch + yaw_rad = torch.atan2(x, z) + yaw_deg = yaw_rad * (180.0 / math.pi) + pitch_rad = torch.atan2(y, torch.sqrt(x**2 + z**2)) + pitch_deg = pitch_rad * (180.0 / math.pi) + + return pitch_deg, yaw_deg + + +def is_inside_fov_3d_hv( + points: torch.Tensor, + center: torch.Tensor, + center_pitch: torch.Tensor, + center_yaw: torch.Tensor, + fov_half_h: torch.Tensor, + fov_half_v: torch.Tensor, +) -> torch.Tensor: + """ + Check whether points are inside a 3D view frustum defined by a center coordinate, pitch angle, and yaw angle. + + :param points: Tensor of shape (N, 3) or (N, B, 3) representing the coordinates of the sampled points. + :param center: Tensor of shape (3) or (B, 3) representing the camera center coordinates. + :param center_pitch: Tensor of shape (1) or (B) representing the pitch angle of center view direction. + :param center_yaw: Tensor of shape (1) or (B) representing the yaw angle of the center view direction. + :param fov_half_h: The horizontal half field-of-view angle (in degrees). + :param fov_half_v: The vertical half field-of-view angle (in degrees). + :return: Boolean tensor of shape (N) or (N, B), indicating whether each point is inside the FOV. + """ + if points.ndim == 2: # N, 3 + vectors = points - center[None, :] + C = 1 + elif points.ndim == 3: # N, B, 3 + vectors = points - center[None, ...] + center_pitch = center_pitch[None, :] if center_pitch.ndim == 1 else center_pitch + center_yaw = center_yaw[None, :] if center_yaw.ndim == 1 else center_yaw + else: + raise ValueError("points' shape should be (N, 3) or (N, B, 3)") + + x = vectors[..., 0] + y = vectors[..., 1] + z = vectors[..., 2] + + # Calculate the horizontal angle (yaw/azimuth), assuming the Z-axis is forward. + azimuth = torch.atan2(x, z) * (180 / math.pi) + + # Calculate the vertical angle (pitch/elevation). + elevation = torch.atan2(y, torch.sqrt(x**2 + z**2)) * (180 / math.pi) + + # Calculate the angular difference from the center view direction (handling angle wrapping). + diff_azimuth = azimuth - center_yaw + diff_azimuth = torch.remainder(diff_azimuth + 180, 360) - 180 + + diff_elevation = elevation - center_pitch + diff_elevation = torch.remainder(diff_elevation + 180, 360) - 180 + + # Check if within FOV + in_fov_h = diff_azimuth.abs() < fov_half_h + in_fov_v = diff_elevation.abs() < fov_half_v + + return in_fov_h & in_fov_v + + +def calculate_fov_overlap_similarity( + w2c_matrix_curr: torch.Tensor, + w2c_matrix_hist: torch.Tensor, + fov_h_deg: float = 105.0, + fov_v_deg: float = 75.0, + device=None, + points_local=None, +) -> float: + """ + Calculate the Field-of-View (FOV) overlap similarity between two W2C poses using Monte Carlo sampling. + + Similarity = (Number of points in Curr_FOV ∩ Hist_FOV) / (Number of points in Curr_FOV). + + :param w2c_matrix_curr: The (4, 4) W2C matrix for the current frame. + :param w2c_matrix_hist: The (4, 4) W2C matrix for the historical frame. + :param num_samples, radius, fov_h_deg, fov_v_deg: Sampling and FOV parameters. + :return: The overlap ratio (a float between 0.0 and 1.0). + """ + w2c_matrix_curr = torch.tensor(w2c_matrix_curr, device=device) + w2c_matrix_hist = torch.tensor(w2c_matrix_hist, device=device) + + c2w_matrix_curr = torch.linalg.inv(w2c_matrix_curr) + c2w_matrix_hist = torch.linalg.inv(w2c_matrix_hist) + C_inv = w2c_matrix_curr + + w2c_matrix_curr = torch.linalg.inv(C_inv @ c2w_matrix_curr) + w2c_matrix_hist = torch.linalg.inv(C_inv @ c2w_matrix_hist) + + R_curr, t_curr = w2c_matrix_curr[:3, :3], w2c_matrix_curr[:3, 3] + R_hist, t_hist = w2c_matrix_hist[:3, :3], w2c_matrix_hist[:3, 3] + P_w_curr = -R_curr.T @ t_curr + P_w_hist = -R_hist.T @ t_hist + + # pitch, yaw + pitch_curr, yaw_curr = rotation_matrix_to_angles(R_curr) + pitch_hist, yaw_hist = rotation_matrix_to_angles(R_hist) + + fov_half_h = torch.tensor(fov_h_deg / 2.0, device=device) + fov_half_v = torch.tensor(fov_v_deg / 2.0, device=device) + + # move to P_w_curr (N, 3) + points_world = points_local + P_w_curr[None, :] + + in_fov_curr = is_inside_fov_3d_hv( + points_world, + P_w_curr[None, :], + pitch_curr[None], + yaw_curr[None], + fov_half_h, + fov_half_v, + ) + + # compute based on angle + in_fov_hist = is_inside_fov_3d_hv( + points_world, + P_w_hist[None, :], + pitch_hist[None], + yaw_hist[None], + fov_half_h, + fov_half_v, + ) + + # compute based on distance + dist = torch.norm(points_world - P_w_hist.reshape(1, -1), dim=1) < 8.0 + in_fov_hist = in_fov_hist.bool() & dist.reshape(1, -1).bool() + + overlap_count = (in_fov_curr.bool() & in_fov_hist.bool()).sum().float() + fov_curr_count = in_fov_curr.sum().float() + + if fov_curr_count == 0: + return 0.0 + + overlap_ratio = overlap_count / fov_curr_count + + return overlap_ratio.item() + + +def select_aligned_memory_frames( + w2c_list: List[np.ndarray], + current_frame_idx: int, + memory_frames: int, + temporal_context_size: int, + pred_latent_size: int, + pos_weight: float = 1.0, + ang_weight: float = 1.0, + device=None, + points_local=None, +) -> List[int]: + """ + Selects memory and context frames for a given frame based on a four-frame segment distance calculation. + + :param w2c_list: List of all N 4x4 World-to-Camera (W2C) extrinsic matrices (np.ndarray). + :param current_frame_idx: The index of the current frame to be processed. + :param memory_frames: The total number of memory frames to select. + :param context_size: The total number of context frames to select. + :param pos_weight: The weight applied to the spatial (position) distance component. + :param ang_weight: The weight applied to the angular distance component. + + :return: List[int]: A list containing the indices of the selected memory frames and context frames. + """ + if current_frame_idx <= memory_frames: + return list(range(0, current_frame_idx)) + + num_total_frames = len(w2c_list) + if current_frame_idx >= num_total_frames or current_frame_idx < 3: + raise ValueError( + f"The current frame index must be within the valid range of w2c_list and must be at least 3." + f"{current_frame_idx}, {len(w2c_list)}" + ) + + start_context_idx = max(0, current_frame_idx - temporal_context_size) + context_frames_indices = list(range(start_context_idx, current_frame_idx)) + + candidate_distances = [] + query_clip_indices = list( + range( + current_frame_idx, + ( + current_frame_idx + pred_latent_size + if current_frame_idx + pred_latent_size <= num_total_frames + else num_total_frames + ), + ) + ) + + historical_clip_indices = list( + range(4, current_frame_idx - temporal_context_size, 4) + ) + + memory_frames_indices = [0, 1, 2, 3] # add the first chunk as context + memory_frames = memory_frames - temporal_context_size + + for hist_idx in historical_clip_indices: + total_dist = 0 + hist_w2c_1 = w2c_list[hist_idx] + hist_w2c_2 = w2c_list[hist_idx + 2] + for query_idx in query_clip_indices: + dist_1_for_query_idx = 1.0 - calculate_fov_overlap_similarity( + w2c_list[query_idx], + hist_w2c_1, + fov_h_deg=60.0, + fov_v_deg=35.0, + device=device, + points_local=points_local, + ) + dist_2_for_query_idx = 1.0 - calculate_fov_overlap_similarity( + w2c_list[query_idx], + hist_w2c_2, + fov_h_deg=60.0, + fov_v_deg=35.0, + device=device, + points_local=points_local, + ) + dist_for_query_idx = (dist_1_for_query_idx + dist_2_for_query_idx) / 2.0 + total_dist += dist_for_query_idx + + final_clip_distance = total_dist / len(query_clip_indices) + candidate_distances.append((hist_idx, final_clip_distance)) + + candidate_distances.sort(key=lambda x: x[1]) + + for start_idx, _ in candidate_distances: + # check the memory frame number + if len(memory_frames_indices) >= memory_frames: + break + + if start_idx not in memory_frames_indices: + memory_frames_indices.extend(range(start_idx, start_idx + 4)) + + # exclude the repeated frames + selected_frames_set = set(context_frames_indices) + selected_frames_set.update(memory_frames_indices) + + final_selected_frames = sorted(list(selected_frames_set)) + + return final_selected_frames diff --git a/hyvideo/utils/rewrite/clients.py b/hyvideo/utils/rewrite/clients.py new file mode 100644 index 0000000000000000000000000000000000000000..faee5e60e0d5bfa9883bc874b5c6d23f4eca6334 --- /dev/null +++ b/hyvideo/utils/rewrite/clients.py @@ -0,0 +1,290 @@ +# -*- coding: utf-8 -*- +# Licensed under the TENCENT HUNYUAN COMMUNITY LICENSE AGREEMENT (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://github.com/Tencent-Hunyuan/HunyuanVideo-1.5/blob/main/LICENSE +# +# Unless and only to the extent required by applicable law, the Tencent Hunyuan works and any +# output and results therefrom are provided "AS IS" without any express or implied warranties of +# any kind including any warranties of title, merchantability, noninfringement, course of dealing, +# usage of trade, or fitness for a particular purpose. You are solely responsible for determining the +# appropriateness of using, reproducing, modifying, performing, displaying or distributing any of +# the Tencent Hunyuan works or outputs and assume any and all risks associated with your or a +# third party's use or distribution of any of the Tencent Hunyuan works or outputs and your exercise +# of rights and permissions under this agreement. +# See the License for the specific language governing permissions and limitations under the License. + +import json +import time +import ast +import os +import io +import base64 + +import openai +from loguru import logger + + +class NonStreamResponse(object): + """Wrapper for non-streaming API responses""" + + def __init__(self): + self.response = "" + + def _deserialize(self, obj): + # Convert API response object to JSON string format + self.response = json.dumps(obj) + + +# Client for DeepSeek-v3.1 model with thinking capabilities +class DeepSeekClient(object): + def __init__(self, key_id, key_secret): + from tencentcloud.common.common_client import CommonClient + from tencentcloud.common import credential + from tencentcloud.common.profile.client_profile import ClientProfile + from tencentcloud.common.profile.http_profile import HttpProfile + + # Initialize authentication credentials + cred = credential.Credential(key_id, key_secret) + httpProfile = HttpProfile() + httpProfile.endpoint = "lkeap.tencentcloudapi.com" + # Set longer timeout for streaming responses + httpProfile.reqTimeout = 40000 + clientProfile = ClientProfile() + clientProfile.httpProfile = httpProfile + # Create API client for Tencent Cloud LLM service + self.common_client = CommonClient( + "lkeap", "2024-05-22", cred, "ap-guangzhou", profile=clientProfile + ) + + def run_single_recaption(self, system_prompt, input_prompt): + # Prepare request with system and user prompts + post_dict = { + "Model": "deepseek-v3.1", + "Messages": [ + {"Role": "system", "Content": system_prompt}, + {"Role": "user", "Content": input_prompt}, + ], + "Stream": False, + "Thinking": {"Type": "enabled"}, + } + # Retry loop to handle transient network errors + while True: + try: + resp = self.common_client._call_and_deserialize( + "ChatCompletions", post_dict, NonStreamResponse + ) + break + except Exception as e: + logger.error(e) + time.sleep(1) + # Make the actual API call (duplicate call, may be intentional) + resp = self.common_client._call_and_deserialize( + "ChatCompletions", post_dict, NonStreamResponse + ) + response = resp.response + # Parse JSON response string to extract content + response = ast.literal_eval(response) + content = response["Choices"][0]["Message"]["Content"] + reasoning_content = response["Choices"][0]["Message"]["ReasoningContent"] + print("Initial prompt: ", input_prompt) + print("Recaption prompt: ", content) + + return content, reasoning_content + + +# Text-to-text client using Qwen models for prompt rewriting +class QwenClient(object): + def __init__(self, base_url=None, model_name=None): + # Recommended model: Qwen3-235B-A22B-Thinking-2507 + self.base_url = base_url + self.model_name = model_name + + def qwen_api_call( + self, system_prompt: str, user_input: str, temperature: float, max_tokens: int + ): + """ + Use Qwen API to perform text rewriting, parse ... sections for reasoning content, + and return (thinking, result). + """ + # Initialize OpenAI-compatible client for Qwen API + client = openai.OpenAI(base_url=self.base_url, api_key="None", timeout=600) + messages = [ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": user_input}, + ] + + last_err = None + + # Retry up to 10 times with exponential backoff + for i in range(10): + try: + stream = False + response = client.chat.completions.create( + model=self.model_name, + messages=messages, + temperature=temperature, + max_tokens=max_tokens, + stream=stream, + ) + content = response.choices[0].message.content + thinking = "" + result = content or "" + # Extract thinking process from tags if present + if content and "" in content: + head, tail = content.split("", 1) + thinking = head.replace("", "").strip() + result = tail.strip() + return thinking, result + except Exception as e: + last_err = e + if i < 9: + # Exponential backoff: wait 2^i seconds + time.sleep(2**i) + continue + raise last_err + + def run_single_recaption( + self, system_prompt, input_prompt, temperature=0.1, max_tokens=4096 + ): + thinking, result = self.qwen_api_call( + system_prompt, input_prompt, temperature, max_tokens + ) + return result + + +# Vision-language client using Qwen-VL for image+text processing +class QwenVLClient(object): + + def __init__(self, base_url=None, model_name=None): + self.base_url = base_url + self.model_name = model_name + self.api_key = "None" + # Get max image resolution from env, default 1024x1024 + self.max_image_size = int(os.getenv("I2V_REWRITE_MAX_IMAGE_SIZE", "1024")) + + def _encode_image_to_base64(self, image_path: str, max_dimension: int) -> str: + """ + 参考 hyvideo/utils/rewrite/qwen_vllm.py 的实现： + 加载本地图片，将其按比例缩放到 max_dimension，然后编码为 Base64 data URL。 + """ + try: + from PIL import Image + except ImportError as e: + logger.error( + "Pillow (PIL) is required for QwenVLClient image encoding but is not installed." + ) + raise e + + try: + with Image.open(image_path) as img: + # Resize if image exceeds max dimension while preserving ratio + if img.width > max_dimension or img.height > max_dimension: + img.thumbnail((max_dimension, max_dimension)) + + buffer = io.BytesIO() + + # Convert images with transparency to RGB for JPEG encoding + if img.mode in ("RGBA", "LA") or ( + img.mode == "P" and "transparency" in img.info + ): + img = img.convert("RGB") + + img.save(buffer, format="JPEG") + mime_type = "image/jpeg" + + # Encode image bytes to base64 data URL format + encoded_string = base64.b64encode(buffer.getvalue()).decode("utf-8") + return f"data:{mime_type};base64,{encoded_string}" + except Exception as e: + logger.error(f"Failed to encode image {image_path} to base64: {e}") + raise + + def qwen_api_call( + self, + system_prompt: str, + user_input: str, + temperature: float, + max_tokens: int, + img_path: str = None, + ): + """ + Use Qwen3-VL to perform text rewriting. + """ + client = openai.OpenAI( + base_url=self.base_url, + api_key=self.api_key, + timeout=3600, + ) + + assert "{}" in system_prompt, "system_prompt must contain {{}}" + prompt_text = system_prompt.format(user_input) + + assert img_path is not None, "img_path is required" + base64_image = self._encode_image_to_base64( + img_path, max_dimension=self.max_image_size + ) + messages = [ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": { + "url": base64_image, + }, + }, + { + "type": "text", + "text": prompt_text, + }, + ], + } + ] + + last_err = None + + for i in range(10): + try: + + response = client.chat.completions.create( + model=self.model_name, + messages=messages, + temperature=temperature, + max_tokens=max_tokens, + stream=False, + ) + content = response.choices[0].message.content + thinking = "" + result = content or "" + # 和 QwenClient 一样，兼容 思维模式输出 + if content and "" in content: + head, tail = content.split("", 1) + thinking = head.replace("", "").strip() + result = tail.strip() + return thinking, result + except Exception as e: + last_err = e + logger.error(f"QwenVLClient request failed (attempt {i + 1}/10): {e}") + if i < 9: + time.sleep(2) + else: + raise last_err + + def run_single_recaption( + self, + system_prompt, + input_prompt, + temperature=0.1, + max_tokens=4096, + img_path: str = None, + ): + thinking, result = self.qwen_api_call( + system_prompt, + input_prompt, + temperature, + max_tokens, + img_path=img_path, + ) + return result diff --git a/hyvideo/utils/rewrite/i2v_prompt.py b/hyvideo/utils/rewrite/i2v_prompt.py new file mode 100644 index 0000000000000000000000000000000000000000..1b63e29dd55280276595fdbc75e6b3d94d62e6df --- /dev/null +++ b/hyvideo/utils/rewrite/i2v_prompt.py @@ -0,0 +1,118 @@ +# Licensed under the TENCENT HUNYUAN COMMUNITY LICENSE AGREEMENT (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://github.com/Tencent-Hunyuan/HunyuanVideo-1.5/blob/main/LICENSE +# +# Unless and only to the extent required by applicable law, the Tencent Hunyuan works and any +# output and results therefrom are provided "AS IS" without any express or implied warranties of +# any kind including any warranties of title, merchantability, noninfringement, course of dealing, +# usage of trade, or fitness for a particular purpose. You are solely responsible for determining the +# appropriateness of using, reproducing, modifying, performing, displaying or distributing any of +# the Tencent Hunyuan works or outputs and assume any and all risks associated with your or a +# third party's use or distribution of any of the Tencent Hunyuan works or outputs and your exercise +# of rights and permissions under this agreement. +# See the License for the specific language governing permissions and limitations under the License. + + +i2v_rewrite_system_prompt = """ +## 角色 +你是一位顶级的图生视频（Image-to-Video）Prompt工程师。你的任务不是生成视频，而是将用户输入的自由形式的自然语言，改写成具有丰富视觉细节、精确动态描述、并采用专业影视语言的中文Prompt。改写后的措辞、句式、表达方式等，必须严格遵循并尽可能接近本指令中定义的语言风格和表达习惯。 + +## 任务 +你的核心任务是进行"文本改写"。接收用户的简短或模糊想法，输出一段符合以下规则的、详细、客观、可执行的中文视频脚本式描述。无论用户输入是中文还是英文，你的输出都必须是中文。 + +## 核心改写规则 + +### 1. 镜头语言标准化 (Camera Language Standardization) +当用户指令中包含镜头运镜描述时，尽可能转换为标准表述。如果不能完全对应到标准表述，保留原意。 +* **运镜标准表述**: `镜头缓缓拉远/后拉`, `镜头向前推进`, `镜头上/下/左/右移动`, `镜头摇动/摇移`, `镜头跟随`, `镜头环绕`, `镜头静止不动`, `手持镜头`。 +* **示例1**: （能对应标准）用户输入"镜头慢慢前推，跟踪小鸟飞行"，应改写为"**镜头向前推进**，跟踪小鸟飞行"。 +* **示例2**: （不能完全对应）用户输入"镜头缓缓顺时针旋转并向前推"，应改写为"镜头缓缓顺时针旋转，并**向前推进**"。 +* 改写时绝对禁止补充用户指令中没有明确提出的镜头运镜描述，除非为了解释或标准化用户已明确提到的运镜所必需；不得擅自联想新增任何镜头运动方式或运镜效果。特别地，在用户未明确说明“镜头静止”或具有同等含义的表述时，严禁擅自添加“镜头静止不动”等描述。 + +### 2. 动态化与时序性 (Dynamic & Sequential) +将用户的静态描述分解成一个微小的时间序列。使用连接词来串联连续发生的、或同时发生的动作，构建出清晰的叙事流。 +* **结构**: 动作A发生，**随后/然后**，动作B发生，**同时**，动作C发生。 +* **常用连接词**: `随后`, `然后`, `接着`, `同时`, `之后`。 +* **示例1**: 用户输入"两个人见面"，应改写为"...画面左侧走进来一个男子，画面右侧走进来一个女子，他们微笑着在中间的心形前停下，**随后**两个人双手握在一起对视，**接着**男子和女子接吻..."。 +* **示例2**: 用户输入"女孩跳舞"，应改写为"**女孩身体开始左右轻轻摇摆,同时双手缓缓举过头顶**。。。"。 + +### 3. 遵循“主体-动作-细节”的客观描述模式 +使用客观、中立的语言，像摄影师一样记录画面中发生的一切。避免使用主观或情感化的词语（如“美丽的”、“悲伤的”），而是通过描述具体的行为来暗示情感（如用“嘴角露出微笑”代替“开心的”）。 +* **句式结构**: `[主体] + [方式状语(如缓缓地)] + [动作(如转动头部)]`。 +* **示例**: "**黄色头的蜥蜴** **转动着头** **向前探出身子**。" 这是一个完美的客观描述链。 + +### 4. 空间与方位的精确化 +明确物体和人物在画面中的位置及其移动方向,如果有新加入物体，需要描述已有物体加以区分。 +* **方位词**: `画面左/右/上/下侧`, `从...上方/下方伸入`, `背景中`, `前景处`, `向...方向移动`。 +* **示例1**: 用户输入"有只手伸进来"，应改写为"**一只手从画面右侧伸出**，摸了摸黑色衣服上的吊牌...**从下方消失在画面中**"。 +* **示例2**: 参考图像中间有一条美人鱼，用户输入"旁边游来两条粉色鱼尾的美人鱼"，应改写为"**画面中间有一条美人鱼，两条粉色鱼尾的美人鱼从画面右侧游入**..."。 + + +### 5. 指代关系清晰 + *模糊指代必须显式化（针对用户输入）**：当用户输入中出现模糊指代（如“他们”“它”“这个/那个”“其”等），在改写中必须替换为明确的实体称谓，并最小合理补充主体类别、数量与性别，但不引入与原意无关的新实体。 + * **示例**： + * 文本指令“他们跳舞。” → 你的输出中写为“一对男女在舞池中跳着华尔兹...” + * 文本指令“把它拿起来。” → 你的输出中写为“一只手从画面左侧伸入，拿起绿色的洗面奶管子...” + * 文本指令“把炸弹递给他。” → 你的输出中写为“黑猫把手中的炸弹递给灰猫。” + +### 6. 过度克制联想 +在不改变用户意图与叙事目标的前提下，仅进行必要的镜头化与轻度细节扩展。 +* 不得新增用户未要求的关键事件或动作（如坐下、拿起、递交、拥抱等）。 +* 可适度补充环境"微动态"，但不得擅自改变主体、方向、数量与时序。 +* **避免补充光影相关描述**：，除非用户指令明确要求,绝对不要添加任何光影相关内容，包括但不限于"光线"、"阴影"、"光影斑驳"、"阳光照射"、"灯光变化"等。 + 同时，绝对严禁输出"无任何光影变化"、"无额外光影渲染"、"整体画面无额外光影变化"等强调不存在光影的表述，否则将视为严重错误！！！ +* 输出长度不必与样例等长，应与用户输入的信息量匹配；避免冗长与过度联想。 + +### 7. 对齐用户意图 +* 结合参考图像和用户指令，正确理解用户的指定的范畴 + - 例1： 参考图像是：三个漂浮在水面上的甜甜圈 + 用户指令是：“美味的甜甜圈从水上慢慢沉下去消失不见，水波浮动” + 不应该改写成："画面中央的巧克力甜甜圈缓缓下沉...旁边的粉色和白色甜甜圈仍在水上轻轻浮动..." + 应该改写成："画面中的三个甜甜圈缓缓下沉..."（因为这里用户指代的是参考图像当中所有的甜甜圈） + - 例2: 参考图像是：狗人、兔子人、猫人三个人形动物头的角色在故宫的背景，面向镜头奔跑 + 用户指令是：兔子人加速向前跑，然后剩下的两个追上去，镜头右摇，拍摄它们跑远宫殿的背影 + 不应该改写成：画面中，一只狗、一只白兔和一只猫正并排在石桥上向前奔跑。突然，中间的白兔加速向前冲刺。随后，它左侧的狗和右侧的猫也加快速度追了上去。镜头向右摇移，跟随拍摄它们跑向远处宫殿的背影。 + 应该改写成：画面中，一只狗、一只白兔和一只猫正并排在石桥上向前奔跑。突然，中间的白兔加速向前冲刺。随后，它左侧的狗和右侧的猫也加快速度追了上去。镜头向右摇移，跟随拍摄它们奔跑远离宫殿的背影。 + + +## 文字处理规则 +如果用户输入包含引用文字（如书名、品牌名、标语等），必须在改写后的Prompt中保持原文和原语言，并一定要用中文双引号包围。不要在没有原文引用的情况下自行添加引用文字。例如，如果用户提到 写着"GROOVY MANGO"，你必须改写为 写着“GROOVY MANGO”。 +注意：绝对不要在指令中让完整的一段文字，逐个字词的出现，例如： + 用户指令：男人在纸上写"iPhone 15" + 不要改写成：...依次写上“i”、“P”、“h”.../依次形成字母“i”、“P”、“h”.../按顺序写上文字“i”、“P”、“h”... （等等任何把文字拆开的描述方式） + 应该改写成：...写上文字“iPhone 15” + +## 改写范例 +以下是一些改写样例，请严格模仿“改写后”的风格。 + +**范例 1** +* **参考图像**：傍晚的海边沙滩，一位长发女子站在近景位置，身体微侧，远处海浪与天际线清晰可见，海风将发丝轻轻吹起。 +* **文本指令**: 在海边的沙滩上站着的一位女人，面朝大海凝望。 +* **你的输出**: 女人的发丝随风轻轻飘动。女人眨了一下眼，向画面右侧转头凝视前方。背景中海浪拍打着沙滩。 + +**范例 2** +* **参考图像**：户外阴天光线下，一朵花的近景特写，花瓣边缘附着细小水珠，背景为柔和的绿色叶片虚化。 +* **文本指令**: 一朵在小雨中被细密雨丝打湿的花朵，花瓣上挂着水珠。 +* **你的输出**: 雨水滴落在花朵上，水珠在花瓣间滚动，花瓣轻微摇动。 + +**范例 3** +* **参考图像**：室内柔和日光下，一个圆形绒面猫窝里蜷着一只橘猫，窗边可见绿色树叶的模糊影子落在地面上。 +* **文本指令**: The cat stretches its body and wags its tail. Camera pulls back. The cat opens and blinks its eyes. +* **你的输出**: 橘猫在柔软的猫窝中舒展身体，把头和一只爪子伸出猫窝外，不停摇晃着尾巴。镜头微微向后拉远，露出窗外更多的树叶景色。猫咪睁开眼睛看了一下前方，随后又眯上了眼睛，尾巴也渐渐停止摇晃。阳光照下的影子也随之动着。 + +**范例 4** +* **参考图像**：灯光温暖的舞池中央，一对男女面对面站立，男士穿深色西装，女士穿深色礼服，周围有零散观众与舞台乐队背景虚化。 +* **文本指令**: A man and a woman are dancing together as the camera follows their movements. +* **你的输出**: 一对男女在舞池中跳着华尔兹，男士穿着黑色西装轻握穿着深色礼服女士的手，扶着她的腰，女子一只手搭在他的肩上。两人眼神交流，面带微笑。女士裙摆随舞步轻轻摆动。镜头跟随拍摄，捕捉他们的动作和表情。后面的人们拍着手，舞台上的乐队在演奏。 + + +## 输出格式要求 +请注意： +1. 无论用户输入是中文还是英文，你的最终输出都必须是中文。 +2. 在你的最终输出中，不要包含任何星号（**）符号。上述示例中的星号仅用于强调重要概念，你的实际输出应该是干净的、不带任何格式标记的纯文本。 + +现在给定参考图像和文本指令: {} +请按照上述规则进行改写，输出改写后的文本: +""" diff --git a/hyvideo/utils/rewrite/rewrite_utils.py b/hyvideo/utils/rewrite/rewrite_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..9f4390efced622e3f82f96b0674a92a500bf86a9 --- /dev/null +++ b/hyvideo/utils/rewrite/rewrite_utils.py @@ -0,0 +1,76 @@ +# Licensed under the TENCENT HUNYUAN COMMUNITY LICENSE AGREEMENT (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://github.com/Tencent-Hunyuan/HunyuanVideo-1.5/blob/main/LICENSE +# +# Unless and only to the extent required by applicable law, the Tencent Hunyuan works and any +# output and results therefrom are provided "AS IS" without any express or implied warranties of +# any kind including any warranties of title, merchantability, noninfringement, course of dealing, +# usage of trade, or fitness for a particular purpose. You are solely responsible for determining the +# appropriateness of using, reproducing, modifying, performing, displaying or distributing any of +# the Tencent Hunyuan works or outputs and assume any and all risks associated with your or a +# third party's use or distribution of any of the Tencent Hunyuan works or outputs and your exercise +# of rights and permissions under this agreement. +# See the License for the specific language governing permissions and limitations under the License. + +import os +from hyvideo.utils.rewrite.clients import QwenClient, QwenVLClient +from hyvideo.utils.rewrite.t2v_prompt import t2v_rewrite_system_prompt +from hyvideo.utils.rewrite.i2v_prompt import i2v_rewrite_system_prompt + + +def t2v_rewrite(user_prompt, rewrite_client=None): + base_url = os.getenv("T2V_REWRITE_BASE_URL") + model_name = os.getenv("T2V_REWRITE_MODEL_NAME") + if not base_url or not model_name: + raise EnvironmentError( + "T2V_REWRITE_BASE_URL and T2V_REWRITE_MODEL_NAME must be set in the environment variables " + "when prompt rewriting is enabled. Please configure the rewrite service correctly." + ) + if rewrite_client is None: + rewrite_client = QwenClient(base_url, model_name) + try: + rewritten_prompt = rewrite_client.run_single_recaption( + t2v_rewrite_system_prompt, user_prompt + ) + except Exception as e: + raise ValueError( + f"Failed to rewrite prompt using {type(rewrite_client).__name__}: {e}" + ) + return rewritten_prompt + + +def i2v_rewrite(user_input, img_path, rewrite_client=None): + """ + Use a rewrite client to generate a rewritten prompt for image-to-video. + """ + i2v_base_url = os.getenv("I2V_REWRITE_BASE_URL") + i2v_model_name = os.getenv("I2V_REWRITE_MODEL_NAME") + if not i2v_base_url or not i2v_model_name: + raise EnvironmentError( + "I2V_REWRITE_BASE_URL and I2V_REWRITE_MODEL_NAME must be set in the environment variables " + "for image-to-video prompt rewriting. Please set them before running, e.g.:\n" + 'export I2V_REWRITE_BASE_URL="YOUR_I2V_REWRITE_BASE_URL"\n' + 'export I2V_REWRITE_MODEL_NAME="YOUR_I2V_REWRITE_MODEL_NAME"\n' + ) + if rewrite_client is None: + rewrite_client = QwenVLClient(i2v_base_url, i2v_model_name) + try: + rewritten_prompt = rewrite_client.run_single_recaption( + i2v_rewrite_system_prompt, user_input, img_path=img_path + ) + except Exception as e: + raise ValueError( + f"Failed to rewrite prompt using {type(rewrite_client).__name__}: {e}" + ) + return rewritten_prompt + + +def run_prompt_rewrite(user_prompt, img_path, task_type): + if task_type == "i2v": + return i2v_rewrite(user_prompt, img_path) + elif task_type == "t2v": + return t2v_rewrite(user_prompt) + else: + raise ValueError(f"Unsupported task_type: {task_type}. Must be 'i2v' or 't2v'") diff --git a/hyvideo/utils/rewrite/t2v_prompt.py b/hyvideo/utils/rewrite/t2v_prompt.py new file mode 100644 index 0000000000000000000000000000000000000000..68f740854f3993f1ec04a174c4e306b07fcb73d1 --- /dev/null +++ b/hyvideo/utils/rewrite/t2v_prompt.py @@ -0,0 +1,250 @@ +# Licensed under the TENCENT HUNYUAN COMMUNITY LICENSE AGREEMENT (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://github.com/Tencent-Hunyuan/HunyuanVideo-1.5/blob/main/LICENSE +# +# Unless and only to the extent required by applicable law, the Tencent Hunyuan works and any +# output and results therefrom are provided "AS IS" without any express or implied warranties of +# any kind including any warranties of title, merchantability, noninfringement, course of dealing, +# usage of trade, or fitness for a particular purpose. You are solely responsible for determining the +# appropriateness of using, reproducing, modifying, performing, displaying or distributing any of +# the Tencent Hunyuan works or outputs and assume any and all risks associated with your or a +# third party's use or distribution of any of the Tencent Hunyuan works or outputs and your exercise +# of rights and permissions under this agreement. +# See the License for the specific language governing permissions and limitations under the License. + +# Configuration for the prompt rewriting agent: +# 1. Role: Video generation prompt expert ("Hunyuan Video Rewriter 1.5"). +# 2. Mission: Transform simple user inputs into structured, objective, and exhaustive descriptions. +# 3. Standards: Enforce strict logical ordering (general to specific) and use professional vocabulary +# to ensure high-quality composition and logical physical consistency in the generated video. +# According to: I. Core General Principles (Description Flow and Grammar) +# II. Rules and Features for Different Video Types (Realistic Video, Animation/CGI) +# III. Camera Control Guidelines +# IV. Standard Generation Process (Language, Art Style, Details) +# V. Final Output Requirements (Prompt Only, Word Count) +t2v_rewrite_system_prompt = """你是世界级的视频生成提示词撰写专家，你的名字是"Hunyuan Video Rewriter 1.5"。 + 你的核心使命是将用户提供的简单句子扩展为详细、**结构化、客观且详尽的**视频生成提示词。 + 最终的提示词将遵循严格的逻辑顺序，从一般到具体，使用精确的专业词汇来指导AI模型生成物理逻辑合理、构图精美的高质量视频。 + +## **一、核心通用原则** + +在构建任何提示词时，你必须遵守以下基本原则： + +### I. 通用句子结构和语法规则（适用于所有视频类型） + +这些规则构成了描述任何视频的基础结构，无论其风格如何。描述始终以客观、详细且易于解析的方式组织。 + +**A. 整体描述结构** +描述遵循逻辑和层次化的流程，从一般到具体。 + +1. **主体与场景介绍：**描述几乎总是以介绍主要对象和直接场景开始。 + * *示例：*"一只棕色皮肤的巨大霸王龙正在穿越广阔的荒漠平原。" + * *示例：*"在一个灯光昏暗、以砖墙为背景的舞台上，一群年轻的东亚表演者正在表演同步舞蹈。" + +2. **主体细节描述：**然后提供关于主体外观、服装和显著特征的具体细节。 + * *示例：*"她穿着一件华丽的红金刺绣上衣，一条带有银色花卉图案的深蓝色连衣裙，以及一条配套的红金项圈。" + +3. **按时间顺序的动作序列：**动作按其发生的顺序进行描述。这一叙事部分使用过渡词，如**"最初，""然后，""接下来，""随着，"**和**"最后，"**来引导读者了解事件的顺序，注意这一部分需要详细描述，用来指导视频生成模型。 + * *示例：*"**最初，**她睁大眼睛看向左边。**然后**镜头绕着她旋转。**接下来，**她的嘴巴张开又闭上……" + +4. **环境和背景细节：**在描述主要动作之后，焦点通常转向周围环境、背景元素和次要角色。 + * *示例：*"在背景中，一块巨大的层状岩石矗立在蓝绿色的天空中，上面有许多大片的白色积云。" + +5. **技术与风格总结：**描述以一个独特的部分结束，详细说明技术方面，如镜头运动、拍摄类型、氛围和照明。这些通常以简短的陈述句或短语呈现。 + * *示例：*"镜头向右平移……低角度。冒险感。" + +**B. 核心语法规则** + +1. **时态：**主要使用的时态是**现在时**（一般现在时或现在进行时）。这使描述感觉即时和主动，就像在播放视频时进行描述一样。 + * *示例：*"一个小女孩……**正在跑**……"，"他**穿着**黑色运动背心……"，"镜头**跟随**她……" + +2. **句子结构：** + * 句子主要是**陈述性的**，陈述关于场景的事实。 + * 结构通常以主语开始，然后是动词：`[主语] + [动词] + [细节]`。 + * **介词短语**被广泛用于添加关于位置（`在土路上`）、外观（`有着苍白的皮肤`）和关系（`在瓢虫后面`）的细节。 + * **分词短语**经常用于简洁地组合动作或描述。 + * *示例：*"一位年轻女性……**戴着棕色帽子**，从马上下来，**微笑着等待**。" + +3. **词汇和语气：** + * **形容词：**语言中充满描述性形容词，用于指定颜色、大小、纹理、情感和外观（例如，"广阔的"，"卷曲的"，"同步的"，"充满活力的"）。 + * **动词：**动作动词精确而动态（例如，"跳跃"，"投掷"，"摇摆"，"攀爬"）。 + * **语气：**语气是客观和事实性的。它描述视觉上呈现的内容，而不做过度主观的解释（除非陈述情绪，例如，"冒险的"，"欢乐的重逢"）。 + +--- + +### II. 不同视频类型的规则和特征 + +虽然上述通用规则适用于所有视频，但某些视频类型在其描述中具有独特的特征。 + +**A. 写实/真人视频** + +1. **关注人物细节：**描述优先考虑人物特征：年龄、种族、肤色、发型/颜色和具体服装项目。情绪状态通过面部表情和肢体语言来描述。 + * *示例：*"这位女性，她的脸上带着**担忧的表情**，抬起头与他对视……她的表情突然转变为**震惊和恐慌**……" + +2. **真实世界的动作和互动：**所描述的动作以现实为基础，关注人与人之间或与物理环境的互动。 + * *示例：*"当跑步者到达她身边时，他们立即**紧紧地拥抱在一起**。" + * *示例：*"他用右手撑起自己，**努力站起来**……" + +3. **电影术语：**描述通常包括暗示物理摄像机的特定电影制作术语。 + * *示例：*"**手持拍摄**向前移动，""**中近景**，""**平视角度**，""镜头**以中景稍微移动拍摄**。" + +4. **照明描述：**照明通常用自然光源（"来自阴天的柔和、漫射自然光"）或刻意的电影布光设置（"高调、漫射的背光"，"高对比度、戏剧性的舞台照明"）来描述。 + +**B. 动画/CGI视频** + +1. **强调风格化：**描述突出动画风格和非写实特征。主体通常是奇幻的（恐龙、蓝精灵）或拟人化的（拿着炸药的北极熊）。 + * *示例：*"四只**动画蓝精灵**骑在一只**毛茸茸的大黄兔子**上。" + * *示例：*"一只有橙色翅膀的瓢虫正在高速飞行……在瓢虫后面，有五只黑色苍蝇，它们有**显眼的大红眼睛**。" + +2. **违反物理和夸张的动作：**所描述的动作通常超越现实世界的限制，反映了动画的创作自由。 + * *示例：*"熊用后腿站立，转身，**将点燃的炸药高高地投入拱形门道**。" + * *示例：*"她突然**被甩出车外**……**安全落地**。" + +3. **明确的风格识别：**技术总结通常明确命名动画风格。 + * *示例：*"一个动态的**风格化昆虫的3D CGI动画**集成在照片级真实的自然环境中……" + * *示例：*"视觉风格是**高质量的3D电脑动画电影**……" + +### **II. 镜头控制指南** +以下是镜头控制系统。你应该使用镜头控制系统来描述提示词中的镜头运动。如果遇到下面的类型，就参考对应的描述。 +* **镜头360度旋转**: ["镜头旋转360度", "镜头进行完整旋转", "镜头绕一圈旋转"] +* **镜头第一人称视角FPV**: ["镜头显示第一人称视角", "场景从第一人称视角拍摄", "镜头采用FPV角度", "镜头处于第一人称视角"] +* **镜头向上移动**: ["镜头向上移动", "镜头上升", "镜头升起"] +* **镜头向下移动**: ["镜头向下移动", "镜头下降", "镜头落下"] +* **镜头低角度/仰拍**: ["镜头从低角度拍摄", "镜头从下方捕捉场景", "镜头位于低视点"] +* **镜头向上倾斜**: ["镜头向上倾斜", "镜头进行向上倾斜运动"] +* **镜头向下倾斜**: ["镜头向下倾斜", "镜头进行向下倾斜运动"] +* **镜头地面拍摄**: ["镜头在地面高度", "镜头从地面拍摄", "镜头从地面视角捕捉场景"] +* **镜头向前推进**: ["镜头向前推进", "镜头向前移动", "镜头向前移"] +* **镜头向右平移**: ["镜头向右移动", "镜头向右移"] +* **镜头向后拉**: ["镜头向后拉", "镜头后退", "镜头向后移"] +* **镜头向左平移**: ["镜头向左移动", "镜头向左移"] +* **镜头延时摄影**: ["镜头捕捉延时摄影", "使用延时拍摄", "场景以延时方式显示"] +* **镜头微距拍摄**: ["镜头进行微距拍摄", "使用微距视角"] +* **镜头慢动作**: ["镜头以慢动作记录", "显示慢动作镜头", "场景以慢动作捕捉"] +* **镜头拉远**: ["镜头拉远", "镜头向后拉", "镜头远离主体"] +* **镜头推近**: ["镜头推近", "镜头向前推", "镜头接近主体"] +* **镜头向右平移**: ["镜头向右平移", "镜头向右摆动", "镜头进行向右平移运动"] +* **镜头向左平移**: ["镜头向左平移", "镜头向左摆动", "镜头进行向左平移运动"] +* **镜头无人机视角**: ["镜头显示无人机视角", "场景从无人机视角拍摄", "镜头采用空中无人机角度"] +* **镜头环绕**: ["镜头环绕主体", "镜头围绕主体旋转"] +* **镜头跟随拍摄**: ["镜头跟随主体", "镜头跟踪运动", "镜头与主体一起移动"] +* **镜头过肩拍摄**: ["镜头使用过肩镜头", "镜头位于主体肩膀后方"] +* **镜头逆时针旋转**: ["镜头逆时针旋转", "镜头以逆时针方向旋转"] +* **镜头静止**: ["镜头保持静止", "镜头静止不动", "镜头保持静态"] +* **镜头顺时针旋转**: ["镜头顺时针旋转", "镜头以顺时针方向旋转"] +* **镜头高角度/俯拍**: ["镜头从高角度拍摄", "镜头从上方捕捉场景", "镜头位于高视点"] +* **镜头鱼眼镜头**: ["镜头使用鱼眼镜头", "场景以鱼眼效果显示", "应用鱼眼透视"] +* **镜头鸟瞰视角**: ["镜头显示鸟瞰视角", "场景从正上方拍摄"] + +** 相机运动也可以是动态的，如"最初，摄像机跟随...然后，焦点平滑地转移，向后拉..." + +## **三、标准生成流程** + +在生成最终提示词之前，你必须在思考和构建时遵循以下步骤： + +0. **语言规则**: + * 整体输出的提示词保持为英文。 + * 文本渲染内容应与用户输入的语言相同。例如，如果用户希望视频显示文本"Hello"，则扩展的提示词应该是英文，但渲染的文本应该是"Hello"。如果是中文，文本渲染内容总是包含在“”内，如果是英文则总是放置在""内。 + * **宝可梦IP的特殊规则**：如果用户输入包含宝可梦的IP角色，始终使用英文的IP名称。（例如，使用Jigglypuff而不是胖丁，因为胖丁来自宝可梦） + +1. **分析核心元素并评估风险**: + * **摘要**：提示词以视频故事的摘要开始。所有主要主体都必须在摘要中描述。摘要应该是一个简洁的句子，并放在提示词的开头。 + * **识别核心元素**：从用户输入中清晰地识别主体（人物、对象）、关键动作/事件、运动、环境和整体叙事弧线。 + * **确定实体数量**：如果用户输入包含多个实体，首先检查用户是否给出了精确的实体数量（例如，"六口之家"，"四个朋友"，"大约五名舞者"），那么你必须严格遵循用户的原始提示词并使用相同的数量描述。否则，如果用户使用模糊的词语（例如，"一群警官"）并且没有精确给出实体数量，那么你应该严格将人物或对象的数量限制在三个或更少，以保持场景清晰度。 + * **识别高风险概念**：特别注意复杂的物理互动（例如，动态体育序列）、随时间展开的抽象概念（例如，"能量"脉动或流动）以及动态排版，需要用简单、可渲染的描述来呈现。 + * **可视化概念**：将非视觉概念转换为视觉可用的序列。例如，"发令枪的声音"暗示比赛的开始，运动员从起跑线冲出并沿着跑道移动。你可以从用户提供的这种非视觉描述中推断事件序列。或出现鸟鸣声，那么场景中就需要出现鸟。 + +2. **确定摄影和构图**: + * **摄影和镜头运动**：使用以下规则决定摄影： + * 如果用户输入包含摄影描述（例如，广角镜头、航拍镜头、跟踪镜头、变焦镜头），你必须严格遵循用户输入，不要更改它。 + * 如果用户没有提供描述，基于场景和核心事件，你应该使用你的摄影知识来选择合适的镜头工作。优先选择能完全捕捉关键动作和所有核心元素的镜头和运动（例如，**缓慢平移穿过场景的略高角度镜头**）。 + * **构图和场景调度**：应用构图技术，如三分法或对称，以确保每一帧都经过专业构图。至关重要的是，考虑主体和元素随时间在画面中的移动方式（调度）——它们的进入、退出和运动路径。 + +3. **选择艺术风格**: + * **风格选择**：如果用户指定了风格（例如，油画动画、动漫、动态图形），严格遵守它。否则，首先根据用户输入推断合适的风格。如果没有明确的风格偏好，默认为**电影感写实风格**。 + +4. **确定镜头运动**: + * **镜头运动**：使用以下规则决定镜头运动： + * 如果用户输入包含镜头运动描述（例如，向前推进、推近、向上倾斜），你必须根据输入描述使用镜头运动关键字。（例如，如果用户输入是"镜头向上移动"，你必须推断它对应关键字"camera upward"，并在"camera upward"的值列表中选择一个词，即["The camera moves upward", "The camera rises", "The camera ascends"]） + * *示例：*输入："镜头向上移动"，使用词语"The camera rises"， + * *示例：*输入："高角度镜头"，使用词语"The camera captures the scene from above"。 + * 如果用户没有提供镜头运动描述。镜头拍摄必须是"The shot is at an eye-level angle with the main subject."。 + 并且镜头运动应该从镜头静止、镜头向前推进、镜头向后拉、镜头向右平移、镜头向左平移、镜头推近、镜头拉远中选择。你需要分析主体的运动和背景的运动，然后选择最合适的镜头运动。首选轻微的运动，以防止画面变化过快产生畸变。 + +4. **填充细节和审查**: + * **时长**：视频时长固定为5秒。填充的细节必须能在5秒内完成。 + * **细节填充**：遵循结构化顺序，描述主体的材料、纹理、动作、手势以及随时间变化的表情，以及环境中的任何次要元素及其变化方式。注意添加恰到好处的细节量；除非汗水是视频叙事的关键，否则不要过分强调汗水等元素。火花、雷电也是容易生成错误的内容，需要注意。**镜头或手机屏幕的正反面**：当场景中出现相机镜头、手机屏幕等物体时，必须明确描述其朝向（正面/背面/侧面），以及观众看到的是屏幕内容还是设备背面，避免产生歧义。例如："手机屏幕朝向镜头，显示......"或"相机的镜头正对着......"。 + * **运动填充**：视频中主体的运动必须在场景中逻辑合理且一致。并且运动必须清楚说明。主体的运动必须能在5秒内完成。一个坏例子是描述大量无法在5秒内完成的动作。主体的动作需要详细描述。 + * **逻辑审查**：检查描述是否完全符合物理定律、因果关系和场景逻辑贯穿整个视频时长。例如，动态场景中的动作流程是否合理？角色移动时的视线和互动是否一致？剪辑开始和结束之间是否有连续性错误？ + * **完整性审查**：确保所有预期的关键元素和事件（例如，裁判发出信号、背景爆炸）都被明确描述。检查镜头运动或主体运动是否导致不自然的裁剪或遮挡，特别是在动作期间的人体肢体或关键对象。 + * **生成畸形审查**：必须高度警惕任何可能引发动作或身体畸形的描述，严密审查并排除一切可能导致角色动作不自然、关节弯曲异常或肢体结构错误的因素，确保所有人体结构、动作轨迹、关节形态均自然、连贯，完全符合物理规律。 + +5. **最终验证**: + * **用户输入遵循检查**：将最终结果与用户输入进行比较，以确保完全描述了用户的核心内容，例如核心实体及其属性、对象或人物的数量、指定的摄影、时长、节奏和事件顺序。最终提示词不得添加用户输入未暗示的任何新前景对象或主要事件。 + * **检查物理和时间逻辑**：检查提示词中是否存在任何物理或时间逻辑错误。例如，对象大小在移动期间是否一致？运动物理是否可信（例如，加速度、重力）？反应的时间是否合理？事件之间的因果关系是否清晰？ + * **检查宝可梦IP**：检查提示词是否包含任何宝可梦IP；你应该使用宝可梦角色的英文名称。（例如，使用Jigglypuff而不是胖丁） + +6. **如果验证失败则重试**: + * **从头开始**：如果验证失败，你应该从头开始重新生成扩展的提示词。 + + + +### **四、风格特定创作指南** + +根据确定的艺术风格激活相应的专业知识库。 + +**1. 摄影和写实风格** +* **总体规则**：想象你是一位摄影大师，用户的输入将被你转换为专业摄影师拍摄的照片。假设你正在查看用户描述的图片，你将使用你的专业摄影知识将用户输入转换为具有专业构图和专业照明的视觉图片。 +* **摄影风格视频的专业照明**：你应该使用你的专业摄影照明技术来增强真实感。你应该根据用户的输入选择合适的照明技术，以下是一些可供选择的示例技术，你可以使用你的世界知识来获得更好的选择。 + * 使用戏剧性照明，强调光影之间的高对比度以创造深度感。使用伦勃朗照明从45度角照亮主体的面部，在脸颊上形成三角形高光，并在面部的另一侧投下深深的阴影。这应该以强烈的维度感突出面部特征。 + * 对于整体场景，应用黄金时段照明，具有温暖的色温，光线在过渡到阴影时逐渐柔化。柔和的侧光应该照亮主体的身体，在光影之间创造柔和的渐变，增强三维形式。背景应该具有渐进的阴影过渡，唤起平静、诱人的氛围。 + * 为了增加真实感，使用背光在主体周围创造剪影效果，强光位于主体后面以突出轮廓。这应该将背景投入深深的阴影中，边缘由附近表面的反射光柔化。确保主体的轮廓清晰，但周围的阴影增加了一种神秘感。 + * 从左侧加入冷光，与温暖的背景光形成对比，创造视觉张力。光强度应该变化，在关键特征上使用强烈的高光，在其他区域使用柔和的阴影。光影之间的过渡应该感觉自然和平衡，焦点区域由锐利的硬光照亮，阴影中的区域更柔和。 + * 为了增加额外的纹理和真实感，在水或玻璃等反射表面上包括反射，光线应该反弹以柔化阴影并在表面上创造光斑。直射光和反射光之间的这种相互作用将为整体构图增加复杂性和趣味性。 + * 确保光线在引导观众的眼睛穿过视频中起着关键作用，无论是通过阴影创造的引导线，还是通过突出场景关键元素的光线定向流动。 + * 调整以获得更多控制的关键参数： + * 照明风格：（例如，伦勃朗、柔和、硬、背光、侧光） + * 光线方向：（例如，45度角、自上而下、侧光） + * 光线质量：（例如，柔和、刺眼、漫射、聚光灯） + * 阴影细节：（例如，深阴影、柔和渐变、高对比度） + * 色温：（例如，温暖的黄金时段、凉爽的日光） + * 反射：（例如，水、玻璃或金属表面上的反射光） + * 剪影和轮廓：（例如，主体背光，创造戏剧性的轮廓） + * 可定制元素： + * 照明情绪：为场景定义整体照明情绪（例如，戏剧性、柔和、高对比度、微妙渐变）。 + * 背景照明：调整光线与背景的相互作用，例如柔和渐变或强烈阴影区域。 + * 柔和对比硬阴影：指定阴影应该有多刺眼或多漫射。 + * 高光细节：关注光线应该突出关键特征的区域（例如，面部、眼睛、纹理）。 + * 氛围：（例如，忧郁、宁静、戏剧性、和平） + * **重要规则**：如果光源不可见，只需描述光效，不要提及视频中未出现的任何光源，一个坏例子是"一个次要的、看不见的温暖光源，可能是画面外的台灯"，像这个例子的描述是被禁止的。 +* **镜头效果**：使用专业术语来描述镜头效果（例如，广角透视、长焦压缩、浅景深），你应该使用你的世界知识为用户的输入选择最佳镜头效果。 +* **构图**：使用专业术语为用户的输入选择最佳构图（例如，引导线、框架构图、三分法），**但不要直接使用构图技术名称，你应该安排人物/对象/环境来反映构图本身**。 +* **极致细节**：深入描述材料纹理（例如，木纹反射、织物纤维）、角色细节（例如，眼睛中的眼神光、皮肤毛孔）和环境氛围（例如，空气中的灰尘颗粒）。 +* **多实体场景**：如果用户的输入表明视频中有多个实体，例如一群人、一个团队或场景中的多个人，如果用户给出了精确的实体数量，则严格遵守用户的输入，不要更改数量。但如果用户没有给出具体的人物或对象数量，严格将人物或对象的数量限制在三个或更少。详细描述每个人/对象，并将它们放在中景到前景，确保他们的面部和肢体清晰、未变形，并且在关节处没有被裁剪。**这一条非常重要** +* **电影摄影写实**：如果用户的输入表明视频是写实风格，风格必须是"cinematic realistic style"（电影摄影写实风格）。 + +**2. 插画和绘画风格（卡通、油画、水彩等）** +* **定义类型**：精确定义风格（例如，"日本赛璐珞动画风格"，"厚涂油画"，"湿画法水彩"，"印象派点彩画法"）。 +* **媒介特定特征**：专注于描述风格的独特视觉语言，例如线条的粗细（"G笔线稿"）、笔触的纹理（"可见的、三维的笔触"）和颜料的特性（"水彩边缘的自然水渍"）。 +* **角色设计**：强调夸张的特征（例如，"Q版身体比例"，"占据脸部三分之一的大眼睛"）和富有表现力的姿势。 + +**3. 字体艺术** +* **主体优先**：描述必须以类似`The words "[Text Content]" rendered as...`的短语开始，以将文本确立为绝对核心主体。 +* **安全透视和构图**：强制使用安全的正面或俯视图（`Front view`，`Top-down`），并将文本主体放在画面中心，使用简单或广阔的背景进行对比，从根本上防止裁剪。 +* **形式而非形成**：描述文本构成的最终"形式"，而不是其"形成过程"。（错误："两个GPU相互倾斜形成一个A"；正确："由GPU制成的金字塔形状的字母A"）。 +* **完整性保险**：在提示词末尾添加强制性指令，例如`The entire [object/phrase] is fully visible`，作为防止裁剪的最后防线。 +* **禁止高风险词汇**：避免使用"巨大的"、"特写"、"复杂的"和"精致的"等词，因为它们可能诱导AI放大主体的一部分，导致裁剪。 + +### **五、最终输出要求** + +1. **仅输出最终提示词**：不要显示任何思考过程、Markdown格式或与文本到视频提示词无关的任何表达（如"摘要显示"）。 +2. **输出语言**：扩展的提示词应该是英文，同时根据语言规则保持文本渲染内容与用户输入的语言相同。 +3. **忠实于输入**：你必须保留用户输入句子中的核心概念、属性、数量和文本渲染内容。 +4. **风格强化**：在提示词中提及核心风格3-5次，并以风格声明句结束（例如，"整个视频是电影感写实风格"）。 +5. **字数控制**：描述主要主体的长度应该在140个单词左右，这里面主体的动作是重要部分。描述背景的长度应该是70个单词。描述其他属性（包括构图、光线、风格、氛围、拍摄角度、拍摄类型）的总长度应该在140个单词以内。 +6. **避免自我引用**：在开头直接描述视频内容，删除冗余短语，如"这个视频"或"这个视频显示了"。 + +接下来，我将提供输入句子，你将提供扩展的提示词。 +""" diff --git a/logs/aborted_4gpu_20260903/atc_global_chunk_4gpu_20260903.log b/logs/aborted_4gpu_20260903/atc_global_chunk_4gpu_20260903.log new file mode 100644 index 0000000000000000000000000000000000000000..7e9d8a3133b23ddfb7fa4d04d84e6ab8ca7a0801 --- /dev/null +++ b/logs/aborted_4gpu_20260903/atc_global_chunk_4gpu_20260903.log @@ -0,0 +1,137 @@ +/mnt/local_nvme/cz/envs/hy_worldplay_july20260720/lib/python3.10/site-packages/transformers/utils/hub.py:111: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +/mnt/local_nvme/cz/envs/hy_worldplay_july20260720/lib/python3.10/site-packages/transformers/utils/hub.py:111: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +/mnt/local_nvme/cz/envs/hy_worldplay_july20260720/lib/python3.10/site-packages/transformers/utils/hub.py:111: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +/mnt/local_nvme/cz/envs/hy_worldplay_july20260720/lib/python3.10/site-packages/transformers/utils/hub.py:111: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +{ + "manifest": "/mnt/local_nvme/zoubin/cz/projects/HY-WorldPlay-DEV-Predictor/datasets/hyworldplay_predictor_prefeature_v4/train_eval_manifest.jsonl", + "teacher_checkpoint": "/mnt/local_nvme/zoubin/cz/checkpoints/hy_worldplay/huggingface/hub/models--tencent--HY-WorldPlay/snapshots/f4c29235647707b571479a69b569e4166f9f5bf8/ar_distilled_action_model/diffusion_pytorch_model.safetensors", + "output_dir": "/mnt/local_nvme/zoubin/cz/projects/HY-WorldPlay-DEV-Predictor/checkpoints/predictor_v4_atc-global-chunk_blocks1-52_fulltrain100_fp32master_4gpu_flr1e-4_blr1e-5_steps2000", + "source_blocks": [ + 1, + 52 + ], + "predictor_variant": "v4", + "previous_condition_mode": "same_step", + "predictor_fusion_type": "concat", + "gate_hidden_dim": 256, + "gate_initial_probability": 0.8, + "predictor_arch": "atc", + "atc_variant": "full", + "atc_mlp_hidden_dim": 4096, + "atc_gate_hidden_dim": 512, + "atc_gate_initial_probability": 0.3, + "atc_transport_heads": 16, + "atc_transport_window": 3, + "atc_transport_query_chunk_size": 128, + "atc_transport_initial_scale": 0.1, + "atc_transport_mode": "global", + "atc_previous_scope": "chunk", + "atc_mlp_init": "xavier", + "max_steps": 2000, + "batch_size": 8, + "gradient_accumulation_steps": 4, + "block_warmup_steps": 100, + "block_warmup_mode": "delayed", + "scheduler_warmup_steps": 100, + "fusion_lr": 0.0001, + "blocks_lr": 1e-05, + "weight_decay": 0.01, + "hidden_weight": 0.1, + "velocity_weight": 1.0, + "grad_clip": 1.0, + "log_every": 10, + "save_every": 100, + "keep_snapshots": 20, + "seed": 0, + "num_workers": 0, + "max_records": null, + "bucket_by_context": true, + "gradient_checkpointing": true, + "fp32_trainable_params": true, + "train_delta_only": false, + "train_fusion_residual_only": false, + "save_optimizer_state": true, + "init_training_state": null, + "resume": null, + "swanlab_project": "HY-WorldPlay-Predictor-v4", + "swanlab_workspace": null, + "swanlab_experiment": "predictor-v4-atc-global-chunk-blocks1-52-fulltrain100-4gpu", + "swanlab_mode": "disabled", + "predictor_version": "v4_prefeature", + "schema_version": "predictor_context_prefeature_bf16_v1", + "supervision_pairs": [ + [ + 0, + 1 + ], + [ + 1, + 2 + ], + [ + 2, + 3 + ] + ], + "previous_condition_source": "offline_full_teacher_hidden", + "records": 2800, + "dataset_pairs": 8400, + "steps_per_epoch_per_rank": 260, + "world_size": 4, + "per_rank_batch_size": 8, + "global_batch_size": 128, + "trainable_parameters": 291649026, + "frozen_projector_parameters": 16785664, + "dtype": "bfloat16", + "activation_dtype": "bfloat16", + "trainable_parameter_dtype": "float32", + "optimizer_moment_dtype": "float32", + "inference_checkpoint_dtype": "bfloat16", + "gpu": "NVIDIA H100 80GB HBM3" +} +{"swanlab_run_id": "kxclcw5j"} +/mnt/local_nvme/zoubin/cz/projects/HY-WorldPlay-DEV-Predictor/hyvideo/commons/__init__.py:216: UserWarning: flash is not available. Falling back to torch attention. + warnings.warn("flash is not available. Falling back to torch attention.") +/mnt/local_nvme/zoubin/cz/projects/HY-WorldPlay-DEV-Predictor/hyvideo/commons/__init__.py:216: UserWarning: flash is not available. Falling back to torch attention. + warnings.warn("flash is not available. Falling back to torch attention.") +/mnt/local_nvme/zoubin/cz/projects/HY-WorldPlay-DEV-Predictor/hyvideo/commons/__init__.py:216: UserWarning: flash is not available. Falling back to torch attention. + warnings.warn("flash is not available. Falling back to torch attention.") +/mnt/local_nvme/zoubin/cz/projects/HY-WorldPlay-DEV-Predictor/hyvideo/commons/__init__.py:216: UserWarning: flash is not available. Falling back to torch attention. + warnings.warn("flash is not available. Falling back to torch attention.") +{"anchor_distance_mean": 120.77734375, "avg_context_frames": 18.0, "avg_micro_time_s": 7.172496337624921, "delta_h_c_norm": 0.0, "delta_h_d_norm": 0.0, "epoch": 1, "gate_p_mean": 0.31068896874785423, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.0, "gate_p_p10": 0.2998046875, "gate_p_p50": 0.3101806640625, "gate_p_p90": 0.3223876953125, "gate_p_std": 0.008523388823959976, "gated_delta_h_c_norm": 0.0, "global_step": 1, "grad_norm_max": 0.10651213675737381, "hidden_mse": 0.017482993425801396, "loss": 0.014502726262435317, "lr_blocks": 0.0, "lr_fusion": 2.0000000000000003e-06, "peak_memory_gib": 40.43218183517456, "transport_argmax_displacement": 23.30183243751526, "transport_attention_entropy": 8.701961100101471, "transport_attention_entropy_normalized": 0.9957918263971806, "transport_residual_norm": 0.025858936365693808, "velocity_mse": 0.012754426948959008} +{"anchor_distance_mean": 124.90538194444444, "avg_context_frames": 13.888888888888888, "avg_micro_time_s": 6.015072300826369, "delta_h_c_norm": 0.010159098375879694, "delta_h_d_norm": 0.0053419652459625565, "epoch": 1, "gate_p_mean": 0.30971763427886695, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.0, "gate_p_p10": 0.3002522786458333, "gate_p_p50": 0.3096516927083333, "gate_p_p90": 0.3192681206597222, "gate_p_std": 0.0073172886849432765, "gated_delta_h_c_norm": 0.0031429872671853644, "global_step": 10, "grad_norm_max": 0.17691870033740997, "hidden_mse": 0.017065480445227068, "loss": 0.01447371335234493, "lr_blocks": 0.0, "lr_fusion": 1.1000000000000001e-05, "peak_memory_gib": 42.605140209198, "transport_argmax_displacement": 22.777525199784172, "transport_attention_entropy": 8.701640506585438, "transport_attention_entropy_normalized": 0.9957551355991098, "transport_residual_norm": 0.02567638120510512, "velocity_mse": 0.01276716533013516} +W0903 07:11:19.161000 186936 /mnt/local_nvme/zoubin/cz/envs/hy_worldplay_july20260720/lib/python3.10/site-packages/torch/distributed/elastic/agent/server/api.py:719] Received Signals.SIGTERM death signal, shutting down workers +W0903 07:11:19.167000 186936 /mnt/local_nvme/zoubin/cz/envs/hy_worldplay_july20260720/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/api.py:897] Sending process 186977 closing signal SIGTERM +W0903 07:11:19.169000 186936 /mnt/local_nvme/zoubin/cz/envs/hy_worldplay_july20260720/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/api.py:897] Sending process 186978 closing signal SIGTERM +W0903 07:11:19.172000 186936 /mnt/local_nvme/zoubin/cz/envs/hy_worldplay_july20260720/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/api.py:897] Sending process 186979 closing signal SIGTERM +W0903 07:11:19.173000 186936 /mnt/local_nvme/zoubin/cz/envs/hy_worldplay_july20260720/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/api.py:897] Sending process 186980 closing signal SIGTERM +Traceback (most recent call last): + File "/mnt/local_nvme/cz/envs/hy_worldplay_runtime/lib/python3.10/runpy.py", line 196, in _run_module_as_main + return _run_code(code, main_globals, None, + File "/mnt/local_nvme/cz/envs/hy_worldplay_runtime/lib/python3.10/runpy.py", line 86, in _run_code + exec(code, run_globals) + File "/mnt/local_nvme/cz/envs/hy_worldplay_july20260720/lib/python3.10/site-packages/torch/distributed/run.py", line 922, in + main() + File "/mnt/local_nvme/cz/envs/hy_worldplay_july20260720/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + File "/mnt/local_nvme/cz/envs/hy_worldplay_july20260720/lib/python3.10/site-packages/torch/distributed/run.py", line 918, in main + run(args) + File "/mnt/local_nvme/cz/envs/hy_worldplay_july20260720/lib/python3.10/site-packages/torch/distributed/run.py", line 909, in run + elastic_launch( + File "/mnt/local_nvme/cz/envs/hy_worldplay_july20260720/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 138, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + File "/mnt/local_nvme/cz/envs/hy_worldplay_july20260720/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 260, in launch_agent + result = agent.run() + File "/mnt/local_nvme/cz/envs/hy_worldplay_july20260720/lib/python3.10/site-packages/torch/distributed/elastic/metrics/api.py", line 137, in wrapper + result = f(*args, **kwargs) + File "/mnt/local_nvme/cz/envs/hy_worldplay_july20260720/lib/python3.10/site-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + File "/mnt/local_nvme/cz/envs/hy_worldplay_july20260720/lib/python3.10/site-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/mnt/local_nvme/cz/envs/hy_worldplay_july20260720/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 186936 got signal: 15 diff --git a/logs/aborted_4gpu_20260903/atc_global_last_frame_4gpu_20260903.log b/logs/aborted_4gpu_20260903/atc_global_last_frame_4gpu_20260903.log new file mode 100644 index 0000000000000000000000000000000000000000..69ef2ebf6a354ba70ec8a8c390fb3a9d58cfdbe7 --- /dev/null +++ b/logs/aborted_4gpu_20260903/atc_global_last_frame_4gpu_20260903.log @@ -0,0 +1,137 @@ +/mnt/local_nvme/cz/envs/hy_worldplay_july20260720/lib/python3.10/site-packages/transformers/utils/hub.py:111: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +/mnt/local_nvme/cz/envs/hy_worldplay_july20260720/lib/python3.10/site-packages/transformers/utils/hub.py:111: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +/mnt/local_nvme/cz/envs/hy_worldplay_july20260720/lib/python3.10/site-packages/transformers/utils/hub.py:111: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +/mnt/local_nvme/cz/envs/hy_worldplay_july20260720/lib/python3.10/site-packages/transformers/utils/hub.py:111: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +{ + "manifest": "/mnt/local_nvme/zoubin/cz/projects/HY-WorldPlay-DEV-Predictor/datasets/hyworldplay_predictor_prefeature_v4/train_eval_manifest.jsonl", + "teacher_checkpoint": "/mnt/local_nvme/zoubin/cz/checkpoints/hy_worldplay/huggingface/hub/models--tencent--HY-WorldPlay/snapshots/f4c29235647707b571479a69b569e4166f9f5bf8/ar_distilled_action_model/diffusion_pytorch_model.safetensors", + "output_dir": "/mnt/local_nvme/zoubin/cz/projects/HY-WorldPlay-DEV-Predictor/checkpoints/predictor_v4_atc-global-last_frame_blocks1-52_fulltrain100_fp32master_4gpu_flr1e-4_blr1e-5_steps2000", + "source_blocks": [ + 1, + 52 + ], + "predictor_variant": "v4", + "previous_condition_mode": "same_step", + "predictor_fusion_type": "concat", + "gate_hidden_dim": 256, + "gate_initial_probability": 0.8, + "predictor_arch": "atc", + "atc_variant": "full", + "atc_mlp_hidden_dim": 4096, + "atc_gate_hidden_dim": 512, + "atc_gate_initial_probability": 0.3, + "atc_transport_heads": 16, + "atc_transport_window": 3, + "atc_transport_query_chunk_size": 128, + "atc_transport_initial_scale": 0.1, + "atc_transport_mode": "global", + "atc_previous_scope": "last_frame", + "atc_mlp_init": "xavier", + "max_steps": 2000, + "batch_size": 8, + "gradient_accumulation_steps": 4, + "block_warmup_steps": 100, + "block_warmup_mode": "delayed", + "scheduler_warmup_steps": 100, + "fusion_lr": 0.0001, + "blocks_lr": 1e-05, + "weight_decay": 0.01, + "hidden_weight": 0.1, + "velocity_weight": 1.0, + "grad_clip": 1.0, + "log_every": 10, + "save_every": 100, + "keep_snapshots": 20, + "seed": 0, + "num_workers": 0, + "max_records": null, + "bucket_by_context": true, + "gradient_checkpointing": true, + "fp32_trainable_params": true, + "train_delta_only": false, + "train_fusion_residual_only": false, + "save_optimizer_state": true, + "init_training_state": null, + "resume": null, + "swanlab_project": "HY-WorldPlay-Predictor-v4", + "swanlab_workspace": null, + "swanlab_experiment": "predictor-v4-atc-global-last_frame-blocks1-52-fulltrain100-4gpu", + "swanlab_mode": "disabled", + "predictor_version": "v4_prefeature", + "schema_version": "predictor_context_prefeature_bf16_v1", + "supervision_pairs": [ + [ + 0, + 1 + ], + [ + 1, + 2 + ], + [ + 2, + 3 + ] + ], + "previous_condition_source": "offline_full_teacher_hidden", + "records": 2800, + "dataset_pairs": 8400, + "steps_per_epoch_per_rank": 260, + "world_size": 4, + "per_rank_batch_size": 8, + "global_batch_size": 128, + "trainable_parameters": 291649026, + "frozen_projector_parameters": 16785664, + "dtype": "bfloat16", + "activation_dtype": "bfloat16", + "trainable_parameter_dtype": "float32", + "optimizer_moment_dtype": "float32", + "inference_checkpoint_dtype": "bfloat16", + "gpu": "NVIDIA H100 80GB HBM3" +} +{"swanlab_run_id": "9h4wngo3"} +/mnt/local_nvme/zoubin/cz/projects/HY-WorldPlay-DEV-Predictor/hyvideo/commons/__init__.py:216: UserWarning: flash is not available. Falling back to torch attention. + warnings.warn("flash is not available. Falling back to torch attention.") +/mnt/local_nvme/zoubin/cz/projects/HY-WorldPlay-DEV-Predictor/hyvideo/commons/__init__.py:216: UserWarning: flash is not available. Falling back to torch attention. + warnings.warn("flash is not available. Falling back to torch attention.") +/mnt/local_nvme/zoubin/cz/projects/HY-WorldPlay-DEV-Predictor/hyvideo/commons/__init__.py:216: UserWarning: flash is not available. Falling back to torch attention. + warnings.warn("flash is not available. Falling back to torch attention.") +/mnt/local_nvme/zoubin/cz/projects/HY-WorldPlay-DEV-Predictor/hyvideo/commons/__init__.py:216: UserWarning: flash is not available. Falling back to torch attention. + warnings.warn("flash is not available. Falling back to torch attention.") +{"anchor_distance_mean": 120.77734375, "avg_context_frames": 18.0, "avg_micro_time_s": 6.410048525749744, "delta_h_c_norm": 0.0, "delta_h_d_norm": 0.0, "epoch": 1, "gate_p_mean": 0.3107036128640175, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.0, "gate_p_p10": 0.2999267578125, "gate_p_p50": 0.310302734375, "gate_p_p90": 0.3223876953125, "gate_p_std": 0.008500846422975883, "gated_delta_h_c_norm": 0.0, "global_step": 1, "grad_norm_max": 0.10649573057889938, "hidden_mse": 0.017482993425801396, "loss": 0.014502726262435317, "lr_blocks": 0.0, "lr_fusion": 2.0000000000000003e-06, "peak_memory_gib": 40.414329051971436, "transport_argmax_displacement": 23.205328345298767, "transport_attention_entropy": 7.316348195075989, "transport_attention_entropy_normalized": 0.9950910024344921, "transport_residual_norm": 0.6862669661641121, "velocity_mse": 0.012754426948959008} +{"anchor_distance_mean": 124.90538194444444, "avg_context_frames": 13.888888888888888, "avg_micro_time_s": 5.319033601972276, "delta_h_c_norm": 0.0101734617023289, "delta_h_d_norm": 0.005345858901111771, "epoch": 1, "gate_p_mean": 0.3096200991421938, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.0, "gate_p_p10": 0.3002794053819444, "gate_p_p50": 0.3095431857638889, "gate_p_p90": 0.31901041666666663, "gate_p_std": 0.0072460693417168735, "gated_delta_h_c_norm": 0.0031463142153774644, "global_step": 10, "grad_norm_max": 0.17813482880592346, "hidden_mse": 0.01706588228504794, "loss": 0.014473357558017597, "lr_blocks": 0.0, "lr_fusion": 1.1000000000000001e-05, "peak_memory_gib": 42.58728742599487, "transport_argmax_displacement": 22.915527754359776, "transport_attention_entropy": 7.316413223743439, "transport_attention_entropy_normalized": 0.9950998495850297, "transport_residual_norm": 0.6949102551572852, "velocity_mse": 0.012766769290060943} +W0903 07:11:19.162000 187023 /mnt/local_nvme/zoubin/cz/envs/hy_worldplay_july20260720/lib/python3.10/site-packages/torch/distributed/elastic/agent/server/api.py:719] Received Signals.SIGTERM death signal, shutting down workers +W0903 07:11:19.163000 187023 /mnt/local_nvme/zoubin/cz/envs/hy_worldplay_july20260720/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/api.py:897] Sending process 187073 closing signal SIGTERM +W0903 07:11:19.168000 187023 /mnt/local_nvme/zoubin/cz/envs/hy_worldplay_july20260720/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/api.py:897] Sending process 187074 closing signal SIGTERM +W0903 07:11:19.169000 187023 /mnt/local_nvme/zoubin/cz/envs/hy_worldplay_july20260720/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/api.py:897] Sending process 187075 closing signal SIGTERM +W0903 07:11:19.169000 187023 /mnt/local_nvme/zoubin/cz/envs/hy_worldplay_july20260720/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/api.py:897] Sending process 187076 closing signal SIGTERM +Traceback (most recent call last): + File "/mnt/local_nvme/cz/envs/hy_worldplay_runtime/lib/python3.10/runpy.py", line 196, in _run_module_as_main + return _run_code(code, main_globals, None, + File "/mnt/local_nvme/cz/envs/hy_worldplay_runtime/lib/python3.10/runpy.py", line 86, in _run_code + exec(code, run_globals) + File "/mnt/local_nvme/cz/envs/hy_worldplay_july20260720/lib/python3.10/site-packages/torch/distributed/run.py", line 922, in + main() + File "/mnt/local_nvme/cz/envs/hy_worldplay_july20260720/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + File "/mnt/local_nvme/cz/envs/hy_worldplay_july20260720/lib/python3.10/site-packages/torch/distributed/run.py", line 918, in main + run(args) + File "/mnt/local_nvme/cz/envs/hy_worldplay_july20260720/lib/python3.10/site-packages/torch/distributed/run.py", line 909, in run + elastic_launch( + File "/mnt/local_nvme/cz/envs/hy_worldplay_july20260720/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 138, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + File "/mnt/local_nvme/cz/envs/hy_worldplay_july20260720/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 260, in launch_agent + result = agent.run() + File "/mnt/local_nvme/cz/envs/hy_worldplay_july20260720/lib/python3.10/site-packages/torch/distributed/elastic/metrics/api.py", line 137, in wrapper + result = f(*args, **kwargs) + File "/mnt/local_nvme/cz/envs/hy_worldplay_july20260720/lib/python3.10/site-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + File "/mnt/local_nvme/cz/envs/hy_worldplay_july20260720/lib/python3.10/site-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/mnt/local_nvme/cz/envs/hy_worldplay_july20260720/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 187023 got signal: 15 diff --git a/logs/atc_global_sequential_8gpu_20260903.log b/logs/atc_global_sequential_8gpu_20260903.log new file mode 100644 index 0000000000000000000000000000000000000000..8860671235c71cdc47558cd50cbdf55ad6f93cb7 --- /dev/null +++ b/logs/atc_global_sequential_8gpu_20260903.log @@ -0,0 +1,646 @@ +[2026-09-03T07:11:44+00:00] start scope=chunk +/mnt/local_nvme/cz/envs/hy_worldplay_july20260720/lib/python3.10/site-packages/transformers/utils/hub.py:111: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +/mnt/local_nvme/cz/envs/hy_worldplay_july20260720/lib/python3.10/site-packages/transformers/utils/hub.py:111: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +/mnt/local_nvme/cz/envs/hy_worldplay_july20260720/lib/python3.10/site-packages/transformers/utils/hub.py:111: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +/mnt/local_nvme/cz/envs/hy_worldplay_july20260720/lib/python3.10/site-packages/transformers/utils/hub.py:111: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +/mnt/local_nvme/cz/envs/hy_worldplay_july20260720/lib/python3.10/site-packages/transformers/utils/hub.py:111: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +/mnt/local_nvme/cz/envs/hy_worldplay_july20260720/lib/python3.10/site-packages/transformers/utils/hub.py:111: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +/mnt/local_nvme/cz/envs/hy_worldplay_july20260720/lib/python3.10/site-packages/transformers/utils/hub.py:111: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +/mnt/local_nvme/cz/envs/hy_worldplay_july20260720/lib/python3.10/site-packages/transformers/utils/hub.py:111: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +{ + "manifest": "/mnt/local_nvme/zoubin/cz/projects/HY-WorldPlay-DEV-Predictor/datasets/hyworldplay_predictor_prefeature_v4/train_eval_manifest.jsonl", + "teacher_checkpoint": "/mnt/local_nvme/zoubin/cz/checkpoints/hy_worldplay/huggingface/hub/models--tencent--HY-WorldPlay/snapshots/f4c29235647707b571479a69b569e4166f9f5bf8/ar_distilled_action_model/diffusion_pytorch_model.safetensors", + "output_dir": "/mnt/local_nvme/zoubin/cz/projects/HY-WorldPlay-DEV-Predictor/checkpoints/predictor_v4_atc-global-chunk_blocks1-52_fulltrain100_fp32master_8gpu_flr1e-4_blr1e-5_steps2000", + "source_blocks": [ + 1, + 52 + ], + "predictor_variant": "v4", + "previous_condition_mode": "same_step", + "predictor_fusion_type": "concat", + "gate_hidden_dim": 256, + "gate_initial_probability": 0.8, + "predictor_arch": "atc", + "atc_variant": "full", + "atc_mlp_hidden_dim": 4096, + "atc_gate_hidden_dim": 512, + "atc_gate_initial_probability": 0.3, + "atc_transport_heads": 16, + "atc_transport_window": 3, + "atc_transport_query_chunk_size": 128, + "atc_transport_initial_scale": 0.1, + "atc_transport_mode": "global", + "atc_previous_scope": "chunk", + "atc_mlp_init": "xavier", + "max_steps": 2000, + "batch_size": 8, + "gradient_accumulation_steps": 1, + "block_warmup_steps": 100, + "block_warmup_mode": "delayed", + "scheduler_warmup_steps": 100, + "fusion_lr": 0.0001, + "blocks_lr": 1e-05, + "weight_decay": 0.01, + "hidden_weight": 0.1, + "velocity_weight": 1.0, + "grad_clip": 1.0, + "log_every": 10, + "save_every": 100, + "keep_snapshots": 20, + "seed": 0, + "num_workers": 0, + "max_records": null, + "bucket_by_context": true, + "gradient_checkpointing": true, + "fp32_trainable_params": true, + "train_delta_only": false, + "train_fusion_residual_only": false, + "save_optimizer_state": true, + "init_training_state": null, + "resume": null, + "swanlab_project": "HY-WorldPlay-Predictor-v4", + "swanlab_workspace": null, + "swanlab_experiment": "predictor-v4-atc-global-chunk-blocks1-52-fulltrain100-8gpu", + "swanlab_mode": "disabled", + "predictor_version": "v4_prefeature", + "schema_version": "predictor_context_prefeature_bf16_v1", + "supervision_pairs": [ + [ + 0, + 1 + ], + [ + 1, + 2 + ], + [ + 2, + 3 + ] + ], + "previous_condition_source": "offline_full_teacher_hidden", + "records": 2800, + "dataset_pairs": 8400, + "steps_per_epoch_per_rank": 128, + "world_size": 8, + "per_rank_batch_size": 8, + "global_batch_size": 64, + "trainable_parameters": 291649026, + "frozen_projector_parameters": 16785664, + "dtype": "bfloat16", + "activation_dtype": "bfloat16", + "trainable_parameter_dtype": "float32", + "optimizer_moment_dtype": "float32", + "inference_checkpoint_dtype": "bfloat16", + "gpu": "NVIDIA H100 80GB HBM3" +} +{"swanlab_run_id": "wnow4n3i"} +/mnt/local_nvme/zoubin/cz/projects/HY-WorldPlay-DEV-Predictor/hyvideo/commons/__init__.py:216: UserWarning: flash is not available. Falling back to torch attention. + warnings.warn("flash is not available. Falling back to torch attention.") +/mnt/local_nvme/zoubin/cz/projects/HY-WorldPlay-DEV-Predictor/hyvideo/commons/__init__.py:216: UserWarning: flash is not available. Falling back to torch attention. + warnings.warn("flash is not available. Falling back to torch attention.") +/mnt/local_nvme/zoubin/cz/projects/HY-WorldPlay-DEV-Predictor/hyvideo/commons/__init__.py:216: UserWarning: flash is not available. Falling back to torch attention. + warnings.warn("flash is not available. Falling back to torch attention.") +/mnt/local_nvme/zoubin/cz/projects/HY-WorldPlay-DEV-Predictor/hyvideo/commons/__init__.py:216: UserWarning: flash is not available. Falling back to torch attention. + warnings.warn("flash is not available. Falling back to torch attention.") +/mnt/local_nvme/zoubin/cz/projects/HY-WorldPlay-DEV-Predictor/hyvideo/commons/__init__.py:216: UserWarning: flash is not available. Falling back to torch attention. + warnings.warn("flash is not available. Falling back to torch attention.") +/mnt/local_nvme/zoubin/cz/projects/HY-WorldPlay-DEV-Predictor/hyvideo/commons/__init__.py:216: UserWarning: flash is not available. Falling back to torch attention. + warnings.warn("flash is not available. Falling back to torch attention.") +/mnt/local_nvme/zoubin/cz/projects/HY-WorldPlay-DEV-Predictor/hyvideo/commons/__init__.py:216: UserWarning: flash is not available. Falling back to torch attention. + warnings.warn("flash is not available. Falling back to torch attention.") +/mnt/local_nvme/zoubin/cz/projects/HY-WorldPlay-DEV-Predictor/hyvideo/commons/__init__.py:216: UserWarning: flash is not available. Falling back to torch attention. + warnings.warn("flash is not available. Falling back to torch attention.") +{"anchor_distance_mean": 136.5234375, "avg_context_frames": 8.0, "avg_micro_time_s": 5.3208711744996435, "delta_h_c_norm": 0.0, "delta_h_d_norm": 0.0, "epoch": 1, "gate_p_mean": 0.3125024065375328, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.0, "gate_p_p10": 0.304443359375, "gate_p_p50": 0.312744140625, "gate_p_p90": 0.320556640625, "gate_p_std": 0.006253311817999929, "gated_delta_h_c_norm": 0.0, "global_step": 1, "grad_norm_max": 0.11027199029922485, "hidden_mse": 0.01441731525119394, "loss": 0.013514157384634018, "lr_blocks": 0.0, "lr_fusion": 2.0000000000000003e-06, "peak_memory_gib": 30.172312259674072, "transport_argmax_displacement": 23.634273529052734, "transport_attention_entropy": 8.702587008476257, "transport_attention_entropy_normalized": 0.9958634451031685, "transport_residual_norm": 0.025913803605362773, "velocity_mse": 0.012072425859514624} +{"anchor_distance_mean": 122.63107638888889, "avg_context_frames": 15.11111111111111, "avg_micro_time_s": 6.455038835694469, "delta_h_c_norm": 0.009588007671279935, "delta_h_d_norm": 0.005017348784248396, "epoch": 1, "gate_p_mean": 0.3088861981199847, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.0, "gate_p_p10": 0.2994249131944444, "gate_p_p50": 0.3088107638888889, "gate_p_p90": 0.3187120225694444, "gate_p_std": 0.007424518045607126, "gated_delta_h_c_norm": 0.0029557292509707622, "global_step": 10, "grad_norm_max": 0.34026044607162476, "hidden_mse": 0.01640209179216375, "loss": 0.013537007092963904, "lr_blocks": 0.0, "lr_fusion": 1.1000000000000001e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 22.736745595932007, "transport_attention_entropy": 8.701729549301994, "transport_attention_entropy_normalized": 0.995765326751603, "transport_residual_norm": 0.0256477455712027, "velocity_mse": 0.011896797944144862} +{"anchor_distance_mean": 126.8140625, "avg_context_frames": 14.8, "avg_micro_time_s": 6.376130198362557, "delta_h_c_norm": 0.01751654405379668, "delta_h_d_norm": 0.007075755251571537, "epoch": 1, "gate_p_mean": 0.2796135265380144, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.0, "gate_p_p10": 0.27102050781250003, "gate_p_p50": 0.2794677734375, "gate_p_p90": 0.2884521484375, "gate_p_std": 0.006793491751886905, "gated_delta_h_c_norm": 0.004864464333513752, "global_step": 20, "grad_norm_max": 0.24026724696159363, "hidden_mse": 0.01621361756697297, "loss": 0.013984380842885002, "lr_blocks": 0.0, "lr_fusion": 2.1e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 22.895590782165527, "transport_attention_entropy": 8.701626515388488, "transport_attention_entropy_normalized": 0.9957535311579705, "transport_residual_norm": 0.025732909864746037, "velocity_mse": 0.01236301908211317} +{"anchor_distance_mean": 123.59921875, "avg_context_frames": 16.0, "avg_micro_time_s": 6.678203227287418, "delta_h_c_norm": 0.025017744163051248, "delta_h_d_norm": 0.008076838066335768, "epoch": 1, "gate_p_mean": 0.22309377547353507, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.0, "gate_p_p10": 0.2137939453125, "gate_p_p50": 0.22272949218750002, "gate_p_p90": 0.2328125, "gate_p_std": 0.0074750799743924295, "gated_delta_h_c_norm": 0.005570541112683714, "global_step": 30, "grad_norm_max": 0.05483506619930267, "hidden_mse": 0.01702611342770979, "loss": 0.013904614077182487, "lr_blocks": 0.0, "lr_fusion": 3.1e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 22.50489537715912, "transport_attention_entropy": 8.701992821693421, "transport_attention_entropy_normalized": 0.9957954533398152, "transport_residual_norm": 0.026720128837041557, "velocity_mse": 0.012202002701815219} +{"anchor_distance_mean": 129.18437500000002, "avg_context_frames": 10.4, "avg_micro_time_s": 5.2241132539375945, "delta_h_c_norm": 0.032111336034722625, "delta_h_d_norm": 0.011161955411080272, "epoch": 1, "gate_p_mean": 0.1816113503649831, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.0, "gate_p_p10": 0.16990966796875, "gate_p_p50": 0.18077392578125, "gate_p_p90": 0.1945812702178955, "gate_p_std": 0.009686951048206539, "gated_delta_h_c_norm": 0.005905326647916809, "global_step": 40, "grad_norm_max": 0.09652803838253021, "hidden_mse": 0.015255597676150501, "loss": 0.01327847913489677, "lr_blocks": 0.0, "lr_fusion": 4.1e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 22.599883580207827, "transport_attention_entropy": 8.701554334163665, "transport_attention_entropy_normalized": 0.9957452714443207, "transport_residual_norm": 0.027581474464386704, "velocity_mse": 0.011752919398713858} +{"anchor_distance_mean": 126.4578125, "avg_context_frames": 12.8, "avg_micro_time_s": 5.8377561225376216, "delta_h_c_norm": 0.03944072318263352, "delta_h_d_norm": 0.015282023244071753, "epoch": 1, "gate_p_mean": 0.16360807344317438, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.0, "gate_p_p10": 0.14627685546875, "gate_p_p50": 0.16151123046875002, "gate_p_p90": 0.18447265625, "gate_p_std": 0.014698829263215885, "gated_delta_h_c_norm": 0.006727325514657424, "global_step": 50, "grad_norm_max": 0.10679689794778824, "hidden_mse": 0.014650038070976736, "loss": 0.012502860464155675, "lr_blocks": 0.0, "lr_fusion": 5.1000000000000006e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 22.593650412559512, "transport_attention_entropy": 8.702318596839905, "transport_attention_entropy_normalized": 0.9958327293395997, "transport_residual_norm": 0.028153949230909348, "velocity_mse": 0.011037856701295824} +{"anchor_distance_mean": 128.471875, "avg_context_frames": 13.600000000000001, "avg_micro_time_s": 6.029425299849777, "delta_h_c_norm": 0.040839216648601, "delta_h_d_norm": 0.01575244190171361, "epoch": 1, "gate_p_mean": 0.1331408793106675, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.0, "gate_p_p10": 0.10902099609375, "gate_p_p50": 0.12907104492187502, "gate_p_p90": 0.163665771484375, "gate_p_std": 0.021008776861708614, "gated_delta_h_c_norm": 0.006045044324127957, "global_step": 60, "grad_norm_max": 0.0794331356883049, "hidden_mse": 0.014848871948197485, "loss": 0.013288636383367703, "lr_blocks": 0.0, "lr_fusion": 6.1e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 22.642056918144227, "transport_attention_entropy": 8.70212243795395, "transport_attention_entropy_normalized": 0.9958102844655514, "transport_residual_norm": 0.028639437863603234, "velocity_mse": 0.011803749136743136} +{"anchor_distance_mean": 124.93281250000001, "avg_context_frames": 16.8, "avg_micro_time_s": 6.921830055837472, "delta_h_c_norm": 0.03563937547150999, "delta_h_d_norm": 0.016541946225333958, "epoch": 1, "gate_p_mean": 0.07906353641301395, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.0895144761761685, "gate_p_p10": 0.0563507080078125, "gate_p_p50": 0.074383544921875, "gate_p_p90": 0.10953369140625001, "gate_p_std": 0.02109818358439952, "gated_delta_h_c_norm": 0.003495262862998061, "global_step": 70, "grad_norm_max": 0.0845075473189354, "hidden_mse": 0.015803745604353027, "loss": 0.013193020137259738, "lr_blocks": 0.0, "lr_fusion": 7.1e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 22.363682198524476, "transport_attention_entropy": 8.70220627784729, "transport_attention_entropy_normalized": 0.99581987708807, "transport_residual_norm": 0.029960562149062755, "velocity_mse": 0.011612645525019616} +{"anchor_distance_mean": 125.22187500000001, "avg_context_frames": 16.0, "avg_micro_time_s": 6.7319117139624725, "delta_h_c_norm": 0.03945494424551726, "delta_h_d_norm": 0.02100885092513636, "epoch": 1, "gate_p_mean": 0.06527812071144581, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.34707032272126526, "gate_p_p10": 0.040606689453125004, "gate_p_p50": 0.05887451171875, "gate_p_p90": 0.10077514648437501, "gate_p_std": 0.02393486751243472, "gated_delta_h_c_norm": 0.003555230793426745, "global_step": 80, "grad_norm_max": 0.07990196347236633, "hidden_mse": 0.01381204069184605, "loss": 0.012383744688122533, "lr_blocks": 0.0, "lr_fusion": 8.1e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 22.57236306667328, "transport_attention_entropy": 8.70215734243393, "transport_attention_entropy_normalized": 0.9958142794668675, "transport_residual_norm": 0.02996563396882266, "velocity_mse": 0.011002540642220993} +{"anchor_distance_mean": 120.7078125, "avg_context_frames": 14.8, "avg_micro_time_s": 6.389231161625026, "delta_h_c_norm": 0.04998268363997341, "delta_h_d_norm": 0.02340932837687433, "epoch": 1, "gate_p_mean": 0.07314352015964687, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.34460788359865546, "gate_p_p10": 0.0384185791015625, "gate_p_p50": 0.0632476806640625, "gate_p_p90": 0.126348876953125, "gate_p_std": 0.03570886431261897, "gated_delta_h_c_norm": 0.00558549965498969, "global_step": 90, "grad_norm_max": 0.07973039895296097, "hidden_mse": 0.014822595653822646, "loss": 0.012982645820011386, "lr_blocks": 0.0, "lr_fusion": 9.1e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 22.24722893238068, "transport_attention_entropy": 8.702274644374848, "transport_attention_entropy_normalized": 0.9958277016878129, "transport_residual_norm": 0.02923904529307038, "velocity_mse": 0.011500386276748032} +{"anchor_distance_mean": 123.20937500000001, "avg_context_frames": 15.200000000000001, "avg_micro_time_s": 6.484632173324986, "delta_h_c_norm": 0.05143033242784441, "delta_h_d_norm": 0.02546379475388676, "epoch": 1, "gate_p_mean": 0.055811060382984584, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.5938426684035222, "gate_p_p10": 0.0254852294921875, "gate_p_p50": 0.04417495727539063, "gate_p_p90": 0.10473752021789551, "gate_p_std": 0.03384404595708475, "gated_delta_h_c_norm": 0.0049568957372684965, "global_step": 100, "grad_norm_max": 0.06912844628095627, "hidden_mse": 0.015084139129612595, "loss": 0.013565984202432447, "lr_blocks": 1e-05, "lr_fusion": 0.0001, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 22.323157024383548, "transport_attention_entropy": 8.7019233584404, "transport_attention_entropy_normalized": 0.9957875043153763, "transport_residual_norm": 0.029419793095439673, "velocity_mse": 0.012057570306933486} +{"anchor_distance_mean": 124.93125, "avg_context_frames": 11.600000000000001, "avg_micro_time_s": 5.534451920887523, "delta_h_c_norm": 0.047528030630201105, "delta_h_d_norm": 0.02610907731577754, "epoch": 1, "gate_p_mean": 0.033742743707261984, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8192435637116433, "gate_p_p10": 0.013689422607421876, "gate_p_p50": 0.025067901611328127, "gate_p_p90": 0.06615474224090577, "gate_p_std": 0.02451263812254183, "gated_delta_h_c_norm": 0.0033023418836819474, "global_step": 110, "grad_norm_max": 0.09098286926746368, "hidden_mse": 0.015365160536020994, "loss": 0.01390361994272098, "lr_blocks": 9.999316524962347e-06, "lr_fusion": 9.999316524962345e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 22.201581406593323, "transport_attention_entropy": 8.702307569980622, "transport_attention_entropy_normalized": 0.9958314679563046, "transport_residual_norm": 0.030719654518179595, "velocity_mse": 0.012367103851283902} +{"anchor_distance_mean": 119.571875, "avg_context_frames": 13.600000000000001, "avg_micro_time_s": 6.066075811599967, "delta_h_c_norm": 0.05234821396879852, "delta_h_d_norm": 0.028146385331638157, "epoch": 1, "gate_p_mean": 0.042467458220198755, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.7236623823642732, "gate_p_p10": 0.0154266357421875, "gate_p_p50": 0.030514526367187503, "gate_p_p90": 0.08671875000000001, "gate_p_std": 0.03321820767596364, "gated_delta_h_c_norm": 0.004824904684210197, "global_step": 120, "grad_norm_max": 0.17972183227539062, "hidden_mse": 0.013651348254643382, "loss": 0.01289966416079551, "lr_blocks": 9.99726628670463e-06, "lr_fusion": 9.997266286704631e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 22.017486405372622, "transport_attention_entropy": 8.701823246479035, "transport_attention_entropy_normalized": 0.9957760460674763, "transport_residual_norm": 0.030043354188092054, "velocity_mse": 0.011534529313212262} +{"anchor_distance_mean": 126.49062500000001, "avg_context_frames": 18.0, "avg_micro_time_s": 7.265871057025061, "delta_h_c_norm": 0.046917987242341044, "delta_h_d_norm": 0.02576414484065026, "epoch": 2, "gate_p_mean": 0.024562701757531613, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.9015507578849793, "gate_p_p10": 0.0092987060546875, "gate_p_p50": 0.017314910888671875, "gate_p_p90": 0.048622578382492065, "gate_p_std": 0.020496825175359848, "gated_delta_h_c_norm": 0.0026249849885061852, "global_step": 130, "grad_norm_max": 0.08160524815320969, "hidden_mse": 0.012775966461049393, "loss": 0.011975678353337571, "lr_blocks": 9.993849845741525e-06, "lr_fusion": 9.993849845741524e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 22.43492066860199, "transport_attention_entropy": 8.702471590042114, "transport_attention_entropy_normalized": 0.9958502367138863, "transport_residual_norm": 0.030835979082621635, "velocity_mse": 0.010698081739246846} +{"anchor_distance_mean": 122.88203125000001, "avg_context_frames": 13.600000000000001, "avg_micro_time_s": 6.034283136024943, "delta_h_c_norm": 0.05061854403465987, "delta_h_d_norm": 0.026810636068694296, "epoch": 2, "gate_p_mean": 0.030545008543413132, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8341739512979984, "gate_p_p10": 0.010544586181640626, "gate_p_p50": 0.02143707275390625, "gate_p_p90": 0.061953735351562504, "gate_p_std": 0.02706727085169405, "gated_delta_h_c_norm": 0.003848242875392316, "global_step": 140, "grad_norm_max": 0.0962262973189354, "hidden_mse": 0.013833333662478254, "loss": 0.013094112253747882, "lr_blocks": 9.989068136093873e-06, "lr_fusion": 9.989068136093873e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 22.28401141166687, "transport_attention_entropy": 8.702119338512421, "transport_attention_entropy_normalized": 0.995809929817915, "transport_residual_norm": 0.02948194076307118, "velocity_mse": 0.011710778862470761} +{"anchor_distance_mean": 128.665625, "avg_context_frames": 14.4, "avg_micro_time_s": 6.300304284149957, "delta_h_c_norm": 0.051449044980108743, "delta_h_d_norm": 0.02808853457681835, "epoch": 2, "gate_p_mean": 0.033971538196783516, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8210534647107125, "gate_p_p10": 0.01040802001953125, "gate_p_p50": 0.021673583984375, "gate_p_p90": 0.07389492988586426, "gate_p_std": 0.033077854767907415, "gated_delta_h_c_norm": 0.004521497844689293, "global_step": 150, "grad_norm_max": 0.0691492110490799, "hidden_mse": 0.0142989503685385, "loss": 0.013231889292364941, "lr_blocks": 9.98292246503335e-06, "lr_fusion": 9.98292246503335e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 22.396534514427188, "transport_attention_entropy": 8.702228581905365, "transport_attention_entropy_normalized": 0.9958224311470986, "transport_residual_norm": 0.028980433638207615, "velocity_mse": 0.01180199423688464} +{"anchor_distance_mean": 126.65234375, "avg_context_frames": 12.4, "avg_micro_time_s": 5.686137850362593, "delta_h_c_norm": 0.048704533977434045, "delta_h_d_norm": 0.028038916899822654, "epoch": 2, "gate_p_mean": 0.01808773261727765, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.9435389406979084, "gate_p_p10": 0.006087684631347657, "gate_p_p50": 0.012697982788085939, "gate_p_p90": 0.03602752685546875, "gate_p_std": 0.016600663735880518, "gated_delta_h_c_norm": 0.001997639447654365, "global_step": 160, "grad_norm_max": 0.03859546408057213, "hidden_mse": 0.014029564923839645, "loss": 0.012659193901345135, "lr_blocks": 9.975414512725058e-06, "lr_fusion": 9.975414512725057e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 22.18011054992676, "transport_attention_entropy": 8.701997530460359, "transport_attention_entropy_normalized": 0.9957959905266762, "transport_residual_norm": 0.030132608767598868, "velocity_mse": 0.011256237394991332} +{"anchor_distance_mean": 124.34609375000001, "avg_context_frames": 14.4, "avg_micro_time_s": 6.250604225362508, "delta_h_c_norm": 0.057821279903873805, "delta_h_d_norm": 0.030710336985066533, "epoch": 2, "gate_p_mean": 0.03638124961871654, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.7968044139444829, "gate_p_p10": 0.013212585449218751, "gate_p_p50": 0.02694549560546875, "gate_p_p90": 0.072955322265625, "gate_p_std": 0.029067916178610178, "gated_delta_h_c_norm": 0.004451515349501278, "global_step": 170, "grad_norm_max": 0.02936680242419243, "hidden_mse": 0.013449536019470543, "loss": 0.011619642568985001, "lr_blocks": 9.966546331768192e-06, "lr_fusion": 9.966546331768191e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 22.145761585235597, "transport_attention_entropy": 8.701554763317109, "transport_attention_entropy_normalized": 0.9957453198730946, "transport_residual_norm": 0.02780253209639341, "velocity_mse": 0.010274688925710507} +{"anchor_distance_mean": 127.30390625000001, "avg_context_frames": 15.600000000000001, "avg_micro_time_s": 6.611219087624864, "delta_h_c_norm": 0.057035144371911886, "delta_h_d_norm": 0.03527430887334049, "epoch": 2, "gate_p_mean": 0.02628964532632381, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8847761660814286, "gate_p_p10": 0.008547210693359375, "gate_p_p50": 0.018067169189453124, "gate_p_p90": 0.053478932380676275, "gate_p_std": 0.02411940349265933, "gated_delta_h_c_norm": 0.0036425872007384897, "global_step": 180, "grad_norm_max": 0.037702776491642, "hidden_mse": 0.013762461248552427, "loss": 0.013091804180294276, "lr_blocks": 9.956320346634877e-06, "lr_fusion": 9.956320346634876e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 22.04848077297211, "transport_attention_entropy": 8.70200228691101, "transport_attention_entropy_normalized": 0.9957965321838856, "transport_residual_norm": 0.0282786856405437, "velocity_mse": 0.011715558083960787} +{"anchor_distance_mean": 127.39687500000001, "avg_context_frames": 16.400000000000002, "avg_micro_time_s": 6.797986262125097, "delta_h_c_norm": 0.05820246245712042, "delta_h_d_norm": 0.03474258026108146, "epoch": 2, "gate_p_mean": 0.02571729933843017, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.890408681333065, "gate_p_p10": 0.009246444702148438, "gate_p_p50": 0.017609405517578124, "gate_p_p90": 0.05130031108856201, "gate_p_std": 0.023058228084119038, "gated_delta_h_c_norm": 0.0034903921055956747, "global_step": 190, "grad_norm_max": 0.030609235167503357, "hidden_mse": 0.011767141899326816, "loss": 0.011019324281369336, "lr_blocks": 9.944739353007344e-06, "lr_fusion": 9.944739353007344e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 21.879258012771608, "transport_attention_entropy": 8.702265703678131, "transport_attention_entropy_normalized": 0.995826679468155, "transport_residual_norm": 0.027882483182474972, "velocity_mse": 0.009842610120540485} +{"anchor_distance_mean": 126.19609375, "avg_context_frames": 12.4, "avg_micro_time_s": 5.723151511349989, "delta_h_c_norm": 0.06220030719414354, "delta_h_d_norm": 0.038560013868846, "epoch": 2, "gate_p_mean": 0.039473543292842805, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.7859467878937721, "gate_p_p10": 0.012944793701171875, "gate_p_p50": 0.026692962646484377, "gate_p_p90": 0.08194427490234375, "gate_p_std": 0.03600760590052232, "gated_delta_h_c_norm": 0.005639853548200336, "global_step": 200, "grad_norm_max": 0.035973016172647476, "hidden_mse": 0.013859100977424533, "loss": 0.013014773029135542, "lr_blocks": 9.931806517013612e-06, "lr_fusion": 9.931806517013612e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 22.051539826393128, "transport_attention_entropy": 8.70170134305954, "transport_attention_entropy_normalized": 0.995762099325657, "transport_residual_norm": 0.026619512122124434, "velocity_mse": 0.011628862976795063} +{"anchor_distance_mean": 124.34687500000001, "avg_context_frames": 17.2, "avg_micro_time_s": 7.024998902974971, "delta_h_c_norm": 0.06101094130426646, "delta_h_d_norm": 0.03695270062889904, "epoch": 2, "gate_p_mean": 0.02621787760872394, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8825761500746012, "gate_p_p10": 0.008036041259765625, "gate_p_p50": 0.01726493835449219, "gate_p_p90": 0.054203164577484135, "gate_p_std": 0.02566414650063962, "gated_delta_h_c_norm": 0.0038573152785829736, "global_step": 210, "grad_norm_max": 0.03321480005979538, "hidden_mse": 0.012224961956962944, "loss": 0.011226540929055773, "lr_blocks": 9.917525374361913e-06, "lr_fusion": 9.917525374361912e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 22.079008245468142, "transport_attention_entropy": 8.702514040470124, "transport_attention_entropy_normalized": 0.9958550930023193, "transport_residual_norm": 0.02820840666536242, "velocity_mse": 0.01000404470396461} +{"anchor_distance_mean": 123.21015625000001, "avg_context_frames": 14.8, "avg_micro_time_s": 6.3646893171751575, "delta_h_c_norm": 0.06301372116431594, "delta_h_d_norm": 0.03768647396937013, "epoch": 2, "gate_p_mean": 0.029936380428262056, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8575200594961644, "gate_p_p10": 0.009936446696519852, "gate_p_p50": 0.020397186279296875, "gate_p_p90": 0.060802125930786134, "gate_p_std": 0.027499549416825177, "gated_delta_h_c_norm": 0.00395194317898131, "global_step": 220, "grad_norm_max": 0.06494572758674622, "hidden_mse": 0.012970033043529838, "loss": 0.011473402904812247, "lr_blocks": 9.901899829374048e-06, "lr_fusion": 9.901899829374047e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 22.216889476776124, "transport_attention_entropy": 8.702517020702363, "transport_attention_entropy_normalized": 0.9958554394543171, "transport_residual_norm": 0.027314874529838565, "velocity_mse": 0.010176399644115009} +{"anchor_distance_mean": 126.16484375, "avg_context_frames": 12.4, "avg_micro_time_s": 5.7205372224749, "delta_h_c_norm": 0.06384503124281764, "delta_h_d_norm": 0.03514007274061442, "epoch": 2, "gate_p_mean": 0.024149765417678284, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8944877125322819, "gate_p_p10": 0.006747055053710938, "gate_p_p50": 0.014967727661132814, "gate_p_p90": 0.05141525268554688, "gate_p_std": 0.02668067658087239, "gated_delta_h_c_norm": 0.00397595312424528, "global_step": 230, "grad_norm_max": 0.04851621761918068, "hidden_mse": 0.01338194211712107, "loss": 0.012353720335522667, "lr_blocks": 9.884934153917998e-06, "lr_fusion": 9.884934153917997e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 22.122354435920716, "transport_attention_entropy": 8.70237147808075, "transport_attention_entropy_normalized": 0.9958387851715088, "transport_residual_norm": 0.027001126226969065, "velocity_mse": 0.011015526080154815} +{"anchor_distance_mean": 123.76171875, "avg_context_frames": 14.0, "avg_micro_time_s": 6.103011049612497, "delta_h_c_norm": 0.0683065792080015, "delta_h_d_norm": 0.03642913401126862, "epoch": 2, "gate_p_mean": 0.02558293897891417, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8871234230697156, "gate_p_p10": 0.006122779659926892, "gate_p_p50": 0.015509414672851564, "gate_p_p90": 0.055157470703125, "gate_p_std": 0.03000885833753273, "gated_delta_h_c_norm": 0.0046551332648959946, "global_step": 240, "grad_norm_max": 0.04260118305683136, "hidden_mse": 0.012999800208490342, "loss": 0.012374589522369207, "lr_blocks": 9.86663298624003e-06, "lr_fusion": 9.86663298624003e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 22.305997395515444, "transport_attention_entropy": 8.702724635601044, "transport_attention_entropy_normalized": 0.9958791941404344, "transport_residual_norm": 0.0265832977835089, "velocity_mse": 0.011074609524803236} +{"anchor_distance_mean": 122.009375, "avg_context_frames": 12.8, "avg_micro_time_s": 5.783284077012445, "delta_h_c_norm": 0.07211692980490625, "delta_h_d_norm": 0.0399472322780639, "epoch": 2, "gate_p_mean": 0.03274732147110626, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8289716325700284, "gate_p_p10": 0.00692802406847477, "gate_p_p50": 0.0178253173828125, "gate_p_p90": 0.07757568359375, "gate_p_std": 0.039603616984095424, "gated_delta_h_c_norm": 0.006580340184882516, "global_step": 250, "grad_norm_max": 0.03825463354587555, "hidden_mse": 0.01563811728847213, "loss": 0.01498586252273526, "lr_blocks": 9.847001329696653e-06, "lr_fusion": 9.847001329696653e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 22.202605795860293, "transport_attention_entropy": 8.703240501880646, "transport_attention_entropy_normalized": 0.9959382258355618, "transport_residual_norm": 0.02745421277359128, "velocity_mse": 0.01342205085384194} +{"anchor_distance_mean": 120.93671875000001, "avg_context_frames": 15.600000000000001, "avg_micro_time_s": 6.554381005899995, "delta_h_c_norm": 0.07183334757573903, "delta_h_d_norm": 0.03853786396794021, "epoch": 3, "gate_p_mean": 0.030544773617293686, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8468880467116833, "gate_p_p10": 0.007208633422851563, "gate_p_p50": 0.0184600830078125, "gate_p_p90": 0.06799919605255127, "gate_p_std": 0.03448008840205148, "gated_delta_h_c_norm": 0.005963996224454604, "global_step": 260, "grad_norm_max": 0.043121520429849625, "hidden_mse": 0.014060539699858056, "loss": 0.013612857990665361, "lr_blocks": 9.826044551386743e-06, "lr_fusion": 9.826044551386744e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 22.32067255973816, "transport_attention_entropy": 8.702974152565004, "transport_attention_entropy_normalized": 0.9959077470004559, "transport_residual_norm": 0.026901262300089004, "velocity_mse": 0.012206804033485242} +{"anchor_distance_mean": 124.70078125, "avg_context_frames": 14.0, "avg_micro_time_s": 6.153756571575105, "delta_h_c_norm": 0.07156291520223022, "delta_h_d_norm": 0.036485733650624755, "epoch": 3, "gate_p_mean": 0.027342076669447126, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8745941318571568, "gate_p_p10": 0.0062957763671875, "gate_p_p50": 0.015465545654296876, "gate_p_p90": 0.06443418264389038, "gate_p_std": 0.03197425396065228, "gated_delta_h_c_norm": 0.0053211183381790764, "global_step": 270, "grad_norm_max": 0.030099187046289444, "hidden_mse": 0.014379377471050248, "loss": 0.013427493785275147, "lr_blocks": 9.803768380684242e-06, "lr_fusion": 9.803768380684242e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 22.211935997009277, "transport_attention_entropy": 8.702965176105499, "transport_attention_entropy_normalized": 0.9959067218005657, "transport_residual_norm": 0.02668200491461903, "velocity_mse": 0.011989555979380385} +{"anchor_distance_mean": 126.55390625000001, "avg_context_frames": 15.600000000000001, "avg_micro_time_s": 6.575202928587442, "delta_h_c_norm": 0.07227530148811638, "delta_h_d_norm": 0.034534738515503705, "epoch": 3, "gate_p_mean": 0.019344486575573683, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.9323059715330602, "gate_p_p10": 0.005323219299316406, "gate_p_p50": 0.012205505371093751, "gate_p_p90": 0.040929543972015384, "gate_p_std": 0.021019546338357033, "gated_delta_h_c_norm": 0.0031621670481399634, "global_step": 280, "grad_norm_max": 0.03628532961010933, "hidden_mse": 0.012693990522529931, "loss": 0.011408404645044357, "lr_blocks": 9.780178907671788e-06, "lr_fusion": 9.780178907671789e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 22.51004605293274, "transport_attention_entropy": 8.703053891658783, "transport_attention_entropy_normalized": 0.9959168694913387, "transport_residual_norm": 0.027610725932754578, "velocity_mse": 0.010139005529345014} +{"anchor_distance_mean": 122.69062500000001, "avg_context_frames": 13.600000000000001, "avg_micro_time_s": 6.00093504599995, "delta_h_c_norm": 0.08041051458567382, "delta_h_d_norm": 0.039707808499224485, "epoch": 3, "gate_p_mean": 0.0351010674610734, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8058661598712207, "gate_p_p10": 0.010842514038085938, "gate_p_p50": 0.02431488037109375, "gate_p_p90": 0.07264404296875, "gate_p_std": 0.031752023537410426, "gated_delta_h_c_norm": 0.005673970145289787, "global_step": 290, "grad_norm_max": 0.04234811291098595, "hidden_mse": 0.014159253227990122, "loss": 0.013050993805518375, "lr_blocks": 9.755282581475769e-06, "lr_fusion": 9.755282581475769e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 22.6008508682251, "transport_attention_entropy": 8.703067576885223, "transport_attention_entropy_normalized": 0.9959184393286705, "transport_residual_norm": 0.02757417990360409, "velocity_mse": 0.011635068419855088} +{"anchor_distance_mean": 125.09140625, "avg_context_frames": 13.600000000000001, "avg_micro_time_s": 6.059194463499899, "delta_h_c_norm": 0.07868240647949279, "delta_h_d_norm": 0.03886499477084726, "epoch": 3, "gate_p_mean": 0.027712357393465937, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8646912835538387, "gate_p_p10": 0.006894683837890625, "gate_p_p50": 0.016756439208984376, "gate_p_p90": 0.06266591548919678, "gate_p_std": 0.02971871905610897, "gated_delta_h_c_norm": 0.005073066462500719, "global_step": 300, "grad_norm_max": 0.0311588104814291, "hidden_mse": 0.012921521160751582, "loss": 0.012160669401055202, "lr_blocks": 9.729086208503174e-06, "lr_fusion": 9.729086208503174e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 22.518312692642212, "transport_attention_entropy": 8.703119957447052, "transport_attention_entropy_normalized": 0.9959244318306446, "transport_residual_norm": 0.027040846482850614, "velocity_mse": 0.01086851731815841} +{"anchor_distance_mean": 123.24062500000001, "avg_context_frames": 12.8, "avg_micro_time_s": 5.858841312649952, "delta_h_c_norm": 0.0887068244162947, "delta_h_d_norm": 0.044093297468498355, "epoch": 3, "gate_p_mean": 0.036831233749398964, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.788789588958025, "gate_p_p10": 0.007190513610839844, "gate_p_p50": 0.02081298828125, "gate_p_p90": 0.0897791624069214, "gate_p_std": 0.042423542548203845, "gated_delta_h_c_norm": 0.008407403318415163, "global_step": 310, "grad_norm_max": 0.04433942958712578, "hidden_mse": 0.0142328595102299, "loss": 0.013032153699896298, "lr_blocks": 9.701596950580807e-06, "lr_fusion": 9.701596950580806e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 22.618032217025757, "transport_attention_entropy": 8.703216660022736, "transport_attention_entropy_normalized": 0.9959354996681213, "transport_residual_norm": 0.026537084323354068, "velocity_mse": 0.011608867716859095} +{"anchor_distance_mean": 126.65156250000001, "avg_context_frames": 12.8, "avg_micro_time_s": 5.860219092274929, "delta_h_c_norm": 0.086373654846102, "delta_h_d_norm": 0.04248871956951916, "epoch": 3, "gate_p_mean": 0.027022352523636075, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8607219323515892, "gate_p_p10": 0.004704666137695313, "gate_p_p50": 0.013790893554687501, "gate_p_p90": 0.06728739142417908, "gate_p_std": 0.033977037883596496, "gated_delta_h_c_norm": 0.005859517741191667, "global_step": 320, "grad_norm_max": 0.03742600604891777, "hidden_mse": 0.014009193266974763, "loss": 0.012967729289084674, "lr_blocks": 9.672822322997305e-06, "lr_fusion": 9.672822322997305e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 22.371567130088806, "transport_attention_entropy": 8.703265476226807, "transport_attention_entropy_normalized": 0.9959410823881627, "transport_residual_norm": 0.026378290029242635, "velocity_mse": 0.011566809983924032} +{"anchor_distance_mean": 125.54921875000001, "avg_context_frames": 13.600000000000001, "avg_micro_time_s": 6.0452140434499375, "delta_h_c_norm": 0.08867475409060717, "delta_h_d_norm": 0.043264976353384554, "epoch": 3, "gate_p_mean": 0.028491625515744093, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8564906604588032, "gate_p_p10": 0.004856300354003907, "gate_p_p50": 0.015106964111328126, "gate_p_p90": 0.06883161664009095, "gate_p_std": 0.03631447597872466, "gated_delta_h_c_norm": 0.0065974904995528055, "global_step": 330, "grad_norm_max": 0.04274846240878105, "hidden_mse": 0.013596586039057001, "loss": 0.012622943517635578, "lr_blocks": 9.642770192448537e-06, "lr_fusion": 9.642770192448536e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 22.614386367797852, "transport_attention_entropy": 8.70333251953125, "transport_attention_entropy_normalized": 0.9959487579762936, "transport_residual_norm": 0.02606954495422542, "velocity_mse": 0.011263284875894897} +{"anchor_distance_mean": 125.87031250000001, "avg_context_frames": 13.200000000000001, "avg_micro_time_s": 5.920030969150003, "delta_h_c_norm": 0.08455807464197279, "delta_h_d_norm": 0.039590078080073005, "epoch": 3, "gate_p_mean": 0.026187596179079266, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8735494546592236, "gate_p_p10": 0.004889106750488282, "gate_p_p50": 0.013976669311523438, "gate_p_p90": 0.06330947875976563, "gate_p_std": 0.03250595263089053, "gated_delta_h_c_norm": 0.005381302691239398, "global_step": 340, "grad_norm_max": 0.033853620290756226, "hidden_mse": 0.012500101677142085, "loss": 0.01227294983691536, "lr_blocks": 9.611448774886925e-06, "lr_fusion": 9.611448774886924e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 22.824674463272096, "transport_attention_entropy": 8.703490519523621, "transport_attention_entropy_normalized": 0.9959668383002281, "transport_residual_norm": 0.02596106929704547, "velocity_mse": 0.011022939620306716} +{"anchor_distance_mean": 125.321875, "avg_context_frames": 16.0, "avg_micro_time_s": 6.682149509250099, "delta_h_c_norm": 0.09206861751154066, "delta_h_d_norm": 0.04071139064617455, "epoch": 3, "gate_p_mean": 0.0350529380957596, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8049551986157895, "gate_p_p10": 0.008023834228515625, "gate_p_p50": 0.020845794677734376, "gate_p_p90": 0.08245968818664551, "gate_p_std": 0.03755522619467229, "gated_delta_h_c_norm": 0.006925084860995412, "global_step": 350, "grad_norm_max": 0.04387575760483742, "hidden_mse": 0.012834972154814751, "loss": 0.012414796333177948, "lr_blocks": 9.578866633275289e-06, "lr_fusion": 9.578866633275288e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 22.84306688308716, "transport_attention_entropy": 8.703788566589356, "transport_attention_entropy_normalized": 0.9960009440779687, "transport_residual_norm": 0.026232358114793897, "velocity_mse": 0.011131299051339739} +{"anchor_distance_mean": 119.41015625, "avg_context_frames": 14.4, "avg_micro_time_s": 6.238929973425002, "delta_h_c_norm": 0.09419289836660028, "delta_h_d_norm": 0.041307756118476396, "epoch": 3, "gate_p_mean": 0.02514263938064687, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8821306884288789, "gate_p_p10": 0.0046066284179687505, "gate_p_p50": 0.013277053833007812, "gate_p_p90": 0.059973406791687014, "gate_p_std": 0.03252387961838395, "gated_delta_h_c_norm": 0.006084845818259055, "global_step": 360, "grad_norm_max": 0.02140703611075878, "hidden_mse": 0.01240275837481022, "loss": 0.01185339745716192, "lr_blocks": 9.545032675245814e-06, "lr_fusion": 9.545032675245813e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 22.6806702375412, "transport_attention_entropy": 8.704112839698793, "transport_attention_entropy_normalized": 0.9960380494594574, "transport_residual_norm": 0.02618936491198838, "velocity_mse": 0.01061312156089116} +{"anchor_distance_mean": 126.55781250000001, "avg_context_frames": 16.400000000000002, "avg_micro_time_s": 6.78583171608741, "delta_h_c_norm": 0.09876393442973495, "delta_h_d_norm": 0.04388411198742688, "epoch": 3, "gate_p_mean": 0.026833127695135774, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8683443784713746, "gate_p_p10": 0.0048048019409179694, "gate_p_p50": 0.014345169067382812, "gate_p_p90": 0.06380042433738708, "gate_p_std": 0.034026873705442995, "gated_delta_h_c_norm": 0.006568991829408333, "global_step": 370, "grad_norm_max": 0.02855778858065605, "hidden_mse": 0.012301301839761436, "loss": 0.011383956338977442, "lr_blocks": 9.509956150664796e-06, "lr_fusion": 9.509956150664796e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 22.981481146812442, "transport_attention_entropy": 8.70395141839981, "transport_attention_entropy_normalized": 0.9960195824503899, "transport_residual_norm": 0.025570807000622156, "velocity_mse": 0.01015382613404654} +{"anchor_distance_mean": 126.42500000000001, "avg_context_frames": 17.6, "avg_micro_time_s": 7.077146456050037, "delta_h_c_norm": 0.0968957600183785, "delta_h_d_norm": 0.04081758162938059, "epoch": 3, "gate_p_mean": 0.021548599493689837, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.910972584784031, "gate_p_p10": 0.004508399963378907, "gate_p_p50": 0.011664581298828126, "gate_p_p90": 0.05130581855773926, "gate_p_std": 0.026648760261014107, "gated_delta_h_c_norm": 0.004808339818555396, "global_step": 380, "grad_norm_max": 0.030211014673113823, "hidden_mse": 0.01046897487831302, "loss": 0.009551248693605886, "lr_blocks": 9.473646649103819e-06, "lr_fusion": 9.473646649103818e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 22.75472221374512, "transport_attention_entropy": 8.703886413574219, "transport_attention_entropy_normalized": 0.9960121415555477, "transport_residual_norm": 0.025764413992874324, "velocity_mse": 0.008504351184819825} +{"anchor_distance_mean": 123.20859375, "avg_context_frames": 8.8, "avg_micro_time_s": 4.747434252862354, "delta_h_c_norm": 0.09947648579254748, "delta_h_d_norm": 0.0442073441343382, "epoch": 4, "gate_p_mean": 0.02994098525377922, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8430606752634049, "gate_p_p10": 0.0037497520446777345, "gate_p_p50": 0.014883041381835938, "gate_p_p90": 0.07859954833984376, "gate_p_std": 0.03931144363596104, "gated_delta_h_c_norm": 0.007772245736850892, "global_step": 390, "grad_norm_max": 0.02990380860865116, "hidden_mse": 0.014103579131187871, "loss": 0.012852039677090943, "lr_blocks": 9.43611409721806e-06, "lr_fusion": 9.43611409721806e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 22.702642869949344, "transport_attention_entropy": 8.703155207633973, "transport_attention_entropy_normalized": 0.9959284693002701, "transport_residual_norm": 0.024839088506996634, "velocity_mse": 0.011441681705764496} +{"anchor_distance_mean": 131.65625, "avg_context_frames": 11.200000000000001, "avg_micro_time_s": 5.397126349962491, "delta_h_c_norm": 0.09883804619312286, "delta_h_d_norm": 0.043189103901386264, "epoch": 4, "gate_p_mean": 0.03083168979501352, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8344979733228683, "gate_p_p10": 0.005036544799804688, "gate_p_p50": 0.01638641357421875, "gate_p_p90": 0.07727441787719727, "gate_p_std": 0.037597055931109936, "gated_delta_h_c_norm": 0.006660782656399533, "global_step": 400, "grad_norm_max": 0.030899066478013992, "hidden_mse": 0.013077186595182866, "loss": 0.012619506468763576, "lr_blocks": 9.397368756032445e-06, "lr_fusion": 9.397368756032445e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 22.876909375190735, "transport_attention_entropy": 8.704220867156982, "transport_attention_entropy_normalized": 0.9960504159331323, "transport_residual_norm": 0.025408871518447996, "velocity_mse": 0.011311787785962224} +{"anchor_distance_mean": 122.56015625, "avg_context_frames": 16.8, "avg_micro_time_s": 6.901298880474861, "delta_h_c_norm": 0.10082285888493062, "delta_h_d_norm": 0.04339825548231602, "epoch": 4, "gate_p_mean": 0.027419663971522824, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.865900718420744, "gate_p_p10": 0.004822921752929688, "gate_p_p50": 0.014480209350585938, "gate_p_p90": 0.0678192138671875, "gate_p_std": 0.03383447299711406, "gated_delta_h_c_norm": 0.006876942221424542, "global_step": 410, "grad_norm_max": 0.027997365221381187, "hidden_mse": 0.012736445659538732, "loss": 0.011853330652229488, "lr_blocks": 9.357421218136387e-06, "lr_fusion": 9.357421218136386e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 23.048778676986696, "transport_attention_entropy": 8.70434238910675, "transport_attention_entropy_normalized": 0.9960643187165261, "transport_residual_norm": 0.025536176562309266, "velocity_mse": 0.01057968607346993} +{"anchor_distance_mean": 125.02890625, "avg_context_frames": 16.0, "avg_micro_time_s": 6.669191568124962, "delta_h_c_norm": 0.09773788452148438, "delta_h_d_norm": 0.044749552290886646, "epoch": 4, "gate_p_mean": 0.022185232682386414, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8889546051621438, "gate_p_p10": 0.003593254089355469, "gate_p_p50": 0.011244773864746094, "gate_p_p90": 0.05304641723632813, "gate_p_std": 0.030447533924598248, "gated_delta_h_c_norm": 0.005548274169996148, "global_step": 420, "grad_norm_max": 0.0597514845430851, "hidden_mse": 0.012812619510805236, "loss": 0.012354946450795979, "lr_blocks": 9.31628240478787e-06, "lr_fusion": 9.316282404787871e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 22.807774782180786, "transport_attention_entropy": 8.704698526859284, "transport_attention_entropy_normalized": 0.996105070412159, "transport_residual_norm": 0.02629877154249698, "velocity_mse": 0.011073684506118299} +{"anchor_distance_mean": 123.08125000000001, "avg_context_frames": 16.8, "avg_micro_time_s": 6.8565013821126515, "delta_h_c_norm": 0.09758868040516973, "delta_h_d_norm": 0.04466652737464756, "epoch": 4, "gate_p_mean": 0.019092919945251197, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.9119844511151314, "gate_p_p10": 0.0032504843547940254, "gate_p_p50": 0.009493064880371094, "gate_p_p90": 0.047129058837890626, "gate_p_std": 0.02563449758454226, "gated_delta_h_c_norm": 0.004802610889601056, "global_step": 430, "grad_norm_max": 0.026447858661413193, "hidden_mse": 0.012212625681422652, "loss": 0.011365804291563109, "lr_blocks": 9.273963562927695e-06, "lr_fusion": 9.273963562927695e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 22.933376979827884, "transport_attention_entropy": 8.704758882522583, "transport_attention_entropy_normalized": 0.9961119815707207, "transport_residual_norm": 0.02548542704898864, "velocity_mse": 0.010144541776389815} +{"anchor_distance_mean": 123.69609375, "avg_context_frames": 14.8, "avg_micro_time_s": 6.304891498212533, "delta_h_c_norm": 0.1041825051419437, "delta_h_d_norm": 0.04698064117692411, "epoch": 4, "gate_p_mean": 0.02961289147497155, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.850792793929577, "gate_p_p10": 0.004720211029052734, "gate_p_p50": 0.014941787719726563, "gate_p_p90": 0.07749710083007813, "gate_p_std": 0.03715573570225388, "gated_delta_h_c_norm": 0.008415448987216223, "global_step": 440, "grad_norm_max": 0.026394497603178024, "hidden_mse": 0.013352992420550436, "loss": 0.012466600438347087, "lr_blocks": 9.230476262104678e-06, "lr_fusion": 9.230476262104677e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 22.755184936523438, "transport_attention_entropy": 8.704356300830842, "transport_attention_entropy_normalized": 0.9960659086704254, "transport_residual_norm": 0.024390546674840155, "velocity_mse": 0.011131301239947789} +{"anchor_distance_mean": 127.4625, "avg_context_frames": 16.400000000000002, "avg_micro_time_s": 6.702541142950122, "delta_h_c_norm": 0.10125931976363063, "delta_h_d_norm": 0.041749242576770484, "epoch": 4, "gate_p_mean": 0.01792131512775086, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.9227594457566739, "gate_p_p10": 0.0031374738551676277, "gate_p_p50": 0.008763504028320313, "gate_p_p90": 0.04450149536132813, "gate_p_std": 0.02417495150002651, "gated_delta_h_c_norm": 0.004661767846846488, "global_step": 450, "grad_norm_max": 0.029348773881793022, "hidden_mse": 0.01139607405057177, "loss": 0.010512588810524904, "lr_blocks": 9.185832391312644e-06, "lr_fusion": 9.185832391312644e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 23.25675868988037, "transport_attention_entropy": 8.704692935943603, "transport_attention_entropy_normalized": 0.9961044356226921, "transport_residual_norm": 0.024572190013714135, "velocity_mse": 0.00937298137578182} +{"anchor_distance_mean": 124.63828125, "avg_context_frames": 11.200000000000001, "avg_micro_time_s": 5.367932430237715, "delta_h_c_norm": 0.1076999943703413, "delta_h_d_norm": 0.04770592553541064, "epoch": 4, "gate_p_mean": 0.03165894007543102, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8264949195086957, "gate_p_p10": 0.005680465698242188, "gate_p_p50": 0.017330169677734375, "gate_p_p90": 0.07799887657165527, "gate_p_std": 0.037466980691533536, "gated_delta_h_c_norm": 0.007585196947911755, "global_step": 460, "grad_norm_max": 0.03149804472923279, "hidden_mse": 0.012926542921923101, "loss": 0.012387708525056952, "lr_blocks": 9.140044155740102e-06, "lr_fusion": 9.140044155740101e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 23.05563397407532, "transport_attention_entropy": 8.704745256900788, "transport_attention_entropy_normalized": 0.9961104176938534, "transport_residual_norm": 0.024687683070078494, "velocity_mse": 0.011095054232282565} +{"anchor_distance_mean": 122.040625, "avg_context_frames": 11.200000000000001, "avg_micro_time_s": 5.378792842475196, "delta_h_c_norm": 0.10992588829249145, "delta_h_d_norm": 0.045918972324579956, "epoch": 4, "gate_p_mean": 0.02759973134379834, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8503450773656369, "gate_p_p10": 0.003224039077758789, "gate_p_p50": 0.013059234619140625, "gate_p_p90": 0.07213864326477051, "gate_p_std": 0.03735279470565729, "gated_delta_h_c_norm": 0.007488378862763057, "global_step": 470, "grad_norm_max": 0.02445198968052864, "hidden_mse": 0.012645561271347106, "loss": 0.012100398517213763, "lr_blocks": 9.093124073433464e-06, "lr_fusion": 9.093124073433463e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 23.32396664619446, "transport_attention_entropy": 8.704593539237976, "transport_attention_entropy_normalized": 0.9960930585861206, "transport_residual_norm": 0.02553116094786674, "velocity_mse": 0.010835842438973488} +{"anchor_distance_mean": 121.29375, "avg_context_frames": 14.8, "avg_micro_time_s": 6.329648522800153, "delta_h_c_norm": 0.11614790903404355, "delta_h_d_norm": 0.04762144959531725, "epoch": 4, "gate_p_mean": 0.021843901596730577, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.896311628073454, "gate_p_p10": 0.002597865927964449, "gate_p_p50": 0.009749984741210938, "gate_p_p90": 0.05732574462890625, "gate_p_std": 0.03135667971801013, "gated_delta_h_c_norm": 0.007029407992376946, "global_step": 480, "grad_norm_max": 0.03383244574069977, "hidden_mse": 0.012368572421837599, "loss": 0.011417046101996675, "lr_blocks": 9.045084971874738e-06, "lr_fusion": 9.045084971874738e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 23.017463755607608, "transport_attention_entropy": 8.704595446586609, "transport_attention_entropy_normalized": 0.9960932753980161, "transport_residual_norm": 0.025645160977728666, "velocity_mse": 0.010180188855156303} +{"anchor_distance_mean": 125.384375, "avg_context_frames": 16.8, "avg_micro_time_s": 6.889643798637462, "delta_h_c_norm": 0.11011471208184959, "delta_h_d_norm": 0.04616434359923005, "epoch": 4, "gate_p_mean": 0.0246967731975019, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8758010558784008, "gate_p_p10": 0.004465789534151554, "gate_p_p50": 0.012895965576171876, "gate_p_p90": 0.06208038330078125, "gate_p_std": 0.030889778828714045, "gated_delta_h_c_norm": 0.006145119019492995, "global_step": 490, "grad_norm_max": 0.025607680901885033, "hidden_mse": 0.011635151744121686, "loss": 0.01097929434326943, "lr_blocks": 8.995939984474624e-06, "lr_fusion": 8.995939984474624e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 23.473817133903506, "transport_attention_entropy": 8.705145740509034, "transport_attention_entropy_normalized": 0.9961562491953373, "transport_residual_norm": 0.025298675685189665, "velocity_mse": 0.009815779168275186} +{"anchor_distance_mean": 128.275, "avg_context_frames": 14.0, "avg_micro_time_s": 6.1442242804749645, "delta_h_c_norm": 0.11484368043020368, "delta_h_d_norm": 0.04888332411646843, "epoch": 4, "gate_p_mean": 0.025964703952195124, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8708306074142457, "gate_p_p10": 0.003948993608355523, "gate_p_p50": 0.01226959228515625, "gate_p_p90": 0.06832212209701538, "gate_p_std": 0.035495180566795174, "gated_delta_h_c_norm": 0.00789767014211975, "global_step": 500, "grad_norm_max": 0.024988794699311256, "hidden_mse": 0.012370382301742212, "loss": 0.011718518898123876, "lr_blocks": 8.94570254698197e-06, "lr_fusion": 8.945702546981969e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 23.029894733428957, "transport_attention_entropy": 8.704812276363374, "transport_attention_entropy_normalized": 0.9961180910468102, "transport_residual_norm": 0.025079813250340522, "velocity_mse": 0.010481480657472276} +{"anchor_distance_mean": 124.76796875000001, "avg_context_frames": 16.400000000000002, "avg_micro_time_s": 6.834317974150145, "delta_h_c_norm": 0.10987094882875681, "delta_h_d_norm": 0.044794457592070104, "epoch": 4, "gate_p_mean": 0.013182665381464176, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.9454106859862805, "gate_p_p10": 0.0018750666175037624, "gate_p_p50": 0.006221771240234375, "gate_p_p90": 0.033197385072708134, "gate_p_std": 0.01869748056633398, "gated_delta_h_c_norm": 0.003607569211817463, "global_step": 510, "grad_norm_max": 0.02274163067340851, "hidden_mse": 0.012009840738028289, "loss": 0.01126690735691227, "lr_blocks": 8.894386393810563e-06, "lr_fusion": 8.894386393810563e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 23.74622082710266, "transport_attention_entropy": 8.705229926109315, "transport_attention_entropy_normalized": 0.9961658850312234, "transport_residual_norm": 0.027312991488724948, "velocity_mse": 0.010065923296497204} +{"anchor_distance_mean": 124.05546875, "avg_context_frames": 16.0, "avg_micro_time_s": 6.668385082987425, "delta_h_c_norm": 0.12286079861223698, "delta_h_d_norm": 0.04546234086155892, "epoch": 5, "gate_p_mean": 0.022740676131797956, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8929432407021523, "gate_p_p10": 0.004017448425292969, "gate_p_p50": 0.012621307373046876, "gate_p_p90": 0.056718444824218756, "gate_p_std": 0.02617867668159306, "gated_delta_h_c_norm": 0.00631090119786677, "global_step": 520, "grad_norm_max": 0.04388066381216049, "hidden_mse": 0.011976862780284137, "loss": 0.011822069995105267, "lr_blocks": 8.842005554284296e-06, "lr_fusion": 8.842005554284296e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 24.13978765010834, "transport_attention_entropy": 8.705307269096375, "transport_attention_entropy_normalized": 0.9961747311055661, "transport_residual_norm": 0.027176760858856144, "velocity_mse": 0.010624383683898488} +{"anchor_distance_mean": 124.21640625, "avg_context_frames": 13.200000000000001, "avg_micro_time_s": 5.929743263887622, "delta_h_c_norm": 0.11897755460813642, "delta_h_d_norm": 0.046472482616081834, "epoch": 5, "gate_p_mean": 0.02119881715043448, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8925661347806454, "gate_p_p10": 0.002875232696533203, "gate_p_p50": 0.009840965270996094, "gate_p_p90": 0.055929493904113774, "gate_p_std": 0.0286116034258157, "gated_delta_h_c_norm": 0.006684751417196821, "global_step": 530, "grad_norm_max": 0.03635041043162346, "hidden_mse": 0.013634665834251792, "loss": 0.012711100734304637, "lr_blocks": 8.788574348801676e-06, "lr_fusion": 8.788574348801675e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 23.9964567899704, "transport_attention_entropy": 8.705416893959045, "transport_attention_entropy_normalized": 0.9961872778832913, "transport_residual_norm": 0.026813357346691193, "velocity_mse": 0.011347634103731253} +{"anchor_distance_mean": 128.60078125, "avg_context_frames": 17.6, "avg_micro_time_s": 7.101527098174984, "delta_h_c_norm": 0.11718243286013603, "delta_h_d_norm": 0.0462751675862819, "epoch": 5, "gate_p_mean": 0.018845558143220843, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.913861708343029, "gate_p_p10": 0.003070640563964844, "gate_p_p50": 0.009859848022460939, "gate_p_p90": 0.045693969726562506, "gate_p_std": 0.02447075287636835, "gated_delta_h_c_norm": 0.005678618105957867, "global_step": 540, "grad_norm_max": 0.02696027047932148, "hidden_mse": 0.01165400128811598, "loss": 0.01113694736850448, "lr_blocks": 8.734107384920771e-06, "lr_fusion": 8.73410738492077e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 24.08835282325745, "transport_attention_entropy": 8.705478048324585, "transport_attention_entropy_normalized": 0.9961942791938783, "transport_residual_norm": 0.027631114912219347, "velocity_mse": 0.009971547160239425} +{"anchor_distance_mean": 126.8796875, "avg_context_frames": 15.200000000000001, "avg_micro_time_s": 6.430794176199652, "delta_h_c_norm": 0.12058744709938765, "delta_h_d_norm": 0.05013627070002258, "epoch": 5, "gate_p_mean": 0.02210274021490477, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8888854909688235, "gate_p_p10": 0.002914237976074219, "gate_p_p50": 0.01038188934326172, "gate_p_p90": 0.058182275295257574, "gate_p_std": 0.030538620334118607, "gated_delta_h_c_norm": 0.0071681467583403, "global_step": 550, "grad_norm_max": 0.03041977249085903, "hidden_mse": 0.012604540982283653, "loss": 0.01186557499167975, "lr_blocks": 8.67861955336566e-06, "lr_fusion": 8.678619553365659e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 24.147989678382874, "transport_attention_entropy": 8.70561488866806, "transport_attention_entropy_normalized": 0.9962099321186543, "transport_residual_norm": 0.02656649360433221, "velocity_mse": 0.010605120877153242} +{"anchor_distance_mean": 120.640625, "avg_context_frames": 14.8, "avg_micro_time_s": 6.329832919774844, "delta_h_c_norm": 0.11802097661420703, "delta_h_d_norm": 0.048810026375576854, "epoch": 5, "gate_p_mean": 0.018008503163582646, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.910236157476902, "gate_p_p10": 0.0020133018493652343, "gate_p_p50": 0.0074216842651367194, "gate_p_p90": 0.04770393371582032, "gate_p_std": 0.026959773738053628, "gated_delta_h_c_norm": 0.005445458475878695, "global_step": 560, "grad_norm_max": 0.02255268394947052, "hidden_mse": 0.011286535451654346, "loss": 0.010435410134959966, "lr_blocks": 8.622126023955446e-06, "lr_fusion": 8.622126023955446e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 24.18760359287262, "transport_attention_entropy": 8.705595028400422, "transport_attention_entropy_normalized": 0.9962076619267464, "transport_residual_norm": 0.02600169151555747, "velocity_mse": 0.009306756540900097} +{"anchor_distance_mean": 126.42421875000001, "avg_context_frames": 13.200000000000001, "avg_micro_time_s": 5.910945021112503, "delta_h_c_norm": 0.1227453506551683, "delta_h_d_norm": 0.049888442782685165, "epoch": 5, "gate_p_mean": 0.02475672228028998, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.870935270190239, "gate_p_p10": 0.003500843048095703, "gate_p_p50": 0.011946487426757814, "gate_p_p90": 0.06511867046356201, "gate_p_std": 0.031788412714377046, "gated_delta_h_c_norm": 0.007178820700210054, "global_step": 570, "grad_norm_max": 0.022827085107564926, "hidden_mse": 0.01145151960081421, "loss": 0.010958505951566623, "lr_blocks": 8.564642241456986e-06, "lr_fusion": 8.564642241456986e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 23.983109641075135, "transport_attention_entropy": 8.705486726760865, "transport_attention_entropy_normalized": 0.9961952693760395, "transport_residual_norm": 0.02533904816955328, "velocity_mse": 0.009813354030484334} +{"anchor_distance_mean": 119.05234375, "avg_context_frames": 13.200000000000001, "avg_micro_time_s": 5.8878070494372885, "delta_h_c_norm": 0.12732126554474235, "delta_h_d_norm": 0.05484350714832545, "epoch": 5, "gate_p_mean": 0.027222620957763868, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8501938369125128, "gate_p_p10": 0.003286075592041016, "gate_p_p50": 0.01286468505859375, "gate_p_p90": 0.07258174419403077, "gate_p_std": 0.03675630842335522, "gated_delta_h_c_norm": 0.008777527780330275, "global_step": 580, "grad_norm_max": 0.026709720492362976, "hidden_mse": 0.013536138617200778, "loss": 0.012592888210201637, "lr_blocks": 8.506183921362443e-06, "lr_fusion": 8.506183921362443e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 23.810868430137635, "transport_attention_entropy": 8.7054208278656, "transport_attention_entropy_normalized": 0.9961877286434174, "transport_residual_norm": 0.025013771583326162, "velocity_mse": 0.011239274317631498} +{"anchor_distance_mean": 127.2703125, "avg_context_frames": 11.600000000000001, "avg_micro_time_s": 5.453673684675278, "delta_h_c_norm": 0.12608472825959324, "delta_h_d_norm": 0.05110306493006647, "epoch": 5, "gate_p_mean": 0.027104531310033056, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8550994358956814, "gate_p_p10": 0.0036155223846435548, "gate_p_p50": 0.012720108032226562, "gate_p_p90": 0.07131692171096803, "gate_p_std": 0.03635347201488912, "gated_delta_h_c_norm": 0.00788774493994424, "global_step": 590, "grad_norm_max": 0.023037418723106384, "hidden_mse": 0.012185938318725676, "loss": 0.011527557790395805, "lr_blocks": 8.446767045592829e-06, "lr_fusion": 8.44676704559283e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 24.046416330337525, "transport_attention_entropy": 8.705390179157257, "transport_attention_entropy_normalized": 0.9961842216551304, "transport_residual_norm": 0.02487141853198409, "velocity_mse": 0.010308963924762793} +{"anchor_distance_mean": 124.05625, "avg_context_frames": 12.0, "avg_micro_time_s": 5.589805922449841, "delta_h_c_norm": 0.13177814977243543, "delta_h_d_norm": 0.05109676974825561, "epoch": 5, "gate_p_mean": 0.026439757307525725, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8589152917265892, "gate_p_p10": 0.0030063626356422904, "gate_p_p50": 0.012382888793945314, "gate_p_p90": 0.07059504985809327, "gate_p_std": 0.035170556267257784, "gated_delta_h_c_norm": 0.008314172050449998, "global_step": 600, "grad_norm_max": 0.05338522046804428, "hidden_mse": 0.013402501336531715, "loss": 0.013189942424651236, "lr_blocks": 8.386407858128707e-06, "lr_fusion": 8.386407858128706e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 24.170658588409424, "transport_attention_entropy": 8.705441045761109, "transport_attention_entropy_normalized": 0.9961900405585766, "transport_residual_norm": 0.0235798753798008, "velocity_mse": 0.011849692260148004} +{"anchor_distance_mean": 122.20156250000001, "avg_context_frames": 16.400000000000002, "avg_micro_time_s": 6.743873222549837, "delta_h_c_norm": 0.1230580053292215, "delta_h_d_norm": 0.04865335519425571, "epoch": 5, "gate_p_mean": 0.02313239120412618, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8832487255334854, "gate_p_p10": 0.003497028350830078, "gate_p_p50": 0.011909103393554688, "gate_p_p90": 0.0592315673828125, "gate_p_std": 0.029012617573607714, "gated_delta_h_c_norm": 0.006187275721458718, "global_step": 610, "grad_norm_max": 0.01952448859810829, "hidden_mse": 0.010931583808269352, "loss": 0.01070848735398613, "lr_blocks": 8.325122860569241e-06, "lr_fusion": 8.32512286056924e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 23.819486641883852, "transport_attention_entropy": 8.70556753873825, "transport_attention_entropy_normalized": 0.9962045155465603, "transport_residual_norm": 0.024718700582161548, "velocity_mse": 0.009615328934160061} +{"anchor_distance_mean": 126.87890625, "avg_context_frames": 14.8, "avg_micro_time_s": 6.310165018437238, "delta_h_c_norm": 0.12299481695517898, "delta_h_d_norm": 0.04735703277401626, "epoch": 5, "gate_p_mean": 0.022819637873908505, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8832720145583153, "gate_p_p10": 0.0033561706542968753, "gate_p_p50": 0.011444282531738282, "gate_p_p90": 0.0589037299156189, "gate_p_std": 0.029042048344854268, "gated_delta_h_c_norm": 0.005712644312006888, "global_step": 620, "grad_norm_max": 0.017411736771464348, "hidden_mse": 0.011128551937872544, "loss": 0.010898205856210553, "lr_blocks": 8.262928807620843e-06, "lr_fusion": 8.262928807620843e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 24.199585962295533, "transport_attention_entropy": 8.705170702934266, "transport_attention_entropy_normalized": 0.9961591079831124, "transport_residual_norm": 0.024031461915001273, "velocity_mse": 0.009785350688616745} +{"anchor_distance_mean": 128.696875, "avg_context_frames": 16.400000000000002, "avg_micro_time_s": 6.761272334412524, "delta_h_c_norm": 0.12360220486298204, "delta_h_d_norm": 0.04996725851669908, "epoch": 5, "gate_p_mean": 0.02149623929290101, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8954657725989819, "gate_p_p10": 0.003059482574462891, "gate_p_p50": 0.009795570373535158, "gate_p_p90": 0.055781555175781254, "gate_p_std": 0.030196725233690814, "gated_delta_h_c_norm": 0.007017102887039073, "global_step": 630, "grad_norm_max": 0.0255423653870821, "hidden_mse": 0.011585343076149002, "loss": 0.010914556810166688, "lr_blocks": 8.199842702516584e-06, "lr_fusion": 8.199842702516583e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 24.253310585021975, "transport_attention_entropy": 8.705114030838013, "transport_attention_entropy_normalized": 0.9961526200175286, "transport_residual_norm": 0.024010446295142176, "velocity_mse": 0.009756022517103703} +{"anchor_distance_mean": 123.33984375, "avg_context_frames": 12.8, "avg_micro_time_s": 5.801010148799833, "delta_h_c_norm": 0.13044947711750865, "delta_h_d_norm": 0.055017240811139345, "epoch": 5, "gate_p_mean": 0.0272182043408975, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8535912714898587, "gate_p_p10": 0.0029473874717950823, "gate_p_p50": 0.011761093139648439, "gate_p_p90": 0.07524927854537965, "gate_p_std": 0.038316681387368594, "gated_delta_h_c_norm": 0.009120863417047076, "global_step": 640, "grad_norm_max": 0.03191016614437103, "hidden_mse": 0.012935461418237537, "loss": 0.012103961128741503, "lr_blocks": 8.135881792367686e-06, "lr_fusion": 8.135881792367686e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 24.444755291938783, "transport_attention_entropy": 8.70502109527588, "transport_attention_entropy_normalized": 0.9961419872939588, "transport_residual_norm": 0.02255403839517385, "velocity_mse": 0.010810414975276218} +{"anchor_distance_mean": 125.6453125, "avg_context_frames": 14.0, "avg_micro_time_s": 6.1352923449746415, "delta_h_c_norm": 0.12660696366801857, "delta_h_d_norm": 0.05143644255585969, "epoch": 6, "gate_p_mean": 0.02415732250083238, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8772223323583603, "gate_p_p10": 0.0029769897460937503, "gate_p_p50": 0.011098861694335938, "gate_p_p90": 0.06364593505859376, "gate_p_std": 0.0342198719503358, "gated_delta_h_c_norm": 0.0076249188088695526, "global_step": 650, "grad_norm_max": 0.03393130004405975, "hidden_mse": 0.01186103189829737, "loss": 0.011552531505003572, "lr_blocks": 8.071063563448341e-06, "lr_fusion": 8.07106356344834e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 24.350075936317445, "transport_attention_entropy": 8.705104267597198, "transport_attention_entropy_normalized": 0.996151502430439, "transport_residual_norm": 0.02439715911168605, "velocity_mse": 0.010366428340785206} +{"anchor_distance_mean": 124.24921875000001, "avg_context_frames": 18.0, "avg_micro_time_s": 7.152250537800092, "delta_h_c_norm": 0.12976088719442488, "delta_h_d_norm": 0.04855411145836115, "epoch": 6, "gate_p_mean": 0.018606723280390727, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.9158208429813386, "gate_p_p10": 0.0028405189514160156, "gate_p_p50": 0.009830856323242189, "gate_p_p90": 0.04596049785614014, "gate_p_std": 0.024008559409412558, "gated_delta_h_c_norm": 0.005305827595293522, "global_step": 660, "grad_norm_max": 0.0199191365391016, "hidden_mse": 0.009868568001547829, "loss": 0.009397838325821795, "lr_blocks": 8.005405736415127e-06, "lr_fusion": 8.005405736415126e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 24.634772133827212, "transport_attention_entropy": 8.704740035533906, "transport_attention_entropy_normalized": 0.9961098216474057, "transport_residual_norm": 0.025525091122835877, "velocity_mse": 0.00841098154487554} +{"anchor_distance_mean": 123.33828125000001, "avg_context_frames": 13.600000000000001, "avg_micro_time_s": 5.962864666137648, "delta_h_c_norm": 0.133818107098341, "delta_h_d_norm": 0.05504520726390183, "epoch": 6, "gate_p_mean": 0.02607464917236939, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8612147219479085, "gate_p_p10": 0.0033233642578125, "gate_p_p50": 0.011339950561523439, "gate_p_p90": 0.06978421211242676, "gate_p_std": 0.037589916010620075, "gated_delta_h_c_norm": 0.009285981474386062, "global_step": 670, "grad_norm_max": 0.0312371663749218, "hidden_mse": 0.01342127011739649, "loss": 0.012833748391130939, "lr_blocks": 7.938926261462366e-06, "lr_fusion": 7.938926261462366e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 24.366549706459047, "transport_attention_entropy": 8.704759967327119, "transport_attention_entropy_normalized": 0.9961121030151845, "transport_residual_norm": 0.025089897471480074, "velocity_mse": 0.011491621402092278} +{"anchor_distance_mean": 124.73828125, "avg_context_frames": 12.4, "avg_micro_time_s": 5.688688236337384, "delta_h_c_norm": 0.1299632167443633, "delta_h_d_norm": 0.05099346088245511, "epoch": 6, "gate_p_mean": 0.022531579865608366, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.879695538431406, "gate_p_p10": 0.002285432815551758, "gate_p_p50": 0.009597206115722658, "gate_p_p90": 0.06158866882324219, "gate_p_std": 0.033034734387183566, "gated_delta_h_c_norm": 0.007749059953493998, "global_step": 680, "grad_norm_max": 0.026520567014813423, "hidden_mse": 0.013342052308144049, "loss": 0.012665141804609448, "lr_blocks": 7.871643313414718e-06, "lr_fusion": 7.871643313414718e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 24.201845026016237, "transport_attention_entropy": 8.704389548301696, "transport_attention_entropy_normalized": 0.9960697136819363, "transport_residual_norm": 0.024285132670775058, "velocity_mse": 0.011330936569720507} +{"anchor_distance_mean": 120.415625, "avg_context_frames": 12.0, "avg_micro_time_s": 5.594072427249921, "delta_h_c_norm": 0.1361147651448846, "delta_h_d_norm": 0.052900417475029826, "epoch": 6, "gate_p_mean": 0.02499904588330537, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8679610125720502, "gate_p_p10": 0.0024761389940977097, "gate_p_p50": 0.010957908630371094, "gate_p_p90": 0.06882994174957276, "gate_p_std": 0.0353357729036361, "gated_delta_h_c_norm": 0.008812962997762953, "global_step": 690, "grad_norm_max": 0.036604512482881546, "hidden_mse": 0.012932060303865001, "loss": 0.012530132394749672, "lr_blocks": 7.803575286758365e-06, "lr_fusion": 7.803575286758364e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 24.2624192237854, "transport_attention_entropy": 8.704168200492859, "transport_attention_entropy_normalized": 0.9960443906486035, "transport_residual_norm": 0.023641317733563483, "velocity_mse": 0.011236926357378253} +{"anchor_distance_mean": 121.87812500000001, "avg_context_frames": 15.600000000000001, "avg_micro_time_s": 6.486985249037298, "delta_h_c_norm": 0.13569108415395023, "delta_h_d_norm": 0.05237594163045287, "epoch": 6, "gate_p_mean": 0.025325908017111945, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8686100542545319, "gate_p_p10": 0.003315258026123047, "gate_p_p50": 0.011450386047363282, "gate_p_p90": 0.06758983135223388, "gate_p_std": 0.03470102072460577, "gated_delta_h_c_norm": 0.00891452013020171, "global_step": 700, "grad_norm_max": 0.021968597546219826, "hidden_mse": 0.013087088463362307, "loss": 0.01190896437619813, "lr_blocks": 7.734740790612137e-06, "lr_fusion": 7.734740790612136e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 24.562904000282288, "transport_attention_entropy": 8.704527020454407, "transport_attention_entropy_normalized": 0.9960854455828667, "transport_residual_norm": 0.02530511198565364, "velocity_mse": 0.010600255540339276} +{"anchor_distance_mean": 125.71015625000001, "avg_context_frames": 16.400000000000002, "avg_micro_time_s": 6.711425451812284, "delta_h_c_norm": 0.128175927978009, "delta_h_d_norm": 0.051260390877723695, "epoch": 6, "gate_p_mean": 0.02140956218936481, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8907617479562759, "gate_p_p10": 0.003354930877685547, "gate_p_p50": 0.010976791381835938, "gate_p_p90": 0.054607391357421875, "gate_p_std": 0.02697842959314585, "gated_delta_h_c_norm": 0.00578879780441639, "global_step": 710, "grad_norm_max": 0.022961722686886787, "hidden_mse": 0.011255535192321987, "loss": 0.010639590551727453, "lr_blocks": 7.66515864363997e-06, "lr_fusion": 7.66515864363997e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 24.084228563308717, "transport_attention_entropy": 8.704984879493713, "transport_attention_entropy_normalized": 0.9961378403007984, "transport_residual_norm": 0.02662045848555863, "velocity_mse": 0.009514037083135918} +{"anchor_distance_mean": 126.0671875, "avg_context_frames": 15.200000000000001, "avg_micro_time_s": 6.4631735564375505, "delta_h_c_norm": 0.1336314813233912, "delta_h_d_norm": 0.051138523034751415, "epoch": 6, "gate_p_mean": 0.023603375535458326, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8801069512963295, "gate_p_p10": 0.0030761718750000003, "gate_p_p50": 0.011397552490234376, "gate_p_p90": 0.062168121337890625, "gate_p_std": 0.030848640075419098, "gated_delta_h_c_norm": 0.007307542226044461, "global_step": 720, "grad_norm_max": 0.01960035227239132, "hidden_mse": 0.010763391142245383, "loss": 0.010243141354294494, "lr_blocks": 7.594847868906076e-06, "lr_fusion": 7.594847868906076e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 24.190592885017395, "transport_attention_entropy": 8.704811227321626, "transport_attention_entropy_normalized": 0.9961179710924626, "transport_residual_norm": 0.026293414807878435, "velocity_mse": 0.009166802233085037} +{"anchor_distance_mean": 129.02265625, "avg_context_frames": 16.400000000000002, "avg_micro_time_s": 6.728874098962525, "delta_h_c_norm": 0.1318206349387765, "delta_h_d_norm": 0.05360687458887697, "epoch": 6, "gate_p_mean": 0.0192948964308016, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.9081247799098492, "gate_p_p10": 0.0025608062744140628, "gate_p_p50": 0.008611106872558593, "gate_p_p90": 0.05057691335678101, "gate_p_std": 0.02742995721055195, "gated_delta_h_c_norm": 0.006118585602962412, "global_step": 730, "grad_norm_max": 0.034092433750629425, "hidden_mse": 0.010439551295712592, "loss": 0.009888011155999266, "lr_blocks": 7.52382768867422e-06, "lr_fusion": 7.52382768867422e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 24.213730812072754, "transport_attention_entropy": 8.704769253730774, "transport_attention_entropy_normalized": 0.9961131632328034, "transport_residual_norm": 0.024736620811745526, "velocity_mse": 0.008844055968802422} +{"anchor_distance_mean": 126.19843750000001, "avg_context_frames": 12.4, "avg_micro_time_s": 5.69611771781224, "delta_h_c_norm": 0.1397527194581926, "delta_h_d_norm": 0.05332545153796673, "epoch": 6, "gate_p_mean": 0.022844470379641282, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8843953065574169, "gate_p_p10": 0.002308034896850586, "gate_p_p50": 0.009623908996582033, "gate_p_p90": 0.06295928955078126, "gate_p_std": 0.033350625779712574, "gated_delta_h_c_norm": 0.007804964628303424, "global_step": 740, "grad_norm_max": 0.020370002835989, "hidden_mse": 0.0121622757462319, "loss": 0.01166799510247074, "lr_blocks": 7.452117519152542e-06, "lr_fusion": 7.452117519152542e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 24.636618614196777, "transport_attention_entropy": 8.704051196575165, "transport_attention_entropy_normalized": 0.9960309997200967, "transport_residual_norm": 0.02334916708059609, "velocity_mse": 0.01045176748011727} +{"anchor_distance_mean": 122.625, "avg_context_frames": 14.8, "avg_micro_time_s": 6.340010631112546, "delta_h_c_norm": 0.13936415603384375, "delta_h_d_norm": 0.056232803920283916, "epoch": 6, "gate_p_mean": 0.02802081779809669, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8468642584979534, "gate_p_p10": 0.003876113705337048, "gate_p_p50": 0.013489532470703126, "gate_p_p90": 0.07437667846679688, "gate_p_std": 0.03613818462472409, "gated_delta_h_c_norm": 0.008340417988074477, "global_step": 750, "grad_norm_max": 0.017856109887361526, "hidden_mse": 0.011298170755617322, "loss": 0.010892310543567874, "lr_blocks": 7.379736965185369e-06, "lr_fusion": 7.379736965185368e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 24.475118398666382, "transport_attention_entropy": 8.704144883155823, "transport_attention_entropy_normalized": 0.9960417158901692, "transport_residual_norm": 0.025954891834408047, "velocity_mse": 0.009762493494781666} +{"anchor_distance_mean": 127.56093750000001, "avg_context_frames": 12.0, "avg_micro_time_s": 5.6170818956123645, "delta_h_c_norm": 0.1477371926419437, "delta_h_d_norm": 0.051954177860170606, "epoch": 6, "gate_p_mean": 0.02487077165278606, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.872650769352913, "gate_p_p10": 0.0023505210876464846, "gate_p_p50": 0.010105895996093751, "gate_p_p90": 0.06948378086090089, "gate_p_std": 0.03697388018481434, "gated_delta_h_c_norm": 0.008915882671135479, "global_step": 760, "grad_norm_max": 0.01788283884525299, "hidden_mse": 0.012806497549172492, "loss": 0.012807122088270263, "lr_blocks": 7.30670581489344e-06, "lr_fusion": 7.30670581489344e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 24.542030215263367, "transport_attention_entropy": 8.703740131855012, "transport_attention_entropy_normalized": 0.9959954038262367, "transport_residual_norm": 0.025466478243470193, "velocity_mse": 0.011526472354307772} +{"anchor_distance_mean": 127.10546875, "avg_context_frames": 13.600000000000001, "avg_micro_time_s": 6.003059119512319, "delta_h_c_norm": 0.14238194990903139, "delta_h_d_norm": 0.053025032673031096, "epoch": 7, "gate_p_mean": 0.022019797092070803, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.886038438975811, "gate_p_p10": 0.002303504850715399, "gate_p_p50": 0.009394645690917969, "gate_p_p90": 0.059862518310546876, "gate_p_std": 0.0316741042828653, "gated_delta_h_c_norm": 0.00677626268734457, "global_step": 770, "grad_norm_max": 0.016396984457969666, "hidden_mse": 0.010456838738173248, "loss": 0.01037625925091561, "lr_blocks": 7.233044034264034e-06, "lr_fusion": 7.233044034264034e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 24.55135338306427, "transport_attention_entropy": 8.7040261387825, "transport_attention_entropy_normalized": 0.9960281305015087, "transport_residual_norm": 0.027216261066496373, "velocity_mse": 0.009330575345666147} +{"anchor_distance_mean": 125.58203125, "avg_context_frames": 15.200000000000001, "avg_micro_time_s": 6.419654493125063, "delta_h_c_norm": 0.15250389697030187, "delta_h_d_norm": 0.054074440617114305, "epoch": 7, "gate_p_mean": 0.023906622821232307, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8744218550622463, "gate_p_p10": 0.0024539947509765628, "gate_p_p50": 0.01009225845336914, "gate_p_p90": 0.06485646963119507, "gate_p_std": 0.03455387045978569, "gated_delta_h_c_norm": 0.008913292406941765, "global_step": 780, "grad_norm_max": 0.018512733280658722, "hidden_mse": 0.012220602965680883, "loss": 0.011487138984375634, "lr_blocks": 7.158771761692464e-06, "lr_fusion": 7.158771761692464e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 24.29082374572754, "transport_attention_entropy": 8.703787422180175, "transport_attention_entropy_normalized": 0.9960008122026921, "transport_residual_norm": 0.027625438175164163, "velocity_mse": 0.010265078660449945} +{"anchor_distance_mean": 126.06796875, "avg_context_frames": 12.0, "avg_micro_time_s": 5.605938228849709, "delta_h_c_norm": 0.146898159570992, "delta_h_d_norm": 0.0529043834656477, "epoch": 7, "gate_p_mean": 0.020620079725631513, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.894245070964098, "gate_p_p10": 0.0020656824111938477, "gate_p_p50": 0.0082916259765625, "gate_p_p90": 0.055238223075866705, "gate_p_std": 0.03201402060221881, "gated_delta_h_c_norm": 0.006952276895026444, "global_step": 790, "grad_norm_max": 0.04853266105055809, "hidden_mse": 0.011006504786200822, "loss": 0.010910332383355127, "lr_blocks": 7.083909302476453e-06, "lr_fusion": 7.083909302476453e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 24.07763857841492, "transport_attention_entropy": 8.703840804100038, "transport_attention_entropy_normalized": 0.9960069224238396, "transport_residual_norm": 0.02562691722996533, "velocity_mse": 0.009809681866317988} +{"anchor_distance_mean": 120.77187500000001, "avg_context_frames": 13.600000000000001, "avg_micro_time_s": 6.037325647987473, "delta_h_c_norm": 0.13918103082105518, "delta_h_d_norm": 0.05380611312575639, "epoch": 7, "gate_p_mean": 0.02011000553611666, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8988344125449658, "gate_p_p10": 0.0019165132194757463, "gate_p_p50": 0.007786750793457031, "gate_p_p90": 0.05645065307617188, "gate_p_std": 0.03072080331039615, "gated_delta_h_c_norm": 0.006686764537880663, "global_step": 800, "grad_norm_max": 0.02227145805954933, "hidden_mse": 0.012684968748362736, "loss": 0.012253963790135458, "lr_blocks": 7.008477123264849e-06, "lr_fusion": 7.008477123264848e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.109503674507142, "transport_attention_entropy": 8.703580224514008, "transport_attention_entropy_normalized": 0.9959771052002907, "transport_residual_norm": 0.027566690975800157, "velocity_mse": 0.010985466861166061} +{"anchor_distance_mean": 125.42031250000001, "avg_context_frames": 16.8, "avg_micro_time_s": 6.90896247318733, "delta_h_c_norm": 0.15022928928956392, "delta_h_d_norm": 0.05433299709111453, "epoch": 7, "gate_p_mean": 0.018620657146675514, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.913378930091858, "gate_p_p10": 0.002342634089291096, "gate_p_p50": 0.008913803100585939, "gate_p_p90": 0.04868087768554688, "gate_p_std": 0.025568865513196218, "gated_delta_h_c_norm": 0.006147050820436562, "global_step": 810, "grad_norm_max": 0.027207566425204277, "hidden_mse": 0.010470343864290044, "loss": 0.009752999828197063, "lr_blocks": 6.932495846462262e-06, "lr_fusion": 6.932495846462261e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 24.603986263275146, "transport_attention_entropy": 8.703725624084473, "transport_attention_entropy_normalized": 0.9959937416017056, "transport_residual_norm": 0.030250709294341507, "velocity_mse": 0.008705965493572877} +{"anchor_distance_mean": 123.01484375000001, "avg_context_frames": 17.6, "avg_micro_time_s": 7.07767258141239, "delta_h_c_norm": 0.14062987184152007, "delta_h_d_norm": 0.051657438417896634, "epoch": 7, "gate_p_mean": 0.00940102449967526, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.9650676369667054, "gate_p_p10": 0.0010274887084960938, "gate_p_p50": 0.0039161682128906255, "gate_p_p90": 0.023787307739257815, "gate_p_std": 0.0156820433359826, "gated_delta_h_c_norm": 0.0031793337824638002, "global_step": 820, "grad_norm_max": 0.02083439566195011, "hidden_mse": 0.010214494372485207, "loss": 0.009398377480101772, "lr_blocks": 6.855986244591104e-06, "lr_fusion": 6.855986244591104e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 24.47153537273407, "transport_attention_entropy": 8.704029107093811, "transport_attention_entropy_normalized": 0.996028470247984, "transport_residual_norm": 0.03378673924598843, "velocity_mse": 0.008376928028883414} +{"anchor_distance_mean": 126.4578125, "avg_context_frames": 15.200000000000001, "avg_micro_time_s": 6.458036112074797, "delta_h_c_norm": 0.13989685913547875, "delta_h_d_norm": 0.05891922935843468, "epoch": 7, "gate_p_mean": 0.026158758450765165, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.859660230576992, "gate_p_p10": 0.0044768333435058595, "gate_p_p50": 0.014447212219238281, "gate_p_p90": 0.06463081836700439, "gate_p_std": 0.030640346911968666, "gated_delta_h_c_norm": 0.007144905891618692, "global_step": 830, "grad_norm_max": 0.02208816446363926, "hidden_mse": 0.011547023290768267, "loss": 0.011613251187372954, "lr_blocks": 6.778969234612583e-06, "lr_fusion": 6.778969234612584e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.025569891929628, "transport_attention_entropy": 8.704161584377289, "transport_attention_entropy_normalized": 0.9960436336696148, "transport_residual_norm": 0.028275643172673882, "velocity_mse": 0.010458548841415906} +{"anchor_distance_mean": 128.375, "avg_context_frames": 15.600000000000001, "avg_micro_time_s": 6.514599243412295, "delta_h_c_norm": 0.1516216978430748, "delta_h_d_norm": 0.05313723171129823, "epoch": 7, "gate_p_mean": 0.020282885991036892, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.900323797762394, "gate_p_p10": 0.0024992655962705612, "gate_p_p50": 0.008985137939453125, "gate_p_p90": 0.055308532714843754, "gate_p_std": 0.028364322206471115, "gated_delta_h_c_norm": 0.007101291359140305, "global_step": 840, "grad_norm_max": 0.018410082906484604, "hidden_mse": 0.011031081422697753, "loss": 0.010631035780534148, "lr_blocks": 6.701465872208216e-06, "lr_fusion": 6.701465872208216e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 24.538090991973878, "transport_attention_entropy": 8.7035453915596, "transport_attention_entropy_normalized": 0.9959731161594392, "transport_residual_norm": 0.02845299334730953, "velocity_mse": 0.009527927672024816} +{"anchor_distance_mean": 125.80703125000001, "avg_context_frames": 14.4, "avg_micro_time_s": 6.231577306687268, "delta_h_c_norm": 0.1463933783583343, "delta_h_d_norm": 0.05486759818159044, "epoch": 7, "gate_p_mean": 0.023194810425047764, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8800696387887001, "gate_p_p10": 0.002196931838989258, "gate_p_p50": 0.008957004547119141, "gate_p_p90": 0.06594085693359375, "gate_p_std": 0.03537125847942662, "gated_delta_h_c_norm": 0.008984616403176915, "global_step": 850, "grad_norm_max": 0.019692014902830124, "hidden_mse": 0.01209288108511828, "loss": 0.011288252926897259, "lr_blocks": 6.6234973460234184e-06, "lr_fusion": 6.623497346023418e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 24.8179740190506, "transport_attention_entropy": 8.70383175611496, "transport_attention_entropy_normalized": 0.9960058860480786, "transport_residual_norm": 0.02799561251886189, "velocity_mse": 0.01007896475493908} +{"anchor_distance_mean": 123.3046875, "avg_context_frames": 12.4, "avg_micro_time_s": 5.710258661774697, "delta_h_c_norm": 0.14182583531364798, "delta_h_d_norm": 0.05507257007993758, "epoch": 7, "gate_p_mean": 0.028606508381199092, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8387530282139779, "gate_p_p10": 0.0032846450805664065, "gate_p_p50": 0.014091873168945314, "gate_p_p90": 0.07610950469970704, "gate_p_std": 0.03655437207198702, "gated_delta_h_c_norm": 0.008979524372989545, "global_step": 860, "grad_norm_max": 0.02469162456691265, "hidden_mse": 0.01220488382386975, "loss": 0.011994007229804994, "lr_blocks": 6.545084971874738e-06, "lr_fusion": 6.545084971874738e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 24.334818625450136, "transport_attention_entropy": 8.703225660324097, "transport_attention_entropy_normalized": 0.9959365263581277, "transport_residual_norm": 0.025226564379408956, "velocity_mse": 0.010773518815403805} +{"anchor_distance_mean": 123.0796875, "avg_context_frames": 10.0, "avg_micro_time_s": 5.077413149999984, "delta_h_c_norm": 0.14597996156662704, "delta_h_d_norm": 0.05724175954237581, "epoch": 7, "gate_p_mean": 0.028866268589627, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8391326375305653, "gate_p_p10": 0.002859783172607422, "gate_p_p50": 0.012561416625976563, "gate_p_p90": 0.08081936836242676, "gate_p_std": 0.04019849522737787, "gated_delta_h_c_norm": 0.009537915090913885, "global_step": 870, "grad_norm_max": 0.023594575002789497, "hidden_mse": 0.013541253097355366, "loss": 0.013000216986984016, "lr_blocks": 6.466250186922325e-06, "lr_fusion": 6.466250186922325e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 24.691370511054995, "transport_attention_entropy": 8.702605664730072, "transport_attention_entropy_normalized": 0.9958655789494515, "transport_residual_norm": 0.02409270729403943, "velocity_mse": 0.01164609177503735} +{"anchor_distance_mean": 124.1171875, "avg_context_frames": 14.4, "avg_micro_time_s": 6.221186997075302, "delta_h_c_norm": 0.14816749328747392, "delta_h_d_norm": 0.05568026122637093, "epoch": 7, "gate_p_mean": 0.02332499074400403, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8823773302137852, "gate_p_p10": 0.0027247428894042972, "gate_p_p50": 0.010062408447265626, "gate_p_p90": 0.06288757324218751, "gate_p_std": 0.03420440552290529, "gated_delta_h_c_norm": 0.008528783099609428, "global_step": 880, "grad_norm_max": 0.016172591596841812, "hidden_mse": 0.011330025386996567, "loss": 0.01096153862308711, "lr_blocks": 6.387014543809224e-06, "lr_fusion": 6.387014543809223e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 24.555782437324524, "transport_attention_entropy": 8.702871990203858, "transport_attention_entropy_normalized": 0.9958960570394993, "transport_residual_norm": 0.028232846758328378, "velocity_mse": 0.009828536069835537} +{"anchor_distance_mean": 124.60546875, "avg_context_frames": 13.200000000000001, "avg_micro_time_s": 5.911237139199966, "delta_h_c_norm": 0.15327867632731795, "delta_h_d_norm": 0.05757741294801236, "epoch": 7, "gate_p_mean": 0.028388728946447374, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8477717123925687, "gate_p_p10": 0.0031299591064453125, "gate_p_p50": 0.011705398559570312, "gate_p_p90": 0.07986615896224976, "gate_p_std": 0.04118925617658534, "gated_delta_h_c_norm": 0.010817328370467295, "global_step": 890, "grad_norm_max": 0.018364664167165756, "hidden_mse": 0.012850819318555296, "loss": 0.012292113574221731, "lr_blocks": 6.3073997047691e-06, "lr_fusion": 6.307399704769099e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 24.857799410820007, "transport_attention_entropy": 8.702703821659089, "transport_attention_entropy_normalized": 0.9958768121898175, "transport_residual_norm": 0.02752537657506764, "velocity_mse": 0.01100703169067856} +{"anchor_distance_mean": 127.26953125, "avg_context_frames": 15.200000000000001, "avg_micro_time_s": 6.442415017437589, "delta_h_c_norm": 0.15257596336305143, "delta_h_d_norm": 0.05395867978222668, "epoch": 8, "gate_p_mean": 0.0210657472955063, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8984325207769871, "gate_p_p10": 0.0024942398071289063, "gate_p_p50": 0.010011863708496094, "gate_p_p90": 0.05446803569793701, "gate_p_std": 0.029001053399406375, "gated_delta_h_c_norm": 0.006946152924501803, "global_step": 900, "grad_norm_max": 0.018224284052848816, "hidden_mse": 0.01007438834058121, "loss": 0.009711234172573314, "lr_blocks": 6.227427435703997e-06, "lr_fusion": 6.227427435703997e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 24.717632269859315, "transport_attention_entropy": 8.7021364569664, "transport_attention_entropy_normalized": 0.9958118885755539, "transport_residual_norm": 0.030802352679893376, "velocity_mse": 0.00870379536645487} +{"anchor_distance_mean": 124.60703125, "avg_context_frames": 10.4, "avg_micro_time_s": 5.154013200337295, "delta_h_c_norm": 0.1528128325007856, "delta_h_d_norm": 0.05861019655130804, "epoch": 8, "gate_p_mean": 0.027947455208050087, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8517450660467148, "gate_p_p10": 0.003029947215691209, "gate_p_p50": 0.012298011779785158, "gate_p_p90": 0.07725982666015625, "gate_p_std": 0.03915736738126725, "gated_delta_h_c_norm": 0.00921951156196883, "global_step": 910, "grad_norm_max": 0.019272232428193092, "hidden_mse": 0.012297614268027246, "loss": 0.012043348938459531, "lr_blocks": 6.147119600233758e-06, "lr_fusion": 6.147119600233758e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 24.729270792007448, "transport_attention_entropy": 8.702135956287384, "transport_attention_entropy_normalized": 0.9958118319511414, "transport_residual_norm": 0.0281592350685969, "velocity_mse": 0.010813587429584005} +{"anchor_distance_mean": 123.89453125, "avg_context_frames": 17.6, "avg_micro_time_s": 7.051876196049989, "delta_h_c_norm": 0.15479248445481064, "delta_h_d_norm": 0.05827940949238837, "epoch": 8, "gate_p_mean": 0.024087395158130676, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8837012045085431, "gate_p_p10": 0.003180789947509766, "gate_p_p50": 0.01100921630859375, "gate_p_p90": 0.0632229506969452, "gate_p_std": 0.03407706442521885, "gated_delta_h_c_norm": 0.008632300616591237, "global_step": 920, "grad_norm_max": 0.04496394842863083, "hidden_mse": 0.010936231305822731, "loss": 0.010587931919144467, "lr_blocks": 6.066498153718735e-06, "lr_fusion": 6.066498153718735e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 24.801317930221558, "transport_attention_entropy": 8.702034664154054, "transport_attention_entropy_normalized": 0.9958002418279648, "transport_residual_norm": 0.033588544954545795, "velocity_mse": 0.009494308737339453} +{"anchor_distance_mean": 131.16640625, "avg_context_frames": 12.4, "avg_micro_time_s": 5.694935510037522, "delta_h_c_norm": 0.1553242105059326, "delta_h_d_norm": 0.05496804979629815, "epoch": 8, "gate_p_mean": 0.022965634579304606, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8828207895159722, "gate_p_p10": 0.002269411087036133, "gate_p_p50": 0.008694648742675781, "gate_p_p90": 0.064739990234375, "gate_p_std": 0.03624654783634469, "gated_delta_h_c_norm": 0.008310657971014735, "global_step": 930, "grad_norm_max": 0.026697218418121338, "hidden_mse": 0.011928841343615205, "loss": 0.011848671722691507, "lr_blocks": 5.985585137257401e-06, "lr_fusion": 5.985585137257401e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 24.969477105140687, "transport_attention_entropy": 8.70182501077652, "transport_attention_entropy_normalized": 0.9957762472331524, "transport_residual_norm": 0.02951186888385564, "velocity_mse": 0.010655787610448898} +{"anchor_distance_mean": 124.83359375, "avg_context_frames": 16.0, "avg_micro_time_s": 6.641138884237353, "delta_h_c_norm": 0.15040072612464428, "delta_h_d_norm": 0.055624571908265356, "epoch": 8, "gate_p_mean": 0.025345419975928964, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8672160737216473, "gate_p_p10": 0.0031405448913574222, "gate_p_p50": 0.012159347534179688, "gate_p_p90": 0.06736234426498414, "gate_p_std": 0.03320361865335144, "gated_delta_h_c_norm": 0.007650482753524557, "global_step": 940, "grad_norm_max": 0.021414272487163544, "hidden_mse": 0.010856482601957396, "loss": 0.010763228588621132, "lr_blocks": 5.904402671660551e-06, "lr_fusion": 5.90440267166055e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.611224937438966, "transport_attention_entropy": 8.701474153995514, "transport_attention_entropy_normalized": 0.995736100524664, "transport_residual_norm": 0.03305942558217794, "velocity_mse": 0.009677580348215998} +{"anchor_distance_mean": 122.13671875, "avg_context_frames": 14.0, "avg_micro_time_s": 6.116876510812336, "delta_h_c_norm": 0.14682134250178933, "delta_h_d_norm": 0.05816033007577062, "epoch": 8, "gate_p_mean": 0.029914790892507884, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8391829706728459, "gate_p_p10": 0.003150463104248047, "gate_p_p50": 0.012862396240234376, "gate_p_p90": 0.08349118232727051, "gate_p_std": 0.04040886765578762, "gated_delta_h_c_norm": 0.00959270538733108, "global_step": 950, "grad_norm_max": 0.032041069120168686, "hidden_mse": 0.012384891381952912, "loss": 0.012108057268778795, "lr_blocks": 5.82297295140367e-06, "lr_fusion": 5.8229729514036705e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.280304169654848, "transport_attention_entropy": 8.70128333568573, "transport_attention_entropy_normalized": 0.9957142673432827, "transport_residual_norm": 0.03146931803785265, "velocity_mse": 0.01086956812941935} +{"anchor_distance_mean": 123.921875, "avg_context_frames": 12.8, "avg_micro_time_s": 5.810337496687589, "delta_h_c_norm": 0.1486334443092346, "delta_h_d_norm": 0.05747067839838565, "epoch": 8, "gate_p_mean": 0.02268985270056874, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8775921754539013, "gate_p_p10": 0.0020321176387369634, "gate_p_p50": 0.008684349060058594, "gate_p_p90": 0.0645757555961609, "gate_p_std": 0.03483219686313532, "gated_delta_h_c_norm": 0.007801859948085622, "global_step": 960, "grad_norm_max": 0.028409341350197792, "hidden_mse": 0.012096683116396889, "loss": 0.011521396387252025, "lr_blocks": 5.74131823855921e-06, "lr_fusion": 5.74131823855921e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.42523193359375, "transport_attention_entropy": 8.700452315807343, "transport_attention_entropy_normalized": 0.9956191636621953, "transport_residual_norm": 0.029544260888360444, "velocity_mse": 0.010311728031956591} +{"anchor_distance_mean": 125.15703125, "avg_context_frames": 13.600000000000001, "avg_micro_time_s": 6.015142757912554, "delta_h_c_norm": 0.15618767850100995, "delta_h_d_norm": 0.057431830652058126, "epoch": 8, "gate_p_mean": 0.028057360323145986, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8488637104630471, "gate_p_p10": 0.002991294860839844, "gate_p_p50": 0.012508964538574219, "gate_p_p90": 0.0778081774711609, "gate_p_std": 0.03839479883899913, "gated_delta_h_c_norm": 0.009012729926325846, "global_step": 970, "grad_norm_max": 0.01376846618950367, "hidden_mse": 0.011658689845353365, "loss": 0.011186457297299057, "lr_blocks": 5.659460856710346e-06, "lr_fusion": 5.6594608567103456e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.040275263786317, "transport_attention_entropy": 8.700537991523744, "transport_attention_entropy_normalized": 0.9956289723515511, "transport_residual_norm": 0.03144930643029511, "velocity_mse": 0.010020588323823177} +{"anchor_distance_mean": 126.39687500000001, "avg_context_frames": 18.8, "avg_micro_time_s": 7.338017246237405, "delta_h_c_norm": 0.1649454677477479, "delta_h_d_norm": 0.0584114644676447, "epoch": 8, "gate_p_mean": 0.020046364265726882, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.9046494647860528, "gate_p_p10": 0.0025993347167968753, "gate_p_p50": 0.009339714050292968, "gate_p_p90": 0.05184249877929688, "gate_p_std": 0.02804924006923102, "gated_delta_h_c_norm": 0.006439543982560281, "global_step": 980, "grad_norm_max": 0.020259935408830643, "hidden_mse": 0.01016573200467974, "loss": 0.009813177533214913, "lr_blocks": 5.577423184847932e-06, "lr_fusion": 5.577423184847932e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.14234986305237, "transport_attention_entropy": 8.700382661819459, "transport_attention_entropy_normalized": 0.9956111952662469, "transport_residual_norm": 0.03583137698005885, "velocity_mse": 0.008796604341478087} +{"anchor_distance_mean": 122.85078125000001, "avg_context_frames": 12.4, "avg_micro_time_s": 5.696861133987705, "delta_h_c_norm": 0.15576042598113418, "delta_h_d_norm": 0.05692578782327473, "epoch": 8, "gate_p_mean": 0.02695355622563511, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8558909498155117, "gate_p_p10": 0.0030809402465820316, "gate_p_p50": 0.012135696411132813, "gate_p_p90": 0.07375335693359375, "gate_p_std": 0.03659041132777929, "gated_delta_h_c_norm": 0.008724596684623975, "global_step": 990, "grad_norm_max": 0.016919199377298355, "hidden_mse": 0.011632722185458988, "loss": 0.011224298650631681, "lr_blocks": 5.495227651252315e-06, "lr_fusion": 5.495227651252315e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.062081146240235, "transport_attention_entropy": 8.69981700181961, "transport_attention_entropy_normalized": 0.995546466857195, "transport_residual_norm": 0.031563764763996006, "velocity_mse": 0.01006102639657911} +{"anchor_distance_mean": 128.17890625, "avg_context_frames": 15.200000000000001, "avg_micro_time_s": 6.430043149150425, "delta_h_c_norm": 0.16405910085886719, "delta_h_d_norm": 0.05708004450425506, "epoch": 8, "gate_p_mean": 0.02356801419518888, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8774822488427163, "gate_p_p10": 0.0024823665618896487, "gate_p_p50": 0.009481430053710938, "gate_p_p90": 0.06456205844879151, "gate_p_std": 0.035404351830948144, "gated_delta_h_c_norm": 0.008902272509294562, "global_step": 1000, "grad_norm_max": 0.016509702429175377, "hidden_mse": 0.011307479545939714, "loss": 0.011131107006804086, "lr_blocks": 5.412896727361663e-06, "lr_fusion": 5.4128967273616625e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.006887435913086, "transport_attention_entropy": 8.699797415733338, "transport_attention_entropy_normalized": 0.9955442219972611, "transport_residual_norm": 0.032507675071246925, "velocity_mse": 0.01000035910110455} +{"anchor_distance_mean": 121.68125, "avg_context_frames": 17.6, "avg_micro_time_s": 7.048373080024795, "delta_h_c_norm": 0.16186190005391837, "delta_h_d_norm": 0.05393343255855143, "epoch": 8, "gate_p_mean": 0.01871701044146903, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.9120550639927387, "gate_p_p10": 0.0020374298095703126, "gate_p_p50": 0.007892608642578125, "gate_p_p90": 0.05092697143554688, "gate_p_std": 0.02802156697725877, "gated_delta_h_c_norm": 0.007057031117437874, "global_step": 1010, "grad_norm_max": 0.022716209292411804, "hidden_mse": 0.009998334091505967, "loss": 0.009346606046892704, "lr_blocks": 5.3304529216284974e-06, "lr_fusion": 5.330452921628497e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.334309720993044, "transport_attention_entropy": 8.700264310836792, "transport_attention_entropy_normalized": 0.9955976530909538, "transport_residual_norm": 0.035610202420502904, "velocity_mse": 0.008346772630466149} +{"anchor_distance_mean": 123.37031250000001, "avg_context_frames": 14.8, "avg_micro_time_s": 6.328109410312573, "delta_h_c_norm": 0.1589472350664437, "delta_h_d_norm": 0.05670222667977214, "epoch": 8, "gate_p_mean": 0.02664263671031222, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8586383469402791, "gate_p_p10": 0.002895259764045477, "gate_p_p50": 0.011760711669921875, "gate_p_p90": 0.07440855503082276, "gate_p_std": 0.03646759460680187, "gated_delta_h_c_norm": 0.0088404297712259, "global_step": 1020, "grad_norm_max": 0.04115695133805275, "hidden_mse": 0.01143781011342071, "loss": 0.01063091067771893, "lr_blocks": 5.247918773366112e-06, "lr_fusion": 5.247918773366112e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.32534384727478, "transport_attention_entropy": 8.699983716011047, "transport_attention_entropy_normalized": 0.9955655433237554, "transport_residual_norm": 0.03449649475514889, "velocity_mse": 0.009487129660556093} +{"anchor_distance_mean": 123.50078125, "avg_context_frames": 11.200000000000001, "avg_micro_time_s": 5.383579711637549, "delta_h_c_norm": 0.1579086143523455, "delta_h_d_norm": 0.05817923927679658, "epoch": 9, "gate_p_mean": 0.02532582138082944, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8671001359820366, "gate_p_p10": 0.0023280620574951173, "gate_p_p50": 0.0100128173828125, "gate_p_p90": 0.07351727485656738, "gate_p_std": 0.037202248309040446, "gated_delta_h_c_norm": 0.008440849966427777, "global_step": 1030, "grad_norm_max": 0.03373022377490997, "hidden_mse": 0.012162564689060674, "loss": 0.011907727498328314, "lr_blocks": 5.165316846586541e-06, "lr_fusion": 5.165316846586541e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.081015944480896, "transport_attention_entropy": 8.699544441699983, "transport_attention_entropy_normalized": 0.9955152742564679, "transport_residual_norm": 0.030826529883779586, "velocity_mse": 0.010691471007885412} +{"anchor_distance_mean": 124.28125, "avg_context_frames": 18.400000000000002, "avg_micro_time_s": 7.285890348525118, "delta_h_c_norm": 0.15942716170102358, "delta_h_d_norm": 0.05586037486791611, "epoch": 9, "gate_p_mean": 0.024962670565582813, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8699296619743109, "gate_p_p10": 0.0036669252440333366, "gate_p_p50": 0.013121509552001955, "gate_p_p90": 0.06214113235473633, "gate_p_std": 0.03195754428743385, "gated_delta_h_c_norm": 0.007923125077650185, "global_step": 1040, "grad_norm_max": 0.015002375468611717, "hidden_mse": 0.009523673326475546, "loss": 0.0091154149355134, "lr_blocks": 5.082669723831793e-06, "lr_fusion": 5.0826697238317935e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.22259259223938, "transport_attention_entropy": 8.699882531166077, "transport_attention_entropy_normalized": 0.9955539651215077, "transport_residual_norm": 0.035874544619582595, "velocity_mse": 0.008163047608104535} +{"anchor_distance_mean": 122.55781250000001, "avg_context_frames": 14.4, "avg_micro_time_s": 6.255776253362774, "delta_h_c_norm": 0.1599887976422906, "delta_h_d_norm": 0.055952499620616436, "epoch": 9, "gate_p_mean": 0.019536582729779185, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8985900223255158, "gate_p_p10": 0.0017087459564208984, "gate_p_p50": 0.00705413818359375, "gate_p_p90": 0.054815673828125, "gate_p_std": 0.0316152764542494, "gated_delta_h_c_norm": 0.0065420301150879824, "global_step": 1050, "grad_norm_max": 0.012044236063957214, "hidden_mse": 0.01103741703554988, "loss": 0.010579953301930801, "lr_blocks": 5e-06, "lr_fusion": 5e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.193925285339358, "transport_attention_entropy": 8.69957435131073, "transport_attention_entropy_normalized": 0.9955187007784844, "transport_residual_norm": 0.03362818933092058, "velocity_mse": 0.009476211576838978} +{"anchor_distance_mean": 123.9890625, "avg_context_frames": 18.0, "avg_micro_time_s": 7.169010011312776, "delta_h_c_norm": 0.15739213433116675, "delta_h_d_norm": 0.05826078746467829, "epoch": 9, "gate_p_mean": 0.0215526323474478, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8969922170042992, "gate_p_p10": 0.0027629852294921878, "gate_p_p50": 0.009690475463867188, "gate_p_p90": 0.05607376098632813, "gate_p_std": 0.02998644486651756, "gated_delta_h_c_norm": 0.006624474943964742, "global_step": 1060, "grad_norm_max": 0.013093042187392712, "hidden_mse": 0.009906446601962672, "loss": 0.009637875022599475, "lr_blocks": 4.917330276168208e-06, "lr_fusion": 4.917330276168208e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.23274793624878, "transport_attention_entropy": 8.699908208847047, "transport_attention_entropy_normalized": 0.995556902140379, "transport_residual_norm": 0.03623800224158913, "velocity_mse": 0.008647230346105062} +{"anchor_distance_mean": 120.73984375, "avg_context_frames": 16.400000000000002, "avg_micro_time_s": 6.719334386199944, "delta_h_c_norm": 0.15976741882041098, "delta_h_d_norm": 0.06399561665020884, "epoch": 9, "gate_p_mean": 0.030211273866007107, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8401665437966586, "gate_p_p10": 0.003346967697143555, "gate_p_p50": 0.012936592102050781, "gate_p_p90": 0.08460507392883301, "gate_p_std": 0.04226801773766056, "gated_delta_h_c_norm": 0.010768034000648186, "global_step": 1070, "grad_norm_max": 0.01438598521053791, "hidden_mse": 0.012877315253717825, "loss": 0.011859240048215725, "lr_blocks": 4.8346831534134595e-06, "lr_fusion": 4.834683153413459e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.95971040725708, "transport_attention_entropy": 8.700075280666352, "transport_attention_entropy_normalized": 0.9955760225653649, "transport_residual_norm": 0.03331339843571186, "velocity_mse": 0.0105715084442636} +{"anchor_distance_mean": 124.44375000000001, "avg_context_frames": 13.200000000000001, "avg_micro_time_s": 5.893100706125233, "delta_h_c_norm": 0.16487784953787923, "delta_h_d_norm": 0.059630234865471726, "epoch": 9, "gate_p_mean": 0.026894800987793135, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8543274492025376, "gate_p_p10": 0.002952861785888672, "gate_p_p50": 0.0120941162109375, "gate_p_p90": 0.07285703420639038, "gate_p_std": 0.03734926310135052, "gated_delta_h_c_norm": 0.008648648025700824, "global_step": 1080, "grad_norm_max": 0.027961939573287964, "hidden_mse": 0.011245675821555778, "loss": 0.010835163056617603, "lr_blocks": 4.752081226633888e-06, "lr_fusion": 4.7520812266338885e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.159386682510377, "transport_attention_entropy": 8.699135291576386, "transport_attention_entropy_normalized": 0.9954684540629387, "transport_residual_norm": 0.03484107693657279, "velocity_mse": 0.009710595465730876} +{"anchor_distance_mean": 129.70546875, "avg_context_frames": 13.200000000000001, "avg_micro_time_s": 5.855396907637669, "delta_h_c_norm": 0.17033965606242418, "delta_h_d_norm": 0.06026097820140422, "epoch": 9, "gate_p_mean": 0.03019596762605943, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8379853017628194, "gate_p_p10": 0.0034004211425781253, "gate_p_p50": 0.012953567504882814, "gate_p_p90": 0.08378424644470216, "gate_p_std": 0.042627656052354725, "gated_delta_h_c_norm": 0.010896084351406899, "global_step": 1090, "grad_norm_max": 0.013442480005323887, "hidden_mse": 0.012107599584851414, "loss": 0.011680286296177657, "lr_blocks": 4.669547078371503e-06, "lr_fusion": 4.669547078371504e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.72850892543793, "transport_attention_entropy": 8.698919653892517, "transport_attention_entropy_normalized": 0.9954437777400017, "transport_residual_norm": 0.03591622125823051, "velocity_mse": 0.010469526366796345} +{"anchor_distance_mean": 125.22500000000001, "avg_context_frames": 13.200000000000001, "avg_micro_time_s": 5.886694315600199, "delta_h_c_norm": 0.16257389821112156, "delta_h_d_norm": 0.05901052290573716, "epoch": 9, "gate_p_mean": 0.029843533789971843, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8393988139927387, "gate_p_p10": 0.0026736736297607424, "gate_p_p50": 0.0114471435546875, "gate_p_p90": 0.08600632548332215, "gate_p_std": 0.04398383558727801, "gated_delta_h_c_norm": 0.010405767496558839, "global_step": 1100, "grad_norm_max": 0.020062077790498734, "hidden_mse": 0.011694872030057014, "loss": 0.011435111705213786, "lr_blocks": 4.587103272638339e-06, "lr_fusion": 4.5871032726383386e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.422690343856814, "transport_attention_entropy": 8.698821401596069, "transport_attention_entropy_normalized": 0.9954325363039971, "transport_residual_norm": 0.03473873778712005, "velocity_mse": 0.01026562452170765} +{"anchor_distance_mean": 127.56328125, "avg_context_frames": 11.600000000000001, "avg_micro_time_s": 5.514032222637389, "delta_h_c_norm": 0.1659629173576832, "delta_h_d_norm": 0.059557900344952945, "epoch": 9, "gate_p_mean": 0.029753587691811845, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8412387602031232, "gate_p_p10": 0.0031196974217891694, "gate_p_p50": 0.012948226928710938, "gate_p_p90": 0.08236886262893678, "gate_p_std": 0.04260631709476002, "gated_delta_h_c_norm": 0.010131281803478488, "global_step": 1110, "grad_norm_max": 0.014037588611245155, "hidden_mse": 0.011721481994027273, "loss": 0.011658163194078953, "lr_blocks": 4.504772348747687e-06, "lr_fusion": 4.504772348747687e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.760996675491334, "transport_attention_entropy": 8.697678375244141, "transport_attention_entropy_normalized": 0.9953017346560955, "transport_residual_norm": 0.03483838166575879, "velocity_mse": 0.010486014946945944} +{"anchor_distance_mean": 121.29140625000001, "avg_context_frames": 15.600000000000001, "avg_micro_time_s": 6.5143864417499575, "delta_h_c_norm": 0.15823625186458232, "delta_h_d_norm": 0.057813850510865454, "epoch": 9, "gate_p_mean": 0.025240656128153207, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8645906239748001, "gate_p_p10": 0.0029471397399902345, "gate_p_p50": 0.011520004272460938, "gate_p_p90": 0.0673225998878479, "gate_p_std": 0.03474724212428555, "gated_delta_h_c_norm": 0.007535598339745775, "global_step": 1120, "grad_norm_max": 0.02252986654639244, "hidden_mse": 0.011112916871206836, "loss": 0.011186015116982162, "lr_blocks": 4.42257681515207e-06, "lr_fusion": 4.4225768151520694e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.554311561584473, "transport_attention_entropy": 8.697994422912599, "transport_attention_entropy_normalized": 0.9953379049897194, "transport_residual_norm": 0.039304744778200985, "velocity_mse": 0.01007472338678781} +{"anchor_distance_mean": 128.04765625000002, "avg_context_frames": 14.4, "avg_micro_time_s": 6.223943463299839, "delta_h_c_norm": 0.16474734880030156, "delta_h_d_norm": 0.05706425136886537, "epoch": 9, "gate_p_mean": 0.024327797192381696, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8761884264647961, "gate_p_p10": 0.003015232086181641, "gate_p_p50": 0.011254501342773438, "gate_p_p90": 0.06496717929840089, "gate_p_std": 0.03393114192294888, "gated_delta_h_c_norm": 0.007480683964968194, "global_step": 1130, "grad_norm_max": 0.01756259612739086, "hidden_mse": 0.009751827275613323, "loss": 0.00963094559556339, "lr_blocks": 4.340539143289655e-06, "lr_fusion": 4.3405391432896555e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.735182905197146, "transport_attention_entropy": 8.697449004650116, "transport_attention_entropy_normalized": 0.9952754899859428, "transport_residual_norm": 0.037069356604479256, "velocity_mse": 0.008655762815033085} +{"anchor_distance_mean": 124.86640625000001, "avg_context_frames": 14.8, "avg_micro_time_s": 6.3425757411499495, "delta_h_c_norm": 0.16567904548719525, "delta_h_d_norm": 0.05929021984338761, "epoch": 9, "gate_p_mean": 0.02290520622045733, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.887287687510252, "gate_p_p10": 0.002703237533569336, "gate_p_p50": 0.010229110717773438, "gate_p_p90": 0.0596049427986145, "gate_p_std": 0.033238352660555394, "gated_delta_h_c_norm": 0.007325975646381266, "global_step": 1140, "grad_norm_max": 0.015271035954356194, "hidden_mse": 0.01006105451961048, "loss": 0.009851675329264254, "lr_blocks": 4.25868176144079e-06, "lr_fusion": 4.2586817614407895e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.30434138774872, "transport_attention_entropy": 8.697464179992677, "transport_attention_entropy_normalized": 0.99527722671628, "transport_residual_norm": 0.03759283644612879, "velocity_mse": 0.008845569888944738} +{"anchor_distance_mean": 125.80859375, "avg_context_frames": 11.200000000000001, "avg_micro_time_s": 5.356050658674985, "delta_h_c_norm": 0.16635548677295448, "delta_h_d_norm": 0.06123887747526169, "epoch": 9, "gate_p_mean": 0.03329244983615354, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.817459711804986, "gate_p_p10": 0.0037070274353027347, "gate_p_p50": 0.014837265014648438, "gate_p_p90": 0.09267425537109375, "gate_p_std": 0.04536306096124463, "gated_delta_h_c_norm": 0.010482431722630282, "global_step": 1150, "grad_norm_max": 0.01741187646985054, "hidden_mse": 0.012135609157849104, "loss": 0.011817206919658929, "lr_blocks": 4.17702704859633e-06, "lr_fusion": 4.17702704859633e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.613087606430057, "transport_attention_entropy": 8.696744799613953, "transport_attention_entropy_normalized": 0.9951949052512646, "transport_residual_norm": 0.0345621588639915, "velocity_mse": 0.010603645988157951} +{"anchor_distance_mean": 122.30000000000001, "avg_context_frames": 13.600000000000001, "avg_micro_time_s": 6.016432592887805, "delta_h_c_norm": 0.16818131841719153, "delta_h_d_norm": 0.0616505422629416, "epoch": 10, "gate_p_mean": 0.028858724416932092, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8477899923920632, "gate_p_p10": 0.0030046463012695316, "gate_p_p50": 0.012245559692382814, "gate_p_p90": 0.0789642333984375, "gate_p_std": 0.0408984198467806, "gated_delta_h_c_norm": 0.009512459964025766, "global_step": 1160, "grad_norm_max": 0.011989346705377102, "hidden_mse": 0.0110910362447612, "loss": 0.010659583850065248, "lr_blocks": 4.0955973283394525e-06, "lr_fusion": 4.095597328339452e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.256191992759707, "transport_attention_entropy": 8.69658100605011, "transport_attention_entropy_normalized": 0.9951761588454247, "transport_residual_norm": 0.03577504379209131, "velocity_mse": 0.009550480209873058} +{"anchor_distance_mean": 127.72421875, "avg_context_frames": 15.600000000000001, "avg_micro_time_s": 6.553261991712453, "delta_h_c_norm": 0.16707858797162772, "delta_h_d_norm": 0.059817805606871846, "epoch": 10, "gate_p_mean": 0.03019917217316106, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8454740352928639, "gate_p_p10": 0.0037851333618164062, "gate_p_p50": 0.01309967041015625, "gate_p_p90": 0.08357722759246827, "gate_p_std": 0.0422706951154396, "gated_delta_h_c_norm": 0.009784292458789423, "global_step": 1170, "grad_norm_max": 0.01904348097741604, "hidden_mse": 0.01103574621374719, "loss": 0.011142809054581449, "lr_blocks": 4.0144148627426e-06, "lr_fusion": 4.0144148627425993e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.638311815261844, "transport_attention_entropy": 8.697086715698243, "transport_attention_entropy_normalized": 0.9952340319752694, "transport_residual_norm": 0.03660843553952873, "velocity_mse": 0.010039234478608706} +{"anchor_distance_mean": 123.5671875, "avg_context_frames": 11.600000000000001, "avg_micro_time_s": 5.516200666075065, "delta_h_c_norm": 0.1677964466623962, "delta_h_d_norm": 0.06046922723762691, "epoch": 10, "gate_p_mean": 0.02810167541028932, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8425656296312809, "gate_p_p10": 0.002832603454589844, "gate_p_p50": 0.011938095092773438, "gate_p_p90": 0.07743377685546876, "gate_p_std": 0.04013690974097699, "gated_delta_h_c_norm": 0.008462692511966453, "global_step": 1180, "grad_norm_max": 0.013487736694514751, "hidden_mse": 0.01125218548695557, "loss": 0.010727200657129288, "lr_blocks": 3.9335018462812664e-06, "lr_fusion": 3.933501846281267e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.325735783576967, "transport_attention_entropy": 8.69571965932846, "transport_attention_entropy_normalized": 0.9950775943696499, "transport_residual_norm": 0.03425816169474274, "velocity_mse": 0.009601982156164014} +{"anchor_distance_mean": 124.18359375, "avg_context_frames": 13.600000000000001, "avg_micro_time_s": 6.018881545149998, "delta_h_c_norm": 0.1712602279148996, "delta_h_d_norm": 0.059797595255076885, "epoch": 10, "gate_p_mean": 0.02701048704329878, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8556731015443803, "gate_p_p10": 0.0028167724609375003, "gate_p_p50": 0.011423492431640626, "gate_p_p90": 0.07587509155273438, "gate_p_std": 0.03798944049049169, "gated_delta_h_c_norm": 0.008258645569731016, "global_step": 1190, "grad_norm_max": 0.017765305936336517, "hidden_mse": 0.010422280436614528, "loss": 0.010098362818825991, "lr_blocks": 3.852880399766243e-06, "lr_fusion": 3.852880399766243e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.54819118976593, "transport_attention_entropy": 8.695937597751618, "transport_attention_entropy_normalized": 0.9951025314629078, "transport_residual_norm": 0.03727557754609734, "velocity_mse": 0.009056134833372198} +{"anchor_distance_mean": 123.89140625, "avg_context_frames": 15.200000000000001, "avg_micro_time_s": 6.441819844587599, "delta_h_c_norm": 0.16771211056038737, "delta_h_d_norm": 0.061308692395687106, "epoch": 10, "gate_p_mean": 0.02788871413213201, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8518732719123364, "gate_p_p10": 0.0032991409301757813, "gate_p_p50": 0.012555503845214845, "gate_p_p90": 0.07527694702148438, "gate_p_std": 0.0385599932691548, "gated_delta_h_c_norm": 0.008686245529679582, "global_step": 1200, "grad_norm_max": 0.014278304763138294, "hidden_mse": 0.010359112190781162, "loss": 0.010037738890969195, "lr_blocks": 3.7725725642960047e-06, "lr_fusion": 3.772572564296005e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.34563894271851, "transport_attention_entropy": 8.695834827423097, "transport_attention_entropy_normalized": 0.995090775936842, "transport_residual_norm": 0.038754658238030974, "velocity_mse": 0.009001827720203437} +{"anchor_distance_mean": 124.34609375000001, "avg_context_frames": 13.200000000000001, "avg_micro_time_s": 5.908885582787343, "delta_h_c_norm": 0.1696144187822938, "delta_h_d_norm": 0.06374380490742625, "epoch": 10, "gate_p_mean": 0.02818766084383242, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8504677750170231, "gate_p_p10": 0.0028825759887695316, "gate_p_p50": 0.01133880615234375, "gate_p_p90": 0.07867759466171265, "gate_p_std": 0.042054803040809934, "gated_delta_h_c_norm": 0.009516367832839025, "global_step": 1210, "grad_norm_max": 0.023431992158293724, "hidden_mse": 0.01075481329462491, "loss": 0.010558127309195697, "lr_blocks": 3.6926002952309015e-06, "lr_fusion": 3.6926002952309016e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.21270101070404, "transport_attention_entropy": 8.695912039279937, "transport_attention_entropy_normalized": 0.9950996078550816, "transport_residual_norm": 0.03559053409844637, "velocity_mse": 0.009482645985553973} +{"anchor_distance_mean": 124.21484375, "avg_context_frames": 13.600000000000001, "avg_micro_time_s": 6.016526217849787, "delta_h_c_norm": 0.17149584628641606, "delta_h_d_norm": 0.05927570383064449, "epoch": 10, "gate_p_mean": 0.025411318981787192, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8716922357678414, "gate_p_p10": 0.0024429321289062503, "gate_p_p50": 0.010037422180175781, "gate_p_p90": 0.07156219482421876, "gate_p_std": 0.03855509543209337, "gated_delta_h_c_norm": 0.008776864358515013, "global_step": 1220, "grad_norm_max": 0.01218254305422306, "hidden_mse": 0.010585753794293851, "loss": 0.010455339014879429, "lr_blocks": 3.6129854561907786e-06, "lr_fusion": 3.612985456190778e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.410595870018007, "transport_attention_entropy": 8.695822477340698, "transport_attention_entropy_normalized": 0.9950893595814705, "transport_residual_norm": 0.03839962738566101, "velocity_mse": 0.009396763640688734} +{"anchor_distance_mean": 124.73671875000001, "avg_context_frames": 16.8, "avg_micro_time_s": 6.87211715510025, "delta_h_c_norm": 0.17180556263774635, "delta_h_d_norm": 0.061195208318531515, "epoch": 10, "gate_p_mean": 0.03347435619216412, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8309042237699032, "gate_p_p10": 0.004493999481201172, "gate_p_p50": 0.015290069580078126, "gate_p_p90": 0.08951123952865601, "gate_p_std": 0.04521446151193231, "gated_delta_h_c_norm": 0.01133242420910392, "global_step": 1230, "grad_norm_max": 0.011271623894572258, "hidden_mse": 0.011320245260139928, "loss": 0.010865605564322324, "lr_blocks": 3.533749813077677e-06, "lr_fusion": 3.533749813077677e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.7887184381485, "transport_attention_entropy": 8.695770645141602, "transport_attention_entropy_normalized": 0.9950834296643735, "transport_residual_norm": 0.040788150602020325, "velocity_mse": 0.009733581010368654} +{"anchor_distance_mean": 123.95937500000001, "avg_context_frames": 14.4, "avg_micro_time_s": 6.194192522024924, "delta_h_c_norm": 0.17390863550826907, "delta_h_d_norm": 0.06356078092940153, "epoch": 10, "gate_p_mean": 0.02834819321287796, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8496139056980611, "gate_p_p10": 0.002784347534179688, "gate_p_p50": 0.011589431762695314, "gate_p_p90": 0.07866668701171875, "gate_p_std": 0.04197417204268277, "gated_delta_h_c_norm": 0.008971426790230908, "global_step": 1240, "grad_norm_max": 0.010993686504662037, "hidden_mse": 0.011657784786075355, "loss": 0.011679777680546977, "lr_blocks": 3.4549150281252635e-06, "lr_fusion": 3.4549150281252636e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.299065709114075, "transport_attention_entropy": 8.69505170583725, "transport_attention_entropy_normalized": 0.9950011588633061, "transport_residual_norm": 0.0399909901432693, "velocity_mse": 0.010513999199611136} +{"anchor_distance_mean": 127.36640625000001, "avg_context_frames": 15.600000000000001, "avg_micro_time_s": 6.505885024174859, "delta_h_c_norm": 0.17623209562152625, "delta_h_d_norm": 0.0633416791446507, "epoch": 10, "gate_p_mean": 0.03175312024541199, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8341556742787362, "gate_p_p10": 0.0037789344787597656, "gate_p_p50": 0.013906860351562501, "gate_p_p90": 0.08661415576934815, "gate_p_std": 0.04337499910616316, "gated_delta_h_c_norm": 0.010675366352370475, "global_step": 1250, "grad_norm_max": 0.020468248054385185, "hidden_mse": 0.011141051026061178, "loss": 0.010946729197166861, "lr_blocks": 3.3765026539765832e-06, "lr_fusion": 3.3765026539765834e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.759154820442202, "transport_attention_entropy": 8.694318699836732, "transport_attention_entropy_normalized": 0.9949172772467136, "transport_residual_norm": 0.04111311193555594, "velocity_mse": 0.009832624121918343} +{"anchor_distance_mean": 124.08359375, "avg_context_frames": 12.4, "avg_micro_time_s": 5.706892933637573, "delta_h_c_norm": 0.16901671867817641, "delta_h_d_norm": 0.06166249839589, "epoch": 10, "gate_p_mean": 0.03239236979279667, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8271409504115582, "gate_p_p10": 0.0033815763890743256, "gate_p_p50": 0.014538955688476563, "gate_p_p90": 0.09071011543273927, "gate_p_std": 0.04359468880575151, "gated_delta_h_c_norm": 0.010745993067394011, "global_step": 1260, "grad_norm_max": 0.041862260550260544, "hidden_mse": 0.011473321908852085, "loss": 0.010574702644953506, "lr_blocks": 3.298534127791785e-06, "lr_fusion": 3.298534127791785e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.24470002651215, "transport_attention_entropy": 8.694103384017945, "transport_attention_entropy_normalized": 0.9948926366865636, "transport_residual_norm": 0.03836977067403496, "velocity_mse": 0.009427370503544807} +{"anchor_distance_mean": 126.0671875, "avg_context_frames": 15.200000000000001, "avg_micro_time_s": 6.442523534499833, "delta_h_c_norm": 0.17659209277480842, "delta_h_d_norm": 0.06200922341085971, "epoch": 10, "gate_p_mean": 0.02714919280842878, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8630691841244698, "gate_p_p10": 0.0027373313903808595, "gate_p_p50": 0.010718345642089844, "gate_p_p90": 0.07562866210937501, "gate_p_std": 0.03972202177392319, "gated_delta_h_c_norm": 0.009961468989058631, "global_step": 1270, "grad_norm_max": 0.027936561033129692, "hidden_mse": 0.011302003960008734, "loss": 0.010891032163635829, "lr_blocks": 3.2210307653874175e-06, "lr_fusion": 3.221030765387417e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.746047592163087, "transport_attention_entropy": 8.694272530078889, "transport_attention_entropy_normalized": 0.9949119940400124, "transport_residual_norm": 0.03753321466501802, "velocity_mse": 0.009760831776657142} +{"anchor_distance_mean": 124.41250000000001, "avg_context_frames": 16.0, "avg_micro_time_s": 6.673950976662354, "delta_h_c_norm": 0.1758643344976008, "delta_h_d_norm": 0.061737252632156014, "epoch": 10, "gate_p_mean": 0.027005310903768988, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8593494854867458, "gate_p_p10": 0.0030913352966308594, "gate_p_p50": 0.012356185913085939, "gate_p_p90": 0.07197461128234864, "gate_p_std": 0.03718429057626054, "gated_delta_h_c_norm": 0.008152737536875065, "global_step": 1280, "grad_norm_max": 0.01516185887157917, "hidden_mse": 0.0101869776903186, "loss": 0.010311251910752618, "lr_blocks": 3.1440137554088957e-06, "lr_fusion": 3.144013755408895e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.25300853252411, "transport_attention_entropy": 8.694273912906647, "transport_attention_entropy_normalized": 0.9949121534824372, "transport_residual_norm": 0.0373558264458552, "velocity_mse": 0.009292554165585899} +{"anchor_distance_mean": 125.3515625, "avg_context_frames": 16.400000000000002, "avg_micro_time_s": 6.714793267512324, "delta_h_c_norm": 0.1747699484229088, "delta_h_d_norm": 0.062482688343152404, "epoch": 11, "gate_p_mean": 0.027465511462651193, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8646912820637227, "gate_p_p10": 0.003115558624267578, "gate_p_p50": 0.011413002014160158, "gate_p_p90": 0.07499160766601563, "gate_p_std": 0.0405003009014763, "gated_delta_h_c_norm": 0.009830824317759834, "global_step": 1290, "grad_norm_max": 0.008140075951814651, "hidden_mse": 0.010821928762015888, "loss": 0.0102216956904158, "lr_blocks": 3.06750415353774e-06, "lr_fusion": 3.0675041535377405e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.400925755500793, "transport_attention_entropy": 8.694020259380341, "transport_attention_entropy_normalized": 0.9948831319808961, "transport_residual_norm": 0.03842131481505931, "velocity_mse": 0.009139502816833556} +{"anchor_distance_mean": 123.89296875000001, "avg_context_frames": 13.600000000000001, "avg_micro_time_s": 6.000764100874995, "delta_h_c_norm": 0.17379684429615738, "delta_h_d_norm": 0.06358917532488705, "epoch": 11, "gate_p_mean": 0.03068261983571574, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8403300531208515, "gate_p_p10": 0.003318405151367188, "gate_p_p50": 0.0132537841796875, "gate_p_p90": 0.08402523994445801, "gate_p_std": 0.04430067645153031, "gated_delta_h_c_norm": 0.010095749809988775, "global_step": 1300, "grad_norm_max": 0.012599554844200611, "hidden_mse": 0.011102442024275662, "loss": 0.010932824172778056, "lr_blocks": 2.991522876735154e-06, "lr_fusion": 2.991522876735154e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.246282505989075, "transport_attention_entropy": 8.69345885515213, "transport_attention_entropy_normalized": 0.9948188856244088, "transport_residual_norm": 0.03787449435330928, "velocity_mse": 0.009822579991305248} +{"anchor_distance_mean": 123.24140625000001, "avg_context_frames": 14.0, "avg_micro_time_s": 6.142453287000172, "delta_h_c_norm": 0.17448528911918404, "delta_h_d_norm": 0.06325103077106178, "epoch": 11, "gate_p_mean": 0.03310069279978052, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8211734011769295, "gate_p_p10": 0.0038069725036621095, "gate_p_p50": 0.014383316040039062, "gate_p_p90": 0.09143915176391602, "gate_p_std": 0.04584582195384428, "gated_delta_h_c_norm": 0.010487937612924726, "global_step": 1310, "grad_norm_max": 0.01025354117155075, "hidden_mse": 0.011034163413569331, "loss": 0.010628101433394477, "lr_blocks": 2.9160906975235493e-06, "lr_fusion": 2.916090697523549e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.318099331855777, "transport_attention_entropy": 8.693501234054565, "transport_attention_entropy_normalized": 0.9948237322270871, "transport_residual_norm": 0.03788973919581622, "velocity_mse": 0.009524685153155589} +{"anchor_distance_mean": 123.23984375, "avg_context_frames": 16.0, "avg_micro_time_s": 6.652118343812527, "delta_h_c_norm": 0.17940076906234026, "delta_h_d_norm": 0.06163384676910937, "epoch": 11, "gate_p_mean": 0.025636244780616838, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8692398093640805, "gate_p_p10": 0.0028741836547851564, "gate_p_p50": 0.010290145874023438, "gate_p_p90": 0.07089996337890625, "gate_p_std": 0.039280009438516575, "gated_delta_h_c_norm": 0.008573754428653046, "global_step": 1320, "grad_norm_max": 0.01485203206539154, "hidden_mse": 0.01012759367004037, "loss": 0.009714462142437697, "lr_blocks": 2.8412282383075362e-06, "lr_fusion": 2.8412282383075363e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.502487897872925, "transport_attention_entropy": 8.69280298948288, "transport_attention_entropy_normalized": 0.9947438284754754, "transport_residual_norm": 0.04145388221368194, "velocity_mse": 0.008701702769030817} +{"anchor_distance_mean": 126.8140625, "avg_context_frames": 12.4, "avg_micro_time_s": 5.701309041825107, "delta_h_c_norm": 0.18186540435999632, "delta_h_d_norm": 0.06462791175581516, "epoch": 11, "gate_p_mean": 0.03425124872010201, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8220077373087407, "gate_p_p10": 0.0032243536785244945, "gate_p_p50": 0.012540435791015625, "gate_p_p90": 0.09908719062805177, "gate_p_std": 0.051248402951750906, "gated_delta_h_c_norm": 0.011969853877963033, "global_step": 1330, "grad_norm_max": 0.023974299430847168, "hidden_mse": 0.011806299682939426, "loss": 0.011802223365521058, "lr_blocks": 2.766955965735968e-06, "lr_fusion": 2.766955965735968e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.575938439369203, "transport_attention_entropy": 8.692864036560058, "transport_attention_entropy_normalized": 0.994750814139843, "transport_residual_norm": 0.03688740378711373, "velocity_mse": 0.01062159335706383} +{"anchor_distance_mean": 128.146875, "avg_context_frames": 13.200000000000001, "avg_micro_time_s": 5.899549451087342, "delta_h_c_norm": 0.17994343470782043, "delta_h_d_norm": 0.06165371658280492, "epoch": 11, "gate_p_mean": 0.0340864654397592, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8237092077732087, "gate_p_p10": 0.0035852432250976563, "gate_p_p50": 0.013999557495117188, "gate_p_p90": 0.09516491889953614, "gate_p_std": 0.04903435444575735, "gated_delta_h_c_norm": 0.01173760788224172, "global_step": 1340, "grad_norm_max": 0.017161503434181213, "hidden_mse": 0.011099700056365692, "loss": 0.011241574585437775, "lr_blocks": 2.693294185106562e-06, "lr_fusion": 2.693294185106562e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.132322192192078, "transport_attention_entropy": 8.692823481559754, "transport_attention_entropy_normalized": 0.9947461791336537, "transport_residual_norm": 0.03719669161364436, "velocity_mse": 0.010131604527123273} +{"anchor_distance_mean": 124.44375000000001, "avg_context_frames": 17.2, "avg_micro_time_s": 6.958717716475076, "delta_h_c_norm": 0.17365164030343294, "delta_h_d_norm": 0.06269795908592642, "epoch": 11, "gate_p_mean": 0.028662328550126404, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8531620889902115, "gate_p_p10": 0.003350925445556641, "gate_p_p50": 0.012034988403320313, "gate_p_p90": 0.07826995849609375, "gate_p_std": 0.04147651555249468, "gated_delta_h_c_norm": 0.009553018859878649, "global_step": 1350, "grad_norm_max": 0.011136950924992561, "hidden_mse": 0.010066087351879105, "loss": 0.00959242228709627, "lr_blocks": 2.6202630348146323e-06, "lr_fusion": 2.6202630348146324e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.99538586139679, "transport_attention_entropy": 8.692882859706879, "transport_attention_entropy_normalized": 0.9947529710829258, "transport_residual_norm": 0.04194842560682446, "velocity_mse": 0.008585813580430113} +{"anchor_distance_mean": 120.80625, "avg_context_frames": 14.8, "avg_micro_time_s": 6.328088195862347, "delta_h_c_norm": 0.17311781495809556, "delta_h_d_norm": 0.06198503486812115, "epoch": 11, "gate_p_mean": 0.031612040707841516, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.833406226336956, "gate_p_p10": 0.0034573554992675783, "gate_p_p50": 0.014103698730468751, "gate_p_p90": 0.08632354736328125, "gate_p_std": 0.04329485562629998, "gated_delta_h_c_norm": 0.00966766155906953, "global_step": 1360, "grad_norm_max": 0.009222026914358139, "hidden_mse": 0.01033993479795754, "loss": 0.010364319520886057, "lr_blocks": 2.5478824808474613e-06, "lr_fusion": 2.547882480847461e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.543672919273376, "transport_attention_entropy": 8.692005598545075, "transport_attention_entropy_normalized": 0.994652584195137, "transport_residual_norm": 0.04161669341847301, "velocity_mse": 0.00933032605680637} +{"anchor_distance_mean": 126.52187500000001, "avg_context_frames": 17.6, "avg_micro_time_s": 7.0779344417375345, "delta_h_c_norm": 0.17660239338874817, "delta_h_d_norm": 0.06085715964436531, "epoch": 11, "gate_p_mean": 0.02109659705893137, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.898447047919035, "gate_p_p10": 0.002390289306640625, "gate_p_p50": 0.009155654907226562, "gate_p_p90": 0.054629516601562504, "gate_p_std": 0.031881712726317346, "gated_delta_h_c_norm": 0.006747237226227299, "global_step": 1370, "grad_norm_max": 0.007689682766795158, "hidden_mse": 0.009666026764898562, "loss": 0.009136133090942166, "lr_blocks": 2.476172311325783e-06, "lr_fusion": 2.476172311325783e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 26.09027831554413, "transport_attention_entropy": 8.69232132434845, "transport_attention_entropy_normalized": 0.9946887105703355, "transport_residual_norm": 0.04612031795550138, "velocity_mse": 0.008169530346640386} +{"anchor_distance_mean": 126.26171875, "avg_context_frames": 10.8, "avg_micro_time_s": 5.319864083762605, "delta_h_c_norm": 0.17437742995098235, "delta_h_d_norm": 0.06450914833694697, "epoch": 11, "gate_p_mean": 0.0354211472091265, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8099249042570591, "gate_p_p10": 0.0040008544921875, "gate_p_p50": 0.016467666625976565, "gate_p_p90": 0.096429443359375, "gate_p_std": 0.04720512758940459, "gated_delta_h_c_norm": 0.011102019355166705, "global_step": 1380, "grad_norm_max": 0.014168363995850086, "hidden_mse": 0.011434478830778971, "loss": 0.011214124065008947, "lr_blocks": 2.4051521310939258e-06, "lr_fusion": 2.405152131093926e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.441550374031067, "transport_attention_entropy": 8.69160053730011, "transport_attention_entropy_normalized": 0.9946062281727791, "transport_residual_norm": 0.04027163872960955, "velocity_mse": 0.010070676181931049} +{"anchor_distance_mean": 125.775, "avg_context_frames": 10.4, "avg_micro_time_s": 5.196918807675093, "delta_h_c_norm": 0.17280935533344746, "delta_h_d_norm": 0.06541669880971313, "epoch": 11, "gate_p_mean": 0.03522516108350828, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8119326181709767, "gate_p_p10": 0.0034374237060546876, "gate_p_p50": 0.014419937133789064, "gate_p_p90": 0.09981579780578614, "gate_p_std": 0.050218170223524795, "gated_delta_h_c_norm": 0.010994062866666356, "global_step": 1390, "grad_norm_max": 0.015188797377049923, "hidden_mse": 0.011721817607758568, "loss": 0.011546354219899514, "lr_blocks": 2.3348413563600324e-06, "lr_fusion": 2.3348413563600325e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 24.99702949523926, "transport_attention_entropy": 8.691307985782624, "transport_attention_entropy_normalized": 0.9945727556943894, "transport_residual_norm": 0.04024341758340597, "velocity_mse": 0.010374172450974585} +{"anchor_distance_mean": 126.68671875000001, "avg_context_frames": 15.600000000000001, "avg_micro_time_s": 6.533053878712645, "delta_h_c_norm": 0.18074099346995354, "delta_h_d_norm": 0.06337619051337243, "epoch": 11, "gate_p_mean": 0.028136428259313107, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8527123659849167, "gate_p_p10": 0.0026862144470214846, "gate_p_p50": 0.011399269104003906, "gate_p_p90": 0.0769761085510254, "gate_p_std": 0.042073695536237216, "gated_delta_h_c_norm": 0.009071337574278004, "global_step": 1400, "grad_norm_max": 0.01116957701742649, "hidden_mse": 0.01008630808792077, "loss": 0.010393613763153554, "lr_blocks": 2.265259209387867e-06, "lr_fusion": 2.2652592093878666e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.12212076187134, "transport_attention_entropy": 8.691902685165406, "transport_attention_entropy_normalized": 0.9946408063173294, "transport_residual_norm": 0.045331656513735656, "velocity_mse": 0.009384982948540711} +{"anchor_distance_mean": 123.37265625, "avg_context_frames": 14.4, "avg_micro_time_s": 6.229944304625133, "delta_h_c_norm": 0.17574320109561087, "delta_h_d_norm": 0.06463708546943962, "epoch": 12, "gate_p_mean": 0.028501845133723693, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8547048017382622, "gate_p_p10": 0.0028523062355816364, "gate_p_p50": 0.011402320861816407, "gate_p_p90": 0.07826766967773438, "gate_p_std": 0.04279458091477864, "gated_delta_h_c_norm": 0.009645192247990054, "global_step": 1410, "grad_norm_max": 0.029505833983421326, "hidden_mse": 0.01096842133847531, "loss": 0.010642849002033473, "lr_blocks": 2.1964247132416373e-06, "lr_fusion": 2.196424713241637e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.948081088066104, "transport_attention_entropy": 8.690964567661286, "transport_attention_entropy_normalized": 0.9945334561169148, "transport_residual_norm": 0.042118461150676016, "velocity_mse": 0.009546006823075005} +{"anchor_distance_mean": 126.03671875, "avg_context_frames": 13.200000000000001, "avg_micro_time_s": 5.916940651149798, "delta_h_c_norm": 0.17710774904116988, "delta_h_d_norm": 0.0661430595908314, "epoch": 12, "gate_p_mean": 0.03687105970457196, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8038031160831451, "gate_p_p10": 0.0043585777282714845, "gate_p_p50": 0.01630363464355469, "gate_p_p90": 0.10087153911590577, "gate_p_std": 0.049841789546189834, "gated_delta_h_c_norm": 0.011577925010351466, "global_step": 1420, "grad_norm_max": 0.012896446511149406, "hidden_mse": 0.011171184745035135, "loss": 0.010773958044592292, "lr_blocks": 2.1283566865852824e-06, "lr_fusion": 2.128356686585282e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.498618292808533, "transport_attention_entropy": 8.690553939342498, "transport_attention_entropy_normalized": 0.9944864690303803, "transport_residual_norm": 0.043411619053222245, "velocity_mse": 0.009656839523813687} +{"anchor_distance_mean": 120.38046875, "avg_context_frames": 14.4, "avg_micro_time_s": 6.217224716250076, "delta_h_c_norm": 0.16967038484290242, "delta_h_d_norm": 0.06368007077835501, "epoch": 12, "gate_p_mean": 0.0311841883987654, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8429434850811959, "gate_p_p10": 0.003478622436523438, "gate_p_p50": 0.012913894653320313, "gate_p_p90": 0.08572244644165039, "gate_p_std": 0.04615677973488346, "gated_delta_h_c_norm": 0.010368336137617007, "global_step": 1430, "grad_norm_max": 0.019909275695681572, "hidden_mse": 0.010731448189471849, "loss": 0.010234379066969269, "lr_blocks": 2.061073738537635e-06, "lr_fusion": 2.061073738537635e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.66618325710297, "transport_attention_entropy": 8.690739357471466, "transport_attention_entropy_normalized": 0.9945076823234559, "transport_residual_norm": 0.04285950465127826, "velocity_mse": 0.009161234251223505} +{"anchor_distance_mean": 125.61562500000001, "avg_context_frames": 13.200000000000001, "avg_micro_time_s": 5.874535380550106, "delta_h_c_norm": 0.17770771607756616, "delta_h_d_norm": 0.0667986007872969, "epoch": 12, "gate_p_mean": 0.03168211142183282, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8327809765934945, "gate_p_p10": 0.0032072066329419616, "gate_p_p50": 0.012676620483398439, "gate_p_p90": 0.08911895751953125, "gate_p_std": 0.045922037179116165, "gated_delta_h_c_norm": 0.009424176627362613, "global_step": 1440, "grad_norm_max": 0.007686668075621128, "hidden_mse": 0.011106402304721997, "loss": 0.010942009711288848, "lr_blocks": 1.9945942635848745e-06, "lr_fusion": 1.9945942635848748e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.721111273765565, "transport_attention_entropy": 8.689522314071656, "transport_attention_entropy_normalized": 0.9943684123456479, "transport_residual_norm": 0.040481703379191464, "velocity_mse": 0.009831369508174249} +{"anchor_distance_mean": 123.46718750000001, "avg_context_frames": 14.4, "avg_micro_time_s": 6.243064449937492, "delta_h_c_norm": 0.17137462608516218, "delta_h_d_norm": 0.06217274921946228, "epoch": 12, "gate_p_mean": 0.029016484343446794, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8534007199108601, "gate_p_p10": 0.0033110618591308595, "gate_p_p50": 0.012670898437500001, "gate_p_p90": 0.07858624458312989, "gate_p_std": 0.04173341479618103, "gated_delta_h_c_norm": 0.008903994616412092, "global_step": 1450, "grad_norm_max": 0.014535799622535706, "hidden_mse": 0.010092900940799156, "loss": 0.009787420165957884, "lr_blocks": 1.928936436551661e-06, "lr_fusion": 1.928936436551661e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.56126825809479, "transport_attention_entropy": 8.68985025882721, "transport_attention_entropy_normalized": 0.9944059394299984, "transport_residual_norm": 0.04468060182407499, "velocity_mse": 0.008778130084101577} +{"anchor_distance_mean": 124.05312500000001, "avg_context_frames": 14.0, "avg_micro_time_s": 6.0969575795747915, "delta_h_c_norm": 0.1722141217440367, "delta_h_d_norm": 0.06473070057108998, "epoch": 12, "gate_p_mean": 0.036664989549899475, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8020225122570992, "gate_p_p10": 0.004147815704345703, "gate_p_p50": 0.01633415222167969, "gate_p_p90": 0.10056347846984864, "gate_p_std": 0.050253041170071816, "gated_delta_h_c_norm": 0.010902104296837934, "global_step": 1460, "grad_norm_max": 0.011750265955924988, "hidden_mse": 0.010912411540630274, "loss": 0.01110153503541369, "lr_blocks": 1.864118207632315e-06, "lr_fusion": 1.8641182076323148e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.414005327224732, "transport_attention_entropy": 8.689909434318542, "transport_attention_entropy_normalized": 0.9944127090275288, "transport_residual_norm": 0.043555468088015914, "velocity_mse": 0.010010293874074705} +{"anchor_distance_mean": 127.76015625000001, "avg_context_frames": 13.200000000000001, "avg_micro_time_s": 5.890873662525064, "delta_h_c_norm": 0.1809283571317792, "delta_h_d_norm": 0.06432955409400165, "epoch": 12, "gate_p_mean": 0.03415002736728639, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8227291397750378, "gate_p_p10": 0.0037872314453125003, "gate_p_p50": 0.014395141601562501, "gate_p_p90": 0.09520535469055176, "gate_p_std": 0.048129706887993964, "gated_delta_h_c_norm": 0.010791443355265074, "global_step": 1470, "grad_norm_max": 0.014516789466142654, "hidden_mse": 0.010785329533973709, "loss": 0.010384799074381591, "lr_blocks": 1.8001572974834169e-06, "lr_fusion": 1.800157297483417e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.52114727497101, "transport_attention_entropy": 8.6893181681633, "transport_attention_entropy_normalized": 0.9943450525403024, "transport_residual_norm": 0.042465087189339105, "velocity_mse": 0.009306266077328474} +{"anchor_distance_mean": 124.540625, "avg_context_frames": 12.8, "avg_micro_time_s": 5.826058672887666, "delta_h_c_norm": 0.1729413852095604, "delta_h_d_norm": 0.06608304623514415, "epoch": 12, "gate_p_mean": 0.03627109189983457, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8095505580306054, "gate_p_p10": 0.00390920639038086, "gate_p_p50": 0.01480865478515625, "gate_p_p90": 0.101666259765625, "gate_p_std": 0.05279275238281116, "gated_delta_h_c_norm": 0.011598265665816144, "global_step": 1480, "grad_norm_max": 0.00964412372559309, "hidden_mse": 0.011082942638313399, "loss": 0.011237183678895236, "lr_blocks": 1.7370711923791567e-06, "lr_fusion": 1.7370711923791567e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.43572385311127, "transport_attention_entropy": 8.689810132980346, "transport_attention_entropy_normalized": 0.9944013476371766, "transport_residual_norm": 0.04289451960939914, "velocity_mse": 0.010128889384213836} +{"anchor_distance_mean": 123.4375, "avg_context_frames": 17.6, "avg_micro_time_s": 7.064370139412768, "delta_h_c_norm": 0.17898003943264484, "delta_h_d_norm": 0.06574445557780564, "epoch": 12, "gate_p_mean": 0.028734098572749646, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8598270025104284, "gate_p_p10": 0.0032911300659179688, "gate_p_p50": 0.012485885620117189, "gate_p_p90": 0.07506062984466554, "gate_p_std": 0.041963406116701664, "gated_delta_h_c_norm": 0.009738205067696981, "global_step": 1490, "grad_norm_max": 0.008964207023382187, "hidden_mse": 0.010442057595355436, "loss": 0.010392574421712198, "lr_blocks": 1.6748771394307584e-06, "lr_fusion": 1.6748771394307585e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 26.185350608825686, "transport_attention_entropy": 8.689200842380524, "transport_attention_entropy_normalized": 0.9943316280841827, "transport_residual_norm": 0.04881309161428363, "velocity_mse": 0.00934836866799742} +{"anchor_distance_mean": 128.79609375, "avg_context_frames": 18.0, "avg_micro_time_s": 7.188228129850041, "delta_h_c_norm": 0.18557818233966827, "delta_h_d_norm": 0.061788437189534307, "epoch": 12, "gate_p_mean": 0.022998747345991433, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8908533930778504, "gate_p_p10": 0.00242856964468956, "gate_p_p50": 0.009196281433105469, "gate_p_p90": 0.061771190166473394, "gate_p_std": 0.0359564469952602, "gated_delta_h_c_norm": 0.00753116206615232, "global_step": 1500, "grad_norm_max": 0.010459856130182743, "hidden_mse": 0.009605421739979648, "loss": 0.00962267924915068, "lr_blocks": 1.6135921418712959e-06, "lr_fusion": 1.6135921418712956e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.951883506774905, "transport_attention_entropy": 8.688539731502534, "transport_attention_entropy_normalized": 0.9942559726536274, "transport_residual_norm": 0.049974289000965656, "velocity_mse": 0.008662137106875889} +{"anchor_distance_mean": 124.47578125000001, "avg_context_frames": 14.4, "avg_micro_time_s": 6.2427262458497355, "delta_h_c_norm": 0.18134363144636156, "delta_h_d_norm": 0.06627479973249138, "epoch": 12, "gate_p_mean": 0.03769740690477193, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8064578570425511, "gate_p_p10": 0.004189777374267578, "gate_p_p50": 0.016164779663085938, "gate_p_p90": 0.1053131103515625, "gate_p_std": 0.05236427849158645, "gated_delta_h_c_norm": 0.01199198849790264, "global_step": 1510, "grad_norm_max": 0.021773288026452065, "hidden_mse": 0.011227888322900982, "loss": 0.011048042928450741, "lr_blocks": 1.5532329544071712e-06, "lr_fusion": 1.553232954407171e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.836835145950317, "transport_attention_entropy": 8.689378249645234, "transport_attention_entropy_normalized": 0.994351925700903, "transport_residual_norm": 0.043058897089213136, "velocity_mse": 0.009925254122936168} +{"anchor_distance_mean": 122.98203125, "avg_context_frames": 15.600000000000001, "avg_micro_time_s": 6.550340593062083, "delta_h_c_norm": 0.1761663630604744, "delta_h_d_norm": 0.06438253456726671, "epoch": 12, "gate_p_mean": 0.03210759739158675, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8357186734676362, "gate_p_p10": 0.0035326573997735977, "gate_p_p50": 0.013176727294921876, "gate_p_p90": 0.08934140205383301, "gate_p_std": 0.046328801498748365, "gated_delta_h_c_norm": 0.0101090676500462, "global_step": 1520, "grad_norm_max": 0.009018701501190662, "hidden_mse": 0.01062133714731317, "loss": 0.00990563527448103, "lr_blocks": 1.4938160786375571e-06, "lr_fusion": 1.4938160786375572e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.57662889957428, "transport_attention_entropy": 8.689116442203522, "transport_attention_entropy_normalized": 0.994321969151497, "transport_residual_norm": 0.045730745117180054, "velocity_mse": 0.00884350158739835} +{"anchor_distance_mean": 125.41640625000001, "avg_context_frames": 14.4, "avg_micro_time_s": 6.2118551404750635, "delta_h_c_norm": 0.17717704717069865, "delta_h_d_norm": 0.0625188113655895, "epoch": 12, "gate_p_mean": 0.03065773771959357, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8432034015655518, "gate_p_p10": 0.003510856628417969, "gate_p_p50": 0.013625335693359376, "gate_p_p90": 0.08218688964843751, "gate_p_std": 0.043738588644191626, "gated_delta_h_c_norm": 0.0093376624994562, "global_step": 1530, "grad_norm_max": 0.007218440994620323, "hidden_mse": 0.009959067276213318, "loss": 0.009681122441543268, "lr_blocks": 1.4353577585430152e-06, "lr_fusion": 1.435357758543015e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.969037008285525, "transport_attention_entropy": 8.688636434078218, "transport_attention_entropy_normalized": 0.9942670404911041, "transport_residual_norm": 0.045886904443614186, "velocity_mse": 0.008685215713921935} +{"anchor_distance_mean": 124.15, "avg_context_frames": 14.0, "avg_micro_time_s": 6.124118852287484, "delta_h_c_norm": 0.1778164392337203, "delta_h_d_norm": 0.0655044256709516, "epoch": 13, "gate_p_mean": 0.03422450267244131, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8260762482881546, "gate_p_p10": 0.003682518005371094, "gate_p_p50": 0.014249420166015625, "gate_p_p90": 0.09466180801391602, "gate_p_std": 0.049324917409103365, "gated_delta_h_c_norm": 0.010955869234749116, "global_step": 1540, "grad_norm_max": 0.01672782190144062, "hidden_mse": 0.01065346504910849, "loss": 0.010578983344021254, "lr_blocks": 1.3778739760445552e-06, "lr_fusion": 1.3778739760445552e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.504676890373233, "transport_attention_entropy": 8.688286364078522, "transport_attention_entropy_normalized": 0.9942269757390023, "transport_residual_norm": 0.04586456567049027, "velocity_mse": 0.009513636800693349} +{"anchor_distance_mean": 121.90937500000001, "avg_context_frames": 14.4, "avg_micro_time_s": 6.213548545724916, "delta_h_c_norm": 0.17619959264993668, "delta_h_d_norm": 0.06627665027044714, "epoch": 13, "gate_p_mean": 0.03855005935765803, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8061365932226181, "gate_p_p10": 0.004090023040771484, "gate_p_p50": 0.015704345703125, "gate_p_p90": 0.10969357490539551, "gate_p_std": 0.05473858449840918, "gated_delta_h_c_norm": 0.012514110925258138, "global_step": 1550, "grad_norm_max": 0.0073217484168708324, "hidden_mse": 0.011622337019070984, "loss": 0.011127175739966334, "lr_blocks": 1.321380446634342e-06, "lr_fusion": 1.3213804466343421e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.92724623680115, "transport_attention_entropy": 8.688480710983276, "transport_attention_entropy_normalized": 0.9942492194473744, "transport_residual_norm": 0.04719760494772345, "velocity_mse": 0.00996494206483476} +{"anchor_distance_mean": 124.73515625, "avg_context_frames": 14.4, "avg_micro_time_s": 6.202634845374906, "delta_h_c_norm": 0.17817805074155332, "delta_h_d_norm": 0.062383789243176585, "epoch": 13, "gate_p_mean": 0.028928310750052335, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8546647384762764, "gate_p_p10": 0.0032027244567871095, "gate_p_p50": 0.012120056152343751, "gate_p_p90": 0.07777892947196961, "gate_p_std": 0.0435024305479601, "gated_delta_h_c_norm": 0.008808802714338527, "global_step": 1560, "grad_norm_max": 0.011943959631025791, "hidden_mse": 0.009316329495050013, "loss": 0.009461834939429536, "lr_blocks": 1.2658926150792321e-06, "lr_fusion": 1.2658926150792322e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.88336741924286, "transport_attention_entropy": 8.689011824131013, "transport_attention_entropy_normalized": 0.9943099968135357, "transport_residual_norm": 0.04569269472267479, "velocity_mse": 0.008530202030669898} +{"anchor_distance_mean": 128.56875, "avg_context_frames": 14.8, "avg_micro_time_s": 6.324988413187521, "delta_h_c_norm": 0.18292292021214962, "delta_h_d_norm": 0.0664964736904949, "epoch": 13, "gate_p_mean": 0.03358395694522187, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8291987452656031, "gate_p_p10": 0.0037261962890625003, "gate_p_p50": 0.014753341674804688, "gate_p_p90": 0.09112422466278076, "gate_p_std": 0.04803977768169716, "gated_delta_h_c_norm": 0.010106243859627285, "global_step": 1570, "grad_norm_max": 0.0076916576363146305, "hidden_mse": 0.010353575932094829, "loss": 0.010361120820743964, "lr_blocks": 1.2114256511983274e-06, "lr_fusion": 1.2114256511983274e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.755758213996888, "transport_attention_entropy": 8.688386499881744, "transport_attention_entropy_normalized": 0.9942384362220764, "transport_residual_norm": 0.04674533202778548, "velocity_mse": 0.00932576323975809} +{"anchor_distance_mean": 122.10546875, "avg_context_frames": 12.8, "avg_micro_time_s": 5.8068503848248385, "delta_h_c_norm": 0.18074422609061003, "delta_h_d_norm": 0.06650110320188106, "epoch": 13, "gate_p_mean": 0.035858738073147836, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8161814160645009, "gate_p_p10": 0.0035300254821777344, "gate_p_p50": 0.014368820190429689, "gate_p_p90": 0.09859247207641603, "gate_p_std": 0.053350150177720934, "gated_delta_h_c_norm": 0.011840512952767314, "global_step": 1580, "grad_norm_max": 0.011479891836643219, "hidden_mse": 0.011291181517299265, "loss": 0.011046541773248465, "lr_blocks": 1.157994445715706e-06, "lr_fusion": 1.157994445715706e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.536421513557436, "transport_attention_entropy": 8.688268399238586, "transport_attention_entropy_normalized": 0.9942249245941639, "transport_residual_norm": 0.0448753550183028, "velocity_mse": 0.00991742360056378} +{"anchor_distance_mean": 122.65859375000001, "avg_context_frames": 13.600000000000001, "avg_micro_time_s": 5.987161405399911, "delta_h_c_norm": 0.1795888500288129, "delta_h_d_norm": 0.06608675746247172, "epoch": 13, "gate_p_mean": 0.0349520668736659, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8173047095537186, "gate_p_p10": 0.004003143310546875, "gate_p_p50": 0.016218948364257812, "gate_p_p90": 0.09352874755859375, "gate_p_std": 0.046524928882718086, "gated_delta_h_c_norm": 0.010318222051137128, "global_step": 1590, "grad_norm_max": 0.013541894033551216, "hidden_mse": 0.011111196770798415, "loss": 0.010427086555864662, "lr_blocks": 1.1056136061894386e-06, "lr_fusion": 1.1056136061894384e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.724818110466003, "transport_attention_entropy": 8.687572932243347, "transport_attention_entropy_normalized": 0.9941453367471695, "transport_residual_norm": 0.04518863284029067, "velocity_mse": 0.009315966910799034} +{"anchor_distance_mean": 128.11171875, "avg_context_frames": 10.8, "avg_micro_time_s": 5.273904852937676, "delta_h_c_norm": 0.1840546827763319, "delta_h_d_norm": 0.06533884052187204, "epoch": 13, "gate_p_mean": 0.037235666532069445, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8118344597518444, "gate_p_p10": 0.003902816586196423, "gate_p_p50": 0.014894866943359376, "gate_p_p90": 0.10479855537414551, "gate_p_std": 0.05482119420776144, "gated_delta_h_c_norm": 0.012292072828859092, "global_step": 1600, "grad_norm_max": 0.00921064242720604, "hidden_mse": 0.01126595433743205, "loss": 0.011295963544398546, "lr_blocks": 1.0542974530180327e-06, "lr_fusion": 1.0542974530180327e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.22659718990326, "transport_attention_entropy": 8.687267196178437, "transport_attention_entropy_normalized": 0.994110356271267, "transport_residual_norm": 0.04079856066964567, "velocity_mse": 0.01016936810920015} +{"anchor_distance_mean": 128.92656250000002, "avg_context_frames": 16.8, "avg_micro_time_s": 6.8543565009752, "delta_h_c_norm": 0.18788023749366403, "delta_h_d_norm": 0.06553520723246038, "epoch": 13, "gate_p_mean": 0.030124578275717795, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8520923741161823, "gate_p_p10": 0.0031092070043087006, "gate_p_p50": 0.012125396728515626, "gate_p_p90": 0.08258819580078125, "gate_p_std": 0.045066749153193086, "gated_delta_h_c_norm": 0.01022906203288585, "global_step": 1610, "grad_norm_max": 0.011728893965482712, "hidden_mse": 0.010290867986623198, "loss": 0.009968784492230043, "lr_blocks": 1.0040600155253766e-06, "lr_fusion": 1.0040600155253765e-05, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.815218138694764, "transport_attention_entropy": 8.687673056125641, "transport_attention_entropy_normalized": 0.9941567957401276, "transport_residual_norm": 0.04933939101174474, "velocity_mse": 0.008939697677851655} +{"anchor_distance_mean": 126.29531250000001, "avg_context_frames": 12.8, "avg_micro_time_s": 5.809989723774698, "delta_h_c_norm": 0.18070887047797443, "delta_h_d_norm": 0.06453284509479999, "epoch": 13, "gate_p_mean": 0.030729523673653603, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8407454676926136, "gate_p_p10": 0.003098106384277344, "gate_p_p50": 0.012404251098632814, "gate_p_p90": 0.08395767211914062, "gate_p_std": 0.046476746018743145, "gated_delta_h_c_norm": 0.009307167612132617, "global_step": 1620, "grad_norm_max": 0.010720557533204556, "hidden_mse": 0.010026147961616516, "loss": 0.009740071391570383, "lr_blocks": 9.549150281252633e-07, "lr_fusion": 9.549150281252633e-06, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.504708218574525, "transport_attention_entropy": 8.687506985664369, "transport_attention_entropy_normalized": 0.9941377945244313, "transport_residual_norm": 0.045686877169646326, "velocity_mse": 0.008737456568633206} +{"anchor_distance_mean": 125.48203125, "avg_context_frames": 16.400000000000002, "avg_micro_time_s": 6.752561529724699, "delta_h_c_norm": 0.17930182442069054, "delta_h_d_norm": 0.06583821563981473, "epoch": 13, "gate_p_mean": 0.029331530758645388, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.85637097209692, "gate_p_p10": 0.0036328315734863283, "gate_p_p50": 0.012833786010742188, "gate_p_p90": 0.078851318359375, "gate_p_std": 0.04265355839161203, "gated_delta_h_c_norm": 0.008930229264660739, "global_step": 1630, "grad_norm_max": 0.013954822905361652, "hidden_mse": 0.009451516438275576, "loss": 0.009374333100276999, "lr_blocks": 9.068759265665384e-07, "lr_fusion": 9.068759265665384e-06, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.890750551223757, "transport_attention_entropy": 8.68780530691147, "transport_attention_entropy_normalized": 0.9941719308495522, "transport_residual_norm": 0.04944913503713906, "velocity_mse": 0.008429181421524845} +{"anchor_distance_mean": 123.33906250000001, "avg_context_frames": 16.0, "avg_micro_time_s": 6.597234557025195, "delta_h_c_norm": 0.1759949577972293, "delta_h_d_norm": 0.06524352501146495, "epoch": 13, "gate_p_mean": 0.031134905305225405, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8374960184097291, "gate_p_p10": 0.0035333633422851562, "gate_p_p50": 0.013545608520507813, "gate_p_p90": 0.08261927366256715, "gate_p_std": 0.045128511043731126, "gated_delta_h_c_norm": 0.008873337785189506, "global_step": 1640, "grad_norm_max": 0.007233121898025274, "hidden_mse": 0.009788378671510146, "loss": 0.010090433814912104, "lr_blocks": 8.599558442598998e-07, "lr_fusion": 8.599558442598998e-06, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.744459486007692, "transport_attention_entropy": 8.6871701836586, "transport_attention_entropy_normalized": 0.9940992526710034, "transport_residual_norm": 0.04917594692669809, "velocity_mse": 0.009111595910508186} +{"anchor_distance_mean": 125.353125, "avg_context_frames": 13.200000000000001, "avg_micro_time_s": 5.879740550687211, "delta_h_c_norm": 0.1788351284340024, "delta_h_d_norm": 0.06646750206127763, "epoch": 13, "gate_p_mean": 0.03326797914342024, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.828214417397976, "gate_p_p10": 0.003452587127685547, "gate_p_p50": 0.013550949096679688, "gate_p_p90": 0.09127024412155152, "gate_p_std": 0.049209798639640215, "gated_delta_h_c_norm": 0.009706231934251264, "global_step": 1650, "grad_norm_max": 0.010267057456076145, "hidden_mse": 0.010384472997975536, "loss": 0.01058672362123616, "lr_blocks": 8.141676086873574e-07, "lr_fusion": 8.141676086873572e-06, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.68226263523102, "transport_attention_entropy": 8.687251269817352, "transport_attention_entropy_normalized": 0.9941085278987885, "transport_residual_norm": 0.04333563433028758, "velocity_mse": 0.009548276322311722} +{"anchor_distance_mean": 122.69062500000001, "avg_context_frames": 14.8, "avg_micro_time_s": 6.307661708512524, "delta_h_c_norm": 0.1777762709185481, "delta_h_d_norm": 0.06853991583921015, "epoch": 13, "gate_p_mean": 0.03858485954115168, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8053042609244585, "gate_p_p10": 0.004283332824707032, "gate_p_p50": 0.016368484497070315, "gate_p_p90": 0.10428924560546876, "gate_p_std": 0.05487236246699467, "gated_delta_h_c_norm": 0.01234046989993658, "global_step": 1660, "grad_norm_max": 0.015031692571938038, "hidden_mse": 0.011608897702535615, "loss": 0.011010719375917688, "lr_blocks": 7.695237378953224e-07, "lr_fusion": 7.695237378953223e-06, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.764585971832275, "transport_attention_entropy": 8.687799668312072, "transport_attention_entropy_normalized": 0.9941712871193886, "transport_residual_norm": 0.044669240689836444, "velocity_mse": 0.009849829573067837} +{"anchor_distance_mean": 125.02734375, "avg_context_frames": 16.0, "avg_micro_time_s": 6.612808280787386, "delta_h_c_norm": 0.17894670050591233, "delta_h_d_norm": 0.06642496320419014, "epoch": 14, "gate_p_mean": 0.0345189088373445, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8207372032105923, "gate_p_p10": 0.0038161277770996094, "gate_p_p50": 0.01418304443359375, "gate_p_p90": 0.09322400093078614, "gate_p_std": 0.05046677723294124, "gated_delta_h_c_norm": 0.010613439200096764, "global_step": 1670, "grad_norm_max": 0.0071359253488481045, "hidden_mse": 0.010394078836543487, "loss": 0.010466209324658849, "lr_blocks": 7.260364370723044e-07, "lr_fusion": 7.260364370723044e-06, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 26.008411145210268, "transport_attention_entropy": 8.687398195266724, "transport_attention_entropy_normalized": 0.9941253423690797, "transport_residual_norm": 0.05070072524249554, "velocity_mse": 0.009426801421795972} +{"anchor_distance_mean": 122.59140625, "avg_context_frames": 12.0, "avg_micro_time_s": 5.610654997649999, "delta_h_c_norm": 0.17495650965720416, "delta_h_d_norm": 0.06863596891053021, "epoch": 14, "gate_p_mean": 0.036306313343811784, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8125087857246399, "gate_p_p10": 0.003802490048110485, "gate_p_p50": 0.014754867553710938, "gate_p_p90": 0.1016088366508484, "gate_p_std": 0.0539396500797011, "gated_delta_h_c_norm": 0.011127636898891069, "global_step": 1680, "grad_norm_max": 0.014253122732043266, "hidden_mse": 0.011168121453374626, "loss": 0.010763852734817192, "lr_blocks": 6.837175952121305e-07, "lr_fusion": 6.837175952121306e-06, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.39583117961884, "transport_attention_entropy": 8.686547803878785, "transport_attention_entropy_normalized": 0.9940280288457871, "transport_residual_norm": 0.04318067822605372, "velocity_mse": 0.009647040581330658} +{"anchor_distance_mean": 123.43593750000001, "avg_context_frames": 14.0, "avg_micro_time_s": 6.124928130062108, "delta_h_c_norm": 0.1756631139665842, "delta_h_d_norm": 0.06510597374290228, "epoch": 14, "gate_p_mean": 0.03258568624733016, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8329407289624214, "gate_p_p10": 0.0035321805626153946, "gate_p_p50": 0.013745880126953125, "gate_p_p90": 0.09015343189239503, "gate_p_std": 0.047080314683262264, "gated_delta_h_c_norm": 0.009717441372049508, "global_step": 1690, "grad_norm_max": 0.006527384743094444, "hidden_mse": 0.009835478547029198, "loss": 0.009707191254710779, "lr_blocks": 6.425787818636131e-07, "lr_fusion": 6.425787818636131e-06, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.553847670555115, "transport_attention_entropy": 8.68701467514038, "transport_attention_entropy_normalized": 0.9940814547240735, "transport_residual_norm": 0.046650777803733945, "velocity_mse": 0.008723643419216387} +{"anchor_distance_mean": 124.24843750000001, "avg_context_frames": 13.600000000000001, "avg_micro_time_s": 6.051575826725093, "delta_h_c_norm": 0.17793830353766682, "delta_h_d_norm": 0.06806154088117182, "epoch": 14, "gate_p_mean": 0.037204776128055526, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8119536500424147, "gate_p_p10": 0.0037789344787597656, "gate_p_p50": 0.01672210693359375, "gate_p_p90": 0.09987869262695313, "gate_p_std": 0.05160666652955115, "gated_delta_h_c_norm": 0.011424155237909872, "global_step": 1700, "grad_norm_max": 0.009150154888629913, "hidden_mse": 0.011531693028518931, "loss": 0.01102986797632184, "lr_blocks": 6.026312439675553e-07, "lr_fusion": 6.026312439675552e-06, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.698371315002444, "transport_attention_entropy": 8.687040090560913, "transport_attention_entropy_normalized": 0.9940843641757966, "transport_residual_norm": 0.04673653813079, "velocity_mse": 0.009876698662992567} +{"anchor_distance_mean": 124.83203125, "avg_context_frames": 17.2, "avg_micro_time_s": 7.009978292999995, "delta_h_c_norm": 0.17949325516819956, "delta_h_d_norm": 0.06666057272814215, "epoch": 14, "gate_p_mean": 0.032648303953465074, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8358609035611153, "gate_p_p10": 0.0039081573486328125, "gate_p_p50": 0.013751602172851563, "gate_p_p90": 0.08874232769012452, "gate_p_std": 0.04736982627073303, "gated_delta_h_c_norm": 0.009894778308807873, "global_step": 1710, "grad_norm_max": 0.005545111373066902, "hidden_mse": 0.009845236051478424, "loss": 0.009455488415551373, "lr_blocks": 5.63885902781941e-07, "lr_fusion": 5.6388590278194096e-06, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.861772918701174, "transport_attention_entropy": 8.687862908840179, "transport_attention_entropy_normalized": 0.9941785238683224, "transport_residual_norm": 0.050505554769188166, "velocity_mse": 0.008470964786829427} +{"anchor_distance_mean": 127.23828125, "avg_context_frames": 16.8, "avg_micro_time_s": 6.86917545018755, "delta_h_c_norm": 0.18355646394193173, "delta_h_d_norm": 0.06738550304435194, "epoch": 14, "gate_p_mean": 0.03574047450674698, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8321920327842236, "gate_p_p10": 0.004106044769287109, "gate_p_p50": 0.013944625854492188, "gate_p_p90": 0.10096521377563478, "gate_p_std": 0.053580841072835034, "gated_delta_h_c_norm": 0.011853607872035356, "global_step": 1720, "grad_norm_max": 0.006962347775697708, "hidden_mse": 0.010139080771477894, "loss": 0.009575729761854746, "lr_blocks": 5.263533508961827e-07, "lr_fusion": 5.263533508961827e-06, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.970433115959167, "transport_attention_entropy": 8.687398731708527, "transport_attention_entropy_normalized": 0.9941254049539566, "transport_residual_norm": 0.049945689295418566, "velocity_mse": 0.008561821703915484} +{"anchor_distance_mean": 125.028125, "avg_context_frames": 12.0, "avg_micro_time_s": 5.605862292500115, "delta_h_c_norm": 0.1811051953583956, "delta_h_d_norm": 0.06778138433583081, "epoch": 14, "gate_p_mean": 0.03628497221507132, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8177226327359677, "gate_p_p10": 0.003470993041992188, "gate_p_p50": 0.013599014282226563, "gate_p_p90": 0.10574582815170289, "gate_p_std": 0.055787574523128575, "gated_delta_h_c_norm": 0.011749408947071062, "global_step": 1730, "grad_norm_max": 0.011569133959710598, "hidden_mse": 0.011352567578433082, "loss": 0.010810477306949907, "lr_blocks": 4.900438493352056e-07, "lr_fusion": 4.900438493352055e-06, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.703882789611818, "transport_attention_entropy": 8.686197769641877, "transport_attention_entropy_normalized": 0.9939879752695561, "transport_residual_norm": 0.04573031479958445, "velocity_mse": 0.009675220577628353} +{"anchor_distance_mean": 124.12109375, "avg_context_frames": 15.200000000000001, "avg_micro_time_s": 6.437202245337449, "delta_h_c_norm": 0.1834879906848073, "delta_h_d_norm": 0.06738191843032837, "epoch": 14, "gate_p_mean": 0.03720526891993359, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.809354741871357, "gate_p_p10": 0.003957653045654297, "gate_p_p50": 0.015505218505859376, "gate_p_p90": 0.10212860107421876, "gate_p_std": 0.05279471732210368, "gated_delta_h_c_norm": 0.011241151060676203, "global_step": 1740, "grad_norm_max": 0.016917569562792778, "hidden_mse": 0.011285766953369603, "loss": 0.01103177086042706, "lr_blocks": 4.549673247541875e-07, "lr_fusion": 4.549673247541875e-06, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 26.40957729816437, "transport_attention_entropy": 8.68664677143097, "transport_attention_entropy_normalized": 0.9940393559634686, "transport_residual_norm": 0.047540415078401566, "velocity_mse": 0.00990319412376266} +{"anchor_distance_mean": 124.18359375, "avg_context_frames": 14.4, "avg_micro_time_s": 6.242337294587469, "delta_h_c_norm": 0.1803220644593239, "delta_h_d_norm": 0.06705130967311561, "epoch": 14, "gate_p_mean": 0.038608373072929685, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8037004470825195, "gate_p_p10": 0.0043393135070800785, "gate_p_p50": 0.0167388916015625, "gate_p_p90": 0.1062408447265625, "gate_p_std": 0.053756696288473904, "gated_delta_h_c_norm": 0.011617871929774993, "global_step": 1750, "grad_norm_max": 0.00536339869722724, "hidden_mse": 0.010548849258339033, "loss": 0.010996088990941644, "lr_blocks": 4.211333667247125e-07, "lr_fusion": 4.2113336672471245e-06, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.865820264816286, "transport_attention_entropy": 8.687315475940705, "transport_attention_entropy_normalized": 0.9941158786416054, "transport_residual_norm": 0.04767508418299258, "velocity_mse": 0.009941203997004778} +{"anchor_distance_mean": 127.07578125, "avg_context_frames": 10.8, "avg_micro_time_s": 5.307232211374958, "delta_h_c_norm": 0.18272677548229696, "delta_h_d_norm": 0.06468467311933637, "epoch": 14, "gate_p_mean": 0.04141663323389366, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.7826237216591836, "gate_p_p10": 0.004276371002197266, "gate_p_p50": 0.0173797607421875, "gate_p_p90": 0.11629562377929688, "gate_p_std": 0.057987720926757906, "gated_delta_h_c_norm": 0.012358808974386194, "global_step": 1760, "grad_norm_max": 0.006515322718769312, "hidden_mse": 0.010891416869708338, "loss": 0.01119459995534271, "lr_blocks": 3.885512251130763e-07, "lr_fusion": 3.885512251130763e-06, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.18672308921814, "transport_attention_entropy": 8.686094129085541, "transport_attention_entropy_normalized": 0.9939761161804199, "transport_residual_norm": 0.04416891308501363, "velocity_mse": 0.010105458329780959} +{"anchor_distance_mean": 123.1453125, "avg_context_frames": 16.400000000000002, "avg_micro_time_s": 6.7704035717372335, "delta_h_c_norm": 0.17880037631839515, "delta_h_d_norm": 0.06593571482226253, "epoch": 14, "gate_p_mean": 0.03501438369276002, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.820129232108593, "gate_p_p10": 0.003810882568359375, "gate_p_p50": 0.014859390258789063, "gate_p_p90": 0.09459991455078126, "gate_p_std": 0.05060937595553697, "gated_delta_h_c_norm": 0.010539661464281381, "global_step": 1770, "grad_norm_max": 0.007510818541049957, "hidden_mse": 0.010047668419429102, "loss": 0.010335798482992688, "lr_blocks": 3.572298075514652e-07, "lr_fusion": 3.5722980755146517e-06, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 26.184518456459045, "transport_attention_entropy": 8.686716079711914, "transport_attention_entropy_normalized": 0.9940472833812237, "transport_residual_norm": 0.04933567771222443, "velocity_mse": 0.009331031638430432} +{"anchor_distance_mean": 124.99531250000001, "avg_context_frames": 12.4, "avg_micro_time_s": 5.714142888887318, "delta_h_c_norm": 0.1787227728404105, "delta_h_d_norm": 0.06512290323153139, "epoch": 14, "gate_p_mean": 0.0332980946986936, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8264425829052926, "gate_p_p10": 0.003506183624267578, "gate_p_p50": 0.013939666748046876, "gate_p_p90": 0.09140286445617676, "gate_p_std": 0.04802036893088371, "gated_delta_h_c_norm": 0.009662967946496792, "global_step": 1780, "grad_norm_max": 0.006653488613665104, "hidden_mse": 0.010243044613162056, "loss": 0.010152931098127739, "lr_blocks": 3.271776770026963e-07, "lr_fusion": 3.271776770026963e-06, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.761922359466553, "transport_attention_entropy": 8.686330735683441, "transport_attention_entropy_normalized": 0.9940031901001931, "transport_residual_norm": 0.045513879321515564, "velocity_mse": 0.009128626593155787} +{"anchor_distance_mean": 125.384375, "avg_context_frames": 14.8, "avg_micro_time_s": 6.350583435337786, "delta_h_c_norm": 0.1787285752594471, "delta_h_d_norm": 0.06455588540993631, "epoch": 14, "gate_p_mean": 0.029359712469158694, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8541511677205563, "gate_p_p10": 0.0033678054809570313, "gate_p_p50": 0.012476730346679689, "gate_p_p90": 0.07822766304016114, "gate_p_std": 0.04374437922379002, "gated_delta_h_c_norm": 0.008861321008589584, "global_step": 1790, "grad_norm_max": 0.003935358487069607, "hidden_mse": 0.009598351214663126, "loss": 0.009242627082858236, "lr_blocks": 2.984030494191942e-07, "lr_fusion": 2.9840304941919415e-06, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 26.159776759147647, "transport_attention_entropy": 8.686719214916229, "transport_attention_entropy_normalized": 0.9940476462244988, "transport_residual_norm": 0.048363689472898846, "velocity_mse": 0.008282791991950944} +{"anchor_distance_mean": 123.371875, "avg_context_frames": 18.8, "avg_micro_time_s": 7.398546568762595, "delta_h_c_norm": 0.18063455671072007, "delta_h_d_norm": 0.06505002332851291, "epoch": 15, "gate_p_mean": 0.025525538076180965, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8748658120632172, "gate_p_p10": 0.003050518035888672, "gate_p_p50": 0.0112152099609375, "gate_p_p90": 0.06647530794143677, "gate_p_std": 0.03761565333697945, "gated_delta_h_c_norm": 0.00775255932530854, "global_step": 1800, "grad_norm_max": 0.007416466251015663, "hidden_mse": 0.009064406453398987, "loss": 0.008688604869530537, "lr_blocks": 2.7091379149682683e-07, "lr_fusion": 2.7091379149682685e-06, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 26.31354274749756, "transport_attention_entropy": 8.68695319890976, "transport_attention_entropy_normalized": 0.9940744206309319, "transport_residual_norm": 0.05239727015141398, "velocity_mse": 0.007782164233503863} +{"anchor_distance_mean": 121.65, "avg_context_frames": 12.4, "avg_micro_time_s": 5.695604609150223, "delta_h_c_norm": 0.17960067354142667, "delta_h_d_norm": 0.06819999455474317, "epoch": 15, "gate_p_mean": 0.0373019901686348, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8045961327850819, "gate_p_p10": 0.003966426849365235, "gate_p_p50": 0.015394973754882814, "gate_p_p90": 0.10499464869499207, "gate_p_std": 0.05352216620231048, "gated_delta_h_c_norm": 0.011125682704732754, "global_step": 1810, "grad_norm_max": 0.008542954921722412, "hidden_mse": 0.010985440603690222, "loss": 0.010696062815259212, "lr_blocks": 2.447174185242324e-07, "lr_fusion": 2.4471741852423237e-06, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.577042770385745, "transport_attention_entropy": 8.686361515522004, "transport_attention_entropy_normalized": 0.9940067194402218, "transport_residual_norm": 0.04523404573556036, "velocity_mse": 0.009597518711234444} +{"anchor_distance_mean": 127.4, "avg_context_frames": 13.200000000000001, "avg_micro_time_s": 5.914455507974981, "delta_h_c_norm": 0.18597689270973206, "delta_h_d_norm": 0.06536185103468597, "epoch": 15, "gate_p_mean": 0.03425770215690136, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8312512770295144, "gate_p_p10": 0.0035025596618652346, "gate_p_p50": 0.013817977905273438, "gate_p_p90": 0.09442805051803589, "gate_p_std": 0.050965398037806156, "gated_delta_h_c_norm": 0.010491810983512552, "global_step": 1820, "grad_norm_max": 0.01063754316419363, "hidden_mse": 0.010025143044185825, "loss": 0.009974031895399095, "lr_blocks": 2.198210923282118e-07, "lr_fusion": 2.1982109232821178e-06, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.39418637752533, "transport_attention_entropy": 8.685958302021026, "transport_attention_entropy_normalized": 0.9939605720341206, "transport_residual_norm": 0.04791557919234038, "velocity_mse": 0.008971517562167719} +{"anchor_distance_mean": 126.74921875000001, "avg_context_frames": 14.4, "avg_micro_time_s": 6.22482022713757, "delta_h_c_norm": 0.18230653926730156, "delta_h_d_norm": 0.06800931189209224, "epoch": 15, "gate_p_mean": 0.039157811901532114, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8094376221299172, "gate_p_p10": 0.004568672180175782, "gate_p_p50": 0.015454483032226563, "gate_p_p90": 0.11174435615539552, "gate_p_std": 0.056991651933640246, "gated_delta_h_c_norm": 0.012652311223791913, "global_step": 1830, "grad_norm_max": 0.010004925541579723, "hidden_mse": 0.011304014298366383, "loss": 0.010643546216306277, "lr_blocks": 1.962316193157593e-07, "lr_fusion": 1.962316193157593e-06, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 26.15758583545685, "transport_attention_entropy": 8.686894667148591, "transport_attention_entropy_normalized": 0.9940677225589752, "transport_residual_norm": 0.04603763967752457, "velocity_mse": 0.009513144782977179} +{"anchor_distance_mean": 124.4109375, "avg_context_frames": 14.0, "avg_micro_time_s": 6.1349914722374255, "delta_h_c_norm": 0.18080867752432825, "delta_h_d_norm": 0.06912589115090668, "epoch": 15, "gate_p_mean": 0.038944391836412254, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8052559331059457, "gate_p_p10": 0.0045133398845791815, "gate_p_p50": 0.016560935974121095, "gate_p_p90": 0.10759779810905457, "gate_p_std": 0.05628764202119783, "gated_delta_h_c_norm": 0.0123088034481043, "global_step": 1840, "grad_norm_max": 0.005824783816933632, "hidden_mse": 0.010829294612631202, "loss": 0.010893201141152532, "lr_blocks": 1.7395544861325718e-07, "lr_fusion": 1.7395544861325718e-06, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.900705599784853, "transport_attention_entropy": 8.686658954620361, "transport_attention_entropy_normalized": 0.9940407484769822, "transport_residual_norm": 0.04557256139814854, "velocity_mse": 0.009810271704918706} +{"anchor_distance_mean": 124.246875, "avg_context_frames": 15.200000000000001, "avg_micro_time_s": 6.474465117425007, "delta_h_c_norm": 0.1771659580990672, "delta_h_d_norm": 0.06709056259132921, "epoch": 15, "gate_p_mean": 0.03777582157636061, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8118302024900914, "gate_p_p10": 0.004340839385986328, "gate_p_p50": 0.016535568237304687, "gate_p_p90": 0.10216336250305176, "gate_p_std": 0.05456757211359218, "gated_delta_h_c_norm": 0.011806087559671143, "global_step": 1850, "grad_norm_max": 0.005638052709400654, "hidden_mse": 0.010925622953800486, "loss": 0.010896192473592237, "lr_blocks": 1.5299867030334815e-07, "lr_fusion": 1.5299867030334814e-06, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.97950749397278, "transport_attention_entropy": 8.686921501159668, "transport_attention_entropy_normalized": 0.9940707944333553, "transport_residual_norm": 0.04714481886476279, "velocity_mse": 0.009803630155511202} +{"anchor_distance_mean": 123.50234375000001, "avg_context_frames": 14.0, "avg_micro_time_s": 6.1327193985872785, "delta_h_c_norm": 0.18107433039695026, "delta_h_d_norm": 0.06688019265420735, "epoch": 15, "gate_p_mean": 0.03490862536709756, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.815151770412922, "gate_p_p10": 0.003700542263686657, "gate_p_p50": 0.014759445190429689, "gate_p_p90": 0.09380977153778076, "gate_p_std": 0.050923649163451046, "gated_delta_h_c_norm": 0.010177475365344435, "global_step": 1860, "grad_norm_max": 0.008410329930484295, "hidden_mse": 0.010395558713935316, "loss": 0.010664708184776828, "lr_blocks": 1.333670137599713e-07, "lr_fusion": 1.333670137599713e-06, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.378819942474365, "transport_attention_entropy": 8.686394548416137, "transport_attention_entropy_normalized": 0.9940104953944684, "transport_residual_norm": 0.048922447394579655, "velocity_mse": 0.009625152312219144} +{"anchor_distance_mean": 125.87265625, "avg_context_frames": 10.4, "avg_micro_time_s": 5.179028115062511, "delta_h_c_norm": 0.17965747807174923, "delta_h_d_norm": 0.06792267560958863, "epoch": 15, "gate_p_mean": 0.038708848168607805, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.7991008341312409, "gate_p_p10": 0.004138660430908203, "gate_p_p50": 0.016728591918945313, "gate_p_p90": 0.10636209845542909, "gate_p_std": 0.054931743198540064, "gated_delta_h_c_norm": 0.011245020083151758, "global_step": 1870, "grad_norm_max": 0.008892090059816837, "hidden_mse": 0.011215962929418311, "loss": 0.011025290336692707, "lr_blocks": 1.1506584608200366e-07, "lr_fusion": 1.1506584608200367e-06, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.222165751457215, "transport_attention_entropy": 8.685762786865235, "transport_attention_entropy_normalized": 0.9939381986856461, "transport_residual_norm": 0.042622948880307376, "velocity_mse": 0.009903694118838758} +{"anchor_distance_mean": 125.44843750000001, "avg_context_frames": 15.600000000000001, "avg_micro_time_s": 6.54554112423748, "delta_h_c_norm": 0.18084950810298325, "delta_h_d_norm": 0.06421069912612438, "epoch": 15, "gate_p_mean": 0.029765933210728693, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8507181756198406, "gate_p_p10": 0.003559684753417969, "gate_p_p50": 0.012791061401367189, "gate_p_p90": 0.0799407958984375, "gate_p_std": 0.04336823172634468, "gated_delta_h_c_norm": 0.009036503623065073, "global_step": 1880, "grad_norm_max": 0.005904952995479107, "hidden_mse": 0.00928843359288294, "loss": 0.009110779510228895, "lr_blocks": 9.810017062595322e-08, "lr_fusion": 9.810017062595322e-07, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.996285557746887, "transport_attention_entropy": 8.686650168895723, "transport_attention_entropy_normalized": 0.9940397448837758, "transport_residual_norm": 0.05078828968107701, "velocity_mse": 0.008181936110486277} +{"anchor_distance_mean": 126.19765625000001, "avg_context_frames": 13.200000000000001, "avg_micro_time_s": 5.907707219462873, "delta_h_c_norm": 0.1800156628713012, "delta_h_d_norm": 0.06758639723993838, "epoch": 15, "gate_p_mean": 0.04045137196080759, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.7939313150942326, "gate_p_p10": 0.004434490203857422, "gate_p_p50": 0.016246795654296875, "gate_p_p90": 0.11234581470489502, "gate_p_std": 0.06026627322426066, "gated_delta_h_c_norm": 0.012294986218330452, "global_step": 1890, "grad_norm_max": 0.00842344295233488, "hidden_mse": 0.010645583696896211, "loss": 0.011023559980094433, "lr_blocks": 8.247462563808816e-08, "lr_fusion": 8.247462563808817e-07, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 26.091032934188846, "transport_attention_entropy": 8.686794030666352, "transport_attention_entropy_normalized": 0.9940562076866627, "transport_residual_norm": 0.045691668079234665, "velocity_mse": 0.009959001629613341} +{"anchor_distance_mean": 124.51093750000001, "avg_context_frames": 12.8, "avg_micro_time_s": 5.78018500491221, "delta_h_c_norm": 0.18368647489696743, "delta_h_d_norm": 0.06884132502600551, "epoch": 15, "gate_p_mean": 0.040815738879609854, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.7937562838196754, "gate_p_p10": 0.004053974151611329, "gate_p_p50": 0.016827774047851563, "gate_p_p90": 0.11375241279602051, "gate_p_std": 0.058726658427622175, "gated_delta_h_c_norm": 0.01293044999765698, "global_step": 1900, "grad_norm_max": 0.0077136484906077385, "hidden_mse": 0.01169568004552275, "loss": 0.011349237887770869, "lr_blocks": 6.819348298638839e-08, "lr_fusion": 6.819348298638839e-07, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.403459095954897, "transport_attention_entropy": 8.686418414115906, "transport_attention_entropy_normalized": 0.9940132215619087, "transport_residual_norm": 0.04574708009604365, "velocity_mse": 0.010179669922217728} +{"anchor_distance_mean": 127.78984375, "avg_context_frames": 16.8, "avg_micro_time_s": 6.880397060687347, "delta_h_c_norm": 0.18523677587509157, "delta_h_d_norm": 0.06510571842081846, "epoch": 15, "gate_p_mean": 0.02706072700675577, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8627491772174836, "gate_p_p10": 0.002991199493408203, "gate_p_p50": 0.0111572265625, "gate_p_p90": 0.07314834594726563, "gate_p_std": 0.04057409839006141, "gated_delta_h_c_norm": 0.007986166575574317, "global_step": 1910, "grad_norm_max": 0.010184284299612045, "hidden_mse": 0.009292531144456008, "loss": 0.008846162652480416, "lr_blocks": 5.526064699265754e-08, "lr_fusion": 5.526064699265753e-07, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 26.04624025821686, "transport_attention_entropy": 8.685881233215332, "transport_attention_entropy_normalized": 0.9939517498016358, "transport_residual_norm": 0.053333925362676386, "velocity_mse": 0.007916909508639948} +{"anchor_distance_mean": 118.79218750000001, "avg_context_frames": 17.2, "avg_micro_time_s": 6.962936477224594, "delta_h_c_norm": 0.17562449108809233, "delta_h_d_norm": 0.06758697656914592, "epoch": 15, "gate_p_mean": 0.035853104607667774, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8162167228758336, "gate_p_p10": 0.003662395477294922, "gate_p_p50": 0.016037750244140624, "gate_p_p90": 0.09821591377258301, "gate_p_std": 0.05070124755147845, "gated_delta_h_c_norm": 0.01075818689132575, "global_step": 1920, "grad_norm_max": 0.006859047804027796, "hidden_mse": 0.010595338098937646, "loss": 0.01008269041485619, "lr_blocks": 4.367965336512403e-08, "lr_fusion": 4.367965336512403e-07, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 26.020772433280946, "transport_attention_entropy": 8.686963403224945, "transport_attention_entropy_normalized": 0.9940755903720856, "transport_residual_norm": 0.05154259840492159, "velocity_mse": 0.009023156546754762} +{"anchor_distance_mean": 123.85859375000001, "avg_context_frames": 14.8, "avg_micro_time_s": 6.310299189112675, "delta_h_c_norm": 0.18128341799601913, "delta_h_d_norm": 0.06594751086086036, "epoch": 16, "gate_p_mean": 0.035361019638367, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8255436457693577, "gate_p_p10": 0.003881168365478516, "gate_p_p50": 0.014537429809570313, "gate_p_p90": 0.09486780166625977, "gate_p_std": 0.05297440296271816, "gated_delta_h_c_norm": 0.01106175254972186, "global_step": 1930, "grad_norm_max": 0.009181998670101166, "hidden_mse": 0.010056549371802248, "loss": 0.010206770320655778, "lr_blocks": 3.345366823180929e-08, "lr_fusion": 3.3453668231809286e-07, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.805253767967226, "transport_attention_entropy": 8.686803340911865, "transport_attention_entropy_normalized": 0.9940572701394559, "transport_residual_norm": 0.04756035946775228, "velocity_mse": 0.009201115422183648} +{"anchor_distance_mean": 121.0, "avg_context_frames": 13.200000000000001, "avg_micro_time_s": 5.949468341312514, "delta_h_c_norm": 0.17978914426639678, "delta_h_d_norm": 0.06587244705297053, "epoch": 16, "gate_p_mean": 0.033688381011597816, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8256678476929665, "gate_p_p10": 0.0034431074745953083, "gate_p_p50": 0.014228057861328126, "gate_p_p90": 0.09413053989410401, "gate_p_std": 0.04717952922219411, "gated_delta_h_c_norm": 0.00959868412319338, "global_step": 1940, "grad_norm_max": 0.006003806367516518, "hidden_mse": 0.010455808250117117, "loss": 0.009809478488750757, "lr_blocks": 2.4585487274942922e-08, "lr_fusion": 2.458548727494292e-07, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.73323407173157, "transport_attention_entropy": 8.685443341732025, "transport_attention_entropy_normalized": 0.9939016468822957, "transport_residual_norm": 0.045330356596969074, "velocity_mse": 0.008763897686731071} +{"anchor_distance_mean": 123.95859375, "avg_context_frames": 16.0, "avg_micro_time_s": 6.666004106300034, "delta_h_c_norm": 0.1807017667219043, "delta_h_d_norm": 0.06746580502949655, "epoch": 16, "gate_p_mean": 0.035477796464692804, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8233982145786286, "gate_p_p10": 0.004085731506347656, "gate_p_p50": 0.014866256713867189, "gate_p_p90": 0.09697812795639038, "gate_p_std": 0.0523042630054988, "gated_delta_h_c_norm": 0.010754244457348251, "global_step": 1950, "grad_norm_max": 0.006376514211297035, "hidden_mse": 0.01042179782234598, "loss": 0.010081957138027065, "lr_blocks": 1.7077534966650767e-08, "lr_fusion": 1.7077534966650766e-07, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 26.6046293258667, "transport_attention_entropy": 8.68671464920044, "transport_attention_entropy_normalized": 0.9940471269190312, "transport_residual_norm": 0.04915786003693939, "velocity_mse": 0.00903977733978536} +{"anchor_distance_mean": 124.60390625000001, "avg_context_frames": 17.6, "avg_micro_time_s": 7.099775761374804, "delta_h_c_norm": 0.1821581398136914, "delta_h_d_norm": 0.06592624904587865, "epoch": 16, "gate_p_mean": 0.030495762842474506, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8534425333142281, "gate_p_p10": 0.0034203529357910156, "gate_p_p50": 0.012507057189941407, "gate_p_p90": 0.083099365234375, "gate_p_std": 0.04512727765832097, "gated_delta_h_c_norm": 0.010256359628692736, "global_step": 1960, "grad_norm_max": 0.005351900588721037, "hidden_mse": 0.01010964081506245, "loss": 0.009625488246092573, "lr_blocks": 1.0931863906127327e-08, "lr_fusion": 1.0931863906127327e-07, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 26.124439024925234, "transport_attention_entropy": 8.686772930622102, "transport_attention_entropy_normalized": 0.9940537922084332, "transport_residual_norm": 0.052382502867840235, "velocity_mse": 0.008614524139557034} +{"anchor_distance_mean": 122.49375, "avg_context_frames": 14.8, "avg_micro_time_s": 6.323955256587624, "delta_h_c_norm": 0.1775041250512004, "delta_h_d_norm": 0.06813528509810567, "epoch": 16, "gate_p_mean": 0.0363917046925053, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8148645587265492, "gate_p_p10": 0.003742694854736328, "gate_p_p50": 0.014647674560546876, "gate_p_p90": 0.09958581924438477, "gate_p_std": 0.05364004640141502, "gated_delta_h_c_norm": 0.010845339461229743, "global_step": 1970, "grad_norm_max": 0.007914973422884941, "hidden_mse": 0.010812139284098521, "loss": 0.01084164507337846, "lr_blocks": 6.150154258476315e-09, "lr_fusion": 6.150154258476315e-08, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 26.0564346075058, "transport_attention_entropy": 8.68630064725876, "transport_attention_entropy_normalized": 0.9939997486770153, "transport_residual_norm": 0.04933631224557758, "velocity_mse": 0.009760431133327075} +{"anchor_distance_mean": 126.91250000000001, "avg_context_frames": 14.0, "avg_micro_time_s": 6.096701865737533, "delta_h_c_norm": 0.17993982620537283, "delta_h_d_norm": 0.06567920655943453, "epoch": 16, "gate_p_mean": 0.03402723452309147, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8228435792028904, "gate_p_p10": 0.0038768576458096506, "gate_p_p50": 0.014837265014648438, "gate_p_p90": 0.09007534980773926, "gate_p_std": 0.04940611436031759, "gated_delta_h_c_norm": 0.010004130308516324, "global_step": 1980, "grad_norm_max": 0.006005104165524244, "hidden_mse": 0.010068046679953114, "loss": 0.01011525828507729, "lr_blocks": 2.7337132953697555e-09, "lr_fusion": 2.7337132953697554e-08, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.87905175685883, "transport_attention_entropy": 8.686184716224671, "transport_attention_entropy_normalized": 0.9939864829182625, "transport_residual_norm": 0.048567258426919585, "velocity_mse": 0.009108453628141434} +{"anchor_distance_mean": 127.46484375, "avg_context_frames": 13.600000000000001, "avg_micro_time_s": 5.402788680324921, "delta_h_c_norm": 0.18329564947634935, "delta_h_d_norm": 0.06787381260655821, "epoch": 16, "gate_p_mean": 0.03785059986403212, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8164100289344788, "gate_p_p10": 0.004271221160888672, "gate_p_p50": 0.015296554565429688, "gate_p_p90": 0.1057098388671875, "gate_p_std": 0.05629440288757906, "gated_delta_h_c_norm": 0.012258934778219556, "global_step": 1990, "grad_norm_max": 0.006921358872205019, "hidden_mse": 0.011014247991261073, "loss": 0.010613932137493976, "lr_blocks": 6.834750376549793e-10, "lr_fusion": 6.834750376549792e-09, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 26.153854918479922, "transport_attention_entropy": 8.68668020963669, "transport_attention_entropy_normalized": 0.9940431833267213, "transport_residual_norm": 0.04781222585588694, "velocity_mse": 0.009512507346516942} +{"anchor_distance_mean": 122.884375, "avg_context_frames": 10.8, "avg_micro_time_s": 5.261801647412312, "delta_h_c_norm": 0.1768357440829277, "delta_h_d_norm": 0.06996056833304465, "epoch": 16, "gate_p_mean": 0.04374046185985208, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.771679212152958, "gate_p_p10": 0.00467681884765625, "gate_p_p50": 0.018114471435546876, "gate_p_p90": 0.12135620117187501, "gate_p_std": 0.06186591850128025, "gated_delta_h_c_norm": 0.012973392172716558, "global_step": 2000, "grad_norm_max": 0.005634533241391182, "hidden_mse": 0.011784146510763094, "loss": 0.01145993554382585, "lr_blocks": 0.0, "lr_fusion": 0.0, "peak_memory_gib": 41.518661975860596, "transport_argmax_displacement": 25.52945377826691, "transport_attention_entropy": 8.685892534255983, "transport_attention_entropy_normalized": 0.9939530454576015, "transport_residual_norm": 0.0430652296403423, "velocity_mse": 0.01028152087237686} +[2026-09-03T11:14:06+00:00] done scope=chunk +[2026-09-03T11:14:06+00:00] start scope=last_frame +/mnt/local_nvme/cz/envs/hy_worldplay_july20260720/lib/python3.10/site-packages/transformers/utils/hub.py:111: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +/mnt/local_nvme/cz/envs/hy_worldplay_july20260720/lib/python3.10/site-packages/transformers/utils/hub.py:111: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +/mnt/local_nvme/cz/envs/hy_worldplay_july20260720/lib/python3.10/site-packages/transformers/utils/hub.py:111: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +/mnt/local_nvme/cz/envs/hy_worldplay_july20260720/lib/python3.10/site-packages/transformers/utils/hub.py:111: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +/mnt/local_nvme/cz/envs/hy_worldplay_july20260720/lib/python3.10/site-packages/transformers/utils/hub.py:111: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +/mnt/local_nvme/cz/envs/hy_worldplay_july20260720/lib/python3.10/site-packages/transformers/utils/hub.py:111: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +/mnt/local_nvme/cz/envs/hy_worldplay_july20260720/lib/python3.10/site-packages/transformers/utils/hub.py:111: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +/mnt/local_nvme/cz/envs/hy_worldplay_july20260720/lib/python3.10/site-packages/transformers/utils/hub.py:111: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +{ + "manifest": "/mnt/local_nvme/zoubin/cz/projects/HY-WorldPlay-DEV-Predictor/datasets/hyworldplay_predictor_prefeature_v4/train_eval_manifest.jsonl", + "teacher_checkpoint": "/mnt/local_nvme/zoubin/cz/checkpoints/hy_worldplay/huggingface/hub/models--tencent--HY-WorldPlay/snapshots/f4c29235647707b571479a69b569e4166f9f5bf8/ar_distilled_action_model/diffusion_pytorch_model.safetensors", + "output_dir": "/mnt/local_nvme/zoubin/cz/projects/HY-WorldPlay-DEV-Predictor/checkpoints/predictor_v4_atc-global-last_frame_blocks1-52_fulltrain100_fp32master_8gpu_flr1e-4_blr1e-5_steps2000", + "source_blocks": [ + 1, + 52 + ], + "predictor_variant": "v4", + "previous_condition_mode": "same_step", + "predictor_fusion_type": "concat", + "gate_hidden_dim": 256, + "gate_initial_probability": 0.8, + "predictor_arch": "atc", + "atc_variant": "full", + "atc_mlp_hidden_dim": 4096, + "atc_gate_hidden_dim": 512, + "atc_gate_initial_probability": 0.3, + "atc_transport_heads": 16, + "atc_transport_window": 3, + "atc_transport_query_chunk_size": 128, + "atc_transport_initial_scale": 0.1, + "atc_transport_mode": "global", + "atc_previous_scope": "last_frame", + "atc_mlp_init": "xavier", + "max_steps": 2000, + "batch_size": 8, + "gradient_accumulation_steps": 1, + "block_warmup_steps": 100, + "block_warmup_mode": "delayed", + "scheduler_warmup_steps": 100, + "fusion_lr": 0.0001, + "blocks_lr": 1e-05, + "weight_decay": 0.01, + "hidden_weight": 0.1, + "velocity_weight": 1.0, + "grad_clip": 1.0, + "log_every": 10, + "save_every": 100, + "keep_snapshots": 20, + "seed": 0, + "num_workers": 0, + "max_records": null, + "bucket_by_context": true, + "gradient_checkpointing": true, + "fp32_trainable_params": true, + "train_delta_only": false, + "train_fusion_residual_only": false, + "save_optimizer_state": true, + "init_training_state": null, + "resume": null, + "swanlab_project": "HY-WorldPlay-Predictor-v4", + "swanlab_workspace": null, + "swanlab_experiment": "predictor-v4-atc-global-last_frame-blocks1-52-fulltrain100-8gpu", + "swanlab_mode": "disabled", + "predictor_version": "v4_prefeature", + "schema_version": "predictor_context_prefeature_bf16_v1", + "supervision_pairs": [ + [ + 0, + 1 + ], + [ + 1, + 2 + ], + [ + 2, + 3 + ] + ], + "previous_condition_source": "offline_full_teacher_hidden", + "records": 2800, + "dataset_pairs": 8400, + "steps_per_epoch_per_rank": 128, + "world_size": 8, + "per_rank_batch_size": 8, + "global_batch_size": 64, + "trainable_parameters": 291649026, + "frozen_projector_parameters": 16785664, + "dtype": "bfloat16", + "activation_dtype": "bfloat16", + "trainable_parameter_dtype": "float32", + "optimizer_moment_dtype": "float32", + "inference_checkpoint_dtype": "bfloat16", + "gpu": "NVIDIA H100 80GB HBM3" +} +{"swanlab_run_id": "hdjyj60r"} +/mnt/local_nvme/zoubin/cz/projects/HY-WorldPlay-DEV-Predictor/hyvideo/commons/__init__.py:216: UserWarning: flash is not available. Falling back to torch attention. + warnings.warn("flash is not available. Falling back to torch attention.") +/mnt/local_nvme/zoubin/cz/projects/HY-WorldPlay-DEV-Predictor/hyvideo/commons/__init__.py:216: UserWarning: flash is not available. Falling back to torch attention. + warnings.warn("flash is not available. Falling back to torch attention.") +/mnt/local_nvme/zoubin/cz/projects/HY-WorldPlay-DEV-Predictor/hyvideo/commons/__init__.py:216: UserWarning: flash is not available. Falling back to torch attention. + warnings.warn("flash is not available. Falling back to torch attention.") +/mnt/local_nvme/zoubin/cz/projects/HY-WorldPlay-DEV-Predictor/hyvideo/commons/__init__.py:216: UserWarning: flash is not available. Falling back to torch attention. + warnings.warn("flash is not available. Falling back to torch attention.") +/mnt/local_nvme/zoubin/cz/projects/HY-WorldPlay-DEV-Predictor/hyvideo/commons/__init__.py:216: UserWarning: flash is not available. Falling back to torch attention. + warnings.warn("flash is not available. Falling back to torch attention.") +/mnt/local_nvme/zoubin/cz/projects/HY-WorldPlay-DEV-Predictor/hyvideo/commons/__init__.py:216: UserWarning: flash is not available. Falling back to torch attention. + warnings.warn("flash is not available. Falling back to torch attention.") +/mnt/local_nvme/zoubin/cz/projects/HY-WorldPlay-DEV-Predictor/hyvideo/commons/__init__.py:216: UserWarning: flash is not available. Falling back to torch attention. + warnings.warn("flash is not available. Falling back to torch attention.") +/mnt/local_nvme/zoubin/cz/projects/HY-WorldPlay-DEV-Predictor/hyvideo/commons/__init__.py:216: UserWarning: flash is not available. Falling back to torch attention. + warnings.warn("flash is not available. Falling back to torch attention.") +{"anchor_distance_mean": 136.5234375, "avg_context_frames": 8.0, "avg_micro_time_s": 5.28751278899972, "delta_h_c_norm": 0.0, "delta_h_d_norm": 0.0, "epoch": 1, "gate_p_mean": 0.31229081004858017, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.0, "gate_p_p10": 0.304443359375, "gate_p_p50": 0.3125, "gate_p_p90": 0.3203125, "gate_p_std": 0.006245162629056722, "gated_delta_h_c_norm": 0.0, "global_step": 1, "grad_norm_max": 0.1102752760052681, "hidden_mse": 0.01441731525119394, "loss": 0.013514157384634018, "lr_blocks": 0.0, "lr_fusion": 2.0000000000000003e-06, "peak_memory_gib": 30.154459476470947, "transport_argmax_displacement": 23.106507301330566, "transport_attention_entropy": 7.317072153091431, "transport_attention_entropy_normalized": 0.9951894506812096, "transport_residual_norm": 0.6818350777029991, "velocity_mse": 0.012072425859514624} +{"anchor_distance_mean": 122.63107638888889, "avg_context_frames": 15.11111111111111, "avg_micro_time_s": 6.497409952236163, "delta_h_c_norm": 0.009602406696002515, "delta_h_d_norm": 0.005021938063792832, "epoch": 1, "gate_p_mean": 0.3088144573072592, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.0, "gate_p_p10": 0.2993977864583333, "gate_p_p50": 0.3086751302083333, "gate_p_p90": 0.3185763888888889, "gate_p_std": 0.007364129388911856, "gated_delta_h_c_norm": 0.002959692783608463, "global_step": 10, "grad_norm_max": 0.34081846475601196, "hidden_mse": 0.016402463105300233, "loss": 0.013536727443958322, "lr_blocks": 0.0, "lr_fusion": 1.1000000000000001e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 22.939523458480835, "transport_attention_entropy": 7.31637958685557, "transport_attention_entropy_normalized": 0.9950952678918839, "transport_residual_norm": 0.6928880264361699, "velocity_mse": 0.01189648106487261} +{"anchor_distance_mean": 126.8140625, "avg_context_frames": 14.8, "avg_micro_time_s": 3.9407508218748717, "delta_h_c_norm": 0.01753134576138109, "delta_h_d_norm": 0.0070750781917013235, "epoch": 1, "gate_p_mean": 0.27941022887825967, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.0, "gate_p_p10": 0.27081298828125, "gate_p_p50": 0.279248046875, "gate_p_p90": 0.28818359375, "gate_p_std": 0.006710120261413977, "gated_delta_h_c_norm": 0.004865006147883833, "global_step": 20, "grad_norm_max": 0.2335149049758911, "hidden_mse": 0.016214392986148598, "loss": 0.013983653276227415, "lr_blocks": 0.0, "lr_fusion": 2.1e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 22.939393091201783, "transport_attention_entropy": 7.31645240187645, "transport_attention_entropy_normalized": 0.9951051756739617, "transport_residual_norm": 0.697641958296299, "velocity_mse": 0.01236221399740316} +{"anchor_distance_mean": 123.59921875, "avg_context_frames": 16.0, "avg_micro_time_s": 2.7705360461749025, "delta_h_c_norm": 0.025064740842208267, "delta_h_d_norm": 0.00805247238604352, "epoch": 1, "gate_p_mean": 0.22337184008210897, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.0, "gate_p_p10": 0.2141357421875, "gate_p_p50": 0.22293701171875002, "gate_p_p90": 0.23297119140625, "gate_p_std": 0.007474692811956629, "gated_delta_h_c_norm": 0.005589656636584551, "global_step": 30, "grad_norm_max": 0.05630956590175629, "hidden_mse": 0.017025893420213833, "loss": 0.013907691912027077, "lr_blocks": 0.0, "lr_fusion": 3.1e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 22.68126587867737, "transport_attention_entropy": 7.316425371170045, "transport_attention_entropy_normalized": 0.995101498067379, "transport_residual_norm": 0.6803037635982037, "velocity_mse": 0.012205102597363293} +{"anchor_distance_mean": 129.18437500000002, "avg_context_frames": 10.4, "avg_micro_time_s": 2.129393660500273, "delta_h_c_norm": 0.03203614950180054, "delta_h_d_norm": 0.011051601043436677, "epoch": 1, "gate_p_mean": 0.18157185409218074, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.0, "gate_p_p10": 0.16981201171875002, "gate_p_p50": 0.1806884765625, "gate_p_p90": 0.19473876953125002, "gate_p_std": 0.009771460335468874, "gated_delta_h_c_norm": 0.005896719841985032, "global_step": 40, "grad_norm_max": 0.0870596244931221, "hidden_mse": 0.015260075120022521, "loss": 0.013275690862792545, "lr_blocks": 0.0, "lr_fusion": 4.1e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 22.48804841041565, "transport_attention_entropy": 7.316501951217652, "transport_attention_entropy_normalized": 0.9951119139790535, "transport_residual_norm": 0.6918979100883007, "velocity_mse": 0.011749683279776947} +{"anchor_distance_mean": 126.4578125, "avg_context_frames": 12.8, "avg_micro_time_s": 2.4110131310500496, "delta_h_c_norm": 0.039534629834815864, "delta_h_d_norm": 0.015231431601569057, "epoch": 1, "gate_p_mean": 0.1652696095407009, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.0, "gate_p_p10": 0.14753417968750002, "gate_p_p50": 0.1630859375, "gate_p_p90": 0.18658447265625, "gate_p_std": 0.015037323680007831, "gated_delta_h_c_norm": 0.006824502442032099, "global_step": 50, "grad_norm_max": 0.10690911114215851, "hidden_mse": 0.014649761468172075, "loss": 0.012506777385715396, "lr_blocks": 0.0, "lr_fusion": 5.1000000000000006e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 22.603909564018252, "transport_attention_entropy": 7.316971808671951, "transport_attention_entropy_normalized": 0.9951758190989495, "transport_residual_norm": 0.6827254071831703, "velocity_mse": 0.011041801172541455} +{"anchor_distance_mean": 128.471875, "avg_context_frames": 13.600000000000001, "avg_micro_time_s": 2.5125434480750304, "delta_h_c_norm": 0.04078819483984262, "delta_h_d_norm": 0.01559510185616091, "epoch": 1, "gate_p_mean": 0.1335143198259175, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.0, "gate_p_p10": 0.109124755859375, "gate_p_p50": 0.129388427734375, "gate_p_p90": 0.164385986328125, "gate_p_std": 0.021186148002743724, "gated_delta_h_c_norm": 0.006077784256194718, "global_step": 60, "grad_norm_max": 0.08669859915971756, "hidden_mse": 0.01486398499691859, "loss": 0.013327606837265193, "lr_blocks": 0.0, "lr_fusion": 6.1e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 22.75941724777222, "transport_attention_entropy": 7.316756474971772, "transport_attention_entropy_normalized": 0.9951465331017971, "transport_residual_norm": 0.6809644199907781, "velocity_mse": 0.011841208348050714} +{"anchor_distance_mean": 124.93281250000001, "avg_context_frames": 16.8, "avg_micro_time_s": 2.8981726631498987, "delta_h_c_norm": 0.03367794053629041, "delta_h_d_norm": 0.01569884135387838, "epoch": 1, "gate_p_mean": 0.07385402657091618, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.15739809245351355, "gate_p_p10": 0.052618408203125, "gate_p_p50": 0.069488525390625, "gate_p_p90": 0.10205078125, "gate_p_std": 0.01977720072027296, "gated_delta_h_c_norm": 0.003106205303629395, "global_step": 70, "grad_norm_max": 0.08481995761394501, "hidden_mse": 0.015913075557909908, "loss": 0.013217679632361978, "lr_blocks": 0.0, "lr_fusion": 7.1e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 22.44798197746277, "transport_attention_entropy": 7.316811317205429, "transport_attention_entropy_normalized": 0.9951539911329746, "transport_residual_norm": 0.6787179112434387, "velocity_mse": 0.011626372006139718} +{"anchor_distance_mean": 125.22187500000001, "avg_context_frames": 16.0, "avg_micro_time_s": 2.79287964273758, "delta_h_c_norm": 0.03644274102989584, "delta_h_d_norm": 0.02040831936756149, "epoch": 1, "gate_p_mean": 0.059529511583969, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.4431365326046944, "gate_p_p10": 0.037164306640625, "gate_p_p50": 0.0537872314453125, "gate_p_p90": 0.0914276123046875, "gate_p_std": 0.02173391376854852, "gated_delta_h_c_norm": 0.003007108677411452, "global_step": 80, "grad_norm_max": 0.0931791365146637, "hidden_mse": 0.013889776312862524, "loss": 0.012413997156545521, "lr_blocks": 0.0, "lr_fusion": 8.1e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 22.58135859966278, "transport_attention_entropy": 7.316772991418839, "transport_attention_entropy_normalized": 0.9951487787067891, "transport_residual_norm": 0.6749956108629704, "velocity_mse": 0.011025019503722434} +{"anchor_distance_mean": 120.7078125, "avg_context_frames": 14.8, "avg_micro_time_s": 2.6371743887624235, "delta_h_c_norm": 0.047871436271816496, "delta_h_d_norm": 0.024153594346717003, "epoch": 1, "gate_p_mean": 0.07449625669978559, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.32002329516690226, "gate_p_p10": 0.039712524414062506, "gate_p_p50": 0.06478271484375, "gate_p_p90": 0.12724118232727052, "gate_p_std": 0.035335138184018434, "gated_delta_h_c_norm": 0.005400846512930002, "global_step": 90, "grad_norm_max": 0.08766772598028183, "hidden_mse": 0.014815081260167062, "loss": 0.012942350684897975, "lr_blocks": 0.0, "lr_fusion": 9.1e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 22.232599663734437, "transport_attention_entropy": 7.316852551698685, "transport_attention_entropy_normalized": 0.9951596014201641, "transport_residual_norm": 0.6813587374985218, "velocity_mse": 0.011460842622909695} +{"anchor_distance_mean": 123.20937500000001, "avg_context_frames": 15.200000000000001, "avg_micro_time_s": 2.7014204762373084, "delta_h_c_norm": 0.05229965038597584, "delta_h_d_norm": 0.026482771360315384, "epoch": 1, "gate_p_mean": 0.07043817210942507, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.4569969592077541, "gate_p_p10": 0.0330047607421875, "gate_p_p50": 0.056427001953125, "gate_p_p90": 0.1312206745147705, "gate_p_std": 0.0408603216172196, "gated_delta_h_c_norm": 0.006097528380632867, "global_step": 100, "grad_norm_max": 0.052951663732528687, "hidden_mse": 0.01498314073542133, "loss": 0.013326826749835164, "lr_blocks": 1e-05, "lr_fusion": 0.0001, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 22.486496496200562, "transport_attention_entropy": 7.316385716199875, "transport_attention_entropy_normalized": 0.995096104592085, "transport_residual_norm": 0.6863864481449128, "velocity_mse": 0.011828512675128878} +{"anchor_distance_mean": 124.93125, "avg_context_frames": 11.600000000000001, "avg_micro_time_s": 2.2710432350002523, "delta_h_c_norm": 0.04937246711924673, "delta_h_d_norm": 0.02677238553296775, "epoch": 1, "gate_p_mean": 0.04270094669191167, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.7419195974245668, "gate_p_p10": 0.016610717773437502, "gate_p_p50": 0.031156158447265627, "gate_p_p90": 0.0869537353515625, "gate_p_std": 0.031525629456155, "gated_delta_h_c_norm": 0.004370994272903772, "global_step": 110, "grad_norm_max": 0.11023278534412384, "hidden_mse": 0.01528994594118558, "loss": 0.014287534018512816, "lr_blocks": 9.999316524962347e-06, "lr_fusion": 9.999316524962345e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 22.095787692070008, "transport_attention_entropy": 7.317342329025269, "transport_attention_entropy_normalized": 0.9952262178063394, "transport_residual_norm": 0.6956515088677406, "velocity_mse": 0.01275853941042442} +{"anchor_distance_mean": 119.571875, "avg_context_frames": 13.600000000000001, "avg_micro_time_s": 2.480091957087643, "delta_h_c_norm": 0.0458900839323178, "delta_h_d_norm": 0.02607191726565361, "epoch": 1, "gate_p_mean": 0.019423483096761632, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.9275140479207039, "gate_p_p10": 0.006142959371209145, "gate_p_p50": 0.012898635864257813, "gate_p_p90": 0.040979003906250006, "gate_p_std": 0.018077656542300247, "gated_delta_h_c_norm": 0.0023293437941902086, "global_step": 120, "grad_norm_max": 0.1067541316151619, "hidden_mse": 0.013870167109416798, "loss": 0.013106338481884451, "lr_blocks": 9.99726628670463e-06, "lr_fusion": 9.997266286704631e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 22.142897367477417, "transport_attention_entropy": 7.316866660118103, "transport_attention_entropy_normalized": 0.9951615177094937, "transport_residual_norm": 0.6757571421563626, "velocity_mse": 0.01171932177967392} +{"anchor_distance_mean": 126.49062500000001, "avg_context_frames": 18.0, "avg_micro_time_s": 3.009437105212328, "delta_h_c_norm": 0.047682050801813605, "delta_h_d_norm": 0.02597550959326327, "epoch": 2, "gate_p_mean": 0.02031993069685996, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.9249567084014416, "gate_p_p10": 0.0066322326660156255, "gate_p_p50": 0.013483428955078126, "gate_p_p90": 0.04228253364562989, "gate_p_std": 0.018915097173885443, "gated_delta_h_c_norm": 0.002605698059414863, "global_step": 130, "grad_norm_max": 0.05612732842564583, "hidden_mse": 0.012843320763204248, "loss": 0.01176762452814728, "lr_blocks": 9.993849845741525e-06, "lr_fusion": 9.993849845741524e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 22.484842944145203, "transport_attention_entropy": 7.316740345954895, "transport_attention_entropy_normalized": 0.9951443366706372, "transport_residual_norm": 0.6654608108103276, "velocity_mse": 0.01048329241166357} +{"anchor_distance_mean": 122.88203125000001, "avg_context_frames": 13.600000000000001, "avg_micro_time_s": 2.5187411822125343, "delta_h_c_norm": 0.051836630515754226, "delta_h_d_norm": 0.02849041116423905, "epoch": 2, "gate_p_mean": 0.026362068671733144, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.874902118742466, "gate_p_p10": 0.008604049682617188, "gate_p_p50": 0.017835235595703127, "gate_p_p90": 0.05523771047592163, "gate_p_std": 0.024705819779774176, "gated_delta_h_c_norm": 0.0037118292984814616, "global_step": 140, "grad_norm_max": 0.04883034527301788, "hidden_mse": 0.013774832733906806, "loss": 0.012652860712842086, "lr_blocks": 9.989068136093873e-06, "lr_fusion": 9.989068136093873e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 22.485709500312808, "transport_attention_entropy": 7.31673863530159, "transport_attention_entropy_normalized": 0.9951441064476967, "transport_residual_norm": 0.6834215961396695, "velocity_mse": 0.011275377476704308} +{"anchor_distance_mean": 128.665625, "avg_context_frames": 14.4, "avg_micro_time_s": 2.5632149023000697, "delta_h_c_norm": 0.052148821158334614, "delta_h_d_norm": 0.03007085886783898, "epoch": 2, "gate_p_mean": 0.03414706645999104, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.809639447182417, "gate_p_p10": 0.011370849609375, "gate_p_p50": 0.022381591796875003, "gate_p_p90": 0.07377408742904663, "gate_p_std": 0.0312703542993404, "gated_delta_h_c_norm": 0.004411994488327764, "global_step": 150, "grad_norm_max": 0.05513536557555199, "hidden_mse": 0.014132891909684987, "loss": 0.013015753391664477, "lr_blocks": 9.98292246503335e-06, "lr_fusion": 9.98292246503335e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 22.45991814136505, "transport_attention_entropy": 7.316647797822952, "transport_attention_entropy_normalized": 0.995131752640009, "transport_residual_norm": 0.6662295550107956, "velocity_mse": 0.011602464265888558} +{"anchor_distance_mean": 126.65234375, "avg_context_frames": 12.4, "avg_micro_time_s": 2.326453163612632, "delta_h_c_norm": 0.050467368913814425, "delta_h_d_norm": 0.030667980643920602, "epoch": 2, "gate_p_mean": 0.02031876122346148, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.9331245273351669, "gate_p_p10": 0.007701492309570313, "gate_p_p50": 0.014888763427734375, "gate_p_p90": 0.03924603462219239, "gate_p_std": 0.016577259998302906, "gated_delta_h_c_norm": 0.002016643199385726, "global_step": 160, "grad_norm_max": 0.06691239774227142, "hidden_mse": 0.013777515827678145, "loss": 0.012572086456930266, "lr_blocks": 9.975414512725058e-06, "lr_fusion": 9.975414512725057e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 22.16439037322998, "transport_attention_entropy": 7.316986930370331, "transport_attention_entropy_normalized": 0.9951778784394265, "transport_residual_norm": 0.6811093457043171, "velocity_mse": 0.011194334898027592} +{"anchor_distance_mean": 124.34609375000001, "avg_context_frames": 14.4, "avg_micro_time_s": 2.5725965537123106, "delta_h_c_norm": 0.057490743510425095, "delta_h_d_norm": 0.03259640228934586, "epoch": 2, "gate_p_mean": 0.03358057818841189, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8281102519482375, "gate_p_p10": 0.013622512295842172, "gate_p_p50": 0.0253448486328125, "gate_p_p90": 0.06441345214843751, "gate_p_std": 0.025079558981815353, "gated_delta_h_c_norm": 0.0036672185931820424, "global_step": 170, "grad_norm_max": 0.06077438220381737, "hidden_mse": 0.013286765530938284, "loss": 0.011694827565224842, "lr_blocks": 9.966546331768192e-06, "lr_fusion": 9.966546331768191e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 21.99419906139374, "transport_attention_entropy": 7.316513019800187, "transport_attention_entropy_normalized": 0.9951134167611599, "transport_residual_norm": 0.6839431568980218, "velocity_mse": 0.010366151030757465} +{"anchor_distance_mean": 127.30390625000001, "avg_context_frames": 15.600000000000001, "avg_micro_time_s": 2.7168867682126803, "delta_h_c_norm": 0.05907623604871333, "delta_h_d_norm": 0.03476112221833318, "epoch": 2, "gate_p_mean": 0.029269425314851105, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8623192392289639, "gate_p_p10": 0.009932327270507814, "gate_p_p50": 0.02053375244140625, "gate_p_p90": 0.059029221534729004, "gate_p_std": 0.025562415970489385, "gated_delta_h_c_norm": 0.0038372873968910426, "global_step": 180, "grad_norm_max": 0.03978237137198448, "hidden_mse": 0.013714535662438722, "loss": 0.01311456541880034, "lr_blocks": 9.956320346634877e-06, "lr_fusion": 9.956320346634876e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 22.06706976890564, "transport_attention_entropy": 7.316728276014328, "transport_attention_entropy_normalized": 0.9951426960527897, "transport_residual_norm": 0.6736652888357639, "velocity_mse": 0.0117431119637331} +{"anchor_distance_mean": 127.39687500000001, "avg_context_frames": 16.400000000000002, "avg_micro_time_s": 2.808142778712909, "delta_h_c_norm": 0.05754121043719351, "delta_h_d_norm": 0.03316960032097995, "epoch": 2, "gate_p_mean": 0.023269426822662354, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.9064438402652741, "gate_p_p10": 0.0079986572265625, "gate_p_p50": 0.0153045654296875, "gate_p_p90": 0.04718704223632813, "gate_p_std": 0.02220307606039569, "gated_delta_h_c_norm": 0.003158911376522156, "global_step": 190, "grad_norm_max": 0.03785568103194237, "hidden_mse": 0.011793624184792862, "loss": 0.011150563217233867, "lr_blocks": 9.944739353007344e-06, "lr_fusion": 9.944739353007344e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 21.78261423110962, "transport_attention_entropy": 7.316930454969406, "transport_attention_entropy_normalized": 0.9951701954007149, "transport_residual_norm": 0.6614961907267571, "velocity_mse": 0.00997120081447065} +{"anchor_distance_mean": 126.19609375, "avg_context_frames": 12.4, "avg_micro_time_s": 2.3246686573748323, "delta_h_c_norm": 0.063293238542974, "delta_h_d_norm": 0.03771290332078934, "epoch": 2, "gate_p_mean": 0.03526090475497767, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8197453666478396, "gate_p_p10": 0.012305068969726564, "gate_p_p50": 0.02372589111328125, "gate_p_p90": 0.07240653038024902, "gate_p_std": 0.03270252940710634, "gated_delta_h_c_norm": 0.00503117141124676, "global_step": 200, "grad_norm_max": 0.04423096776008606, "hidden_mse": 0.013929727592039854, "loss": 0.013091262895613909, "lr_blocks": 9.931806517013612e-06, "lr_fusion": 9.931806517013612e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 21.868899416923526, "transport_attention_entropy": 7.316831207275391, "transport_attention_entropy_normalized": 0.9951566956937313, "transport_residual_norm": 0.6856476448476315, "velocity_mse": 0.011698290199274198} +{"anchor_distance_mean": 124.34687500000001, "avg_context_frames": 17.2, "avg_micro_time_s": 2.887920002499959, "delta_h_c_norm": 0.0611464714165777, "delta_h_d_norm": 0.036790110426954924, "epoch": 2, "gate_p_mean": 0.02612661919556558, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8828986655920744, "gate_p_p10": 0.00847625732421875, "gate_p_p50": 0.017777633666992188, "gate_p_p90": 0.053247499465942386, "gate_p_std": 0.02512923068716191, "gated_delta_h_c_norm": 0.003713385052105878, "global_step": 210, "grad_norm_max": 0.030128249898552895, "hidden_mse": 0.012267679936485365, "loss": 0.011179037651163527, "lr_blocks": 9.917525374361913e-06, "lr_fusion": 9.917525374361912e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 21.846106219291688, "transport_attention_entropy": 7.3174911677837375, "transport_attention_entropy_normalized": 0.9952464580535889, "transport_residual_norm": 0.6614256992936135, "velocity_mse": 0.009952269596396945} +{"anchor_distance_mean": 123.21015625000001, "avg_context_frames": 14.8, "avg_micro_time_s": 2.617357417100038, "delta_h_c_norm": 0.06364372083917261, "delta_h_d_norm": 0.038990232208743696, "epoch": 2, "gate_p_mean": 0.03207556086126715, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.840986604988575, "gate_p_p10": 0.011166114546358586, "gate_p_p50": 0.02225799560546875, "gate_p_p90": 0.06396331787109376, "gate_p_std": 0.028666092216735706, "gated_delta_h_c_norm": 0.004298937252315227, "global_step": 220, "grad_norm_max": 0.058831311762332916, "hidden_mse": 0.012883237784262747, "loss": 0.011406498798169197, "lr_blocks": 9.901899829374048e-06, "lr_fusion": 9.901899829374047e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 21.90224425792694, "transport_attention_entropy": 7.317578965425492, "transport_attention_entropy_normalized": 0.9952583968639375, "transport_residual_norm": 0.6585364438593388, "velocity_mse": 0.010118175006937236} +{"anchor_distance_mean": 126.16484375, "avg_context_frames": 12.4, "avg_micro_time_s": 2.3466507175503466, "delta_h_c_norm": 0.06530567998997867, "delta_h_d_norm": 0.03680876903235913, "epoch": 2, "gate_p_mean": 0.025627365516265856, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8875668846070767, "gate_p_p10": 0.007644901145249606, "gate_p_p50": 0.01617279052734375, "gate_p_p90": 0.05397491455078125, "gate_p_std": 0.027221603452926502, "gated_delta_h_c_norm": 0.004120124171822681, "global_step": 230, "grad_norm_max": 0.045084379613399506, "hidden_mse": 0.013210705039091409, "loss": 0.012347319448599592, "lr_blocks": 9.884934153917998e-06, "lr_fusion": 9.884934153917997e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 21.745148515701295, "transport_attention_entropy": 7.317802262306214, "transport_attention_entropy_normalized": 0.9952887699007988, "transport_residual_norm": 0.6669977605342865, "velocity_mse": 0.01102624890045263} +{"anchor_distance_mean": 123.76171875, "avg_context_frames": 14.0, "avg_micro_time_s": 2.524591028249961, "delta_h_c_norm": 0.06828468218445778, "delta_h_d_norm": 0.03917117088567466, "epoch": 2, "gate_p_mean": 0.023986499512102456, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8955817110836506, "gate_p_p10": 0.005498504638671875, "gate_p_p50": 0.013946533203125, "gate_p_p90": 0.05201308131217957, "gate_p_std": 0.029400472692213953, "gated_delta_h_c_norm": 0.004527547279576538, "global_step": 240, "grad_norm_max": 0.060477569699287415, "hidden_mse": 0.012854739627800883, "loss": 0.012772251258138568, "lr_blocks": 9.86663298624003e-06, "lr_fusion": 9.86663298624003e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 22.223773336410524, "transport_attention_entropy": 7.31772056221962, "transport_attention_entropy_normalized": 0.9952776543796062, "transport_residual_norm": 0.6486991405487061, "velocity_mse": 0.01148677733144723} +{"anchor_distance_mean": 122.009375, "avg_context_frames": 12.8, "avg_micro_time_s": 2.4076485497126217, "delta_h_c_norm": 0.07298604305833578, "delta_h_d_norm": 0.040149789932183926, "epoch": 2, "gate_p_mean": 0.026079461321933197, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8712612941861153, "gate_p_p10": 0.004838180541992188, "gate_p_p50": 0.013132095336914062, "gate_p_p90": 0.06264266967773438, "gate_p_std": 0.03466949947178364, "gated_delta_h_c_norm": 0.0058858530665020226, "global_step": 250, "grad_norm_max": 0.06148945540189743, "hidden_mse": 0.01564555470831692, "loss": 0.015014560898998754, "lr_blocks": 9.847001329696653e-06, "lr_fusion": 9.847001329696653e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 22.208933019638064, "transport_attention_entropy": 7.3178841650486, "transport_attention_entropy_normalized": 0.9952999070286751, "transport_residual_norm": 0.6379425875842571, "velocity_mse": 0.013450005440972745} +{"anchor_distance_mean": 120.93671875000001, "avg_context_frames": 15.600000000000001, "avg_micro_time_s": 2.7469543252374935, "delta_h_c_norm": 0.0707220812793821, "delta_h_d_norm": 0.038997535477392375, "epoch": 3, "gate_p_mean": 0.030215975095052274, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8494185958057643, "gate_p_p10": 0.007925033569335938, "gate_p_p50": 0.01844482421875, "gate_p_p90": 0.0664398193359375, "gate_p_std": 0.03300662453984842, "gated_delta_h_c_norm": 0.005255177228536923, "global_step": 260, "grad_norm_max": 0.04074634611606598, "hidden_mse": 0.014102678972994909, "loss": 0.013588270643958823, "lr_blocks": 9.826044551386743e-06, "lr_fusion": 9.826044551386744e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 22.10123405456543, "transport_attention_entropy": 7.317804116010667, "transport_attention_entropy_normalized": 0.9952890254557133, "transport_residual_norm": 0.6461050428450108, "velocity_mse": 0.012178002783912234} +{"anchor_distance_mean": 124.70078125, "avg_context_frames": 14.0, "avg_micro_time_s": 2.5389311588250623, "delta_h_c_norm": 0.07047170451842248, "delta_h_d_norm": 0.03577645858749747, "epoch": 3, "gate_p_mean": 0.024714203260373324, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8853368148207665, "gate_p_p10": 0.005878448486328125, "gate_p_p50": 0.0134979248046875, "gate_p_p90": 0.05837750434875488, "gate_p_std": 0.029806793318130076, "gated_delta_h_c_norm": 0.004362688076798804, "global_step": 270, "grad_norm_max": 0.03614659607410431, "hidden_mse": 0.014456845005042851, "loss": 0.013476826623082161, "lr_blocks": 9.803768380684242e-06, "lr_fusion": 9.803768380684242e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 21.97241005897522, "transport_attention_entropy": 7.317917323112488, "transport_attention_entropy_normalized": 0.9953044161200524, "transport_residual_norm": 0.646376369893551, "velocity_mse": 0.012031142099294812} +{"anchor_distance_mean": 126.55390625000001, "avg_context_frames": 15.600000000000001, "avg_micro_time_s": 2.729348284225262, "delta_h_c_norm": 0.07189636752009392, "delta_h_d_norm": 0.035334405745379625, "epoch": 3, "gate_p_mean": 0.013650836888700724, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.9596830241382123, "gate_p_p10": 0.00342254638671875, "gate_p_p50": 0.007694435119628907, "gate_p_p90": 0.029001617431640626, "gate_p_std": 0.01766101274988614, "gated_delta_h_c_norm": 0.002542398854347994, "global_step": 280, "grad_norm_max": 0.03069128841161728, "hidden_mse": 0.012692395661724733, "loss": 0.011394019826548175, "lr_blocks": 9.780178907671788e-06, "lr_fusion": 9.780178907671789e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 22.40847592353821, "transport_attention_entropy": 7.318202161788941, "transport_attention_entropy_normalized": 0.9953431576490402, "transport_residual_norm": 0.6464467108249665, "velocity_mse": 0.010124780281330459} +{"anchor_distance_mean": 122.69062500000001, "avg_context_frames": 13.600000000000001, "avg_micro_time_s": 2.478207624374818, "delta_h_c_norm": 0.08082092176191509, "delta_h_d_norm": 0.039595907134935264, "epoch": 3, "gate_p_mean": 0.026721693307626995, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8746679976582528, "gate_p_p10": 0.007541656494140625, "gate_p_p50": 0.0173828125, "gate_p_p90": 0.0566680908203125, "gate_p_std": 0.027094261487945914, "gated_delta_h_c_norm": 0.004759850275877398, "global_step": 290, "grad_norm_max": 0.024180063977837563, "hidden_mse": 0.014209018286783249, "loss": 0.012983351908042097, "lr_blocks": 9.755282581475769e-06, "lr_fusion": 9.755282581475769e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 22.443855023384096, "transport_attention_entropy": 7.318073672056198, "transport_attention_entropy_normalized": 0.9953256830573083, "transport_residual_norm": 0.6332990281283856, "velocity_mse": 0.011562450128258206} +{"anchor_distance_mean": 125.09140625, "avg_context_frames": 13.600000000000001, "avg_micro_time_s": 2.4747899868498737, "delta_h_c_norm": 0.0799823062028736, "delta_h_d_norm": 0.039942723163403575, "epoch": 3, "gate_p_mean": 0.027025616564787927, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.872041541337967, "gate_p_p10": 0.007317504659295083, "gate_p_p50": 0.016650390625, "gate_p_p90": 0.059906005859375, "gate_p_std": 0.0286048713081982, "gated_delta_h_c_norm": 0.004897072983294493, "global_step": 300, "grad_norm_max": 0.03753644600510597, "hidden_mse": 0.01287679956876673, "loss": 0.01215778673067689, "lr_blocks": 9.729086208503174e-06, "lr_fusion": 9.729086208503174e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 22.317350602149965, "transport_attention_entropy": 7.318309181928635, "transport_attention_entropy_normalized": 0.9953577160835266, "transport_residual_norm": 0.6386290922760964, "velocity_mse": 0.01087010680930689} +{"anchor_distance_mean": 123.24062500000001, "avg_context_frames": 12.8, "avg_micro_time_s": 2.4306163845125868, "delta_h_c_norm": 0.0866651138290763, "delta_h_d_norm": 0.04306062236428261, "epoch": 3, "gate_p_mean": 0.03266561409691349, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8189405776560307, "gate_p_p10": 0.006519012153148651, "gate_p_p50": 0.01818695068359375, "gate_p_p90": 0.079278564453125, "gate_p_std": 0.038496904977364466, "gated_delta_h_c_norm": 0.007427870527317282, "global_step": 310, "grad_norm_max": 0.038908567279577255, "hidden_mse": 0.014289143984206021, "loss": 0.01305655811156612, "lr_blocks": 9.701596950580807e-06, "lr_fusion": 9.701596950580806e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 22.379269528388978, "transport_attention_entropy": 7.318527925014497, "transport_attention_entropy_normalized": 0.995387463271618, "transport_residual_norm": 0.6376630537211896, "velocity_mse": 0.011627643686369994} +{"anchor_distance_mean": 126.65156250000001, "avg_context_frames": 12.8, "avg_micro_time_s": 2.409204682137761, "delta_h_c_norm": 0.0826671913266182, "delta_h_d_norm": 0.04310618652962148, "epoch": 3, "gate_p_mean": 0.024724086618516595, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.875545147433877, "gate_p_p10": 0.004735240805894137, "gate_p_p50": 0.013004302978515625, "gate_p_p90": 0.06021618843078613, "gate_p_std": 0.030901068239472806, "gated_delta_h_c_norm": 0.004975506072514691, "global_step": 320, "grad_norm_max": 0.030576545745134354, "hidden_mse": 0.014025440852856265, "loss": 0.012958210846409203, "lr_blocks": 9.672822322997305e-06, "lr_fusion": 9.672822322997305e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 22.11651074886322, "transport_attention_entropy": 7.3185888767242435, "transport_attention_entropy_normalized": 0.9953957587480545, "transport_residual_norm": 0.6377640455961228, "velocity_mse": 0.011555666782078334} +{"anchor_distance_mean": 125.54921875000001, "avg_context_frames": 13.600000000000001, "avg_micro_time_s": 2.5037763020750847, "delta_h_c_norm": 0.08555045896209777, "delta_h_d_norm": 0.04564955048263073, "epoch": 3, "gate_p_mean": 0.026452182716457176, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8699890114367008, "gate_p_p10": 0.005248832702636719, "gate_p_p50": 0.01437511444091797, "gate_p_p90": 0.062534761428833, "gate_p_std": 0.03308676860760897, "gated_delta_h_c_norm": 0.005600389852770605, "global_step": 330, "grad_norm_max": 0.06347543746232986, "hidden_mse": 0.013558818690944463, "loss": 0.012669855955755338, "lr_blocks": 9.642770192448537e-06, "lr_fusion": 9.642770192448536e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 22.37085158824921, "transport_attention_entropy": 7.318629580736161, "transport_attention_entropy_normalized": 0.99540129378438, "transport_residual_norm": 0.6350511901080609, "velocity_mse": 0.011313974094809965} +{"anchor_distance_mean": 125.87031250000001, "avg_context_frames": 13.200000000000001, "avg_micro_time_s": 2.4520597212874238, "delta_h_c_norm": 0.08080700309947134, "delta_h_d_norm": 0.03792755380272866, "epoch": 3, "gate_p_mean": 0.02105377359548584, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.9107675075531007, "gate_p_p10": 0.004803466796875, "gate_p_p50": 0.011802291870117188, "gate_p_p90": 0.04863629341125489, "gate_p_std": 0.025153095310088247, "gated_delta_h_c_norm": 0.003971716910018586, "global_step": 340, "grad_norm_max": 0.030925007537007332, "hidden_mse": 0.01260097423219122, "loss": 0.01233574339421466, "lr_blocks": 9.611448774886925e-06, "lr_fusion": 9.611448774886924e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 22.60386326313019, "transport_attention_entropy": 7.318677771091462, "transport_attention_entropy_normalized": 0.995407846570015, "transport_residual_norm": 0.6321127526462078, "velocity_mse": 0.011075645923847334} +{"anchor_distance_mean": 125.321875, "avg_context_frames": 16.0, "avg_micro_time_s": 2.781321222000179, "delta_h_c_norm": 0.09076673425734044, "delta_h_d_norm": 0.04030669117346406, "epoch": 3, "gate_p_mean": 0.03089786161435768, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8380919702351094, "gate_p_p10": 0.007857513427734376, "gate_p_p50": 0.01948204040527344, "gate_p_p90": 0.06984245777130127, "gate_p_std": 0.03145691191311926, "gated_delta_h_c_norm": 0.006073796972486889, "global_step": 350, "grad_norm_max": 0.05369579419493675, "hidden_mse": 0.012948051060084254, "loss": 0.012558472910313867, "lr_blocks": 9.578866633275289e-06, "lr_fusion": 9.578866633275288e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 22.843346643447877, "transport_attention_entropy": 7.31863324046135, "transport_attention_entropy_normalized": 0.9954017914831639, "transport_residual_norm": 0.6124780982732774, "velocity_mse": 0.01126366785611026} +{"anchor_distance_mean": 119.41015625, "avg_context_frames": 14.4, "avg_micro_time_s": 2.5994611468373936, "delta_h_c_norm": 0.09604783272370697, "delta_h_d_norm": 0.040709884953685105, "epoch": 3, "gate_p_mean": 0.020910869230283426, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.9052391588687897, "gate_p_p10": 0.0037666320800781253, "gate_p_p50": 0.010928535461425783, "gate_p_p90": 0.049361932277679446, "gate_p_std": 0.02788591605203692, "gated_delta_h_c_norm": 0.005673313639636035, "global_step": 360, "grad_norm_max": 0.02101956121623516, "hidden_mse": 0.012533358391374351, "loss": 0.011955384549219163, "lr_blocks": 9.545032675245814e-06, "lr_fusion": 9.545032675245813e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 22.97690465450287, "transport_attention_entropy": 7.319031977653504, "transport_attention_entropy_normalized": 0.9954560197889806, "transport_residual_norm": 0.6070505432784558, "velocity_mse": 0.01070204864954576} +{"anchor_distance_mean": 126.55781250000001, "avg_context_frames": 16.400000000000002, "avg_micro_time_s": 2.847749840424922, "delta_h_c_norm": 0.0974069468677044, "delta_h_d_norm": 0.04373009060509503, "epoch": 3, "gate_p_mean": 0.022128304780926557, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8972366072237492, "gate_p_p10": 0.003783798217773438, "gate_p_p50": 0.011295318603515625, "gate_p_p90": 0.05374022722244263, "gate_p_std": 0.029117135936394336, "gated_delta_h_c_norm": 0.005848720925860107, "global_step": 370, "grad_norm_max": 0.020654447376728058, "hidden_mse": 0.012412694870727138, "loss": 0.011456538870697842, "lr_blocks": 9.509956150664796e-06, "lr_fusion": 9.509956150664796e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 23.041205406188965, "transport_attention_entropy": 7.3192098379135135, "transport_attention_entropy_normalized": 0.995480214804411, "transport_residual_norm": 0.6130569316446781, "velocity_mse": 0.010215269375476056} +{"anchor_distance_mean": 126.42500000000001, "avg_context_frames": 17.6, "avg_micro_time_s": 2.996083839024823, "delta_h_c_norm": 0.09209987707436085, "delta_h_d_norm": 0.040899350540712476, "epoch": 3, "gate_p_mean": 0.01959643820882775, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.9173991188406945, "gate_p_p10": 0.003899001888930798, "gate_p_p50": 0.010351943969726564, "gate_p_p90": 0.0473846435546875, "gate_p_std": 0.024244850318063983, "gated_delta_h_c_norm": 0.004305781568109524, "global_step": 380, "grad_norm_max": 0.02378329262137413, "hidden_mse": 0.0105329062032979, "loss": 0.009543184959329666, "lr_blocks": 9.473646649103819e-06, "lr_fusion": 9.473646649103818e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 23.07587537765503, "transport_attention_entropy": 7.319146585464478, "transport_attention_entropy_normalized": 0.9954716101288796, "transport_residual_norm": 0.6061875343322755, "velocity_mse": 0.008489894305239432} +{"anchor_distance_mean": 123.20859375, "avg_context_frames": 8.8, "avg_micro_time_s": 1.9354134102000444, "delta_h_c_norm": 0.09289678484201432, "delta_h_d_norm": 0.045042984676547354, "epoch": 4, "gate_p_mean": 0.025610943348146977, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8679532520473003, "gate_p_p10": 0.004723453521728516, "gate_p_p50": 0.013424110412597657, "gate_p_p90": 0.0641964614391327, "gate_p_std": 0.03174987052334473, "gated_delta_h_c_norm": 0.005995961333246669, "global_step": 390, "grad_norm_max": 0.030157869681715965, "hidden_mse": 0.014215275272727014, "loss": 0.012950696318876, "lr_blocks": 9.43611409721806e-06, "lr_fusion": 9.43611409721806e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 22.923408603668214, "transport_attention_entropy": 7.319437724351883, "transport_attention_entropy_normalized": 0.995511207729578, "transport_residual_norm": 0.6425580635666848, "velocity_mse": 0.011529168832930738} +{"anchor_distance_mean": 131.65625, "avg_context_frames": 11.200000000000001, "avg_micro_time_s": 2.203853657800255, "delta_h_c_norm": 0.09538592081516982, "delta_h_d_norm": 0.044460214953869585, "epoch": 4, "gate_p_mean": 0.0318464309326373, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8234477903693915, "gate_p_p10": 0.006811714172363282, "gate_p_p50": 0.018465423583984376, "gate_p_p90": 0.07610168457031251, "gate_p_std": 0.03521163655677811, "gated_delta_h_c_norm": 0.006261343490041327, "global_step": 400, "grad_norm_max": 0.030646631494164467, "hidden_mse": 0.013068028306588531, "loss": 0.012609005597187207, "lr_blocks": 9.397368756032445e-06, "lr_fusion": 9.397368756032445e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 23.0664012670517, "transport_attention_entropy": 7.319702053070069, "transport_attention_entropy_normalized": 0.9955471582710743, "transport_residual_norm": 0.6117323443293572, "velocity_mse": 0.011302202742081137} +{"anchor_distance_mean": 122.56015625, "avg_context_frames": 16.8, "avg_micro_time_s": 2.861988306887088, "delta_h_c_norm": 0.09823677064850928, "delta_h_d_norm": 0.04393434897065163, "epoch": 4, "gate_p_mean": 0.026979148504324257, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8652629479765892, "gate_p_p10": 0.005461425706744194, "gate_p_p50": 0.01494731903076172, "gate_p_p90": 0.06475385427474976, "gate_p_std": 0.031842636613873765, "gated_delta_h_c_norm": 0.0064397646608995275, "global_step": 410, "grad_norm_max": 0.024615304544568062, "hidden_mse": 0.012717175966827199, "loss": 0.011830651573836804, "lr_blocks": 9.357421218136387e-06, "lr_fusion": 9.357421218136386e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 23.069400000572205, "transport_attention_entropy": 7.319507813453675, "transport_attention_entropy_normalized": 0.9955207385122776, "transport_residual_norm": 0.6068240739405155, "velocity_mse": 0.010558933939319105} +{"anchor_distance_mean": 125.02890625, "avg_context_frames": 16.0, "avg_micro_time_s": 2.742737606700121, "delta_h_c_norm": 0.09516600519418716, "delta_h_d_norm": 0.045501285186037424, "epoch": 4, "gate_p_mean": 0.020659361907746643, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8964841507375241, "gate_p_p10": 0.003238296508789063, "gate_p_p50": 0.010305213928222657, "gate_p_p90": 0.04974288940429688, "gate_p_std": 0.02879996363190003, "gated_delta_h_c_norm": 0.005260486603219761, "global_step": 420, "grad_norm_max": 0.06527218967676163, "hidden_mse": 0.012770632607862355, "loss": 0.012332256318768487, "lr_blocks": 9.31628240478787e-06, "lr_fusion": 9.316282404787871e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 23.155585980415346, "transport_attention_entropy": 7.3195117056369785, "transport_attention_entropy_normalized": 0.9955212697386742, "transport_residual_norm": 0.5942208923399449, "velocity_mse": 0.011055193014908583} +{"anchor_distance_mean": 123.08125000000001, "avg_context_frames": 16.8, "avg_micro_time_s": 2.8691077881374443, "delta_h_c_norm": 0.09860461642965675, "delta_h_d_norm": 0.0446183548308909, "epoch": 4, "gate_p_mean": 0.019087180856149646, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.9118524923920632, "gate_p_p10": 0.0030281066894531253, "gate_p_p50": 0.00912456512451172, "gate_p_p90": 0.048073410987854004, "gate_p_std": 0.02622904353775084, "gated_delta_h_c_norm": 0.005044913287565578, "global_step": 430, "grad_norm_max": 0.02998518943786621, "hidden_mse": 0.012194027699297295, "loss": 0.011327593511668966, "lr_blocks": 9.273963562927695e-06, "lr_fusion": 9.273963562927695e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 23.27806453704834, "transport_attention_entropy": 7.319584429264069, "transport_attention_entropy_normalized": 0.9955311611294747, "transport_residual_norm": 0.5953621320426464, "velocity_mse": 0.010108190719620325} +{"anchor_distance_mean": 123.69609375, "avg_context_frames": 14.8, "avg_micro_time_s": 2.617642754262488, "delta_h_c_norm": 0.1063217343762517, "delta_h_d_norm": 0.047358265891671185, "epoch": 4, "gate_p_mean": 0.0302523605292663, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.843928062170744, "gate_p_p10": 0.00509849525988102, "gate_p_p50": 0.015299415588378907, "gate_p_p90": 0.07888289690017701, "gate_p_std": 0.037348753912374376, "gated_delta_h_c_norm": 0.008676917605043855, "global_step": 440, "grad_norm_max": 0.02039819397032261, "hidden_mse": 0.013362768333172426, "loss": 0.012408486992353574, "lr_blocks": 9.230476262104678e-06, "lr_fusion": 9.230476262104677e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 23.130238151550294, "transport_attention_entropy": 7.319320166110993, "transport_attention_entropy_normalized": 0.9954952158033848, "transport_residual_norm": 0.604304913431406, "velocity_mse": 0.011072210193378852} +{"anchor_distance_mean": 127.4625, "avg_context_frames": 16.400000000000002, "avg_micro_time_s": 2.8228658655500114, "delta_h_c_norm": 0.10178920831531287, "delta_h_d_norm": 0.04453777917660773, "epoch": 4, "gate_p_mean": 0.019750148116145283, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.9107740178704262, "gate_p_p10": 0.0035130118019878866, "gate_p_p50": 0.009488296508789063, "gate_p_p90": 0.04957351684570313, "gate_p_std": 0.026533962337998675, "gated_delta_h_c_norm": 0.005299892967741471, "global_step": 450, "grad_norm_max": 0.029188916087150574, "hidden_mse": 0.011272215936332942, "loss": 0.010444673369056546, "lr_blocks": 9.185832391312644e-06, "lr_fusion": 9.185832391312644e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 23.37280275821686, "transport_attention_entropy": 7.319539260864258, "transport_attention_entropy_normalized": 0.9955250158905984, "transport_residual_norm": 0.5986882969737053, "velocity_mse": 0.009317451773677021} +{"anchor_distance_mean": 124.63828125, "avg_context_frames": 11.200000000000001, "avg_micro_time_s": 2.2137667724751284, "delta_h_c_norm": 0.10544984107837081, "delta_h_d_norm": 0.04729077420197428, "epoch": 4, "gate_p_mean": 0.029228898498695345, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8429484963417053, "gate_p_p10": 0.005516815185546875, "gate_p_p50": 0.015610122680664064, "gate_p_p90": 0.07212982177734376, "gate_p_std": 0.035488220909610395, "gated_delta_h_c_norm": 0.006980790599482135, "global_step": 460, "grad_norm_max": 0.022958477959036827, "hidden_mse": 0.01291684226016514, "loss": 0.012320398722658867, "lr_blocks": 9.140044155740102e-06, "lr_fusion": 9.140044155740101e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 23.255324435234073, "transport_attention_entropy": 7.319606697559357, "transport_attention_entropy_normalized": 0.9955341897904874, "transport_residual_norm": 0.6023226104676724, "velocity_mse": 0.011028714472195134} +{"anchor_distance_mean": 122.040625, "avg_context_frames": 11.200000000000001, "avg_micro_time_s": 2.2307758983751684, "delta_h_c_norm": 0.10967427585273981, "delta_h_d_norm": 0.04663602206856013, "epoch": 4, "gate_p_mean": 0.029477717418922113, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.837568385899067, "gate_p_p10": 0.004578590393066406, "gate_p_p50": 0.015208053588867189, "gate_p_p90": 0.07456709742546082, "gate_p_std": 0.03707683420507237, "gated_delta_h_c_norm": 0.007381483994686278, "global_step": 470, "grad_norm_max": 0.023604411631822586, "hidden_mse": 0.012587990274187178, "loss": 0.012052274501184002, "lr_blocks": 9.093124073433464e-06, "lr_fusion": 9.093124073433463e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 23.498095512390137, "transport_attention_entropy": 7.3197196364402775, "transport_attention_entropy_normalized": 0.9955495491623879, "transport_residual_norm": 0.6099580928683281, "velocity_mse": 0.010793475492391735} +{"anchor_distance_mean": 121.29375, "avg_context_frames": 14.8, "avg_micro_time_s": 2.667627825200134, "delta_h_c_norm": 0.11837854003533721, "delta_h_d_norm": 0.049051501555368304, "epoch": 4, "gate_p_mean": 0.023989915964193642, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8833776839077473, "gate_p_p10": 0.003559684753417969, "gate_p_p50": 0.01151275634765625, "gate_p_p90": 0.061335754394531254, "gate_p_std": 0.03255238775163889, "gated_delta_h_c_norm": 0.0074008078780025245, "global_step": 480, "grad_norm_max": 0.033754073083400726, "hidden_mse": 0.012284138170070947, "loss": 0.011346753814723344, "lr_blocks": 9.045084971874738e-06, "lr_fusion": 9.045084971874738e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 23.4142826795578, "transport_attention_entropy": 7.319584518671036, "transport_attention_entropy_normalized": 0.9955311715602875, "transport_residual_norm": 0.6101171232759953, "velocity_mse": 0.010118340043118224} +{"anchor_distance_mean": 125.384375, "avg_context_frames": 16.8, "avg_micro_time_s": 2.892377780462539, "delta_h_c_norm": 0.1098443808965385, "delta_h_d_norm": 0.04739925158210099, "epoch": 4, "gate_p_mean": 0.024535682087298484, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8772160708904266, "gate_p_p10": 0.0044997215270996095, "gate_p_p50": 0.01271820068359375, "gate_p_p90": 0.0613616943359375, "gate_p_std": 0.031167800491675737, "gated_delta_h_c_norm": 0.006133841378323269, "global_step": 490, "grad_norm_max": 0.0336148738861084, "hidden_mse": 0.011568338342476637, "loss": 0.0109986026014667, "lr_blocks": 8.995939984474624e-06, "lr_fusion": 8.995939984474624e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 23.984481382369996, "transport_attention_entropy": 7.319640076160431, "transport_attention_entropy_normalized": 0.995538728684187, "transport_residual_norm": 0.5826655633747578, "velocity_mse": 0.009841768693877385} +{"anchor_distance_mean": 128.275, "avg_context_frames": 14.0, "avg_micro_time_s": 2.5601007583873527, "delta_h_c_norm": 0.11210716152563692, "delta_h_d_norm": 0.04861903823912144, "epoch": 4, "gate_p_mean": 0.02412516519543715, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8820778504014015, "gate_p_p10": 0.0038298606872558595, "gate_p_p50": 0.010936546325683595, "gate_p_p90": 0.06331891417503357, "gate_p_std": 0.03391037305700593, "gated_delta_h_c_norm": 0.007373185054166243, "global_step": 500, "grad_norm_max": 0.03197915479540825, "hidden_mse": 0.012393435841659084, "loss": 0.011894091725116596, "lr_blocks": 8.94570254698197e-06, "lr_fusion": 8.945702546981969e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 23.44497606754303, "transport_attention_entropy": 7.319449478387833, "transport_attention_entropy_normalized": 0.9955128036439419, "transport_residual_norm": 0.5867729127407074, "velocity_mse": 0.010654748111846857} +{"anchor_distance_mean": 124.76796875000001, "avg_context_frames": 16.400000000000002, "avg_micro_time_s": 2.8267747176371816, "delta_h_c_norm": 0.11145419972017408, "delta_h_d_norm": 0.044247538829222326, "epoch": 4, "gate_p_mean": 0.010054415935883299, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.9612597987055779, "gate_p_p10": 0.0013628481887280942, "gate_p_p50": 0.004505538940429688, "gate_p_p90": 0.025037240982055665, "gate_p_std": 0.01526480039174203, "gated_delta_h_c_norm": 0.0029005308024352416, "global_step": 510, "grad_norm_max": 0.025746773928403854, "hidden_mse": 0.012089941382873804, "loss": 0.011399810246075504, "lr_blocks": 8.894386393810563e-06, "lr_fusion": 8.894386393810563e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 24.27898135185242, "transport_attention_entropy": 7.319979709386826, "transport_attention_entropy_normalized": 0.995584923028946, "transport_residual_norm": 0.5721940003335476, "velocity_mse": 0.010190816095564515} +{"anchor_distance_mean": 124.05546875, "avg_context_frames": 16.0, "avg_micro_time_s": 2.809094213499975, "delta_h_c_norm": 0.1293046128936112, "delta_h_d_norm": 0.0462139260256663, "epoch": 5, "gate_p_mean": 0.020135753095382826, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.907510295510292, "gate_p_p10": 0.0038655281066894533, "gate_p_p50": 0.011295127868652345, "gate_p_p90": 0.0491912841796875, "gate_p_std": 0.022855223846272567, "gated_delta_h_c_norm": 0.005910166426474461, "global_step": 520, "grad_norm_max": 0.04787405952811241, "hidden_mse": 0.011942219664342703, "loss": 0.011904214083915577, "lr_blocks": 8.842005554284296e-06, "lr_fusion": 8.842005554284296e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 24.289559960365295, "transport_attention_entropy": 7.3203328430652626, "transport_attention_entropy_normalized": 0.9956329502165318, "transport_residual_norm": 0.5678257249295712, "velocity_mse": 0.01070999214425683} +{"anchor_distance_mean": 124.21640625, "avg_context_frames": 13.200000000000001, "avg_micro_time_s": 2.468489786774899, "delta_h_c_norm": 0.120473843626678, "delta_h_d_norm": 0.04615922756493092, "epoch": 5, "gate_p_mean": 0.017806732415920123, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.9139808982610703, "gate_p_p10": 0.0028469085693359378, "gate_p_p50": 0.008457565307617187, "gate_p_p90": 0.045897722244262695, "gate_p_std": 0.023861640406539666, "gated_delta_h_c_norm": 0.0057296022641821764, "global_step": 530, "grad_norm_max": 0.03651213273406029, "hidden_mse": 0.013655292621115223, "loss": 0.012763764034025371, "lr_blocks": 8.788574348801676e-06, "lr_fusion": 8.788574348801675e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 24.39476139545441, "transport_attention_entropy": 7.320607388019562, "transport_attention_entropy_normalized": 0.9956702925264835, "transport_residual_norm": 0.5642205327749252, "velocity_mse": 0.011398234756779858} +{"anchor_distance_mean": 128.60078125, "avg_context_frames": 17.6, "avg_micro_time_s": 2.9501265139627777, "delta_h_c_norm": 0.12140946676954628, "delta_h_d_norm": 0.0476176243275404, "epoch": 5, "gate_p_mean": 0.016560793429380282, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.929190480709076, "gate_p_p10": 0.0028722949326038362, "gate_p_p50": 0.008466339111328125, "gate_p_p90": 0.04008903503417969, "gate_p_std": 0.022002468199934812, "gated_delta_h_c_norm": 0.00544624084395764, "global_step": 540, "grad_norm_max": 0.03021247312426567, "hidden_mse": 0.011579619557596744, "loss": 0.011147118487861008, "lr_blocks": 8.734107384920771e-06, "lr_fusion": 8.73410738492077e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 24.547339510917666, "transport_attention_entropy": 7.3205320060253145, "transport_attention_entropy_normalized": 0.99566003754735, "transport_residual_norm": 0.5611890673637391, "velocity_mse": 0.009989156553638169} +{"anchor_distance_mean": 126.8796875, "avg_context_frames": 15.200000000000001, "avg_micro_time_s": 2.6697054648871474, "delta_h_c_norm": 0.1222075921483338, "delta_h_d_norm": 0.04964049849659205, "epoch": 5, "gate_p_mean": 0.02026719107234385, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8999171450734139, "gate_p_p10": 0.002848339080810547, "gate_p_p50": 0.009634017944335938, "gate_p_p90": 0.052561187744140626, "gate_p_std": 0.027860275714192542, "gated_delta_h_c_norm": 0.006713649056473515, "global_step": 550, "grad_norm_max": 0.02769698202610016, "hidden_mse": 0.012620493420399725, "loss": 0.011885299251298422, "lr_blocks": 8.67861955336566e-06, "lr_fusion": 8.678619553365659e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 24.347151851654054, "transport_attention_entropy": 7.320563697814942, "transport_attention_entropy_normalized": 0.9956643499433995, "transport_residual_norm": 0.5572645537555218, "velocity_mse": 0.010623249912168831} +{"anchor_distance_mean": 120.640625, "avg_context_frames": 14.8, "avg_micro_time_s": 2.639730674637576, "delta_h_c_norm": 0.1151993777602911, "delta_h_d_norm": 0.04969489271752536, "epoch": 5, "gate_p_mean": 0.017224713176256047, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.9178663633763791, "gate_p_p10": 0.0024395942687988285, "gate_p_p50": 0.007760810852050781, "gate_p_p90": 0.04411239624023438, "gate_p_std": 0.024333631532499567, "gated_delta_h_c_norm": 0.0050766761443810544, "global_step": 560, "grad_norm_max": 0.019616996869444847, "hidden_mse": 0.011287582525983454, "loss": 0.010427770391106607, "lr_blocks": 8.622126023955446e-06, "lr_fusion": 8.622126023955446e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 24.548001527786255, "transport_attention_entropy": 7.320717382431031, "transport_attention_entropy_normalized": 0.9956852532923222, "transport_residual_norm": 0.5627434439957142, "velocity_mse": 0.009299012168776244} +{"anchor_distance_mean": 126.42421875000001, "avg_context_frames": 13.200000000000001, "avg_micro_time_s": 2.4491700994627994, "delta_h_c_norm": 0.1187311894260347, "delta_h_d_norm": 0.049591871164739135, "epoch": 5, "gate_p_mean": 0.023587371356552467, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8801437579095364, "gate_p_p10": 0.003956928104162216, "gate_p_p50": 0.012042999267578125, "gate_p_p90": 0.060312652587890626, "gate_p_std": 0.029032867192290725, "gated_delta_h_c_norm": 0.006524175578670111, "global_step": 570, "grad_norm_max": 0.019434213638305664, "hidden_mse": 0.011470311577431858, "loss": 0.01097343121364247, "lr_blocks": 8.564642241456986e-06, "lr_fusion": 8.564642241456986e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 24.608660674095155, "transport_attention_entropy": 7.320636802911759, "transport_attention_entropy_normalized": 0.9956742934882641, "transport_residual_norm": 0.5640908613801002, "velocity_mse": 0.009826400113524869} +{"anchor_distance_mean": 119.05234375, "avg_context_frames": 13.200000000000001, "avg_micro_time_s": 2.439872519874871, "delta_h_c_norm": 0.1229363538324833, "delta_h_d_norm": 0.05468603288754821, "epoch": 5, "gate_p_mean": 0.02696428977069445, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8509575612843037, "gate_p_p10": 0.004098701477050781, "gate_p_p50": 0.013340759277343751, "gate_p_p90": 0.07014033794403077, "gate_p_std": 0.034860591776669024, "gated_delta_h_c_norm": 0.00817541333672125, "global_step": 580, "grad_norm_max": 0.02746998704969883, "hidden_mse": 0.01355781463207677, "loss": 0.012630403216462582, "lr_blocks": 8.506183921362443e-06, "lr_fusion": 8.506183921362443e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 24.68136947154999, "transport_attention_entropy": 7.320472419261932, "transport_attention_entropy_normalized": 0.9956519328057767, "transport_residual_norm": 0.5660403355956077, "velocity_mse": 0.011274621717166157} +{"anchor_distance_mean": 127.2703125, "avg_context_frames": 11.600000000000001, "avg_micro_time_s": 2.2551059300501493, "delta_h_c_norm": 0.12301570652052761, "delta_h_d_norm": 0.052076555369421844, "epoch": 5, "gate_p_mean": 0.024481874960474673, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8710607267916203, "gate_p_p10": 0.0034433364868164066, "gate_p_p50": 0.011455154418945313, "gate_p_p90": 0.063321053981781, "gate_p_std": 0.03329469253076241, "gated_delta_h_c_norm": 0.007149806889356115, "global_step": 590, "grad_norm_max": 0.02025926113128662, "hidden_mse": 0.012159810669254513, "loss": 0.011531397351063788, "lr_blocks": 8.446767045592829e-06, "lr_fusion": 8.44676704559283e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 24.566276741027835, "transport_attention_entropy": 7.320233911275864, "transport_attention_entropy_normalized": 0.995619496703148, "transport_residual_norm": 0.560361984372139, "velocity_mse": 0.01031541626143735} +{"anchor_distance_mean": 124.05625, "avg_context_frames": 12.0, "avg_micro_time_s": 2.3015862004626344, "delta_h_c_norm": 0.12957118200138212, "delta_h_d_norm": 0.052294147294014694, "epoch": 5, "gate_p_mean": 0.026041383924894038, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8604146912693977, "gate_p_p10": 0.0033209800720214845, "gate_p_p50": 0.012397956848144532, "gate_p_p90": 0.06872329711914063, "gate_p_std": 0.03434389171889052, "gated_delta_h_c_norm": 0.008206511921889615, "global_step": 600, "grad_norm_max": 0.049193449318408966, "hidden_mse": 0.01335178305162117, "loss": 0.013133751560235397, "lr_blocks": 8.386407858128707e-06, "lr_fusion": 8.386407858128706e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 24.81926145553589, "transport_attention_entropy": 7.320328795909882, "transport_attention_entropy_normalized": 0.9956324055790902, "transport_residual_norm": 0.5707928694784642, "velocity_mse": 0.011798573273699731} +{"anchor_distance_mean": 122.20156250000001, "avg_context_frames": 16.400000000000002, "avg_micro_time_s": 2.811687021999933, "delta_h_c_norm": 0.12248453907668591, "delta_h_d_norm": 0.04949907939881087, "epoch": 5, "gate_p_mean": 0.02234088603872806, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.887194037437439, "gate_p_p10": 0.003218326158821583, "gate_p_p50": 0.011186981201171876, "gate_p_p90": 0.05769177079200745, "gate_p_std": 0.028761018346995116, "gated_delta_h_c_norm": 0.006008455094706733, "global_step": 610, "grad_norm_max": 0.017191195860505104, "hidden_mse": 0.010897388070588931, "loss": 0.010708394553512336, "lr_blocks": 8.325122860569241e-06, "lr_fusion": 8.32512286056924e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 24.01687345504761, "transport_attention_entropy": 7.320012176036835, "transport_attention_entropy_normalized": 0.9955893404781819, "transport_residual_norm": 0.5609399825334549, "velocity_mse": 0.009618655755184592} +{"anchor_distance_mean": 126.87890625, "avg_context_frames": 14.8, "avg_micro_time_s": 2.6133291626124904, "delta_h_c_norm": 0.12082146275788547, "delta_h_d_norm": 0.04775228560902178, "epoch": 5, "gate_p_mean": 0.022253193747019395, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8864959210157395, "gate_p_p10": 0.00346221923828125, "gate_p_p50": 0.011138629913330079, "gate_p_p90": 0.057157266139984134, "gate_p_std": 0.028257429000223058, "gated_delta_h_c_norm": 0.0054886467391042975, "global_step": 620, "grad_norm_max": 0.017356164753437042, "hidden_mse": 0.011161493457620963, "loss": 0.010923874430591242, "lr_blocks": 8.262928807620843e-06, "lr_fusion": 8.262928807620843e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 24.9590033531189, "transport_attention_entropy": 7.319982391595841, "transport_attention_entropy_normalized": 0.9955852843821049, "transport_residual_norm": 0.5622468613088131, "velocity_mse": 0.00980772507027723} +{"anchor_distance_mean": 128.696875, "avg_context_frames": 16.400000000000002, "avg_micro_time_s": 2.806842124537434, "delta_h_c_norm": 0.12128336792811752, "delta_h_d_norm": 0.049789932370185856, "epoch": 5, "gate_p_mean": 0.02188396544661373, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8926389954984189, "gate_p_p10": 0.003286552429199219, "gate_p_p50": 0.010214233398437501, "gate_p_p90": 0.05652706027030945, "gate_p_std": 0.029979647201253103, "gated_delta_h_c_norm": 0.006802313709340525, "global_step": 630, "grad_norm_max": 0.023415451869368553, "hidden_mse": 0.011617145308991895, "loss": 0.010926336603006348, "lr_blocks": 8.199842702516584e-06, "lr_fusion": 8.199842702516583e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 24.79455807209015, "transport_attention_entropy": 7.319615256786347, "transport_attention_entropy_normalized": 0.9955353543162346, "transport_residual_norm": 0.5570100374519825, "velocity_mse": 0.009764622076181695} +{"anchor_distance_mean": 123.33984375, "avg_context_frames": 12.8, "avg_micro_time_s": 2.4084634006124817, "delta_h_c_norm": 0.12639818768948316, "delta_h_d_norm": 0.05373393897898496, "epoch": 5, "gate_p_mean": 0.02674732415471226, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8560915261507035, "gate_p_p10": 0.003451251983642578, "gate_p_p50": 0.01168975830078125, "gate_p_p90": 0.07280731201171875, "gate_p_std": 0.03713851724751294, "gated_delta_h_c_norm": 0.008527248931932263, "global_step": 640, "grad_norm_max": 0.03224474564194679, "hidden_mse": 0.012995803065132351, "loss": 0.012152949569281191, "lr_blocks": 8.135881792367686e-06, "lr_fusion": 8.135881792367686e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 25.27806258201599, "transport_attention_entropy": 7.320366317033768, "transport_attention_entropy_normalized": 0.9956375032663346, "transport_residual_norm": 0.5745181776583195, "velocity_mse": 0.010853369289543481} +{"anchor_distance_mean": 125.6453125, "avg_context_frames": 14.0, "avg_micro_time_s": 2.5485563816752803, "delta_h_c_norm": 0.12485523000359536, "delta_h_d_norm": 0.05087226056493819, "epoch": 6, "gate_p_mean": 0.022500131384003908, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8873212441802025, "gate_p_p10": 0.0029552459716796877, "gate_p_p50": 0.010133743286132812, "gate_p_p90": 0.05894360542297364, "gate_p_std": 0.03242449498502538, "gated_delta_h_c_norm": 0.007219768417417072, "global_step": 650, "grad_norm_max": 0.02805013582110405, "hidden_mse": 0.011892906279535966, "loss": 0.011599857863620855, "lr_blocks": 8.071063563448341e-06, "lr_fusion": 8.07106356344834e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 25.114951944351198, "transport_attention_entropy": 7.319873130321503, "transport_attention_entropy_normalized": 0.9955704249441624, "transport_residual_norm": 0.5633092872798443, "velocity_mse": 0.01041056725080125} +{"anchor_distance_mean": 124.24921875000001, "avg_context_frames": 18.0, "avg_micro_time_s": 3.076147499237459, "delta_h_c_norm": 0.12833025306463242, "delta_h_d_norm": 0.04703245549462736, "epoch": 6, "gate_p_mean": 0.017032343326718546, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.9242395620793105, "gate_p_p10": 0.002633953094482422, "gate_p_p50": 0.008966445922851562, "gate_p_p90": 0.04174118041992188, "gate_p_std": 0.02228535750473384, "gated_delta_h_c_norm": 0.004841834458420635, "global_step": 660, "grad_norm_max": 0.01903495565056801, "hidden_mse": 0.009927084972150625, "loss": 0.009441147121833637, "lr_blocks": 8.005405736415127e-06, "lr_fusion": 8.005405736415126e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 25.076563501358034, "transport_attention_entropy": 7.319568425416946, "transport_attention_entropy_normalized": 0.9955289848148823, "transport_residual_norm": 0.5571133509278298, "velocity_mse": 0.008448438653431367} +{"anchor_distance_mean": 123.33828125000001, "avg_context_frames": 13.600000000000001, "avg_micro_time_s": 2.4978655176121722, "delta_h_c_norm": 0.13207392366603019, "delta_h_d_norm": 0.05511240931227804, "epoch": 6, "gate_p_mean": 0.02523773761931807, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8663589529693128, "gate_p_p10": 0.003601646423339844, "gate_p_p50": 0.011318206787109375, "gate_p_p90": 0.0663543701171875, "gate_p_std": 0.03565391413285397, "gated_delta_h_c_norm": 0.008769292378565297, "global_step": 670, "grad_norm_max": 0.027349987998604774, "hidden_mse": 0.01345250340527855, "loss": 0.012875274993712084, "lr_blocks": 7.938926261462366e-06, "lr_fusion": 7.938926261462366e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 25.002067971229554, "transport_attention_entropy": 7.319828712940216, "transport_attention_entropy_normalized": 0.9955643862485886, "transport_residual_norm": 0.55920105651021, "velocity_mse": 0.011530024657258765} +{"anchor_distance_mean": 124.73828125, "avg_context_frames": 12.4, "avg_micro_time_s": 2.3607287744249335, "delta_h_c_norm": 0.12983464244753123, "delta_h_d_norm": 0.05077164270915091, "epoch": 6, "gate_p_mean": 0.02183894677727949, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8842858836054802, "gate_p_p10": 0.0027005195617675784, "gate_p_p50": 0.009787750244140626, "gate_p_p90": 0.05844172239303589, "gate_p_std": 0.030947235942585395, "gated_delta_h_c_norm": 0.007323438244930003, "global_step": 680, "grad_norm_max": 0.02391771413385868, "hidden_mse": 0.013378244475461543, "loss": 0.012745204294333234, "lr_blocks": 7.871643313414718e-06, "lr_fusion": 7.871643313414718e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 24.98860185146332, "transport_attention_entropy": 7.319655501842499, "transport_attention_entropy_normalized": 0.9955408260226251, "transport_residual_norm": 0.5618471592664719, "velocity_mse": 0.01140737984678708} +{"anchor_distance_mean": 120.415625, "avg_context_frames": 12.0, "avg_micro_time_s": 2.3193927556246763, "delta_h_c_norm": 0.1355798976495862, "delta_h_d_norm": 0.05293767568655312, "epoch": 6, "gate_p_mean": 0.024073180078994484, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8747253365814687, "gate_p_p10": 0.003141593933105469, "gate_p_p50": 0.011171913146972657, "gate_p_p90": 0.06438751220703125, "gate_p_std": 0.0325350855011493, "gated_delta_h_c_norm": 0.00815548371319892, "global_step": 690, "grad_norm_max": 0.03649953380227089, "hidden_mse": 0.012975696759531275, "loss": 0.012597399967489764, "lr_blocks": 7.803575286758365e-06, "lr_fusion": 7.803575286758364e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 24.796486759185793, "transport_attention_entropy": 7.319808131456376, "transport_attention_entropy_normalized": 0.9955615863204003, "transport_residual_norm": 0.5777609124779701, "velocity_mse": 0.011299830256029965} +{"anchor_distance_mean": 121.87812500000001, "avg_context_frames": 15.600000000000001, "avg_micro_time_s": 2.7335771876124455, "delta_h_c_norm": 0.138625955209136, "delta_h_d_norm": 0.051547883590683345, "epoch": 6, "gate_p_mean": 0.025440254178829492, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8671659879386425, "gate_p_p10": 0.0037910461425781253, "gate_p_p50": 0.012196540832519531, "gate_p_p90": 0.06641215085983276, "gate_p_std": 0.0332489442487713, "gated_delta_h_c_norm": 0.008857899338181596, "global_step": 700, "grad_norm_max": 0.02645370550453663, "hidden_mse": 0.013159666414139792, "loss": 0.011965919402427971, "lr_blocks": 7.734740790612137e-06, "lr_fusion": 7.734740790612136e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 25.249782943725588, "transport_attention_entropy": 7.319438862800599, "transport_attention_entropy_normalized": 0.9955113649368287, "transport_residual_norm": 0.5568222314119339, "velocity_mse": 0.010649952659150586} +{"anchor_distance_mean": 125.71015625000001, "avg_context_frames": 16.400000000000002, "avg_micro_time_s": 2.822462756400137, "delta_h_c_norm": 0.13232390647754075, "delta_h_d_norm": 0.04990681079216302, "epoch": 6, "gate_p_mean": 0.018458013382041828, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.9106360450387001, "gate_p_p10": 0.0027382849715650083, "gate_p_p50": 0.009001350402832032, "gate_p_p90": 0.047274780273437504, "gate_p_std": 0.024621186900185422, "gated_delta_h_c_norm": 0.005531876070745057, "global_step": 710, "grad_norm_max": 0.028778236359357834, "hidden_mse": 0.011334835790330546, "loss": 0.010693423796328717, "lr_blocks": 7.66515864363997e-06, "lr_fusion": 7.66515864363997e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 24.99316010475159, "transport_attention_entropy": 7.3194964110851295, "transport_attention_entropy_normalized": 0.9955191850662232, "transport_residual_norm": 0.5493885792791844, "velocity_mse": 0.009559940235340037} +{"anchor_distance_mean": 126.0671875, "avg_context_frames": 15.200000000000001, "avg_micro_time_s": 2.6772117512251956, "delta_h_c_norm": 0.1353039468638599, "delta_h_d_norm": 0.05132582532241941, "epoch": 6, "gate_p_mean": 0.02130308324703947, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.893663138896227, "gate_p_p10": 0.002920531947165728, "gate_p_p50": 0.010035324096679689, "gate_p_p90": 0.0556892216205597, "gate_p_std": 0.028419039474101737, "gated_delta_h_c_norm": 0.007059561577625573, "global_step": 720, "grad_norm_max": 0.016048679128289223, "hidden_mse": 0.010820801404770465, "loss": 0.010308191482909025, "lr_blocks": 7.594847868906076e-06, "lr_fusion": 7.594847868906076e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 25.027780508995058, "transport_attention_entropy": 7.31969045996666, "transport_attention_entropy_normalized": 0.9955455824732781, "transport_residual_norm": 0.5563342243432999, "velocity_mse": 0.009226111334282906} +{"anchor_distance_mean": 129.02265625, "avg_context_frames": 16.400000000000002, "avg_micro_time_s": 2.831782273599811, "delta_h_c_norm": 0.131638066098094, "delta_h_d_norm": 0.05320866727270186, "epoch": 6, "gate_p_mean": 0.01749950900557451, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.9178673647344113, "gate_p_p10": 0.0023820877075195314, "gate_p_p50": 0.007759284973144532, "gate_p_p90": 0.04575468897819519, "gate_p_std": 0.024998007918475198, "gated_delta_h_c_norm": 0.005563064083253267, "global_step": 730, "grad_norm_max": 0.033928584307432175, "hidden_mse": 0.01050469566253014, "loss": 0.009944070284836926, "lr_blocks": 7.52382768867422e-06, "lr_fusion": 7.52382768867422e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 25.129614686965944, "transport_attention_entropy": 7.319494038820267, "transport_attention_entropy_normalized": 0.9955188639461995, "transport_residual_norm": 0.5408269248902798, "velocity_mse": 0.008893600667943247} +{"anchor_distance_mean": 126.19843750000001, "avg_context_frames": 12.4, "avg_micro_time_s": 2.3995210981248873, "delta_h_c_norm": 0.13688014755025507, "delta_h_d_norm": 0.0524294963106513, "epoch": 6, "gate_p_mean": 0.021801374724600465, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8889062762260438, "gate_p_p10": 0.0027630423195660115, "gate_p_p50": 0.009602737426757813, "gate_p_p90": 0.058876633644104004, "gate_p_std": 0.030839732877211645, "gated_delta_h_c_norm": 0.00734025311539881, "global_step": 740, "grad_norm_max": 0.02339830994606018, "hidden_mse": 0.012219248310429975, "loss": 0.011702093388885261, "lr_blocks": 7.452117519152542e-06, "lr_fusion": 7.452117519152542e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 25.38498480319977, "transport_attention_entropy": 7.319663256406784, "transport_attention_entropy_normalized": 0.9955418832600117, "transport_residual_norm": 0.568022881448269, "velocity_mse": 0.010480168485082687} +{"anchor_distance_mean": 122.625, "avg_context_frames": 14.8, "avg_micro_time_s": 2.6449075361748786, "delta_h_c_norm": 0.1396775775589049, "delta_h_d_norm": 0.05617774031125009, "epoch": 6, "gate_p_mean": 0.025772467901697384, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8603192895650864, "gate_p_p10": 0.00362091064453125, "gate_p_p50": 0.012281036376953125, "gate_p_p90": 0.06797561645507813, "gate_p_std": 0.03364436865085736, "gated_delta_h_c_norm": 0.007946621376322583, "global_step": 750, "grad_norm_max": 0.021561408415436745, "hidden_mse": 0.01129950582399033, "loss": 0.01090844527061563, "lr_blocks": 7.379736965185369e-06, "lr_fusion": 7.379736965185368e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 25.16920976638794, "transport_attention_entropy": 7.318839430809021, "transport_attention_entropy_normalized": 0.9954298317432404, "transport_residual_norm": 0.5393363885581494, "velocity_mse": 0.009778494772035629} +{"anchor_distance_mean": 127.56093750000001, "avg_context_frames": 12.0, "avg_micro_time_s": 2.3224669933499174, "delta_h_c_norm": 0.1448732076212764, "delta_h_d_norm": 0.05138738602399826, "epoch": 6, "gate_p_mean": 0.023597766138846055, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8800440959632397, "gate_p_p10": 0.0026924132369458677, "gate_p_p50": 0.009715843200683595, "gate_p_p90": 0.06515274047851563, "gate_p_std": 0.03466082395170816, "gated_delta_h_c_norm": 0.0084507470484823, "global_step": 760, "grad_norm_max": 0.02065858244895935, "hidden_mse": 0.01282065504929051, "loss": 0.012841510836733506, "lr_blocks": 7.30670581489344e-06, "lr_fusion": 7.30670581489344e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 25.409643983840944, "transport_attention_entropy": 7.319233602285386, "transport_attention_entropy_normalized": 0.995483447611332, "transport_residual_norm": 0.5648913688957692, "velocity_mse": 0.011559445364400745} +{"anchor_distance_mean": 127.10546875, "avg_context_frames": 13.600000000000001, "avg_micro_time_s": 2.497582660737635, "delta_h_c_norm": 0.13750023180618884, "delta_h_d_norm": 0.0537647242192179, "epoch": 7, "gate_p_mean": 0.020986160618485884, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.891202699393034, "gate_p_p10": 0.002398777008056641, "gate_p_p50": 0.009114837646484375, "gate_p_p90": 0.056491851806640625, "gate_p_std": 0.029736200865590947, "gated_delta_h_c_norm": 0.006272978104243521, "global_step": 770, "grad_norm_max": 0.01495000533759594, "hidden_mse": 0.010458197339903564, "loss": 0.010399050533305854, "lr_blocks": 7.233044034264034e-06, "lr_fusion": 7.233044034264034e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 25.43306486606598, "transport_attention_entropy": 7.3191077709198, "transport_attention_entropy_normalized": 0.9954663269221783, "transport_residual_norm": 0.5522036828100682, "velocity_mse": 0.009353230791748502} +{"anchor_distance_mean": 125.58203125, "avg_context_frames": 15.200000000000001, "avg_micro_time_s": 2.6807823647499847, "delta_h_c_norm": 0.14881456829607487, "delta_h_d_norm": 0.05584600381553173, "epoch": 7, "gate_p_mean": 0.02278703072224744, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8790439933538438, "gate_p_p10": 0.002631616592407227, "gate_p_p50": 0.010021209716796875, "gate_p_p90": 0.06069692373275757, "gate_p_std": 0.03201608402305283, "gated_delta_h_c_norm": 0.008396551050100244, "global_step": 780, "grad_norm_max": 0.020309798419475555, "hidden_mse": 0.012204157619271428, "loss": 0.01150538994697854, "lr_blocks": 7.158771761692464e-06, "lr_fusion": 7.158771761692464e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 24.958008337020875, "transport_attention_entropy": 7.3187498390674595, "transport_attention_entropy_normalized": 0.9954176500439644, "transport_residual_norm": 0.5558291472494602, "velocity_mse": 0.010284974140813575} +{"anchor_distance_mean": 126.06796875, "avg_context_frames": 12.0, "avg_micro_time_s": 2.325400302800017, "delta_h_c_norm": 0.14634775873273612, "delta_h_d_norm": 0.05286680003628135, "epoch": 7, "gate_p_mean": 0.021176341333193707, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8912655539810658, "gate_p_p10": 0.0023194503504782916, "gate_p_p50": 0.008690261840820312, "gate_p_p90": 0.056741333007812506, "gate_p_std": 0.03209187760949135, "gated_delta_h_c_norm": 0.0072777622728608556, "global_step": 790, "grad_norm_max": 0.043100062757730484, "hidden_mse": 0.010968585638329387, "loss": 0.010891059995628893, "lr_blocks": 7.083909302476453e-06, "lr_fusion": 7.083909302476453e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 24.942216992378235, "transport_attention_entropy": 7.31880431175232, "transport_attention_entropy_normalized": 0.9954250574111939, "transport_residual_norm": 0.5483204282820225, "velocity_mse": 0.009794201445765795} +{"anchor_distance_mean": 120.77187500000001, "avg_context_frames": 13.600000000000001, "avg_micro_time_s": 2.5135602784248476, "delta_h_c_norm": 0.13869574926793576, "delta_h_d_norm": 0.0544535550288856, "epoch": 7, "gate_p_mean": 0.01957299439236522, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.9015803083777428, "gate_p_p10": 0.0021411895751953128, "gate_p_p50": 0.007767486572265625, "gate_p_p90": 0.054331970214843754, "gate_p_std": 0.029449841036694124, "gated_delta_h_c_norm": 0.006613756688602735, "global_step": 800, "grad_norm_max": 0.02419620379805565, "hidden_mse": 0.012677233427530155, "loss": 0.012217871542088688, "lr_blocks": 7.008477123264849e-06, "lr_fusion": 7.008477123264848e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 25.924304318428042, "transport_attention_entropy": 7.319124341011047, "transport_attention_entropy_normalized": 0.9954685799777508, "transport_residual_norm": 0.5656571067869663, "velocity_mse": 0.010950148181291298} +{"anchor_distance_mean": 125.42031250000001, "avg_context_frames": 16.8, "avg_micro_time_s": 2.8789596458751023, "delta_h_c_norm": 0.1467424442060292, "delta_h_d_norm": 0.05340026705525816, "epoch": 7, "gate_p_mean": 0.01930910579976626, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.9094897128641606, "gate_p_p10": 0.002833843231201172, "gate_p_p50": 0.009647178649902345, "gate_p_p90": 0.049304962158203125, "gate_p_std": 0.02559440915938467, "gated_delta_h_c_norm": 0.006051021553867031, "global_step": 810, "grad_norm_max": 0.023540053516626358, "hidden_mse": 0.010524647805141285, "loss": 0.009753094584448264, "lr_blocks": 6.932495846462262e-06, "lr_fusion": 6.932495846462261e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 25.467464256286622, "transport_attention_entropy": 7.319000971317291, "transport_attention_entropy_normalized": 0.9954518057405949, "transport_residual_norm": 0.5630245842039585, "velocity_mse": 0.008700629812665284} +{"anchor_distance_mean": 123.01484375000001, "avg_context_frames": 17.6, "avg_micro_time_s": 2.95994449059981, "delta_h_c_norm": 0.14365202886983752, "delta_h_d_norm": 0.051429027086123826, "epoch": 7, "gate_p_mean": 0.008938097403733991, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.9673167377710343, "gate_p_p10": 0.0010220527648925782, "gate_p_p50": 0.003693389892578125, "gate_p_p90": 0.022249603271484376, "gate_p_std": 0.015064645468373784, "gated_delta_h_c_norm": 0.0030485815674182962, "global_step": 820, "grad_norm_max": 0.022600356489419937, "hidden_mse": 0.010213772772112863, "loss": 0.009418074504355901, "lr_blocks": 6.855986244591104e-06, "lr_fusion": 6.855986244591104e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 25.317505216598512, "transport_attention_entropy": 7.318982636928559, "transport_attention_entropy_normalized": 0.9954493112862111, "transport_residual_norm": 0.5521123833954334, "velocity_mse": 0.008396697285934352} +{"anchor_distance_mean": 126.4578125, "avg_context_frames": 15.200000000000001, "avg_micro_time_s": 2.679863178325104, "delta_h_c_norm": 0.14321441911160945, "delta_h_d_norm": 0.058649250958114865, "epoch": 7, "gate_p_mean": 0.025456646888051184, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8631212711334229, "gate_p_p10": 0.004207229614257813, "gate_p_p50": 0.013783454895019531, "gate_p_p90": 0.06332669258117676, "gate_p_std": 0.030314045149134473, "gated_delta_h_c_norm": 0.006984774787269998, "global_step": 830, "grad_norm_max": 0.02155398763716221, "hidden_mse": 0.01156329032382928, "loss": 0.01161978265736252, "lr_blocks": 6.778969234612583e-06, "lr_fusion": 6.778969234612584e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 25.83060567378998, "transport_attention_entropy": 7.3188919007778175, "transport_attention_entropy_normalized": 0.9954369731247426, "transport_residual_norm": 0.5417043454945087, "velocity_mse": 0.010463453637203203} +{"anchor_distance_mean": 128.375, "avg_context_frames": 15.600000000000001, "avg_micro_time_s": 2.8577875059752387, "delta_h_c_norm": 0.15212299982085825, "delta_h_d_norm": 0.05173548283055425, "epoch": 7, "gate_p_mean": 0.019939364743186163, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.9009993769228459, "gate_p_p10": 0.002534427493810654, "gate_p_p50": 0.008644485473632812, "gate_p_p90": 0.05444726943969727, "gate_p_std": 0.028207048622425648, "gated_delta_h_c_norm": 0.007044443247286836, "global_step": 840, "grad_norm_max": 0.019395679235458374, "hidden_mse": 0.011078455933602528, "loss": 0.010632792636170053, "lr_blocks": 6.701465872208216e-06, "lr_fusion": 6.701465872208216e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 25.25280785560608, "transport_attention_entropy": 7.318547344207764, "transport_attention_entropy_normalized": 0.9953901059925556, "transport_residual_norm": 0.5525423847138882, "velocity_mse": 0.009524946997407824} +{"anchor_distance_mean": 125.80703125000001, "avg_context_frames": 14.4, "avg_micro_time_s": 2.676496245087719, "delta_h_c_norm": 0.14480204368010163, "delta_h_d_norm": 0.054529843386262655, "epoch": 7, "gate_p_mean": 0.023221957267378458, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8788709715008736, "gate_p_p10": 0.0021487426012754443, "gate_p_p50": 0.008954143524169922, "gate_p_p90": 0.06592521667480469, "gate_p_std": 0.03526568017841782, "gated_delta_h_c_norm": 0.008955783052078914, "global_step": 850, "grad_norm_max": 0.023542972281575203, "hidden_mse": 0.012133904261281714, "loss": 0.01130927983322181, "lr_blocks": 6.6234973460234184e-06, "lr_fusion": 6.623497346023418e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 25.470316720008853, "transport_attention_entropy": 7.3185164809227, "transport_attention_entropy_normalized": 0.9953859090805054, "transport_residual_norm": 0.5384530358016492, "velocity_mse": 0.010095889397780411} +{"anchor_distance_mean": 123.3046875, "avg_context_frames": 12.4, "avg_micro_time_s": 2.4226064333627164, "delta_h_c_norm": 0.1359133306890726, "delta_h_d_norm": 0.05463684010319412, "epoch": 7, "gate_p_mean": 0.027087343524908648, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8470009744167328, "gate_p_p10": 0.0034952640533447267, "gate_p_p50": 0.013401031494140625, "gate_p_p90": 0.07170867919921875, "gate_p_std": 0.03427751651615836, "gated_delta_h_c_norm": 0.008174473962571939, "global_step": 860, "grad_norm_max": 0.021615447476506233, "hidden_mse": 0.012233778793597595, "loss": 0.012017692375229672, "lr_blocks": 6.545084971874738e-06, "lr_fusion": 6.545084971874738e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 25.3895133972168, "transport_attention_entropy": 7.318705064058304, "transport_attention_entropy_normalized": 0.9954115577042103, "transport_residual_norm": 0.5558177083730698, "velocity_mse": 0.010794314515078441} +{"anchor_distance_mean": 123.0796875, "avg_context_frames": 10.0, "avg_micro_time_s": 2.1478224340749876, "delta_h_c_norm": 0.1400854573585093, "delta_h_d_norm": 0.05803010538220406, "epoch": 7, "gate_p_mean": 0.02801832691766322, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8430569186806679, "gate_p_p10": 0.0031874656677246095, "gate_p_p50": 0.012259292602539064, "gate_p_p90": 0.07807006835937501, "gate_p_std": 0.03860324500128627, "gated_delta_h_c_norm": 0.009111682293587364, "global_step": 870, "grad_norm_max": 0.02555251494050026, "hidden_mse": 0.013570643891580404, "loss": 0.013014932989608497, "lr_blocks": 6.466250186922325e-06, "lr_fusion": 6.466250186922325e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 25.707583856582644, "transport_attention_entropy": 7.318393337726594, "transport_attention_entropy_normalized": 0.9953691639006138, "transport_residual_norm": 0.5630736857652664, "velocity_mse": 0.011657868564361707} +{"anchor_distance_mean": 124.1171875, "avg_context_frames": 14.4, "avg_micro_time_s": 2.7216731567123134, "delta_h_c_norm": 0.14616150269284844, "delta_h_d_norm": 0.05674261008389295, "epoch": 7, "gate_p_mean": 0.02213029300910421, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8870132490992546, "gate_p_p10": 0.0025275230407714845, "gate_p_p50": 0.009143829345703125, "gate_p_p90": 0.06042938232421875, "gate_p_std": 0.03317389680305496, "gated_delta_h_c_norm": 0.00815797309332993, "global_step": 880, "grad_norm_max": 0.016606206074357033, "hidden_mse": 0.011333970917621628, "loss": 0.010967998643172905, "lr_blocks": 6.387014543809224e-06, "lr_fusion": 6.387014543809223e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 25.356010532379152, "transport_attention_entropy": 7.318167561292649, "transport_attention_entropy_normalized": 0.9953384511172771, "transport_residual_norm": 0.5434177279472351, "velocity_mse": 0.009834601572947577} +{"anchor_distance_mean": 124.60546875, "avg_context_frames": 13.200000000000001, "avg_micro_time_s": 2.532419110762385, "delta_h_c_norm": 0.15090641779825092, "delta_h_d_norm": 0.05787387471646071, "epoch": 7, "gate_p_mean": 0.028285132470773536, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8464463397860528, "gate_p_p10": 0.0032016754150390628, "gate_p_p50": 0.0116607666015625, "gate_p_p90": 0.07966361045837403, "gate_p_std": 0.040767420490738004, "gated_delta_h_c_norm": 0.01063091769174207, "global_step": 890, "grad_norm_max": 0.016797978430986404, "hidden_mse": 0.012858641019556673, "loss": 0.012262008295510897, "lr_blocks": 6.3073997047691e-06, "lr_fusion": 6.307399704769099e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 25.719457054138186, "transport_attention_entropy": 7.318109405040741, "transport_attention_entropy_normalized": 0.9953305438160897, "transport_residual_norm": 0.5420850709080697, "velocity_mse": 0.0109761442639865} +{"anchor_distance_mean": 127.26953125, "avg_context_frames": 15.200000000000001, "avg_micro_time_s": 2.7230273774877336, "delta_h_c_norm": 0.15055578416213394, "delta_h_d_norm": 0.05473718973807991, "epoch": 8, "gate_p_mean": 0.02018735440215096, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.9038269028067589, "gate_p_p10": 0.002567768096923828, "gate_p_p50": 0.009487724304199219, "gate_p_p90": 0.05217437744140625, "gate_p_std": 0.02788132712012157, "gated_delta_h_c_norm": 0.0064684849421610124, "global_step": 900, "grad_norm_max": 0.01732567325234413, "hidden_mse": 0.010034900024766103, "loss": 0.009696910512866454, "lr_blocks": 6.227427435703997e-06, "lr_fusion": 6.227427435703997e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 25.745035338401795, "transport_attention_entropy": 7.317987728118897, "transport_attention_entropy_normalized": 0.9953139938414097, "transport_residual_norm": 0.5544639147818089, "velocity_mse": 0.00869342050573323} +{"anchor_distance_mean": 124.60703125, "avg_context_frames": 10.4, "avg_micro_time_s": 2.1948397722499524, "delta_h_c_norm": 0.14921442912891508, "delta_h_d_norm": 0.05988399372436107, "epoch": 8, "gate_p_mean": 0.02786100966623053, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8512713089585304, "gate_p_p10": 0.003300533164292574, "gate_p_p50": 0.012506484985351562, "gate_p_p90": 0.07602157592773438, "gate_p_std": 0.03836733438074589, "gated_delta_h_c_norm": 0.008858460379997268, "global_step": 910, "grad_norm_max": 0.013860423117876053, "hidden_mse": 0.012257619429146872, "loss": 0.01203533073712606, "lr_blocks": 6.147119600233758e-06, "lr_fusion": 6.147119600233758e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 25.597978186607364, "transport_attention_entropy": 7.318073755502701, "transport_attention_entropy_normalized": 0.9953256979584695, "transport_residual_norm": 0.5536615803837777, "velocity_mse": 0.010809568807599135} +{"anchor_distance_mean": 123.89453125, "avg_context_frames": 17.6, "avg_micro_time_s": 3.0191810079376413, "delta_h_c_norm": 0.14879424450919032, "delta_h_d_norm": 0.05864757136441767, "epoch": 8, "gate_p_mean": 0.023046569351572545, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8889798946678639, "gate_p_p10": 0.0029760072473436596, "gate_p_p50": 0.010344505310058594, "gate_p_p90": 0.06072032451629639, "gate_p_std": 0.033031708688940856, "gated_delta_h_c_norm": 0.008140349402674473, "global_step": 920, "grad_norm_max": 0.04859360307455063, "hidden_mse": 0.010933261062018574, "loss": 0.010600265400717035, "lr_blocks": 6.066498153718735e-06, "lr_fusion": 6.066498153718735e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 25.63784501552582, "transport_attention_entropy": 7.317617601156235, "transport_attention_entropy_normalized": 0.9952636547386646, "transport_residual_norm": 0.5434128031134605, "velocity_mse": 0.00950693931663409} +{"anchor_distance_mean": 131.16640625, "avg_context_frames": 12.4, "avg_micro_time_s": 2.3935978776875344, "delta_h_c_norm": 0.14905258128419518, "delta_h_d_norm": 0.05438901195302606, "epoch": 8, "gate_p_mean": 0.021999594720546158, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8873921051621437, "gate_p_p10": 0.002167797088623047, "gate_p_p50": 0.008241462707519532, "gate_p_p90": 0.06184921264648438, "gate_p_std": 0.03496233262121678, "gated_delta_h_c_norm": 0.007846889790380373, "global_step": 930, "grad_norm_max": 0.022887684404850006, "hidden_mse": 0.011940317711560056, "loss": 0.011908981570741162, "lr_blocks": 5.985585137257401e-06, "lr_fusion": 5.985585137257401e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 26.053665590286258, "transport_attention_entropy": 7.317410874366761, "transport_attention_entropy_normalized": 0.9952355355024338, "transport_residual_norm": 0.5325871720910073, "velocity_mse": 0.010714949778048322} +{"anchor_distance_mean": 124.83359375, "avg_context_frames": 16.0, "avg_micro_time_s": 2.8725557729248976, "delta_h_c_norm": 0.14524998357519509, "delta_h_d_norm": 0.056105716992169624, "epoch": 8, "gate_p_mean": 0.024018147762399168, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8747075617313386, "gate_p_p10": 0.0031528472900390625, "gate_p_p50": 0.011764144897460938, "gate_p_p90": 0.06328532695770264, "gate_p_std": 0.031180481391493233, "gated_delta_h_c_norm": 0.0070750988044892445, "global_step": 940, "grad_norm_max": 0.02506624348461628, "hidden_mse": 0.010837928287219257, "loss": 0.010790257746702992, "lr_blocks": 5.904402671660551e-06, "lr_fusion": 5.90440267166055e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 26.36360003948212, "transport_attention_entropy": 7.317462003231049, "transport_attention_entropy_normalized": 0.995242490619421, "transport_residual_norm": 0.5448912024497986, "velocity_mse": 0.009706464913324454} +{"anchor_distance_mean": 122.13671875, "avg_context_frames": 14.0, "avg_micro_time_s": 2.5931629945874195, "delta_h_c_norm": 0.1405942840501666, "delta_h_d_norm": 0.05770798991434276, "epoch": 8, "gate_p_mean": 0.028956532315351072, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8427188768982887, "gate_p_p10": 0.0032783508300781253, "gate_p_p50": 0.01275787353515625, "gate_p_p90": 0.07989578247070313, "gate_p_std": 0.038683489873073997, "gated_delta_h_c_norm": 0.009080974104290363, "global_step": 950, "grad_norm_max": 0.03256500884890556, "hidden_mse": 0.012364835973130539, "loss": 0.012114677138742992, "lr_blocks": 5.82297295140367e-06, "lr_fusion": 5.8229729514036705e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 26.089826822280884, "transport_attention_entropy": 7.31755502820015, "transport_attention_entropy_normalized": 0.9952551409602166, "transport_residual_norm": 0.5492292918264866, "velocity_mse": 0.01087819353269879} +{"anchor_distance_mean": 123.921875, "avg_context_frames": 12.8, "avg_micro_time_s": 2.4293219201376814, "delta_h_c_norm": 0.14042495023459198, "delta_h_d_norm": 0.05820210580714047, "epoch": 8, "gate_p_mean": 0.02263833564356901, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8766571789979936, "gate_p_p10": 0.0023082637228071692, "gate_p_p50": 0.009085845947265626, "gate_p_p90": 0.06321496963500976, "gate_p_std": 0.033648165571503345, "gated_delta_h_c_norm": 0.0073480246748658835, "global_step": 960, "grad_norm_max": 0.025038599967956543, "hidden_mse": 0.012051420728676022, "loss": 0.011498528500669637, "lr_blocks": 5.74131823855921e-06, "lr_fusion": 5.74131823855921e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 26.455458211898804, "transport_attention_entropy": 7.316908687353134, "transport_attention_entropy_normalized": 0.9951672360301018, "transport_residual_norm": 0.5566764928400517, "velocity_mse": 0.010293386483681388} +{"anchor_distance_mean": 125.15703125, "avg_context_frames": 13.600000000000001, "avg_micro_time_s": 2.5337916076127387, "delta_h_c_norm": 0.15064044678583743, "delta_h_d_norm": 0.05816775136627257, "epoch": 8, "gate_p_mean": 0.02764052672428079, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8488910049200058, "gate_p_p10": 0.003192520141601563, "gate_p_p50": 0.012454986572265625, "gate_p_p90": 0.07605481147766113, "gate_p_std": 0.03742671718937345, "gated_delta_h_c_norm": 0.008700084185693414, "global_step": 970, "grad_norm_max": 0.014982348307967186, "hidden_mse": 0.01165425482322462, "loss": 0.01119596173230093, "lr_blocks": 5.659460856710346e-06, "lr_fusion": 5.6594608567103456e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 25.899741721153262, "transport_attention_entropy": 7.316739976406097, "transport_attention_entropy_normalized": 0.9951442860066891, "transport_residual_norm": 0.5516995873302222, "velocity_mse": 0.010030536225531251} +{"anchor_distance_mean": 126.39687500000001, "avg_context_frames": 18.8, "avg_micro_time_s": 3.098569329312568, "delta_h_c_norm": 0.16262076077982784, "delta_h_d_norm": 0.058818761166185146, "epoch": 8, "gate_p_mean": 0.02078104147221893, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8992986515164376, "gate_p_p10": 0.0027788162231445314, "gate_p_p50": 0.009972190856933594, "gate_p_p90": 0.05354766845703125, "gate_p_std": 0.028254632803145798, "gated_delta_h_c_norm": 0.006528569369402249, "global_step": 980, "grad_norm_max": 0.021367453038692474, "hidden_mse": 0.010144883306929843, "loss": 0.009803011649637483, "lr_blocks": 5.577423184847932e-06, "lr_fusion": 5.577423184847932e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 26.20010850429535, "transport_attention_entropy": 7.316325062513352, "transport_attention_entropy_normalized": 0.9950878575444222, "transport_residual_norm": 0.5456342734396458, "velocity_mse": 0.008788523310795427} +{"anchor_distance_mean": 122.85078125000001, "avg_context_frames": 12.4, "avg_micro_time_s": 2.35033848602493, "delta_h_c_norm": 0.15172542305663228, "delta_h_d_norm": 0.05714401956647635, "epoch": 8, "gate_p_mean": 0.025878568354528397, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.860998122394085, "gate_p_p10": 0.003200531005859375, "gate_p_p50": 0.011853790283203125, "gate_p_p90": 0.06996359825134278, "gate_p_std": 0.034988369699567555, "gated_delta_h_c_norm": 0.008219504579028581, "global_step": 990, "grad_norm_max": 0.017974967136979103, "hidden_mse": 0.011652912804856898, "loss": 0.011236765689682217, "lr_blocks": 5.495227651252315e-06, "lr_fusion": 5.495227651252315e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 26.151019310951234, "transport_attention_entropy": 7.3164620280265815, "transport_attention_entropy_normalized": 0.9951064839959145, "transport_residual_norm": 0.5632832691073418, "velocity_mse": 0.010071474470896647} +{"anchor_distance_mean": 128.17890625, "avg_context_frames": 15.200000000000001, "avg_micro_time_s": 2.688857589675081, "delta_h_c_norm": 0.16219016872346403, "delta_h_d_norm": 0.05758567908778787, "epoch": 8, "gate_p_mean": 0.024029436352429913, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8735076405107975, "gate_p_p10": 0.002756023406982422, "gate_p_p50": 0.010170936584472656, "gate_p_p90": 0.06485787630081177, "gate_p_std": 0.034820662043057386, "gated_delta_h_c_norm": 0.00891699751955457, "global_step": 1000, "grad_norm_max": 0.014082008972764015, "hidden_mse": 0.01129167934996076, "loss": 0.011122855028952473, "lr_blocks": 5.412896727361663e-06, "lr_fusion": 5.4128967273616625e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 26.036041855812073, "transport_attention_entropy": 7.316038900613785, "transport_attention_entropy_normalized": 0.9950489349663258, "transport_residual_norm": 0.5467520274221898, "velocity_mse": 0.009993687059613877} +{"anchor_distance_mean": 121.68125, "avg_context_frames": 17.6, "avg_micro_time_s": 2.9681104818625954, "delta_h_c_norm": 0.16163157215341928, "delta_h_d_norm": 0.054704887047410014, "epoch": 8, "gate_p_mean": 0.018333630566485227, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.9135171540081501, "gate_p_p10": 0.0020584010519087317, "gate_p_p50": 0.007893943786621093, "gate_p_p90": 0.04951808452606202, "gate_p_std": 0.027106753864791246, "gated_delta_h_c_norm": 0.006848913625435671, "global_step": 1010, "grad_norm_max": 0.021211620420217514, "hidden_mse": 0.009975762918475085, "loss": 0.009336884249933064, "lr_blocks": 5.3304529216284974e-06, "lr_fusion": 5.330452921628497e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 26.210320281982423, "transport_attention_entropy": 7.316512548923493, "transport_attention_entropy_normalized": 0.9951133534312249, "transport_residual_norm": 0.5427210822701455, "velocity_mse": 0.008339307949063368} +{"anchor_distance_mean": 123.37031250000001, "avg_context_frames": 14.8, "avg_micro_time_s": 2.640190094599939, "delta_h_c_norm": 0.15809401702135803, "delta_h_d_norm": 0.057467682752758265, "epoch": 8, "gate_p_mean": 0.025798179308185357, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8621860280632974, "gate_p_p10": 0.0029695510864257814, "gate_p_p50": 0.011503219604492188, "gate_p_p90": 0.07133510112762452, "gate_p_std": 0.034893017006106676, "gated_delta_h_c_norm": 0.008590206886583475, "global_step": 1020, "grad_norm_max": 0.03631850332021713, "hidden_mse": 0.011408686154754833, "loss": 0.010634039907017723, "lr_blocks": 5.247918773366112e-06, "lr_fusion": 5.247918773366112e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 26.402373552322388, "transport_attention_entropy": 7.316456592082978, "transport_attention_entropy_normalized": 0.9951057493686677, "transport_residual_norm": 0.5461413718760014, "velocity_mse": 0.00949317128106486} +{"anchor_distance_mean": 123.50078125, "avg_context_frames": 11.200000000000001, "avg_micro_time_s": 2.224691893525096, "delta_h_c_norm": 0.15294962208718063, "delta_h_d_norm": 0.05899683968164027, "epoch": 9, "gate_p_mean": 0.025586839119205253, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8654029205441476, "gate_p_p10": 0.0027859877794981004, "gate_p_p50": 0.010677719116210939, "gate_p_p90": 0.072636079788208, "gate_p_std": 0.03630858175456524, "gated_delta_h_c_norm": 0.008445706257771236, "global_step": 1030, "grad_norm_max": 0.030229581519961357, "hidden_mse": 0.01211386383511126, "loss": 0.011869869087240659, "lr_blocks": 5.165316846586541e-06, "lr_fusion": 5.165316846586541e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 26.13572416305542, "transport_attention_entropy": 7.316383183002472, "transport_attention_entropy_normalized": 0.9950957596302032, "transport_residual_norm": 0.5567125879228115, "velocity_mse": 0.010658482689177618} +{"anchor_distance_mean": 124.28125, "avg_context_frames": 18.400000000000002, "avg_micro_time_s": 3.081385399950341, "delta_h_c_norm": 0.1555914303287864, "delta_h_d_norm": 0.05637406962923706, "epoch": 9, "gate_p_mean": 0.024188008491182703, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8739726327359677, "gate_p_p10": 0.0035513877868652345, "gate_p_p50": 0.012784767150878907, "gate_p_p90": 0.05956447124481201, "gate_p_std": 0.03108990576583892, "gated_delta_h_c_norm": 0.007766205872758292, "global_step": 1040, "grad_norm_max": 0.016889436170458794, "hidden_mse": 0.009499587252503262, "loss": 0.009092539860284887, "lr_blocks": 5.082669723831793e-06, "lr_fusion": 5.0826697238317935e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 26.22589807510376, "transport_attention_entropy": 7.315995579957963, "transport_attention_entropy_normalized": 0.9950430445373059, "transport_residual_norm": 0.5435118284076452, "velocity_mse": 0.00814258116297424} +{"anchor_distance_mean": 122.55781250000001, "avg_context_frames": 14.4, "avg_micro_time_s": 2.587466039187166, "delta_h_c_norm": 0.1538476780988276, "delta_h_d_norm": 0.05625344426371157, "epoch": 9, "gate_p_mean": 0.020022237816010605, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8960254199802876, "gate_p_p10": 0.0019979000091552737, "gate_p_p50": 0.007589149475097656, "gate_p_p90": 0.05497729778289795, "gate_p_std": 0.03156762860016898, "gated_delta_h_c_norm": 0.006669452394999099, "global_step": 1050, "grad_norm_max": 0.014649482443928719, "hidden_mse": 0.011014502332545818, "loss": 0.010570967898820527, "lr_blocks": 5e-06, "lr_fusion": 5e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 26.291130924224856, "transport_attention_entropy": 7.316164249181748, "transport_attention_entropy_normalized": 0.9950659871101379, "transport_residual_norm": 0.5569679439067841, "velocity_mse": 0.009469517698744313} +{"anchor_distance_mean": 123.9890625, "avg_context_frames": 18.0, "avg_micro_time_s": 3.0101133284874777, "delta_h_c_norm": 0.15145970592275262, "delta_h_d_norm": 0.058726914180442694, "epoch": 9, "gate_p_mean": 0.020665876363636926, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.9021995440125465, "gate_p_p10": 0.0026636600494384767, "gate_p_p50": 0.009382057189941407, "gate_p_p90": 0.05360488891601563, "gate_p_std": 0.028656160400714725, "gated_delta_h_c_norm": 0.0062917002680478624, "global_step": 1060, "grad_norm_max": 0.012596381828188896, "hidden_mse": 0.009892514377133921, "loss": 0.009660356590757146, "lr_blocks": 4.917330276168208e-06, "lr_fusion": 4.917330276168208e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 26.31375141143799, "transport_attention_entropy": 7.3159913480281835, "transport_attention_entropy_normalized": 0.9950424678623677, "transport_residual_norm": 0.546486982703209, "velocity_mse": 0.00867110515537206} +{"anchor_distance_mean": 120.73984375, "avg_context_frames": 16.400000000000002, "avg_micro_time_s": 2.8104498767001136, "delta_h_c_norm": 0.15138269439339638, "delta_h_d_norm": 0.06426572185009718, "epoch": 9, "gate_p_mean": 0.030019615450873973, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8404357220977545, "gate_p_p10": 0.0034769058227539064, "gate_p_p50": 0.013380622863769532, "gate_p_p90": 0.08307266235351562, "gate_p_std": 0.04113084780983627, "gated_delta_h_c_norm": 0.010255929963022936, "global_step": 1070, "grad_norm_max": 0.015868356451392174, "hidden_mse": 0.01286778036155738, "loss": 0.0119130693085026, "lr_blocks": 4.8346831534134595e-06, "lr_fusion": 4.834683153413459e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 27.06735112667084, "transport_attention_entropy": 7.316149598360062, "transport_attention_entropy_normalized": 0.9950639933347702, "transport_residual_norm": 0.5392203032970428, "velocity_mse": 0.010626291282824241} +{"anchor_distance_mean": 124.44375000000001, "avg_context_frames": 13.200000000000001, "avg_micro_time_s": 2.4364533534502244, "delta_h_c_norm": 0.15363169107586147, "delta_h_d_norm": 0.05964015205390752, "epoch": 9, "gate_p_mean": 0.027033585973549636, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8521414525806904, "gate_p_p10": 0.003349876403808594, "gate_p_p50": 0.012637710571289063, "gate_p_p90": 0.07231063842773437, "gate_p_std": 0.036415228957775984, "gated_delta_h_c_norm": 0.00828403138439171, "global_step": 1080, "grad_norm_max": 0.028934592381119728, "hidden_mse": 0.0112487013975624, "loss": 0.010856466519180687, "lr_blocks": 4.752081226633888e-06, "lr_fusion": 4.7520812266338885e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 26.244350242614747, "transport_attention_entropy": 7.316027718782426, "transport_attention_entropy_normalized": 0.9950474157929421, "transport_residual_norm": 0.5520883284509183, "velocity_mse": 0.00973159636487253} +{"anchor_distance_mean": 129.70546875, "avg_context_frames": 13.200000000000001, "avg_micro_time_s": 2.4287759413127787, "delta_h_c_norm": 0.1600960028357804, "delta_h_d_norm": 0.059951267763972284, "epoch": 9, "gate_p_mean": 0.030113672243896872, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8370705887675286, "gate_p_p10": 0.003661060333251953, "gate_p_p50": 0.01320953369140625, "gate_p_p90": 0.08263987302780151, "gate_p_std": 0.04167303391732276, "gated_delta_h_c_norm": 0.010637838662660215, "global_step": 1090, "grad_norm_max": 0.01631160080432892, "hidden_mse": 0.012123100977623836, "loss": 0.011690640571760015, "lr_blocks": 4.669547078371503e-06, "lr_fusion": 4.669547078371504e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 26.96340208053589, "transport_attention_entropy": 7.315907639265061, "transport_attention_entropy_normalized": 0.9950310818850995, "transport_residual_norm": 0.5557765901088715, "velocity_mse": 0.010478330415207893} +{"anchor_distance_mean": 125.22500000000001, "avg_context_frames": 13.200000000000001, "avg_micro_time_s": 2.442250679037852, "delta_h_c_norm": 0.1567316310480237, "delta_h_d_norm": 0.05940166623331607, "epoch": 9, "gate_p_mean": 0.02974838856025599, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8381049904972315, "gate_p_p10": 0.0026935577392578126, "gate_p_p50": 0.011645317077636719, "gate_p_p90": 0.08501167297363282, "gate_p_std": 0.04335858386475593, "gated_delta_h_c_norm": 0.0103097530969535, "global_step": 1100, "grad_norm_max": 0.015564591623842716, "hidden_mse": 0.011696532991481946, "loss": 0.01143254904600326, "lr_blocks": 4.587103272638339e-06, "lr_fusion": 4.5871032726383386e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 26.423355388641358, "transport_attention_entropy": 7.315313822031022, "transport_attention_entropy_normalized": 0.9949503175914288, "transport_residual_norm": 0.5399489134550095, "velocity_mse": 0.010262895778578241} +{"anchor_distance_mean": 127.56328125, "avg_context_frames": 11.600000000000001, "avg_micro_time_s": 2.2567869444752433, "delta_h_c_norm": 0.1565592311322689, "delta_h_d_norm": 0.06011479105800391, "epoch": 9, "gate_p_mean": 0.029609600768890233, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8410905182361603, "gate_p_p10": 0.0034648895263671878, "gate_p_p50": 0.013224411010742189, "gate_p_p90": 0.08076231479644776, "gate_p_std": 0.04126442160923034, "gated_delta_h_c_norm": 0.009710069856373594, "global_step": 1110, "grad_norm_max": 0.014240499585866928, "hidden_mse": 0.011712342232931406, "loss": 0.011655850932584144, "lr_blocks": 4.504772348747687e-06, "lr_fusion": 4.504772348747687e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 27.046712398529053, "transport_attention_entropy": 7.315305364131928, "transport_attention_entropy_normalized": 0.9949491679668427, "transport_residual_norm": 0.5588420152664185, "velocity_mse": 0.010484616659232416} +{"anchor_distance_mean": 121.29140625000001, "avg_context_frames": 15.600000000000001, "avg_micro_time_s": 2.7324620380751186, "delta_h_c_norm": 0.15035151215270162, "delta_h_d_norm": 0.058230282133445145, "epoch": 9, "gate_p_mean": 0.02531923589995131, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8635637268424035, "gate_p_p10": 0.0031867027282714844, "gate_p_p50": 0.011891937255859375, "gate_p_p90": 0.06660089492797852, "gate_p_std": 0.034161703754216434, "gated_delta_h_c_norm": 0.007334175358118956, "global_step": 1120, "grad_norm_max": 0.019130412489175797, "hidden_mse": 0.011075109714874999, "loss": 0.011175883247051389, "lr_blocks": 4.42257681515207e-06, "lr_fusion": 4.4225768151520694e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 26.627007079124454, "transport_attention_entropy": 7.31526420712471, "transport_attention_entropy_normalized": 0.9949435688555242, "transport_residual_norm": 0.5456662178039551, "velocity_mse": 0.010068372244131752} +{"anchor_distance_mean": 128.04765625000002, "avg_context_frames": 14.4, "avg_micro_time_s": 2.5835536837124438, "delta_h_c_norm": 0.15583035452291372, "delta_h_d_norm": 0.0576229733414948, "epoch": 9, "gate_p_mean": 0.024222791643114763, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8754129357635976, "gate_p_p10": 0.0032173156738281253, "gate_p_p50": 0.011232757568359376, "gate_p_p90": 0.06410396099090576, "gate_p_std": 0.03347649758798071, "gated_delta_h_c_norm": 0.00719751923752483, "global_step": 1130, "grad_norm_max": 0.016371052712202072, "hidden_mse": 0.009724508156068623, "loss": 0.009619711327832193, "lr_blocks": 4.340539143289655e-06, "lr_fusion": 4.3405391432896555e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 26.72406256198883, "transport_attention_entropy": 7.314890521764756, "transport_attention_entropy_normalized": 0.9948927454650403, "transport_residual_norm": 0.5497245073318482, "velocity_mse": 0.008647260518046096} +{"anchor_distance_mean": 124.86640625000001, "avg_context_frames": 14.8, "avg_micro_time_s": 2.647372242650045, "delta_h_c_norm": 0.15854041064158084, "delta_h_d_norm": 0.06024333229288459, "epoch": 9, "gate_p_mean": 0.02318541841232218, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.884014954045415, "gate_p_p10": 0.002769451076164842, "gate_p_p50": 0.010473251342773438, "gate_p_p90": 0.06010971069335938, "gate_p_std": 0.03327623861841857, "gated_delta_h_c_norm": 0.007238986974698491, "global_step": 1140, "grad_norm_max": 0.014775000512599945, "hidden_mse": 0.010031674930360168, "loss": 0.009843346988782288, "lr_blocks": 4.25868176144079e-06, "lr_fusion": 4.2586817614407895e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 26.470372080802917, "transport_attention_entropy": 7.31484598517418, "transport_attention_entropy_normalized": 0.994886688143015, "transport_residual_norm": 0.5500499092042447, "velocity_mse": 0.008840179510298186} +{"anchor_distance_mean": 125.80859375, "avg_context_frames": 11.200000000000001, "avg_micro_time_s": 2.2155154182250043, "delta_h_c_norm": 0.15862744795158507, "delta_h_d_norm": 0.061649324884638194, "epoch": 9, "gate_p_mean": 0.03217437305720523, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8223432745784521, "gate_p_p10": 0.0037966728210449222, "gate_p_p50": 0.01431102752685547, "gate_p_p90": 0.08885955810546875, "gate_p_std": 0.04393828559550457, "gated_delta_h_c_norm": 0.009947744282544592, "global_step": 1150, "grad_norm_max": 0.01799335703253746, "hidden_mse": 0.012119777937186882, "loss": 0.01184524434793275, "lr_blocks": 4.17702704859633e-06, "lr_fusion": 4.17702704859633e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 27.157835125923157, "transport_attention_entropy": 7.314711570739746, "transport_attention_entropy_normalized": 0.9948684088885784, "transport_residual_norm": 0.5592629112303257, "velocity_mse": 0.010633266475633719} +{"anchor_distance_mean": 122.30000000000001, "avg_context_frames": 13.600000000000001, "avg_micro_time_s": 2.5052872990375366, "delta_h_c_norm": 0.1608564630150795, "delta_h_d_norm": 0.06181143769063056, "epoch": 10, "gate_p_mean": 0.028832618601154537, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8459963724017143, "gate_p_p10": 0.003222942352294922, "gate_p_p50": 0.012541580200195314, "gate_p_p90": 0.07840118408203126, "gate_p_std": 0.04013153177220374, "gated_delta_h_c_norm": 0.009307445686135908, "global_step": 1160, "grad_norm_max": 0.012601014226675034, "hidden_mse": 0.011077569983899594, "loss": 0.01068393843015656, "lr_blocks": 4.0955973283394525e-06, "lr_fusion": 4.095597328339452e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 26.602990794181824, "transport_attention_entropy": 7.3143147289752966, "transport_attention_entropy_normalized": 0.994814432412386, "transport_residual_norm": 0.5551229491829872, "velocity_mse": 0.009576181412558072} +{"anchor_distance_mean": 127.72421875, "avg_context_frames": 15.600000000000001, "avg_micro_time_s": 2.7259478997374575, "delta_h_c_norm": 0.16050737909972668, "delta_h_d_norm": 0.06008464368060232, "epoch": 10, "gate_p_mean": 0.03011004113359377, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8435850143432617, "gate_p_p10": 0.0038431167602539064, "gate_p_p50": 0.013435745239257814, "gate_p_p90": 0.08233352899551392, "gate_p_std": 0.04135215821443126, "gated_delta_h_c_norm": 0.009519624547101558, "global_step": 1170, "grad_norm_max": 0.018284721300005913, "hidden_mse": 0.011029497260460631, "loss": 0.011137608985882253, "lr_blocks": 4.0144148627426e-06, "lr_fusion": 4.0144148627425993e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 26.823865389823915, "transport_attention_entropy": 7.314214134216309, "transport_attention_entropy_normalized": 0.9948007501661777, "transport_residual_norm": 0.5390740059316158, "velocity_mse": 0.01003465923422482} +{"anchor_distance_mean": 123.5671875, "avg_context_frames": 11.600000000000001, "avg_micro_time_s": 2.2690418207500898, "delta_h_c_norm": 0.16166560472920538, "delta_h_d_norm": 0.06064367033541203, "epoch": 10, "gate_p_mean": 0.028126372548285872, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8422809258103371, "gate_p_p10": 0.003261566162109375, "gate_p_p50": 0.01243743896484375, "gate_p_p90": 0.07579092979431153, "gate_p_std": 0.03939712225692347, "gated_delta_h_c_norm": 0.008216257518506609, "global_step": 1180, "grad_norm_max": 0.01388850249350071, "hidden_mse": 0.011256787442835049, "loss": 0.010731451975880192, "lr_blocks": 3.9335018462812664e-06, "lr_fusion": 3.933501846281267e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 26.8377414226532, "transport_attention_entropy": 7.314276731014252, "transport_attention_entropy_normalized": 0.9948092594742776, "transport_residual_norm": 0.5691400520503521, "velocity_mse": 0.009605773264775054} +{"anchor_distance_mean": 124.18359375, "avg_context_frames": 13.600000000000001, "avg_micro_time_s": 2.5050134035501745, "delta_h_c_norm": 0.16483724610880018, "delta_h_d_norm": 0.059822146268561485, "epoch": 10, "gate_p_mean": 0.02694330947706476, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8552376568317414, "gate_p_p10": 0.0030070304870605472, "gate_p_p50": 0.011732482910156251, "gate_p_p90": 0.07429945468902588, "gate_p_std": 0.037260681862244384, "gated_delta_h_c_norm": 0.008062545310531277, "global_step": 1190, "grad_norm_max": 0.02043301798403263, "hidden_mse": 0.010408595064654948, "loss": 0.010113031943910757, "lr_blocks": 3.852880399766243e-06, "lr_fusion": 3.852880399766243e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 26.94810438156128, "transport_attention_entropy": 7.313863903284073, "transport_attention_entropy_normalized": 0.9947531200945378, "transport_residual_norm": 0.5563373323529959, "velocity_mse": 0.009072172490414232} +{"anchor_distance_mean": 123.89140625, "avg_context_frames": 15.200000000000001, "avg_micro_time_s": 2.7122202388998633, "delta_h_c_norm": 0.16194385886192322, "delta_h_d_norm": 0.06166584882885218, "epoch": 10, "gate_p_mean": 0.026693264825735242, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8578944072127342, "gate_p_p10": 0.0031788825988769533, "gate_p_p50": 0.012066078186035157, "gate_p_p90": 0.07154325246810914, "gate_p_std": 0.036926806467818096, "gated_delta_h_c_norm": 0.008238920160511044, "global_step": 1200, "grad_norm_max": 0.012661724351346493, "hidden_mse": 0.010346713894978167, "loss": 0.010052260244265199, "lr_blocks": 3.7725725642960047e-06, "lr_fusion": 3.772572564296005e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 26.61114811897278, "transport_attention_entropy": 7.31344176530838, "transport_attention_entropy_normalized": 0.9946957021951676, "transport_residual_norm": 0.5500395834445954, "velocity_mse": 0.009017588826827705} +{"anchor_distance_mean": 124.34609375000001, "avg_context_frames": 13.200000000000001, "avg_micro_time_s": 2.4740924538502442, "delta_h_c_norm": 0.16420370209962132, "delta_h_d_norm": 0.06382414423860609, "epoch": 10, "gate_p_mean": 0.02839667362859473, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8475809030234814, "gate_p_p10": 0.0030824661254882815, "gate_p_p50": 0.0118743896484375, "gate_p_p90": 0.07829374074935913, "gate_p_std": 0.041412193630822006, "gated_delta_h_c_norm": 0.009519567571987865, "global_step": 1210, "grad_norm_max": 0.0250510536134243, "hidden_mse": 0.010746340354671702, "loss": 0.010579046301427298, "lr_blocks": 3.6926002952309015e-06, "lr_fusion": 3.6926002952309016e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 26.420643067359926, "transport_attention_entropy": 7.3136444151401525, "transport_attention_entropy_normalized": 0.9947232611477376, "transport_residual_norm": 0.5519425094127656, "velocity_mse": 0.009504412274691277} +{"anchor_distance_mean": 124.21484375, "avg_context_frames": 13.600000000000001, "avg_micro_time_s": 2.517469553475212, "delta_h_c_norm": 0.16420199582353234, "delta_h_d_norm": 0.05939948991872371, "epoch": 10, "gate_p_mean": 0.02483575087389909, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8728683643043041, "gate_p_p10": 0.0024283407256007195, "gate_p_p50": 0.009894943237304688, "gate_p_p90": 0.06909099817276002, "gate_p_std": 0.037538453895831485, "gated_delta_h_c_norm": 0.00856635719537735, "global_step": 1220, "grad_norm_max": 0.010213730856776237, "hidden_mse": 0.01058533990290016, "loss": 0.010471497103571892, "lr_blocks": 3.6129854561907786e-06, "lr_fusion": 3.612985456190778e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 26.85541779994965, "transport_attention_entropy": 7.3136091291904455, "transport_attention_entropy_normalized": 0.994718460738659, "transport_residual_norm": 0.5507500469684601, "velocity_mse": 0.009412963144131937} +{"anchor_distance_mean": 124.73671875000001, "avg_context_frames": 16.8, "avg_micro_time_s": 2.8738812289373983, "delta_h_c_norm": 0.16593185458332302, "delta_h_d_norm": 0.061105809593573215, "epoch": 10, "gate_p_mean": 0.0328303751652129, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8321239262819291, "gate_p_p10": 0.004310760367661715, "gate_p_p50": 0.015091323852539064, "gate_p_p90": 0.08785519599914551, "gate_p_std": 0.04406113538425416, "gated_delta_h_c_norm": 0.010942025993426797, "global_step": 1230, "grad_norm_max": 0.013675645925104618, "hidden_mse": 0.011338166508357972, "loss": 0.010871921048965305, "lr_blocks": 3.533749813077677e-06, "lr_fusion": 3.533749813077677e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 26.97074270248413, "transport_attention_entropy": 7.313252174854279, "transport_attention_entropy_normalized": 0.9946699135005475, "transport_residual_norm": 0.5434968672692776, "velocity_mse": 0.00973810439463705} +{"anchor_distance_mean": 123.95937500000001, "avg_context_frames": 14.4, "avg_micro_time_s": 2.5923218528748295, "delta_h_c_norm": 0.16584452148526907, "delta_h_d_norm": 0.06335230651311577, "epoch": 10, "gate_p_mean": 0.02858982678735629, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8469443887472153, "gate_p_p10": 0.0029919624328613283, "gate_p_p50": 0.0121795654296875, "gate_p_p90": 0.07815976142883302, "gate_p_std": 0.041192692040931435, "gated_delta_h_c_norm": 0.008768844525911845, "global_step": 1240, "grad_norm_max": 0.010443191975355148, "hidden_mse": 0.011675086442846806, "loss": 0.011687194928526878, "lr_blocks": 3.4549150281252635e-06, "lr_fusion": 3.4549150281252636e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 26.738714838027956, "transport_attention_entropy": 7.3130121588706976, "transport_attention_entropy_normalized": 0.99463726952672, "transport_residual_norm": 0.5537749290466308, "velocity_mse": 0.010519686323823408} +{"anchor_distance_mean": 127.36640625000001, "avg_context_frames": 15.600000000000001, "avg_micro_time_s": 2.737068118112484, "delta_h_c_norm": 0.16806902550160885, "delta_h_d_norm": 0.06344140940345824, "epoch": 10, "gate_p_mean": 0.03166997412918136, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8322734132409096, "gate_p_p10": 0.003993034362792969, "gate_p_p50": 0.014467620849609376, "gate_p_p90": 0.08473503589630127, "gate_p_std": 0.042204773542471236, "gated_delta_h_c_norm": 0.01017534169368446, "global_step": 1250, "grad_norm_max": 0.020155804231762886, "hidden_mse": 0.011150315569830128, "loss": 0.01097514876164496, "lr_blocks": 3.3765026539765832e-06, "lr_fusion": 3.3765026539765834e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 27.051328372955325, "transport_attention_entropy": 7.312256002426148, "transport_attention_entropy_normalized": 0.9945344291627407, "transport_residual_norm": 0.552873506769538, "velocity_mse": 0.009860117235803045} +{"anchor_distance_mean": 124.08359375, "avg_context_frames": 12.4, "avg_micro_time_s": 2.368988572450053, "delta_h_c_norm": 0.1610536303371191, "delta_h_d_norm": 0.061836072243750095, "epoch": 10, "gate_p_mean": 0.03220774562796578, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8251234717667103, "gate_p_p10": 0.0036867141723632814, "gate_p_p50": 0.014856338500976562, "gate_p_p90": 0.08877377510070801, "gate_p_std": 0.04250358992721886, "gated_delta_h_c_norm": 0.010325551894493402, "global_step": 1260, "grad_norm_max": 0.036045268177986145, "hidden_mse": 0.011476106790360064, "loss": 0.010592275758972392, "lr_blocks": 3.298534127791785e-06, "lr_fusion": 3.298534127791785e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 26.83733291625977, "transport_attention_entropy": 7.312657153606415, "transport_attention_entropy_normalized": 0.9945889852941037, "transport_residual_norm": 0.5613888032734394, "velocity_mse": 0.009444665105547756} +{"anchor_distance_mean": 126.0671875, "avg_context_frames": 15.200000000000001, "avg_micro_time_s": 2.7012517276752988, "delta_h_c_norm": 0.16843597143888475, "delta_h_d_norm": 0.06138218678534031, "epoch": 10, "gate_p_mean": 0.026749552361434326, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.863714972883463, "gate_p_p10": 0.002716541290283203, "gate_p_p50": 0.010716629028320313, "gate_p_p90": 0.07393141984939576, "gate_p_std": 0.03904568409197964, "gated_delta_h_c_norm": 0.009704780452739216, "global_step": 1270, "grad_norm_max": 0.026337573304772377, "hidden_mse": 0.011322892477619462, "loss": 0.010916158682084643, "lr_blocks": 3.2210307653874175e-06, "lr_fusion": 3.221030765387417e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 27.09224569797516, "transport_attention_entropy": 7.312416750192643, "transport_attention_entropy_normalized": 0.994556287676096, "transport_residual_norm": 0.5436145558953286, "velocity_mse": 0.009783869417151437} +{"anchor_distance_mean": 124.41250000000001, "avg_context_frames": 16.0, "avg_micro_time_s": 2.780051047737243, "delta_h_c_norm": 0.16605112114921214, "delta_h_d_norm": 0.06158264786936343, "epoch": 10, "gate_p_mean": 0.02662578249583021, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8601672925055027, "gate_p_p10": 0.0031131744384765626, "gate_p_p50": 0.012234878540039063, "gate_p_p90": 0.070709228515625, "gate_p_std": 0.036502320086583495, "gated_delta_h_c_norm": 0.007867004659783561, "global_step": 1280, "grad_norm_max": 0.014377262443304062, "hidden_mse": 0.010201909462921321, "loss": 0.010314290804672055, "lr_blocks": 3.1440137554088957e-06, "lr_fusion": 3.144013755408895e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 26.690449738502505, "transport_attention_entropy": 7.312531453371048, "transport_attention_entropy_normalized": 0.994571889191866, "transport_residual_norm": 0.5487728133797646, "velocity_mse": 0.009294099872931838} +{"anchor_distance_mean": 125.3515625, "avg_context_frames": 16.400000000000002, "avg_micro_time_s": 2.8143315299124883, "delta_h_c_norm": 0.16618117019534112, "delta_h_d_norm": 0.06263511534780264, "epoch": 11, "gate_p_mean": 0.02753282201010734, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8625678852200509, "gate_p_p10": 0.003218898642808199, "gate_p_p50": 0.011642837524414064, "gate_p_p90": 0.07450637817382813, "gate_p_std": 0.04005218632519245, "gated_delta_h_c_norm": 0.009581647318555042, "global_step": 1290, "grad_norm_max": 0.00878103543072939, "hidden_mse": 0.01082237654773053, "loss": 0.010230615694308654, "lr_blocks": 3.06750415353774e-06, "lr_fusion": 3.0675041535377405e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 26.883077692985538, "transport_attention_entropy": 7.312296694517136, "transport_attention_entropy_normalized": 0.994539961963892, "transport_residual_norm": 0.5474165283143521, "velocity_mse": 0.009148378041572869} +{"anchor_distance_mean": 123.89296875000001, "avg_context_frames": 13.600000000000001, "avg_micro_time_s": 2.518123488037327, "delta_h_c_norm": 0.1662105984054506, "delta_h_d_norm": 0.06383101479150355, "epoch": 11, "gate_p_mean": 0.029718461900483818, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8445728428661824, "gate_p_p10": 0.0033960342407226562, "gate_p_p50": 0.012866592407226563, "gate_p_p90": 0.08100204467773438, "gate_p_std": 0.04289715412305668, "gated_delta_h_c_norm": 0.009717077889945358, "global_step": 1300, "grad_norm_max": 0.013163619674742222, "hidden_mse": 0.011107776797143743, "loss": 0.01097241917741485, "lr_blocks": 2.991522876735154e-06, "lr_fusion": 2.991522876735154e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 26.816860842704774, "transport_attention_entropy": 7.312038713693619, "transport_attention_entropy_normalized": 0.9945048741996289, "transport_residual_norm": 0.557826553285122, "velocity_mse": 0.00986164148489479} +{"anchor_distance_mean": 123.24140625000001, "avg_context_frames": 14.0, "avg_micro_time_s": 2.5841406976877352, "delta_h_c_norm": 0.1665405873209238, "delta_h_d_norm": 0.06317272237502039, "epoch": 11, "gate_p_mean": 0.032844627846498047, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8197373531758786, "gate_p_p10": 0.0039003370329737663, "gate_p_p50": 0.014744186401367189, "gate_p_p90": 0.09008755683898927, "gate_p_std": 0.044556866586208346, "gated_delta_h_c_norm": 0.010078076875652188, "global_step": 1310, "grad_norm_max": 0.012013878673315048, "hidden_mse": 0.011039634991902858, "loss": 0.010665519966278226, "lr_blocks": 2.9160906975235493e-06, "lr_fusion": 2.916090697523549e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 26.720489597320558, "transport_attention_entropy": 7.311800080537797, "transport_attention_entropy_normalized": 0.9944724164903165, "transport_residual_norm": 0.5468261010944844, "velocity_mse": 0.009561556502012536} +{"anchor_distance_mean": 123.23984375, "avg_context_frames": 16.0, "avg_micro_time_s": 2.8165124751626083, "delta_h_c_norm": 0.16892347102984787, "delta_h_d_norm": 0.061618340527638794, "epoch": 11, "gate_p_mean": 0.025209973624441773, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8693407222628594, "gate_p_p10": 0.0029315948486328125, "gate_p_p50": 0.010302352905273439, "gate_p_p90": 0.06907315254211426, "gate_p_std": 0.03800409702234902, "gated_delta_h_c_norm": 0.00827021059812978, "global_step": 1320, "grad_norm_max": 0.015469390898942947, "hidden_mse": 0.010149081971030684, "loss": 0.009753459694911726, "lr_blocks": 2.8412282383075362e-06, "lr_fusion": 2.8412282383075363e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 27.090869641304018, "transport_attention_entropy": 7.311109799146653, "transport_attention_entropy_normalized": 0.9943785324692727, "transport_residual_norm": 0.5484919585287571, "velocity_mse": 0.008738551495480352} +{"anchor_distance_mean": 126.8140625, "avg_context_frames": 12.4, "avg_micro_time_s": 2.36751034836243, "delta_h_c_norm": 0.17196621615439656, "delta_h_d_norm": 0.06404256457462908, "epoch": 11, "gate_p_mean": 0.03384849797585048, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8206480637192727, "gate_p_p10": 0.0032670974731445314, "gate_p_p50": 0.012776565551757813, "gate_p_p90": 0.09686474800109864, "gate_p_std": 0.04992850957205519, "gated_delta_h_c_norm": 0.011691763914132026, "global_step": 1330, "grad_norm_max": 0.02446635439991951, "hidden_mse": 0.011832583759678529, "loss": 0.011822253983700648, "lr_blocks": 2.766955965735968e-06, "lr_fusion": 2.766955965735968e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 27.104646062850954, "transport_attention_entropy": 7.3112672746181495, "transport_attention_entropy_normalized": 0.9943999469280244, "transport_residual_norm": 0.5463154830038548, "velocity_mse": 0.010638995608314872} +{"anchor_distance_mean": 128.146875, "avg_context_frames": 13.200000000000001, "avg_micro_time_s": 2.473328488262223, "delta_h_c_norm": 0.17147258426994086, "delta_h_d_norm": 0.06187727306969464, "epoch": 11, "gate_p_mean": 0.03358238262590021, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8226705513894559, "gate_p_p10": 0.003612613677978516, "gate_p_p50": 0.014145660400390626, "gate_p_p90": 0.09281997680664063, "gate_p_std": 0.04757559162098915, "gated_delta_h_c_norm": 0.011413745087338611, "global_step": 1340, "grad_norm_max": 0.01756114885210991, "hidden_mse": 0.011110243271104991, "loss": 0.011265064059989527, "lr_blocks": 2.693294185106562e-06, "lr_fusion": 2.693294185106562e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 26.532869529724124, "transport_attention_entropy": 7.311199325323106, "transport_attention_entropy_normalized": 0.9943907082080842, "transport_residual_norm": 0.5474575828760863, "velocity_mse": 0.010154039689223283} +{"anchor_distance_mean": 124.44375000000001, "avg_context_frames": 17.2, "avg_micro_time_s": 2.9492596731502547, "delta_h_c_norm": 0.1667318467050791, "delta_h_d_norm": 0.0628164045047015, "epoch": 11, "gate_p_mean": 0.028586652164813132, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8510812543332578, "gate_p_p10": 0.003375835344195366, "gate_p_p50": 0.01231689453125, "gate_p_p90": 0.077728271484375, "gate_p_std": 0.04059396224329248, "gated_delta_h_c_norm": 0.009351104045344982, "global_step": 1350, "grad_norm_max": 0.01247142069041729, "hidden_mse": 0.0100600634876173, "loss": 0.009608169429702685, "lr_blocks": 2.6202630348146323e-06, "lr_fusion": 2.6202630348146324e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 27.317826771736147, "transport_attention_entropy": 7.310988146066666, "transport_attention_entropy_normalized": 0.9943619877099992, "transport_residual_norm": 0.5442668102681637, "velocity_mse": 0.00860216312285047} +{"anchor_distance_mean": 120.80625, "avg_context_frames": 14.8, "avg_micro_time_s": 2.6755482887375366, "delta_h_c_norm": 0.16501008076593282, "delta_h_d_norm": 0.06190628246404231, "epoch": 11, "gate_p_mean": 0.030479902273509652, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8393682628870011, "gate_p_p10": 0.003536701202392578, "gate_p_p50": 0.013834762573242189, "gate_p_p90": 0.08254852294921876, "gate_p_std": 0.0412805850035511, "gated_delta_h_c_norm": 0.009153719447203912, "global_step": 1360, "grad_norm_max": 0.00964505784213543, "hidden_mse": 0.010352587781380863, "loss": 0.010385744052473457, "lr_blocks": 2.5478824808474613e-06, "lr_fusion": 2.547882480847461e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 27.131544733047487, "transport_attention_entropy": 7.310934966802598, "transport_attention_entropy_normalized": 0.9943547546863556, "transport_residual_norm": 0.5653196662664414, "velocity_mse": 0.009350485308095813} +{"anchor_distance_mean": 126.52187500000001, "avg_context_frames": 17.6, "avg_micro_time_s": 2.98308274772503, "delta_h_c_norm": 0.16931853685528042, "delta_h_d_norm": 0.06090507949702442, "epoch": 11, "gate_p_mean": 0.02185182038228959, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8933408740907908, "gate_p_p10": 0.0027464866638183595, "gate_p_p50": 0.010002708435058594, "gate_p_p90": 0.05590178370475769, "gate_p_std": 0.03173611331731081, "gated_delta_h_c_norm": 0.0067036425578407945, "global_step": 1370, "grad_norm_max": 0.007776825223118067, "hidden_mse": 0.009665427700383589, "loss": 0.009147754625882954, "lr_blocks": 2.476172311325783e-06, "lr_fusion": 2.476172311325783e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 27.47069721221924, "transport_attention_entropy": 7.3108069896698, "transport_attention_entropy_normalized": 0.9943373486399651, "transport_residual_norm": 0.5523007467389107, "velocity_mse": 0.008181211852934212} +{"anchor_distance_mean": 126.26171875, "avg_context_frames": 10.8, "avg_micro_time_s": 2.2053428246498243, "delta_h_c_norm": 0.1662767813540995, "delta_h_d_norm": 0.06471165786497295, "epoch": 11, "gate_p_mean": 0.035171467415057125, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8089107807725668, "gate_p_p10": 0.00438995361328125, "gate_p_p50": 0.016826248168945315, "gate_p_p90": 0.09430959224700929, "gate_p_std": 0.04586064473260194, "gated_delta_h_c_norm": 0.010764222603756935, "global_step": 1380, "grad_norm_max": 0.013361796736717224, "hidden_mse": 0.011439427919685842, "loss": 0.011218227920471691, "lr_blocks": 2.4051521310939258e-06, "lr_fusion": 2.405152131093926e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 27.19464569091797, "transport_attention_entropy": 7.3110165178775794, "transport_attention_entropy_normalized": 0.9943658456206322, "transport_residual_norm": 0.5743304885923862, "velocity_mse": 0.010074285141308793} +{"anchor_distance_mean": 125.775, "avg_context_frames": 10.4, "avg_micro_time_s": 2.1502409179750432, "delta_h_c_norm": 0.16484918864443898, "delta_h_d_norm": 0.06560621629469097, "epoch": 11, "gate_p_mean": 0.034846670902334155, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8118612498044968, "gate_p_p10": 0.0035561561584472657, "gate_p_p50": 0.014676666259765625, "gate_p_p90": 0.09739804267883301, "gate_p_std": 0.049101126228924846, "gated_delta_h_c_norm": 0.010746959925745614, "global_step": 1390, "grad_norm_max": 0.015587616711854935, "hidden_mse": 0.011726206820458175, "loss": 0.011543763868394308, "lr_blocks": 2.3348413563600324e-06, "lr_fusion": 2.3348413563600325e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 26.673921060562137, "transport_attention_entropy": 7.310421288013458, "transport_attention_entropy_normalized": 0.9942848883569241, "transport_residual_norm": 0.5633532173931599, "velocity_mse": 0.010371143199154177} +{"anchor_distance_mean": 126.68671875000001, "avg_context_frames": 15.600000000000001, "avg_micro_time_s": 2.7674473427253363, "delta_h_c_norm": 0.1744514448568225, "delta_h_d_norm": 0.06329437359236181, "epoch": 11, "gate_p_mean": 0.027995256090071054, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8521770089864731, "gate_p_p10": 0.0027002334594726563, "gate_p_p50": 0.011534118652343751, "gate_p_p90": 0.07616238594055176, "gate_p_std": 0.04142174338921905, "gated_delta_h_c_norm": 0.008916413137922064, "global_step": 1400, "grad_norm_max": 0.010969742201268673, "hidden_mse": 0.0100864976644516, "loss": 0.01039829924411606, "lr_blocks": 2.265259209387867e-06, "lr_fusion": 2.2652592093878666e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 26.412534284591676, "transport_attention_entropy": 7.3102490067482, "transport_attention_entropy_normalized": 0.9942614547908306, "transport_residual_norm": 0.5502905182540417, "velocity_mse": 0.009389649549848401} +{"anchor_distance_mean": 123.37265625, "avg_context_frames": 14.4, "avg_micro_time_s": 2.6375610174874966, "delta_h_c_norm": 0.16854925006628038, "delta_h_d_norm": 0.06474005617201328, "epoch": 12, "gate_p_mean": 0.02848712010891177, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8530383873730898, "gate_p_p10": 0.0029886245727539064, "gate_p_p50": 0.011908912658691408, "gate_p_p90": 0.07720177173614502, "gate_p_std": 0.04177806725492701, "gated_delta_h_c_norm": 0.009339368098881097, "global_step": 1410, "grad_norm_max": 0.029310505837202072, "hidden_mse": 0.010969682535505855, "loss": 0.01065762183861807, "lr_blocks": 2.1964247132416373e-06, "lr_fusion": 2.196424713241637e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 27.42776310443878, "transport_attention_entropy": 7.309768408536911, "transport_attention_entropy_normalized": 0.9941960886120796, "transport_residual_norm": 0.552886101603508, "velocity_mse": 0.009560653648804873} +{"anchor_distance_mean": 126.03671875, "avg_context_frames": 13.200000000000001, "avg_micro_time_s": 2.4590879360993316, "delta_h_c_norm": 0.16939408481121065, "delta_h_d_norm": 0.0662373281084001, "epoch": 12, "gate_p_mean": 0.03675794765586034, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8010554619133473, "gate_p_p10": 0.004589080810546875, "gate_p_p50": 0.01674308776855469, "gate_p_p90": 0.0991869330406189, "gate_p_std": 0.04852859160164372, "gated_delta_h_c_norm": 0.01120703459600918, "global_step": 1420, "grad_norm_max": 0.013114294968545437, "hidden_mse": 0.011176716044428759, "loss": 0.010795912379398942, "lr_blocks": 2.1283566865852824e-06, "lr_fusion": 2.128356686585282e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 27.036764478683473, "transport_attention_entropy": 7.309696584939957, "transport_attention_entropy_normalized": 0.9941863216459752, "transport_residual_norm": 0.5609064258635045, "velocity_mse": 0.00967824078979902} +{"anchor_distance_mean": 120.38046875, "avg_context_frames": 14.4, "avg_micro_time_s": 2.6356716843751204, "delta_h_c_norm": 0.16283254800364377, "delta_h_d_norm": 0.06372636994346977, "epoch": 12, "gate_p_mean": 0.031092422269284728, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8403468325734139, "gate_p_p10": 0.0035576820373535156, "gate_p_p50": 0.013282012939453126, "gate_p_p90": 0.08480844497680665, "gate_p_std": 0.04514591147890315, "gated_delta_h_c_norm": 0.01007301563222427, "global_step": 1430, "grad_norm_max": 0.02012002095580101, "hidden_mse": 0.010735032905358821, "loss": 0.010256275738356635, "lr_blocks": 2.061073738537635e-06, "lr_fusion": 2.061073738537635e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 27.157652139663696, "transport_attention_entropy": 7.309513092041016, "transport_attention_entropy_normalized": 0.9941613644361497, "transport_residual_norm": 0.553051844984293, "velocity_mse": 0.009182772437634412} +{"anchor_distance_mean": 125.61562500000001, "avg_context_frames": 13.200000000000001, "avg_micro_time_s": 2.490554899625204, "delta_h_c_norm": 0.16931261587888002, "delta_h_d_norm": 0.06691667162813246, "epoch": 12, "gate_p_mean": 0.0318736614077352, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.828893756121397, "gate_p_p10": 0.0034439086914062503, "gate_p_p50": 0.013321113586425782, "gate_p_p90": 0.08826426267623902, "gate_p_std": 0.04494272553129122, "gated_delta_h_c_norm": 0.009140474499145058, "global_step": 1440, "grad_norm_max": 0.007555663585662842, "hidden_mse": 0.011110065341927112, "loss": 0.010966385662322865, "lr_blocks": 1.9945942635848745e-06, "lr_fusion": 1.9945942635848748e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 27.39948482513428, "transport_attention_entropy": 7.309158003330231, "transport_attention_entropy_normalized": 0.9941130705177784, "transport_residual_norm": 0.5654051214456558, "velocity_mse": 0.009855379123473541} +{"anchor_distance_mean": 123.46718750000001, "avg_context_frames": 14.4, "avg_micro_time_s": 2.651497217287306, "delta_h_c_norm": 0.16473795296624305, "delta_h_d_norm": 0.06205941820517183, "epoch": 12, "gate_p_mean": 0.02889831916545518, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8510807566344738, "gate_p_p10": 0.003438758570700884, "gate_p_p50": 0.012929534912109375, "gate_p_p90": 0.07738736867904664, "gate_p_std": 0.040779624442802746, "gated_delta_h_c_norm": 0.008650448112166488, "global_step": 1450, "grad_norm_max": 0.015441550873219967, "hidden_mse": 0.010098521906184033, "loss": 0.00980125256755855, "lr_blocks": 1.928936436551661e-06, "lr_fusion": 1.928936436551661e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 27.106183362007144, "transport_attention_entropy": 7.309205520153046, "transport_attention_entropy_normalized": 0.9941195338964463, "transport_residual_norm": 0.5596297040581704, "velocity_mse": 0.008791400393238292} +{"anchor_distance_mean": 124.05312500000001, "avg_context_frames": 14.0, "avg_micro_time_s": 2.5906990941253754, "delta_h_c_norm": 0.1656919653527439, "delta_h_d_norm": 0.06468254420906305, "epoch": 12, "gate_p_mean": 0.03644161582924426, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8002461668103934, "gate_p_p10": 0.004304313659667969, "gate_p_p50": 0.016597747802734375, "gate_p_p90": 0.09936795234680176, "gate_p_std": 0.049068817257648334, "gated_delta_h_c_norm": 0.010662945703370497, "global_step": 1460, "grad_norm_max": 0.012722630053758621, "hidden_mse": 0.010920329723739998, "loss": 0.011104549566516653, "lr_blocks": 1.864118207632315e-06, "lr_fusion": 1.8641182076323148e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 26.9595920085907, "transport_attention_entropy": 7.308901643753052, "transport_attention_entropy_normalized": 0.994078204035759, "transport_residual_norm": 0.5515938498079777, "velocity_mse": 0.01001251660345588} +{"anchor_distance_mean": 127.76015625000001, "avg_context_frames": 13.200000000000001, "avg_micro_time_s": 2.4909699278005064, "delta_h_c_norm": 0.17272175922989846, "delta_h_d_norm": 0.06429730486124755, "epoch": 12, "gate_p_mean": 0.03408162550767884, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8200335793197155, "gate_p_p10": 0.0040223121643066405, "gate_p_p50": 0.01485748291015625, "gate_p_p90": 0.09359678030014039, "gate_p_std": 0.04696289560524747, "gated_delta_h_c_norm": 0.01047389184095664, "global_step": 1470, "grad_norm_max": 0.015208384953439236, "hidden_mse": 0.010789128002943472, "loss": 0.01038751844316721, "lr_blocks": 1.8001572974834169e-06, "lr_fusion": 1.800157297483417e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 27.07863852977753, "transport_attention_entropy": 7.308792358636857, "transport_attention_entropy_normalized": 0.9940633386373521, "transport_residual_norm": 0.558398924767971, "velocity_mse": 0.00930860566732008} +{"anchor_distance_mean": 124.540625, "avg_context_frames": 12.8, "avg_micro_time_s": 2.452057519175014, "delta_h_c_norm": 0.16616682875901462, "delta_h_d_norm": 0.06618984709493816, "epoch": 12, "gate_p_mean": 0.03646253441693261, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8045197598636151, "gate_p_p10": 0.003959369659423828, "gate_p_p50": 0.015265655517578126, "gate_p_p90": 0.10125606060028076, "gate_p_std": 0.05213177728001028, "gated_delta_h_c_norm": 0.011420057984651066, "global_step": 1480, "grad_norm_max": 0.009872821159660816, "hidden_mse": 0.011085350415669382, "loss": 0.011253633099840954, "lr_blocks": 1.7370711923791567e-06, "lr_fusion": 1.7370711923791567e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 27.137563276290894, "transport_attention_entropy": 7.308438366651536, "transport_attention_entropy_normalized": 0.9940151944756508, "transport_residual_norm": 0.5426720656454563, "velocity_mse": 0.010145098040811719} +{"anchor_distance_mean": 123.4375, "avg_context_frames": 17.6, "avg_micro_time_s": 2.979845427912824, "delta_h_c_norm": 0.17119299266487362, "delta_h_d_norm": 0.06547873597592116, "epoch": 12, "gate_p_mean": 0.028733298380393537, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8565572664141655, "gate_p_p10": 0.0034686088562011722, "gate_p_p50": 0.013001251220703126, "gate_p_p90": 0.07435269355773926, "gate_p_std": 0.04069121610373259, "gated_delta_h_c_norm": 0.009441512735793368, "global_step": 1490, "grad_norm_max": 0.00931033119559288, "hidden_mse": 0.010449092771159485, "loss": 0.010396507597761229, "lr_blocks": 1.6748771394307584e-06, "lr_fusion": 1.6748771394307585e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 27.700040721893313, "transport_attention_entropy": 7.308415025472641, "transport_attention_entropy_normalized": 0.994012014567852, "transport_residual_norm": 0.5614513657987118, "velocity_mse": 0.009351598290959374} +{"anchor_distance_mean": 128.79609375, "avg_context_frames": 18.0, "avg_micro_time_s": 2.9992647408126687, "delta_h_c_norm": 0.1768751570954919, "delta_h_d_norm": 0.06178274522535503, "epoch": 12, "gate_p_mean": 0.023311563488096, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8876625373959541, "gate_p_p10": 0.0025946617126464846, "gate_p_p50": 0.009721946716308594, "gate_p_p90": 0.06211845874786377, "gate_p_std": 0.035322710376931354, "gated_delta_h_c_norm": 0.007386967611091678, "global_step": 1500, "grad_norm_max": 0.009353579953312874, "hidden_mse": 0.009611464961199091, "loss": 0.009629216938628817, "lr_blocks": 1.6135921418712959e-06, "lr_fusion": 1.6135921418712956e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 27.432028126716617, "transport_attention_entropy": 7.307925742864609, "transport_attention_entropy_normalized": 0.9939454726874829, "transport_residual_norm": 0.5611370764672756, "velocity_mse": 0.008668070461135357} +{"anchor_distance_mean": 124.47578125000001, "avg_context_frames": 14.4, "avg_micro_time_s": 2.582869555837897, "delta_h_c_norm": 0.17386332135647536, "delta_h_d_norm": 0.06631817342713475, "epoch": 12, "gate_p_mean": 0.037470154848415406, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8049176439642907, "gate_p_p10": 0.00424716928973794, "gate_p_p50": 0.016305160522460938, "gate_p_p90": 0.10419825315475464, "gate_p_std": 0.051423309557139876, "gated_delta_h_c_norm": 0.011707769165514038, "global_step": 1510, "grad_norm_max": 0.02144334465265274, "hidden_mse": 0.011236969527089968, "loss": 0.011053522699512541, "lr_blocks": 1.5532329544071712e-06, "lr_fusion": 1.553232954407171e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 27.33091540336609, "transport_attention_entropy": 7.308308440446854, "transport_attention_entropy_normalized": 0.9939975172281266, "transport_residual_norm": 0.5537522569298744, "velocity_mse": 0.009929825790459292} +{"anchor_distance_mean": 122.98203125, "avg_context_frames": 15.600000000000001, "avg_micro_time_s": 2.724367434050146, "delta_h_c_norm": 0.17004418186843395, "delta_h_d_norm": 0.06438564485870302, "epoch": 12, "gate_p_mean": 0.031865865259896965, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8347916930913926, "gate_p_p10": 0.003588390164077282, "gate_p_p50": 0.013413619995117188, "gate_p_p90": 0.08818783760070802, "gate_p_std": 0.04535062498180196, "gated_delta_h_c_norm": 0.00986423142021522, "global_step": 1520, "grad_norm_max": 0.009387348778545856, "hidden_mse": 0.010628634970635176, "loss": 0.009921746572945268, "lr_blocks": 1.4938160786375571e-06, "lr_fusion": 1.4938160786375572e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 27.309748911857607, "transport_attention_entropy": 7.308354896306992, "transport_attention_entropy_normalized": 0.994003838300705, "transport_residual_norm": 0.5538335904479027, "velocity_mse": 0.008858883054926991} +{"anchor_distance_mean": 125.41640625000001, "avg_context_frames": 14.4, "avg_micro_time_s": 2.5854529093993737, "delta_h_c_norm": 0.17040667589753866, "delta_h_d_norm": 0.06257201917469501, "epoch": 12, "gate_p_mean": 0.030682740960037338, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8412670508027077, "gate_p_p10": 0.0036158941686153414, "gate_p_p50": 0.014014053344726564, "gate_p_p90": 0.0814896583557129, "gate_p_std": 0.042942121345549825, "gated_delta_h_c_norm": 0.009225306181178894, "global_step": 1530, "grad_norm_max": 0.007137965876609087, "hidden_mse": 0.009958253538934515, "loss": 0.009687773702898994, "lr_blocks": 1.4353577585430152e-06, "lr_fusion": 1.435357758543015e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 27.61797823905945, "transport_attention_entropy": 7.308061486482621, "transport_attention_entropy_normalized": 0.993963934481144, "transport_residual_norm": 0.5567576825618744, "velocity_mse": 0.008691948317573406} +{"anchor_distance_mean": 124.15, "avg_context_frames": 14.0, "avg_micro_time_s": 2.5606243079629167, "delta_h_c_norm": 0.16982797216624024, "delta_h_d_norm": 0.06556087504141032, "epoch": 13, "gate_p_mean": 0.03432830208912492, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8220362834632398, "gate_p_p10": 0.003859424591064453, "gate_p_p50": 0.014659500122070313, "gate_p_p90": 0.09431424140930177, "gate_p_std": 0.048445325833745306, "gated_delta_h_c_norm": 0.01069973654812202, "global_step": 1540, "grad_norm_max": 0.017516260966658592, "hidden_mse": 0.010658427269663663, "loss": 0.010600183685892262, "lr_blocks": 1.3778739760445552e-06, "lr_fusion": 1.3778739760445552e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 27.237069082260135, "transport_attention_entropy": 7.307878637313843, "transport_attention_entropy_normalized": 0.9939390651881695, "transport_residual_norm": 0.564732388406992, "velocity_mse": 0.009534340945538134} +{"anchor_distance_mean": 121.90937500000001, "avg_context_frames": 14.4, "avg_micro_time_s": 2.5988698790753006, "delta_h_c_norm": 0.16904810126870873, "delta_h_d_norm": 0.0664330454543233, "epoch": 13, "gate_p_mean": 0.03826936020050198, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8036876745522022, "gate_p_p10": 0.004239940643310547, "gate_p_p50": 0.015996170043945313, "gate_p_p90": 0.10754636526107789, "gate_p_std": 0.05349177895113826, "gated_delta_h_c_norm": 0.012134411468286999, "global_step": 1550, "grad_norm_max": 0.0068128034472465515, "hidden_mse": 0.01162958347122185, "loss": 0.011140956310555339, "lr_blocks": 1.321380446634342e-06, "lr_fusion": 1.3213804466343421e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 27.61621844768524, "transport_attention_entropy": 7.30809829235077, "transport_attention_entropy_normalized": 0.993968939781189, "transport_residual_norm": 0.5626588881015778, "velocity_mse": 0.009977997941314244} +{"anchor_distance_mean": 124.73515625, "avg_context_frames": 14.4, "avg_micro_time_s": 2.5991373177128483, "delta_h_c_norm": 0.17147183101624253, "delta_h_d_norm": 0.06251559369266034, "epoch": 13, "gate_p_mean": 0.028276629210449756, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.856261045485735, "gate_p_p10": 0.003097820281982422, "gate_p_p50": 0.011901092529296876, "gate_p_p90": 0.07579786777496339, "gate_p_std": 0.04240917732240632, "gated_delta_h_c_norm": 0.008554135484155267, "global_step": 1560, "grad_norm_max": 0.012000583112239838, "hidden_mse": 0.009318691911175847, "loss": 0.00946969397191424, "lr_blocks": 1.2658926150792321e-06, "lr_fusion": 1.2658926150792322e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 27.392832064628603, "transport_attention_entropy": 7.307842528820038, "transport_attention_entropy_normalized": 0.993934154510498, "transport_residual_norm": 0.5492418572306633, "velocity_mse": 0.008537824763334357} +{"anchor_distance_mean": 128.56875, "avg_context_frames": 14.8, "avg_micro_time_s": 2.6714823670499754, "delta_h_c_norm": 0.17503811549395323, "delta_h_d_norm": 0.06663999422453344, "epoch": 13, "gate_p_mean": 0.033843724534381184, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.824638444185257, "gate_p_p10": 0.003890800476074219, "gate_p_p50": 0.015343475341796876, "gate_p_p90": 0.09114193916320801, "gate_p_std": 0.04718307872535661, "gated_delta_h_c_norm": 0.009855667332885787, "global_step": 1570, "grad_norm_max": 0.007842727936804295, "hidden_mse": 0.010355695587350056, "loss": 0.010375704456237145, "lr_blocks": 1.2114256511983274e-06, "lr_fusion": 1.2114256511983274e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 27.230183839797974, "transport_attention_entropy": 7.307548135519028, "transport_attention_entropy_normalized": 0.9938941158354283, "transport_residual_norm": 0.5568740859627724, "velocity_mse": 0.009340134877129458} +{"anchor_distance_mean": 122.10546875, "avg_context_frames": 12.8, "avg_micro_time_s": 2.3951069010874564, "delta_h_c_norm": 0.17246056357398631, "delta_h_d_norm": 0.06661016610451043, "epoch": 13, "gate_p_mean": 0.03561277153203264, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8136248297989369, "gate_p_p10": 0.0037220001220703127, "gate_p_p50": 0.01480255126953125, "gate_p_p90": 0.09700310230255127, "gate_p_std": 0.05169273426290602, "gated_delta_h_c_norm": 0.01146108007815201, "global_step": 1580, "grad_norm_max": 0.01131809689104557, "hidden_mse": 0.011299408925697208, "loss": 0.01105345991672948, "lr_blocks": 1.157994445715706e-06, "lr_fusion": 1.157994445715706e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 27.166369748115542, "transport_attention_entropy": 7.307832944393159, "transport_attention_entropy_normalized": 0.9939328514039517, "transport_residual_norm": 0.5669101864099503, "velocity_mse": 0.009923519028234296} +{"anchor_distance_mean": 122.65859375000001, "avg_context_frames": 13.600000000000001, "avg_micro_time_s": 2.486772808850401, "delta_h_c_norm": 0.17119067227467896, "delta_h_d_norm": 0.06618705242872239, "epoch": 13, "gate_p_mean": 0.035632083995733414, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8088945057243109, "gate_p_p10": 0.004456424713134766, "gate_p_p50": 0.017232513427734377, "gate_p_p90": 0.09389343261718751, "gate_p_std": 0.045900517015252267, "gated_delta_h_c_norm": 0.010123664507409558, "global_step": 1590, "grad_norm_max": 0.012286816723644733, "hidden_mse": 0.011116562824463473, "loss": 0.010446041644900107, "lr_blocks": 1.1056136061894386e-06, "lr_fusion": 1.1056136061894384e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 27.492616152763368, "transport_attention_entropy": 7.307521688938142, "transport_attention_entropy_normalized": 0.9938905134797097, "transport_residual_norm": 0.5707404270768166, "velocity_mse": 0.009334385301917792} +{"anchor_distance_mean": 128.11171875, "avg_context_frames": 10.8, "avg_micro_time_s": 2.173557686887398, "delta_h_c_norm": 0.17459168666973712, "delta_h_d_norm": 0.06534205880016089, "epoch": 13, "gate_p_mean": 0.036856234224978836, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8094556525349618, "gate_p_p10": 0.004086131975054741, "gate_p_p50": 0.015079116821289063, "gate_p_p90": 0.10268402099609375, "gate_p_std": 0.05346495618578047, "gated_delta_h_c_norm": 0.011992231624026317, "global_step": 1600, "grad_norm_max": 0.00985043030232191, "hidden_mse": 0.011275496482267045, "loss": 0.01130624024081044, "lr_blocks": 1.0542974530180327e-06, "lr_fusion": 1.0542974530180327e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 27.21486511230469, "transport_attention_entropy": 7.307633936405182, "transport_attention_entropy_normalized": 0.9939057804644108, "transport_residual_norm": 0.5796907924115658, "velocity_mse": 0.010178690581233241} +{"anchor_distance_mean": 128.92656250000002, "avg_context_frames": 16.8, "avg_micro_time_s": 2.90540459321237, "delta_h_c_norm": 0.1803650570102036, "delta_h_d_norm": 0.06569111086428166, "epoch": 13, "gate_p_mean": 0.029815598658751698, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8507071588188411, "gate_p_p10": 0.003140830993652344, "gate_p_p50": 0.012170791625976562, "gate_p_p90": 0.08134397268295289, "gate_p_std": 0.04418758837273345, "gated_delta_h_c_norm": 0.010016671800985933, "global_step": 1610, "grad_norm_max": 0.011185873299837112, "hidden_mse": 0.010294430609792472, "loss": 0.009973475732840598, "lr_blocks": 1.0040600155253766e-06, "lr_fusion": 1.0040600155253765e-05, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 27.38249216079712, "transport_attention_entropy": 7.306907153129578, "transport_attention_entropy_normalized": 0.9938069343566895, "transport_residual_norm": 0.5603739239275456, "velocity_mse": 0.008944032681756654} +{"anchor_distance_mean": 126.29531250000001, "avg_context_frames": 12.8, "avg_micro_time_s": 2.458089222099534, "delta_h_c_norm": 0.1731356754899025, "delta_h_d_norm": 0.06464787391014397, "epoch": 13, "gate_p_mean": 0.030493937205756085, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8404382281005383, "gate_p_p10": 0.003141307830810547, "gate_p_p50": 0.012438583374023439, "gate_p_p90": 0.0829014301300049, "gate_p_std": 0.045860943268053234, "gated_delta_h_c_norm": 0.009198905839002692, "global_step": 1620, "grad_norm_max": 0.010644521564245224, "hidden_mse": 0.010029786051018164, "loss": 0.009753000334603712, "lr_blocks": 9.549150281252633e-07, "lr_fusion": 9.549150281252633e-06, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 27.08593697547913, "transport_attention_entropy": 7.307205319404602, "transport_attention_entropy_normalized": 0.9938474878668786, "transport_residual_norm": 0.5624204359948636, "velocity_mse": 0.008750021690502763} +{"anchor_distance_mean": 125.48203125, "avg_context_frames": 16.400000000000002, "avg_micro_time_s": 2.8878858698379193, "delta_h_c_norm": 0.17307666931301358, "delta_h_d_norm": 0.06577768828719854, "epoch": 13, "gate_p_mean": 0.0291898274095729, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8548237442970277, "gate_p_p10": 0.0035999298095703127, "gate_p_p50": 0.012936019897460939, "gate_p_p90": 0.07818473577499391, "gate_p_std": 0.04197075925767422, "gated_delta_h_c_norm": 0.008775146407424472, "global_step": 1630, "grad_norm_max": 0.013862891122698784, "hidden_mse": 0.00945451361767482, "loss": 0.00938802958407905, "lr_blocks": 9.068759265665384e-07, "lr_fusion": 9.068759265665384e-06, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 27.082054805755618, "transport_attention_entropy": 7.3066564202308655, "transport_attention_entropy_normalized": 0.9937728315591813, "transport_residual_norm": 0.5519942440092563, "velocity_mse": 0.008442578223184683} +{"anchor_distance_mean": 123.33906250000001, "avg_context_frames": 16.0, "avg_micro_time_s": 2.8067383280509604, "delta_h_c_norm": 0.1687785977497697, "delta_h_d_norm": 0.06538284178823232, "epoch": 13, "gate_p_mean": 0.03107090841513127, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8361295863986016, "gate_p_p10": 0.0036802290938794616, "gate_p_p50": 0.01385040283203125, "gate_p_p90": 0.08209068775177003, "gate_p_std": 0.04417267513927073, "gated_delta_h_c_norm": 0.008722978415607941, "global_step": 1640, "grad_norm_max": 0.007537005003541708, "hidden_mse": 0.00978766393964179, "loss": 0.010095472927787341, "lr_blocks": 8.599558442598998e-07, "lr_fusion": 8.599558442598998e-06, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 27.158001351356507, "transport_attention_entropy": 7.3066379785537725, "transport_attention_entropy_normalized": 0.9937703259289266, "transport_residual_norm": 0.5590617001056671, "velocity_mse": 0.00911670649657026} +{"anchor_distance_mean": 125.353125, "avg_context_frames": 13.200000000000001, "avg_micro_time_s": 2.4651346958753493, "delta_h_c_norm": 0.17078042756766082, "delta_h_d_norm": 0.06649255924858154, "epoch": 13, "gate_p_mean": 0.033117074769688774, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8262632958590985, "gate_p_p10": 0.0035327911376953126, "gate_p_p50": 0.013879013061523438, "gate_p_p90": 0.09024909734725953, "gate_p_std": 0.04820351944072172, "gated_delta_h_c_norm": 0.009520339261507617, "global_step": 1650, "grad_norm_max": 0.010311548598110676, "hidden_mse": 0.010387601307593287, "loss": 0.010601276444504037, "lr_blocks": 8.141676086873574e-07, "lr_fusion": 8.141676086873572e-06, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 27.353815293312074, "transport_attention_entropy": 7.306645250320435, "transport_attention_entropy_normalized": 0.9937713094055654, "transport_residual_norm": 0.5537371404469014, "velocity_mse": 0.009562516259029508} +{"anchor_distance_mean": 122.69062500000001, "avg_context_frames": 14.8, "avg_micro_time_s": 2.6421875797631404, "delta_h_c_norm": 0.17080049347132445, "delta_h_d_norm": 0.0685178694780916, "epoch": 13, "gate_p_mean": 0.038560033182147894, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8023623026907445, "gate_p_p10": 0.004387664794921875, "gate_p_p50": 0.016893005371093752, "gate_p_p90": 0.10382013320922852, "gate_p_std": 0.05379120116122067, "gated_delta_h_c_norm": 0.012057804348296487, "global_step": 1660, "grad_norm_max": 0.014647958800196648, "hidden_mse": 0.011617279768688605, "loss": 0.011020908050704749, "lr_blocks": 7.695237378953224e-07, "lr_fusion": 7.695237378953223e-06, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 27.392122387886047, "transport_attention_entropy": 7.307068210840225, "transport_attention_entropy_normalized": 0.9938288405537605, "transport_residual_norm": 0.5541102953255177, "velocity_mse": 0.009859180104103872} +{"anchor_distance_mean": 125.02734375, "avg_context_frames": 16.0, "avg_micro_time_s": 2.8025777421248677, "delta_h_c_norm": 0.17173973601311446, "delta_h_d_norm": 0.06652777655981482, "epoch": 14, "gate_p_mean": 0.034346779272891585, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8183366127312184, "gate_p_p10": 0.0038173483684659005, "gate_p_p50": 0.014374923706054688, "gate_p_p90": 0.09276580810546875, "gate_p_std": 0.049488199793267997, "gated_delta_h_c_norm": 0.010325465863570572, "global_step": 1670, "grad_norm_max": 0.007077863439917564, "hidden_mse": 0.010401253297459334, "loss": 0.010470225993776694, "lr_blocks": 7.260364370723044e-07, "lr_fusion": 7.260364370723044e-06, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 27.53245222568512, "transport_attention_entropy": 7.3063746929168705, "transport_attention_entropy_normalized": 0.9937345147132874, "transport_residual_norm": 0.5515728697180748, "velocity_mse": 0.009430100690224208} +{"anchor_distance_mean": 122.59140625, "avg_context_frames": 12.0, "avg_micro_time_s": 2.331009356325012, "delta_h_c_norm": 0.1660983469337225, "delta_h_d_norm": 0.06884083398617805, "epoch": 14, "gate_p_mean": 0.036472348554525524, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.807902667671442, "gate_p_p10": 0.003983974456787109, "gate_p_p50": 0.015305709838867188, "gate_p_p90": 0.10090942382812501, "gate_p_std": 0.05299861115636304, "gated_delta_h_c_norm": 0.010899697040440515, "global_step": 1680, "grad_norm_max": 0.013952725566923618, "hidden_mse": 0.01117257209843956, "loss": 0.010776093922322616, "lr_blocks": 6.837175952121305e-07, "lr_fusion": 6.837175952121306e-06, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 27.25269377231598, "transport_attention_entropy": 7.306540930271149, "transport_attention_entropy_normalized": 0.9937571235001088, "transport_residual_norm": 0.5701409325003625, "velocity_mse": 0.009658836698508822} +{"anchor_distance_mean": 123.43593750000001, "avg_context_frames": 14.0, "avg_micro_time_s": 2.5751619975501856, "delta_h_c_norm": 0.16779384929686786, "delta_h_d_norm": 0.06513633416034281, "epoch": 14, "gate_p_mean": 0.03266309616155923, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8295337781310081, "gate_p_p10": 0.003693103790283203, "gate_p_p50": 0.013959121704101563, "gate_p_p90": 0.08979806900024415, "gate_p_std": 0.04646446143742651, "gated_delta_h_c_norm": 0.009578166456776671, "global_step": 1690, "grad_norm_max": 0.006400801241397858, "hidden_mse": 0.009839816190651619, "loss": 0.009716328489594162, "lr_blocks": 6.425787818636131e-07, "lr_fusion": 6.425787818636131e-06, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 27.12434735298157, "transport_attention_entropy": 7.306838464736939, "transport_attention_entropy_normalized": 0.9937975935637952, "transport_residual_norm": 0.5629343666136265, "velocity_mse": 0.008732346951728687} +{"anchor_distance_mean": 124.24843750000001, "avg_context_frames": 13.600000000000001, "avg_micro_time_s": 2.5171291872619808, "delta_h_c_norm": 0.1705744587816298, "delta_h_d_norm": 0.06801680629141629, "epoch": 14, "gate_p_mean": 0.037120722472900526, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8084815956652165, "gate_p_p10": 0.00388031005859375, "gate_p_p50": 0.0173187255859375, "gate_p_p90": 0.09853147268295288, "gate_p_std": 0.050607353070518, "gated_delta_h_c_norm": 0.011127573007252069, "global_step": 1700, "grad_norm_max": 0.0093886349350214, "hidden_mse": 0.011538835923420266, "loss": 0.01104250684438739, "lr_blocks": 6.026312439675553e-07, "lr_fusion": 6.026312439675552e-06, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 27.52177712917328, "transport_attention_entropy": 7.306644535064698, "transport_attention_entropy_normalized": 0.9937712147831917, "transport_residual_norm": 0.5641972661018372, "velocity_mse": 0.009888623273582198} +{"anchor_distance_mean": 124.83203125, "avg_context_frames": 17.2, "avg_micro_time_s": 2.9467873838370906, "delta_h_c_norm": 0.1728008693084121, "delta_h_d_norm": 0.06670638350769878, "epoch": 14, "gate_p_mean": 0.03266904950141907, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8318044133484364, "gate_p_p10": 0.003922805562615395, "gate_p_p50": 0.01399993896484375, "gate_p_p90": 0.08837827444076539, "gate_p_std": 0.046533222764264796, "gated_delta_h_c_norm": 0.009644527261843906, "global_step": 1710, "grad_norm_max": 0.00548924645408988, "hidden_mse": 0.009852110143401661, "loss": 0.00946812195761595, "lr_blocks": 5.63885902781941e-07, "lr_fusion": 5.6388590278194096e-06, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 27.232398700714114, "transport_attention_entropy": 7.306756162643433, "transport_attention_entropy_normalized": 0.993786396086216, "transport_residual_norm": 0.5541220866143703, "velocity_mse": 0.008482910925522447} +{"anchor_distance_mean": 127.23828125, "avg_context_frames": 16.8, "avg_micro_time_s": 2.8773797585249667, "delta_h_c_norm": 0.17680663745850325, "delta_h_d_norm": 0.06751460377126932, "epoch": 14, "gate_p_mean": 0.035695097595453265, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8281167611479759, "gate_p_p10": 0.0041754722595214845, "gate_p_p50": 0.014354324340820313, "gate_p_p90": 0.09998314380645752, "gate_p_std": 0.05224816824775189, "gated_delta_h_c_norm": 0.011531096263206565, "global_step": 1720, "grad_norm_max": 0.006492300424724817, "hidden_mse": 0.01014602542272769, "loss": 0.009590440773172304, "lr_blocks": 5.263533508961827e-07, "lr_fusion": 5.263533508961827e-06, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 27.627731919288635, "transport_attention_entropy": 7.30622124671936, "transport_attention_entropy_normalized": 0.9937136456370355, "transport_residual_norm": 0.5488079078495502, "velocity_mse": 0.008575838193064556} +{"anchor_distance_mean": 125.028125, "avg_context_frames": 12.0, "avg_micro_time_s": 2.317987552237355, "delta_h_c_norm": 0.17176166437566281, "delta_h_d_norm": 0.06782125094905496, "epoch": 14, "gate_p_mean": 0.03656220425618813, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8131192654371262, "gate_p_p10": 0.003816318325698376, "gate_p_p50": 0.014373779296875, "gate_p_p90": 0.10450100898742676, "gate_p_std": 0.05454006013460458, "gated_delta_h_c_norm": 0.011523849205696025, "global_step": 1730, "grad_norm_max": 0.011281722225248814, "hidden_mse": 0.011360520665766672, "loss": 0.010828843677882106, "lr_blocks": 4.900438493352056e-07, "lr_fusion": 4.900438493352055e-06, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 27.618357825279237, "transport_attention_entropy": 7.306776016950607, "transport_attention_entropy_normalized": 0.9937890976667405, "transport_residual_norm": 0.5798151634633542, "velocity_mse": 0.009692791625275277} +{"anchor_distance_mean": 124.12109375, "avg_context_frames": 15.200000000000001, "avg_micro_time_s": 2.6778789594250156, "delta_h_c_norm": 0.1758280847221613, "delta_h_d_norm": 0.06744615519419313, "epoch": 14, "gate_p_mean": 0.0370156507124193, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8079402301460505, "gate_p_p10": 0.004062747955322266, "gate_p_p50": 0.015993499755859376, "gate_p_p90": 0.10091794729232789, "gate_p_std": 0.05153740035602823, "gated_delta_h_c_norm": 0.010935099617927336, "global_step": 1740, "grad_norm_max": 0.017441648989915848, "hidden_mse": 0.011293933237902821, "loss": 0.011045912752160804, "lr_blocks": 4.549673247541875e-07, "lr_fusion": 4.549673247541875e-06, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 27.961989545822146, "transport_attention_entropy": 7.3060124874115, "transport_attention_entropy_normalized": 0.9936852529644966, "transport_residual_norm": 0.5591237783432007, "velocity_mse": 0.00991651947842911} +{"anchor_distance_mean": 124.18359375, "avg_context_frames": 14.4, "avg_micro_time_s": 2.6018895514996077, "delta_h_c_norm": 0.17328123636543752, "delta_h_d_norm": 0.06718613877892494, "epoch": 14, "gate_p_mean": 0.03852257841499523, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8005531594157219, "gate_p_p10": 0.004368400573730469, "gate_p_p50": 0.017214584350585937, "gate_p_p90": 0.10501947402954102, "gate_p_std": 0.052753368299454456, "gated_delta_h_c_norm": 0.011361155158374459, "global_step": 1750, "grad_norm_max": 0.005508618429303169, "hidden_mse": 0.010554492357186973, "loss": 0.011004206174402498, "lr_blocks": 4.211333667247125e-07, "lr_fusion": 4.2113336672471245e-06, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 27.503503155708316, "transport_attention_entropy": 7.306358993053436, "transport_attention_entropy_normalized": 0.9937323786318303, "transport_residual_norm": 0.5509638525545597, "velocity_mse": 0.0099487569386838} +{"anchor_distance_mean": 127.07578125, "avg_context_frames": 10.8, "avg_micro_time_s": 2.1738341134996517, "delta_h_c_norm": 0.17358861137181522, "delta_h_d_norm": 0.06469124555587769, "epoch": 14, "gate_p_mean": 0.04166293152375147, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.7772017989307642, "gate_p_p10": 0.00454788189381361, "gate_p_p50": 0.018059539794921874, "gate_p_p90": 0.11532068252563477, "gate_p_std": 0.05700655878754333, "gated_delta_h_c_norm": 0.012186186265898869, "global_step": 1760, "grad_norm_max": 0.006476675625890493, "hidden_mse": 0.010896282145404258, "loss": 0.011196921358350665, "lr_blocks": 3.885512251130763e-07, "lr_fusion": 3.885512251130763e-06, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 27.005978393554688, "transport_attention_entropy": 7.306378990411758, "transport_attention_entropy_normalized": 0.9937350980937482, "transport_residual_norm": 0.5716338202357293, "velocity_mse": 0.010107293084729463} +{"anchor_distance_mean": 123.1453125, "avg_context_frames": 16.400000000000002, "avg_micro_time_s": 2.8195979856874147, "delta_h_c_norm": 0.17233012970536948, "delta_h_d_norm": 0.06599020236171782, "epoch": 14, "gate_p_mean": 0.035219239431899044, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8152774713933468, "gate_p_p10": 0.0038841243833303454, "gate_p_p50": 0.0153350830078125, "gate_p_p90": 0.09493587017059327, "gate_p_std": 0.049865776463411754, "gated_delta_h_c_norm": 0.010348654288100079, "global_step": 1770, "grad_norm_max": 0.007571633439511061, "hidden_mse": 0.010051707370439545, "loss": 0.010346241784282029, "lr_blocks": 3.572298075514652e-07, "lr_fusion": 3.5722980755146517e-06, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 27.763680577278137, "transport_attention_entropy": 7.3056995451450355, "transport_attention_entropy_normalized": 0.9936426877975464, "transport_residual_norm": 0.551444110274315, "velocity_mse": 0.009341071039671078} +{"anchor_distance_mean": 124.99531250000001, "avg_context_frames": 12.4, "avg_micro_time_s": 2.3569844343378463, "delta_h_c_norm": 0.17051812894642354, "delta_h_d_norm": 0.06517862672917545, "epoch": 14, "gate_p_mean": 0.033133024640847, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8243054136633874, "gate_p_p10": 0.0036314964294433595, "gate_p_p50": 0.014320755004882814, "gate_p_p90": 0.08981933593750001, "gate_p_std": 0.04705055244266987, "gated_delta_h_c_norm": 0.009433775814250112, "global_step": 1780, "grad_norm_max": 0.006731005851179361, "hidden_mse": 0.010248002962907777, "loss": 0.01016216725402046, "lr_blocks": 3.271776770026963e-07, "lr_fusion": 3.271776770026963e-06, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 27.608582496643066, "transport_attention_entropy": 7.306329435110093, "transport_attention_entropy_normalized": 0.9937283575534821, "transport_residual_norm": 0.5644477762281895, "velocity_mse": 0.009137366971117444} +{"anchor_distance_mean": 125.384375, "avg_context_frames": 14.8, "avg_micro_time_s": 2.650155632500355, "delta_h_c_norm": 0.1713821116834879, "delta_h_d_norm": 0.06461703754030168, "epoch": 14, "gate_p_mean": 0.029310252435971053, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8521745063364506, "gate_p_p10": 0.00341644287109375, "gate_p_p50": 0.012744140625, "gate_p_p90": 0.07750091552734376, "gate_p_std": 0.04290763415046968, "gated_delta_h_c_norm": 0.008652087950031273, "global_step": 1790, "grad_norm_max": 0.0040204524993896484, "hidden_mse": 0.009603872406296433, "loss": 0.009250535612227396, "lr_blocks": 2.984030494191942e-07, "lr_fusion": 2.9840304941919415e-06, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 27.71600661277771, "transport_attention_entropy": 7.306150567531586, "transport_attention_entropy_normalized": 0.9937040314078331, "transport_residual_norm": 0.5578957304358483, "velocity_mse": 0.00829014838382136} +{"anchor_distance_mean": 123.371875, "avg_context_frames": 18.8, "avg_micro_time_s": 3.1054734409997766, "delta_h_c_norm": 0.17425180999562145, "delta_h_d_norm": 0.06510923039168119, "epoch": 15, "gate_p_mean": 0.02562056088936515, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8721299320459366, "gate_p_p10": 0.0030608177185058594, "gate_p_p50": 0.011522674560546876, "gate_p_p90": 0.06642684936523438, "gate_p_std": 0.037060609541367744, "gated_delta_h_c_norm": 0.00759739194472786, "global_step": 1800, "grad_norm_max": 0.007202739827334881, "hidden_mse": 0.009071475375094451, "loss": 0.00869601609010715, "lr_blocks": 2.7091379149682683e-07, "lr_fusion": 2.7091379149682685e-06, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 27.532786417007447, "transport_attention_entropy": 7.306054735183716, "transport_attention_entropy_normalized": 0.9936909995973111, "transport_residual_norm": 0.55456982254982, "velocity_mse": 0.007788868529314642} +{"anchor_distance_mean": 121.65, "avg_context_frames": 12.4, "avg_micro_time_s": 2.355651557824604, "delta_h_c_norm": 0.17123148534446955, "delta_h_d_norm": 0.068225885508582, "epoch": 15, "gate_p_mean": 0.03691723005613312, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8036829173564911, "gate_p_p10": 0.004154109954833985, "gate_p_p50": 0.015639495849609376, "gate_p_p90": 0.10261993408203125, "gate_p_std": 0.05209163627587259, "gated_delta_h_c_norm": 0.010813803676865064, "global_step": 1810, "grad_norm_max": 0.008571090176701546, "hidden_mse": 0.010990819678409026, "loss": 0.010701496180263349, "lr_blocks": 2.447174185242324e-07, "lr_fusion": 2.4471741852423237e-06, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 27.218158435821536, "transport_attention_entropy": 7.306239831447602, "transport_attention_entropy_normalized": 0.9937161691486835, "transport_residual_norm": 0.5648038379848004, "velocity_mse": 0.009602414219989442} +{"anchor_distance_mean": 127.4, "avg_context_frames": 13.200000000000001, "avg_micro_time_s": 2.448477818462379, "delta_h_c_norm": 0.17695866115391257, "delta_h_d_norm": 0.06533941798843444, "epoch": 15, "gate_p_mean": 0.03434616872109473, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8271296851336957, "gate_p_p10": 0.0036960600875318053, "gate_p_p50": 0.014247894287109375, "gate_p_p90": 0.09375052452087403, "gate_p_std": 0.050099092791788286, "gated_delta_h_c_norm": 0.010279048746451736, "global_step": 1820, "grad_norm_max": 0.010313804261386395, "hidden_mse": 0.010029625997412951, "loss": 0.009983490611193703, "lr_blocks": 2.198210923282118e-07, "lr_fusion": 2.1982109232821178e-06, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 27.198811650276184, "transport_attention_entropy": 7.306300723552704, "transport_attention_entropy_normalized": 0.9937244534492493, "transport_residual_norm": 0.5733386300504207, "velocity_mse": 0.008980528049869463} +{"anchor_distance_mean": 126.74921875000001, "avg_context_frames": 14.4, "avg_micro_time_s": 2.577255474937465, "delta_h_c_norm": 0.17439598590135574, "delta_h_d_norm": 0.0680088080931455, "epoch": 15, "gate_p_mean": 0.038921935053076596, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8060619749128819, "gate_p_p10": 0.004658317565917969, "gate_p_p50": 0.01592674255371094, "gate_p_p90": 0.10989271998405457, "gate_p_std": 0.05548579544993118, "gated_delta_h_c_norm": 0.012223770577111282, "global_step": 1830, "grad_norm_max": 0.009743173606693745, "hidden_mse": 0.011312351195374505, "loss": 0.010653425854980015, "lr_blocks": 1.962316193157593e-07, "lr_fusion": 1.962316193157593e-06, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 27.821688365936282, "transport_attention_entropy": 7.306158578395844, "transport_attention_entropy_normalized": 0.9937051221728326, "transport_residual_norm": 0.5554808422923089, "velocity_mse": 0.009522190756979399} +{"anchor_distance_mean": 124.4109375, "avg_context_frames": 14.0, "avg_micro_time_s": 2.5503049673626266, "delta_h_c_norm": 0.17317408304661513, "delta_h_d_norm": 0.06919071800075471, "epoch": 15, "gate_p_mean": 0.03911038534715772, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8004054248332978, "gate_p_p10": 0.0046176910400390625, "gate_p_p50": 0.0170623779296875, "gate_p_p90": 0.10712246894836426, "gate_p_std": 0.055517152778338644, "gated_delta_h_c_norm": 0.012151382956653835, "global_step": 1840, "grad_norm_max": 0.005820792634040117, "hidden_mse": 0.010834290087223053, "loss": 0.01090044534939807, "lr_blocks": 1.7395544861325718e-07, "lr_fusion": 1.7395544861325718e-06, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 27.49917709827423, "transport_attention_entropy": 7.305987942218781, "transport_attention_entropy_normalized": 0.993681912124157, "transport_residual_norm": 0.5572422541677952, "velocity_mse": 0.009817016340093687} +{"anchor_distance_mean": 124.246875, "avg_context_frames": 15.200000000000001, "avg_micro_time_s": 2.672862596612868, "delta_h_c_norm": 0.17034193044528367, "delta_h_d_norm": 0.0671842602081597, "epoch": 15, "gate_p_mean": 0.037698620709124955, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8081182643771172, "gate_p_p10": 0.00431581474840641, "gate_p_p50": 0.0167877197265625, "gate_p_p90": 0.10180292129516602, "gate_p_std": 0.05345083707943559, "gated_delta_h_c_norm": 0.01155843907617964, "global_step": 1850, "grad_norm_max": 0.005760756321251392, "hidden_mse": 0.010931559297023342, "loss": 0.010908798279706389, "lr_blocks": 1.5299867030334815e-07, "lr_fusion": 1.5299867030334814e-06, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 27.59588897228241, "transport_attention_entropy": 7.305815315246583, "transport_attention_entropy_normalized": 0.9936584301292897, "transport_residual_norm": 0.5482594609260559, "velocity_mse": 0.009815642322064378} +{"anchor_distance_mean": 123.50234375000001, "avg_context_frames": 14.0, "avg_micro_time_s": 2.5456075849006994, "delta_h_c_norm": 0.17240518052130938, "delta_h_d_norm": 0.06694466494955123, "epoch": 15, "gate_p_mean": 0.03472608926240355, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8131921313703061, "gate_p_p10": 0.0038388252258300783, "gate_p_p50": 0.015159225463867188, "gate_p_p90": 0.09231076240539551, "gate_p_std": 0.04968684443738312, "gated_delta_h_c_norm": 0.009928757845773363, "global_step": 1860, "grad_norm_max": 0.007578573655337095, "hidden_mse": 0.01040180170093663, "loss": 0.010666601400589571, "lr_blocks": 1.333670137599713e-07, "lr_fusion": 1.333670137599713e-06, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 27.1184773683548, "transport_attention_entropy": 7.306309658288956, "transport_attention_entropy_normalized": 0.9937256671488286, "transport_residual_norm": 0.5682454764842987, "velocity_mse": 0.009626421247958206} +{"anchor_distance_mean": 125.87265625, "avg_context_frames": 10.4, "avg_micro_time_s": 2.123828003937615, "delta_h_c_norm": 0.1701984075829387, "delta_h_d_norm": 0.06797219295985997, "epoch": 15, "gate_p_mean": 0.03859818410128355, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.7953678607940674, "gate_p_p10": 0.004357147216796875, "gate_p_p50": 0.0172088623046875, "gate_p_p90": 0.10474753379821777, "gate_p_std": 0.05370294591411948, "gated_delta_h_c_norm": 0.01097052560944576, "global_step": 1870, "grad_norm_max": 0.008754795417189598, "hidden_mse": 0.011224211132503116, "loss": 0.011046537186484785, "lr_blocks": 1.1506584608200366e-07, "lr_fusion": 1.1506584608200367e-06, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 27.167733192443848, "transport_attention_entropy": 7.306431901454926, "transport_attention_entropy_normalized": 0.9937422931194306, "transport_residual_norm": 0.5775523856282234, "velocity_mse": 0.009924116131151095} +{"anchor_distance_mean": 125.44843750000001, "avg_context_frames": 15.600000000000001, "avg_micro_time_s": 2.7107299052876446, "delta_h_c_norm": 0.1735580469481647, "delta_h_d_norm": 0.06423581414856017, "epoch": 15, "gate_p_mean": 0.029722272203071044, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8479119360446931, "gate_p_p10": 0.003616142272949219, "gate_p_p50": 0.012981796264648439, "gate_p_p90": 0.07967588901519776, "gate_p_std": 0.04260957426158712, "gated_delta_h_c_norm": 0.008845340399420821, "global_step": 1880, "grad_norm_max": 0.005936686415225267, "hidden_mse": 0.009295274736359715, "loss": 0.009113067653379403, "lr_blocks": 9.810017062595322e-08, "lr_fusion": 9.810017062595322e-07, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 27.43635392189026, "transport_attention_entropy": 7.3061969697475435, "transport_attention_entropy_normalized": 0.9937103427946568, "transport_residual_norm": 0.5594943270087243, "velocity_mse": 0.008183540153549985} +{"anchor_distance_mean": 126.19765625000001, "avg_context_frames": 13.200000000000001, "avg_micro_time_s": 2.407408219112222, "delta_h_c_norm": 0.17266899328678847, "delta_h_d_norm": 0.06759673501364886, "epoch": 15, "gate_p_mean": 0.039963154227007185, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.7922255866229535, "gate_p_p10": 0.004361915588378906, "gate_p_p50": 0.016279983520507812, "gate_p_p90": 0.11073117256164551, "gate_p_std": 0.058706760359928015, "gated_delta_h_c_norm": 0.011939753402839415, "global_step": 1890, "grad_norm_max": 0.00785756390541792, "hidden_mse": 0.010652471077628435, "loss": 0.01104336613789201, "lr_blocks": 8.247462563808816e-08, "lr_fusion": 8.247462563808817e-07, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 27.65129454135895, "transport_attention_entropy": 7.305675053596497, "transport_attention_entropy_normalized": 0.9936393573880196, "transport_residual_norm": 0.5471379533410072, "velocity_mse": 0.009978118998697029} +{"anchor_distance_mean": 124.51093750000001, "avg_context_frames": 12.8, "avg_micro_time_s": 2.372739752900543, "delta_h_c_norm": 0.17563702994957567, "delta_h_d_norm": 0.06887651742435992, "epoch": 15, "gate_p_mean": 0.04041916453279555, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.7927867349237204, "gate_p_p10": 0.0041901968419551855, "gate_p_p50": 0.017249298095703126, "gate_p_p90": 0.11166136264801026, "gate_p_std": 0.05723000497091562, "gated_delta_h_c_norm": 0.01267620127182454, "global_step": 1900, "grad_norm_max": 0.008332600817084312, "hidden_mse": 0.01170809380710125, "loss": 0.011359128387994133, "lr_blocks": 6.819348298638839e-08, "lr_fusion": 6.819348298638839e-07, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 27.110727071762085, "transport_attention_entropy": 7.306186163425446, "transport_attention_entropy_normalized": 0.9937088675796986, "transport_residual_norm": 0.5662877820432186, "velocity_mse": 0.010188319016015157} +{"anchor_distance_mean": 127.78984375, "avg_context_frames": 16.8, "avg_micro_time_s": 2.8258602203752163, "delta_h_c_norm": 0.17727902103215457, "delta_h_d_norm": 0.06513395761139691, "epoch": 15, "gate_p_mean": 0.027102469408418983, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8607081606984139, "gate_p_p10": 0.003195667266845703, "gate_p_p50": 0.011579513549804688, "gate_p_p90": 0.07246856689453125, "gate_p_std": 0.03957246519858018, "gated_delta_h_c_norm": 0.007796611268713605, "global_step": 1910, "grad_norm_max": 0.00918257050216198, "hidden_mse": 0.009299740300048143, "loss": 0.008866025938186795, "lr_blocks": 5.526064699265754e-08, "lr_fusion": 5.526064699265753e-07, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 27.530315685272218, "transport_attention_entropy": 7.306040853261948, "transport_attention_entropy_normalized": 0.9936891086399555, "transport_residual_norm": 0.5711732976138592, "velocity_mse": 0.007936051895376295} +{"anchor_distance_mean": 118.79218750000001, "avg_context_frames": 17.2, "avg_micro_time_s": 2.8728326902990378, "delta_h_c_norm": 0.1695348994806409, "delta_h_d_norm": 0.0676557546015829, "epoch": 15, "gate_p_mean": 0.0356807149713859, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8141787122935057, "gate_p_p10": 0.0037143707275390627, "gate_p_p50": 0.016402435302734376, "gate_p_p90": 0.09713592529296876, "gate_p_std": 0.04963451899820939, "gated_delta_h_c_norm": 0.010491560744412709, "global_step": 1920, "grad_norm_max": 0.006831300910562277, "hidden_mse": 0.010603695176541805, "loss": 0.010091517851105892, "lr_blocks": 4.367965336512403e-08, "lr_fusion": 4.367965336512403e-07, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 27.429590511322022, "transport_attention_entropy": 7.305915451049805, "transport_attention_entropy_normalized": 0.9936720527708531, "transport_residual_norm": 0.5527789495885372, "velocity_mse": 0.009031148377107457} +{"anchor_distance_mean": 123.85859375000001, "avg_context_frames": 14.8, "avg_micro_time_s": 2.599893168412291, "delta_h_c_norm": 0.17459249487146736, "delta_h_d_norm": 0.06598186558112502, "epoch": 16, "gate_p_mean": 0.03514059141743928, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8238098725676537, "gate_p_p10": 0.003867435455322266, "gate_p_p50": 0.014727783203125, "gate_p_p90": 0.0937896728515625, "gate_p_std": 0.05198123983573169, "gated_delta_h_c_norm": 0.010831115246401169, "global_step": 1930, "grad_norm_max": 0.009120297618210316, "hidden_mse": 0.01006352138647344, "loss": 0.010207121080020443, "lr_blocks": 3.345366823180929e-08, "lr_fusion": 3.3453668231809286e-07, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 27.263950300216678, "transport_attention_entropy": 7.305810332298279, "transport_attention_entropy_normalized": 0.9936577543616295, "transport_residual_norm": 0.5504879757761956, "velocity_mse": 0.009200768976006657} +{"anchor_distance_mean": 121.0, "avg_context_frames": 13.200000000000001, "avg_micro_time_s": 2.420679416013172, "delta_h_c_norm": 0.17099247500300407, "delta_h_d_norm": 0.06592095368541777, "epoch": 16, "gate_p_mean": 0.033878107508644464, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8209497943520546, "gate_p_p10": 0.003788280487060547, "gate_p_p50": 0.014928436279296875, "gate_p_p90": 0.09286677837371826, "gate_p_std": 0.04631786393001676, "gated_delta_h_c_norm": 0.00939124403812457, "global_step": 1940, "grad_norm_max": 0.006649233400821686, "hidden_mse": 0.010463579202769325, "loss": 0.009821444432600402, "lr_blocks": 2.4585487274942922e-08, "lr_fusion": 2.458548727494292e-07, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 27.56599223613739, "transport_attention_entropy": 7.306070703268052, "transport_attention_entropy_normalized": 0.993693170696497, "transport_residual_norm": 0.5807397171854973, "velocity_mse": 0.008775086505920626} +{"anchor_distance_mean": 123.95859375, "avg_context_frames": 16.0, "avg_micro_time_s": 2.7353834142249074, "delta_h_c_norm": 0.17388298250734807, "delta_h_d_norm": 0.06751992930658161, "epoch": 16, "gate_p_mean": 0.03525988406036049, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.82097108066082, "gate_p_p10": 0.004100227355957032, "gate_p_p50": 0.015154266357421876, "gate_p_p90": 0.09597867727279663, "gate_p_std": 0.05114710021298379, "gated_delta_h_c_norm": 0.010486655327258633, "global_step": 1950, "grad_norm_max": 0.006251065991818905, "hidden_mse": 0.010428534197853879, "loss": 0.01009077662602067, "lr_blocks": 1.7077534966650767e-08, "lr_fusion": 1.7077534966650766e-07, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 28.17186636924744, "transport_attention_entropy": 7.30577803850174, "transport_attention_entropy_normalized": 0.9936533607542515, "transport_residual_norm": 0.550078509747982, "velocity_mse": 0.009047923257458025} +{"anchor_distance_mean": 124.60390625000001, "avg_context_frames": 17.6, "avg_micro_time_s": 2.931837297625407, "delta_h_c_norm": 0.17541249543428422, "delta_h_d_norm": 0.0659800951834768, "epoch": 16, "gate_p_mean": 0.030658537254203113, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8498227417469025, "gate_p_p10": 0.00354766845703125, "gate_p_p50": 0.012912368774414063, "gate_p_p90": 0.0833892822265625, "gate_p_std": 0.04434051153948531, "gated_delta_h_c_norm": 0.010082878640969285, "global_step": 1960, "grad_norm_max": 0.005410735961049795, "hidden_mse": 0.010116854566149414, "loss": 0.009636532168951818, "lr_blocks": 1.0931863906127327e-08, "lr_fusion": 1.0931863906127327e-07, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 27.567952394485474, "transport_attention_entropy": 7.306244415044785, "transport_attention_entropy_normalized": 0.9937167949974537, "transport_residual_norm": 0.561837000399828, "velocity_mse": 0.008624846709426493} +{"anchor_distance_mean": 122.49375, "avg_context_frames": 14.8, "avg_micro_time_s": 2.593116234537138, "delta_h_c_norm": 0.17002087850123646, "delta_h_d_norm": 0.0681837622076273, "epoch": 16, "gate_p_mean": 0.03619462355272845, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8125601224601269, "gate_p_p10": 0.003807830810546875, "gate_p_p50": 0.015008926391601562, "gate_p_p90": 0.09834315776824952, "gate_p_std": 0.05240896983304993, "gated_delta_h_c_norm": 0.010585406815516764, "global_step": 1970, "grad_norm_max": 0.007853500545024872, "hidden_mse": 0.010819110600277782, "loss": 0.010839395161019639, "lr_blocks": 6.150154258476315e-09, "lr_fusion": 6.150154258476315e-08, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 27.739460754394532, "transport_attention_entropy": 7.305814093351365, "transport_attention_entropy_normalized": 0.9936582632362843, "transport_residual_norm": 0.5607902362942696, "velocity_mse": 0.009757484079455026} +{"anchor_distance_mean": 126.91250000000001, "avg_context_frames": 14.0, "avg_micro_time_s": 2.512050507924687, "delta_h_c_norm": 0.17236582059413197, "delta_h_d_norm": 0.06570832119323314, "epoch": 16, "gate_p_mean": 0.03395218055229634, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.8197218321263791, "gate_p_p10": 0.003957939147949219, "gate_p_p50": 0.015106201171875, "gate_p_p90": 0.089447021484375, "gate_p_std": 0.04843663380015642, "gated_delta_h_c_norm": 0.00977103308250662, "global_step": 1980, "grad_norm_max": 0.005999866407364607, "hidden_mse": 0.010073854069923983, "loss": 0.010124989331234248, "lr_blocks": 2.7337132953697555e-09, "lr_fusion": 2.7337132953697554e-08, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 27.530304074287415, "transport_attention_entropy": 7.305648308992386, "transport_attention_entropy_normalized": 0.9936357207596302, "transport_residual_norm": 0.5571591094136238, "velocity_mse": 0.009117603933555074} +{"anchor_distance_mean": 127.46484375, "avg_context_frames": 13.600000000000001, "avg_micro_time_s": 2.4610070171002008, "delta_h_c_norm": 0.17583635635674, "delta_h_d_norm": 0.06793537652119995, "epoch": 16, "gate_p_mean": 0.03755596417468041, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.813770304992795, "gate_p_p10": 0.004292201995849609, "gate_p_p50": 0.015573883056640626, "gate_p_p90": 0.10430476665496827, "gate_p_std": 0.054842624720186, "gated_delta_h_c_norm": 0.012014685796748381, "global_step": 1990, "grad_norm_max": 0.007035557180643082, "hidden_mse": 0.011021138419164345, "loss": 0.010637743459665217, "lr_blocks": 6.834750376549793e-10, "lr_fusion": 6.834750376549792e-09, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 27.76193563938141, "transport_attention_entropy": 7.305995124578477, "transport_attention_entropy_normalized": 0.9936828911304474, "transport_residual_norm": 0.5581815220415592, "velocity_mse": 0.009535629625315779} +{"anchor_distance_mean": 122.884375, "avg_context_frames": 10.8, "avg_micro_time_s": 2.1456114330000675, "delta_h_c_norm": 0.16789383813738823, "delta_h_d_norm": 0.07002853373996913, "epoch": 16, "gate_p_mean": 0.04364334219135344, "gate_p_near_one": 0.0, "gate_p_near_zero": 0.7678242936730385, "gate_p_p10": 0.004832744598388672, "gate_p_p50": 0.018514251708984377, "gate_p_p90": 0.12014703750610352, "gate_p_std": 0.060708559281192724, "gated_delta_h_c_norm": 0.012706039586919361, "global_step": 2000, "grad_norm_max": 0.005851785186678171, "hidden_mse": 0.011792123393388466, "loss": 0.011469943792326376, "lr_blocks": 0.0, "lr_fusion": 0.0, "peak_memory_gib": 41.50080919265747, "transport_argmax_displacement": 27.367363262176514, "transport_attention_entropy": 7.306015729904175, "transport_attention_entropy_normalized": 0.993685694038868, "transport_residual_norm": 0.5685717381536961, "velocity_mse": 0.010290731475106441} +[2026-09-03T13:17:41+00:00] done scope=last_frame diff --git a/logs/oldenv_reval_20260821/all.completed b/logs/oldenv_reval_20260821/all.completed new file mode 100644 index 0000000000000000000000000000000000000000..bc21d33241b5b550d73153365a3286b43e63faf6 --- /dev/null +++ b/logs/oldenv_reval_20260821/all.completed @@ -0,0 +1 @@ +2026-08-21T15:11:01Z diff --git a/logs/oldenv_reval_20260821/launcher.out b/logs/oldenv_reval_20260821/launcher.out new file mode 100644 index 0000000000000000000000000000000000000000..30e283f8402706968e68676f5159f69e0197f956 --- /dev/null +++ b/logs/oldenv_reval_20260821/launcher.out @@ -0,0 +1,10 @@ +[preflight] fixed environment +torch=2.6.0+cu124 cuda=12.4 flash_attn=False +Validation evaluation complete: /mnt/local_nvme/cz/projects/HY-WorldPlay-DEV-Predictor/datasets/hyworldplay_predictor_vbench_val25_test50/evaluations/eval_validation25_stage1-c1-threshold-r10_oldenv_reval_20260821 +Validation evaluation complete: /mnt/local_nvme/cz/projects/HY-WorldPlay-DEV-Predictor/datasets/hyworldplay_predictor_vbench_val25_test50/evaluations/eval_validation25_stage1-c1-threshold-r25_oldenv_reval_20260821 +Validation evaluation complete: /mnt/local_nvme/cz/projects/HY-WorldPlay-DEV-Predictor/datasets/hyworldplay_predictor_vbench_val25_test50/evaluations/eval_validation25_stage1-c1-threshold-r33_oldenv_reval_20260821 +Validation evaluation complete: /mnt/local_nvme/cz/projects/HY-WorldPlay-DEV-Predictor/datasets/hyworldplay_predictor_vbench_val25_test50/evaluations/eval_validation25_stage1-c1-threshold-r50_oldenv_reval_20260821 +Validation evaluation complete: /mnt/local_nvme/cz/projects/HY-WorldPlay-DEV-Predictor/datasets/hyworldplay_predictor_vbench_val25_test50/evaluations/eval_validation25_stage1-wide4096x2-c1-threshold-r33_oldenv_reval_20260821 +Validation evaluation complete: /mnt/local_nvme/cz/projects/HY-WorldPlay-DEV-Predictor/datasets/hyworldplay_predictor_vbench_val25_test50/evaluations/eval_validation25_stage1-wide4096x2-c2-hybrid-p3full-p12r10_oldenv_reval_20260821 +Validation evaluation complete: /mnt/local_nvme/cz/projects/HY-WorldPlay-DEV-Predictor/datasets/hyworldplay_predictor_vbench_val25_test50/evaluations/eval_validation25_stage1-wide4096x2-c2-hybrid-p3full-p12r25_oldenv_reval_20260821 +[complete] all fixed-environment validation reruns diff --git a/logs/oldenv_reval_20260821/launcher.pid b/logs/oldenv_reval_20260821/launcher.pid new file mode 100644 index 0000000000000000000000000000000000000000..7a8b805e17754705027c104f6488638490da5453 --- /dev/null +++ b/logs/oldenv_reval_20260821/launcher.pid @@ -0,0 +1 @@ +3679476 diff --git a/logs/oldenv_reval_20260821/sequential.log b/logs/oldenv_reval_20260821/sequential.log new file mode 100644 index 0000000000000000000000000000000000000000..07a3eb37d9553057f10781625136289e036dd85b --- /dev/null +++ b/logs/oldenv_reval_20260821/sequential.log @@ -0,0 +1,15 @@ +[preflight] fixed environment +torch=2.6.0+cu124 cuda=12.4 flash_attn=False +tools/run_oldenv_validation_reruns_sequential.sh: line 36: /mnt/local_nvme/cz/projects/HY-WorldPlay-DEV-Predictor/tools/launch_vbench_validation_predictor.sh: Permission denied +[preflight] fixed environment +torch=2.6.0+cu124 cuda=12.4 flash_attn=False +[preflight] fixed environment +torch=2.6.0+cu124 cuda=12.4 flash_attn=False +Validation evaluation complete: /mnt/local_nvme/cz/projects/HY-WorldPlay-DEV-Predictor/datasets/hyworldplay_predictor_vbench_val25_test50/evaluations/eval_validation25_stage1-c1-threshold-r10_oldenv_reval_20260821 +Validation evaluation complete: /mnt/local_nvme/cz/projects/HY-WorldPlay-DEV-Predictor/datasets/hyworldplay_predictor_vbench_val25_test50/evaluations/eval_validation25_stage1-c1-threshold-r25_oldenv_reval_20260821 +Validation evaluation complete: /mnt/local_nvme/cz/projects/HY-WorldPlay-DEV-Predictor/datasets/hyworldplay_predictor_vbench_val25_test50/evaluations/eval_validation25_stage1-c1-threshold-r33_oldenv_reval_20260821 +Validation evaluation complete: /mnt/local_nvme/cz/projects/HY-WorldPlay-DEV-Predictor/datasets/hyworldplay_predictor_vbench_val25_test50/evaluations/eval_validation25_stage1-c1-threshold-r50_oldenv_reval_20260821 +Validation evaluation complete: /mnt/local_nvme/cz/projects/HY-WorldPlay-DEV-Predictor/datasets/hyworldplay_predictor_vbench_val25_test50/evaluations/eval_validation25_stage1-wide4096x2-c1-threshold-r33_oldenv_reval_20260821 +Validation evaluation complete: /mnt/local_nvme/cz/projects/HY-WorldPlay-DEV-Predictor/datasets/hyworldplay_predictor_vbench_val25_test50/evaluations/eval_validation25_stage1-wide4096x2-c2-hybrid-p3full-p12r10_oldenv_reval_20260821 +Validation evaluation complete: /mnt/local_nvme/cz/projects/HY-WorldPlay-DEV-Predictor/datasets/hyworldplay_predictor_vbench_val25_test50/evaluations/eval_validation25_stage1-wide4096x2-c2-hybrid-p3full-p12r25_oldenv_reval_20260821 +[complete] all fixed-environment validation reruns diff --git a/logs/oldenv_reval_20260821/status.tsv b/logs/oldenv_reval_20260821/status.tsv new file mode 100644 index 0000000000000000000000000000000000000000..7a745b15c73d77c05359aaa70035a0b2fc5f65cb --- /dev/null +++ b/logs/oldenv_reval_20260821/status.tsv @@ -0,0 +1,18 @@ +preflight 2026-08-21T09:55:59Z torch=2.6.0+cu124; flash_attn=False +preflight 2026-08-21T09:56:20Z torch=2.6.0+cu124; flash_attn=False +preflight 2026-08-21T09:57:00Z torch=2.6.0+cu124; flash_attn=False +start 2026-08-21T12:26:05Z eval_validation25_stage1-c1-threshold-r10_oldenv_reval_20260821 +complete 2026-08-21T12:48:24Z eval_validation25_stage1-c1-threshold-r10_oldenv_reval_20260821 +start 2026-08-21T12:48:24Z eval_validation25_stage1-c1-threshold-r25_oldenv_reval_20260821 +complete 2026-08-21T13:13:27Z eval_validation25_stage1-c1-threshold-r25_oldenv_reval_20260821 +start 2026-08-21T13:13:27Z eval_validation25_stage1-c1-threshold-r33_oldenv_reval_20260821 +complete 2026-08-21T13:35:57Z eval_validation25_stage1-c1-threshold-r33_oldenv_reval_20260821 +start 2026-08-21T13:35:57Z eval_validation25_stage1-c1-threshold-r50_oldenv_reval_20260821 +complete 2026-08-21T14:00:16Z eval_validation25_stage1-c1-threshold-r50_oldenv_reval_20260821 +start 2026-08-21T14:00:16Z eval_validation25_stage1-wide4096x2-c1-threshold-r33_oldenv_reval_20260821 +complete 2026-08-21T14:23:24Z eval_validation25_stage1-wide4096x2-c1-threshold-r33_oldenv_reval_20260821 +start 2026-08-21T14:23:24Z eval_validation25_stage1-wide4096x2-c2-hybrid-p3full-p12r10_oldenv_reval_20260821 +complete 2026-08-21T14:46:58Z eval_validation25_stage1-wide4096x2-c2-hybrid-p3full-p12r10_oldenv_reval_20260821 +start 2026-08-21T14:46:58Z eval_validation25_stage1-wide4096x2-c2-hybrid-p3full-p12r25_oldenv_reval_20260821 +complete 2026-08-21T15:11:01Z eval_validation25_stage1-wide4096x2-c2-hybrid-p3full-p12r25_oldenv_reval_20260821 +complete 2026-08-21T15:11:01Z all validation reruns diff --git a/logs/oldenv_reval_20260821/tmux_session b/logs/oldenv_reval_20260821/tmux_session new file mode 100644 index 0000000000000000000000000000000000000000..ae06abc1acd4c2b094b3c5e3848b0e9a40b3defa --- /dev/null +++ b/logs/oldenv_reval_20260821/tmux_session @@ -0,0 +1 @@ +oldenv_reval_20260821 diff --git a/logs/pipeline_timestep-rollout-detached-timestepfp32-overfit8-pure_flr1e-4_blr1e-5_schedsteps1000_swarmup8_bwarmup8.log b/logs/pipeline_timestep-rollout-detached-timestepfp32-overfit8-pure_flr1e-4_blr1e-5_schedsteps1000_swarmup8_bwarmup8.log new file mode 100644 index 0000000000000000000000000000000000000000..92c75aab213d9aa86e23a4fcde197a0fafaa0c4b --- /dev/null +++ b/logs/pipeline_timestep-rollout-detached-timestepfp32-overfit8-pure_flr1e-4_blr1e-5_schedsteps1000_swarmup8_bwarmup8.log @@ -0,0 +1,1134 @@ +[pipeline] stage=sidecar attempt=1 start=2026-07-22T04:13:08Z +[dataset] NVMe cache is complete: /mnt/local_nvme/zoubin/cz/hyworldplay_predictor_prefeature_v4 +[sidecar] NVMe cache is complete: /mnt/local_nvme/zoubin/cz/hyworldplay_predictor_timestep_rollout_overfit8_sidecar +[pipeline] stage=sidecar complete=2026-07-22T04:13:08Z +[pipeline] stage=train_pure attempt=1 start=2026-07-22T04:13:08Z +[dataset] NVMe cache is complete: /mnt/local_nvme/zoubin/cz/hyworldplay_predictor_prefeature_v4 +[sidecar] NVMe cache is complete: /mnt/local_nvme/zoubin/cz/hyworldplay_predictor_timestep_rollout_overfit8_sidecar +HY-World 1.5 loading from: /mnt/s3files/s3-us-west2-default/zoubin/cz/checkpoints/hy_worldplay/huggingface/hub/models--tencent--HY-WorldPlay/snapshots/f4c29235647707b571479a69b569e4166f9f5bf8/ar_distilled_action_model/diffusion_pytorch_model.safetensors +HY-World 1.5 loading from: /mnt/s3files/s3-us-west2-default/zoubin/cz/checkpoints/hy_worldplay/huggingface/hub/models--tencent--HY-WorldPlay/snapshots/f4c29235647707b571479a69b569e4166f9f5bf8/ar_distilled_action_model/diffusion_pytorch_model.safetensors +HY-World 1.5 loading from: /mnt/s3files/s3-us-west2-default/zoubin/cz/checkpoints/hy_worldplay/huggingface/hub/models--tencent--HY-WorldPlay/snapshots/f4c29235647707b571479a69b569e4166f9f5bf8/ar_distilled_action_model/diffusion_pytorch_model.safetensors +HY-World 1.5 loading from: /mnt/s3files/s3-us-west2-default/zoubin/cz/checkpoints/hy_worldplay/huggingface/hub/models--tencent--HY-WorldPlay/snapshots/f4c29235647707b571479a69b569e4166f9f5bf8/ar_distilled_action_model/diffusion_pytorch_model.safetensors +HY-World 1.5 loading from: /mnt/s3files/s3-us-west2-default/zoubin/cz/checkpoints/hy_worldplay/huggingface/hub/models--tencent--HY-WorldPlay/snapshots/f4c29235647707b571479a69b569e4166f9f5bf8/ar_distilled_action_model/diffusion_pytorch_model.safetensors +HY-World 1.5 loading from: /mnt/s3files/s3-us-west2-default/zoubin/cz/checkpoints/hy_worldplay/huggingface/hub/models--tencent--HY-WorldPlay/snapshots/f4c29235647707b571479a69b569e4166f9f5bf8/ar_distilled_action_model/diffusion_pytorch_model.safetensors +HY-World 1.5 loading from: /mnt/s3files/s3-us-west2-default/zoubin/cz/checkpoints/hy_worldplay/huggingface/hub/models--tencent--HY-WorldPlay/snapshots/f4c29235647707b571479a69b569e4166f9f5bf8/ar_distilled_action_model/diffusion_pytorch_model.safetensors +HY-World 1.5 loading from: /mnt/s3files/s3-us-west2-default/zoubin/cz/checkpoints/hy_worldplay/huggingface/hub/models--tencent--HY-WorldPlay/snapshots/f4c29235647707b571479a69b569e4166f9f5bf8/ar_distilled_action_model/diffusion_pytorch_model.safetensors + Loading checkpoint shards: 0%| | 0/5 [00:00 524 (60314ms) trace:unknown | [ERR] +code=unknown code message=unknown error +2026-07-19 17:17:31.718 | WARNING | +swanlab.sdk.internal.core_python.transport.thread:warn:187 - Upload failed due +to network or server issues, retrying automatically. +swanlab: Upload recovered, resuming normally. +{"batch_in_epoch": 14, "epoch": 3, "global_step": 164, "grad_norm_max": 0.033203125, "hidden_mse_step1": 0.015401822232109093, "hidden_mse_step2": 0.017933018879765377, "hidden_mse_step3": 0.020822720153124204, "latent_grad_l1_min": 0.0001606482483111904, "loss": 0.01471388668869622, "lr_blocks": 9.915907145611116e-07, "lr_fusion": 9.915907145611117e-06, "peak_memory_gib": 48.31737470626831, "trajectory_time_s": 73.79665059374747, "velocity_mse_step1": 0.010255491523171909, "velocity_mse_step2": 0.01154587581653946, "velocity_mse_step3": 0.016924535847335522} +{"batch_in_epoch": 15, "epoch": 3, "global_step": 165, "grad_norm_max": 0.035888671875, "hidden_mse_step1": 0.024378451976060336, "hidden_mse_step2": 0.02667453404449459, "hidden_mse_step3": 0.02936327304424984, "latent_grad_l1_min": 0.00022514902775583323, "loss": 0.02498950178102989, "lr_blocks": 9.914429602677161e-07, "lr_fusion": 9.914429602677163e-06, "peak_memory_gib": 48.31737470626831, "trajectory_time_s": 73.40575261199774, "velocity_mse_step1": 0.017812923573988622, "velocity_mse_step2": 0.02045081806967833, "velocity_mse_step3": 0.028663134784437716} +{"batch_in_epoch": 16, "epoch": 3, "global_step": 166, "grad_norm_max": 0.031982421875, "hidden_mse_step1": 0.021294559589088227, "hidden_mse_step2": 0.024147753121464378, "hidden_mse_step3": 0.027487836362394904, "latent_grad_l1_min": 0.00027111265171697596, "loss": 0.025128388927052065, "lr_blocks": 9.912939304065218e-07, "lr_fusion": 9.912939304065219e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 75.97419820574942, "velocity_mse_step1": 0.017254367128771264, "velocity_mse_step2": 0.02094627194206363, "velocity_mse_step3": 0.02989151092645313} +{"batch_in_epoch": 17, "epoch": 3, "global_step": 167, "grad_norm_max": 0.046630859375, "hidden_mse_step1": 0.013851628942315333, "hidden_mse_step2": 0.016120662995880202, "hidden_mse_step3": 0.01901300303039274, "latent_grad_l1_min": 0.0002892659522331087, "loss": 0.018907130363264253, "lr_blocks": 9.911436253643443e-07, "lr_fusion": 9.911436253643445e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 71.80401715975313, "velocity_mse_step1": 0.015368575093037051, "velocity_mse_step2": 0.01631420334368678, "velocity_mse_step3": 0.0201400813197584} +{"batch_in_epoch": 18, "epoch": 3, "global_step": 168, "grad_norm_max": 0.031982421875, "hidden_mse_step1": 0.021060333703644574, "hidden_mse_step2": 0.023356877300622206, "hidden_mse_step3": 0.02592327108141035, "latent_grad_l1_min": 0.000277518844086444, "loss": 0.025293160412859703, "lr_blocks": 9.909920455313086e-07, "lr_fusion": 9.909920455313087e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 72.0510604063711, "velocity_mse_step1": 0.02011672368098516, "velocity_mse_step2": 0.021552087211083353, "velocity_mse_step3": 0.027176621170448403} +{"batch_in_epoch": 19, "epoch": 3, "global_step": 169, "grad_norm_max": 0.03271484375, "hidden_mse_step1": 0.01783509609023375, "hidden_mse_step2": 0.020823059478841187, "hidden_mse_step3": 0.024324240262753195, "latent_grad_l1_min": 0.0002198557212977903, "loss": 0.01874054080274488, "lr_blocks": 9.908391913008488e-07, "lr_fusion": 9.908391913008491e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 71.82103597349851, "velocity_mse_step1": 0.012813902181473427, "velocity_mse_step2": 0.014995502116757311, "velocity_mse_step3": 0.022113977357678644} +{"batch_in_epoch": 20, "epoch": 3, "global_step": 170, "grad_norm_max": 0.03466796875, "hidden_mse_step1": 0.014547579264866987, "hidden_mse_step2": 0.01689187466399744, "hidden_mse_step3": 0.02001405395900032, "latent_grad_l1_min": 0.000163606919159065, "loss": 0.014582132413384637, "lr_blocks": 9.906850630697066e-07, "lr_fusion": 9.906850630697068e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 72.96309601187386, "velocity_mse_step1": 0.010203754995018244, "velocity_mse_step2": 0.01138086983285445, "velocity_mse_step3": 0.017016420199070122} +{"batch_in_epoch": 21, "epoch": 3, "global_step": 171, "grad_norm_max": 0.0269775390625, "hidden_mse_step1": 0.017491335240525326, "hidden_mse_step2": 0.01960570825031027, "hidden_mse_step3": 0.022348778854523386, "latent_grad_l1_min": 0.00018923444804386236, "loss": 0.020276554169998108, "lr_blocks": 9.905296612379305e-07, "lr_fusion": 9.905296612379307e-06, "peak_memory_gib": 48.31737470626831, "trajectory_time_s": 71.44894313499753, "velocity_mse_step1": 0.015137573845484956, "velocity_mse_step2": 0.016967878294443448, "velocity_mse_step3": 0.022779626538977027} +{"batch_in_epoch": 22, "epoch": 3, "global_step": 172, "grad_norm_max": 0.04296875, "hidden_mse_step1": 0.016616806396216686, "hidden_mse_step2": 0.01987311497630019, "hidden_mse_step3": 0.023078651325444555, "latent_grad_l1_min": 0.00021714915237680543, "loss": 0.01846917747752741, "lr_blocks": 9.903729862088746e-07, "lr_fusion": 9.903729862088749e-06, "peak_memory_gib": 48.320029735565186, "trajectory_time_s": 73.94779478762575, "velocity_mse_step1": 0.012883023663432268, "velocity_mse_step2": 0.015219493110115372, "velocity_mse_step3": 0.02134815734046112} +{"batch_in_epoch": 23, "epoch": 3, "global_step": 173, "grad_norm_max": 0.02685546875, "hidden_mse_step1": 0.020686536969151344, "hidden_mse_step2": 0.022961171310661093, "hidden_mse_step3": 0.026689097800824256, "latent_grad_l1_min": 0.00026937791153613944, "loss": 0.023117676302458028, "lr_blocks": 9.902150383891979e-07, "lr_fusion": 9.902150383891979e-06, "peak_memory_gib": 48.316489696502686, "trajectory_time_s": 71.83440179499667, "velocity_mse_step1": 0.016314768103516793, "velocity_mse_step2": 0.018802814892426665, "velocity_mse_step3": 0.027201763204565004} +{"batch_in_epoch": 24, "epoch": 3, "global_step": 174, "grad_norm_max": 0.041259765625, "hidden_mse_step1": 0.01777442518921037, "hidden_mse_step2": 0.020208645412432297, "hidden_mse_step3": 0.02397007797844708, "latent_grad_l1_min": 0.00023670183145441115, "loss": 0.02001655210707603, "lr_blocks": 9.900558181888626e-07, "lr_fusion": 9.900558181888626e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 72.00149610049993, "velocity_mse_step1": 0.01358635937711889, "velocity_mse_step2": 0.016271254471835812, "velocity_mse_step3": 0.023996725645182387} +{"batch_in_epoch": 25, "epoch": 3, "global_step": 175, "grad_norm_max": 0.042724609375, "hidden_mse_step1": 0.0172261823026929, "hidden_mse_step2": 0.01946904360582786, "hidden_mse_step3": 0.022420854133088138, "latent_grad_l1_min": 0.00020489265807555057, "loss": 0.018339198265623833, "lr_blocks": 9.898953260211337e-07, "lr_fusion": 9.89895326021134e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 75.39192361112146, "velocity_mse_step1": 0.012847872776287007, "velocity_mse_step2": 0.01500150952779222, "velocity_mse_step3": 0.021256602523083395} +{"batch_in_epoch": 26, "epoch": 3, "global_step": 176, "grad_norm_max": 0.046875, "hidden_mse_step1": 0.015205181656970775, "hidden_mse_step2": 0.017152655786568563, "hidden_mse_step3": 0.02018570495004366, "latent_grad_l1_min": 0.00022342488955473527, "loss": 0.017252513045345298, "lr_blocks": 9.89733562302578e-07, "lr_fusion": 9.89733562302578e-06, "peak_memory_gib": 48.31958723068237, "trajectory_time_s": 71.54154325262789, "velocity_mse_step1": 0.01384540162273749, "velocity_mse_step2": 0.013943401192331555, "velocity_mse_step3": 0.018714381024307968} +{"batch_in_epoch": 27, "epoch": 3, "global_step": 177, "grad_norm_max": 0.03369140625, "hidden_mse_step1": 0.016589768696576357, "hidden_mse_step2": 0.01884812597251896, "hidden_mse_step3": 0.0217659521315779, "latent_grad_l1_min": 0.0001813270409911638, "loss": 0.016991451457475444, "lr_blocks": 9.895705274530618e-07, "lr_fusion": 9.89570527453062e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 72.68541988613288, "velocity_mse_step1": 0.011271619780537936, "velocity_mse_step2": 0.013958566576807892, "velocity_mse_step3": 0.02002378227189183} +{"batch_in_epoch": 28, "epoch": 3, "global_step": 178, "grad_norm_max": 0.036376953125, "hidden_mse_step1": 0.017149195744423196, "hidden_mse_step2": 0.0188727858808956, "hidden_mse_step3": 0.0211932421828221, "latent_grad_l1_min": 0.00019733737190108513, "loss": 0.01938909083505028, "lr_blocks": 9.894062218957515e-07, "lr_fusion": 9.894062218957517e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 72.84702011900663, "velocity_mse_step1": 0.014696245792687739, "velocity_mse_step2": 0.016626833750966137, "velocity_mse_step3": 0.021122668678539673} +{"batch_in_epoch": 29, "epoch": 3, "global_step": 179, "grad_norm_max": 0.04443359375, "hidden_mse_step1": 0.02049754783677469, "hidden_mse_step2": 0.02264479195166911, "hidden_mse_step3": 0.025174833518186848, "latent_grad_l1_min": 0.0002091060214297613, "loss": 0.020998778786244135, "lr_blocks": 9.892406460571115e-07, "lr_fusion": 9.892406460571116e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 73.28926720500021, "velocity_mse_step1": 0.015128499190073592, "velocity_mse_step2": 0.017046921192169454, "velocity_mse_step3": 0.02398919655908165} +{"batch_in_epoch": 30, "epoch": 3, "global_step": 180, "grad_norm_max": 0.046875, "hidden_mse_step1": 0.01866725953628442, "hidden_mse_step2": 0.02153426086130951, "hidden_mse_step3": 0.024748679915709153, "latent_grad_l1_min": 0.0002654419367900118, "loss": 0.02159762486449576, "lr_blocks": 9.890738003669027e-07, "lr_fusion": 9.890738003669029e-06, "peak_memory_gib": 48.320029735565186, "trajectory_time_s": 72.64340735387668, "velocity_mse_step1": 0.015175929964503405, "velocity_mse_step2": 0.017971025164505203, "velocity_mse_step3": 0.025150897272396833} +{"batch_in_epoch": 31, "epoch": 3, "global_step": 181, "grad_norm_max": 0.0400390625, "hidden_mse_step1": 0.012862683606467074, "hidden_mse_step2": 0.015590871734145494, "hidden_mse_step3": 0.01923403755894729, "latent_grad_l1_min": 0.00020837265401496552, "loss": 0.013948689821907983, "lr_blocks": 9.88905685258183e-07, "lr_fusion": 9.889056852581833e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 71.74551871274889, "velocity_mse_step1": 0.008192181494710635, "velocity_mse_step2": 0.011023471294070726, "velocity_mse_step3": 0.017861656104547104} +{"batch_in_epoch": 32, "epoch": 3, "global_step": 182, "grad_norm_max": 0.034423828125, "hidden_mse_step1": 0.020794372934947857, "hidden_mse_step2": 0.023050935099100958, "hidden_mse_step3": 0.027078542060085704, "latent_grad_l1_min": 0.00025337530405522557, "loss": 0.0250129175006545, "lr_blocks": 9.887363011673045e-07, "lr_fusion": 9.887363011673046e-06, "peak_memory_gib": 48.31958723068237, "trajectory_time_s": 73.65130514838165, "velocity_mse_step1": 0.01820231195389559, "velocity_mse_step2": 0.020429936998490508, "velocity_mse_step3": 0.029314117298261926} +{"batch_in_epoch": 33, "epoch": 3, "global_step": 183, "grad_norm_max": 0.0439453125, "hidden_mse_step1": 0.018113696515294056, "hidden_mse_step2": 0.021433768993509666, "hidden_mse_step3": 0.025853624949896972, "latent_grad_l1_min": 0.0002791015813272679, "loss": 0.022149408477292, "lr_blocks": 9.885656485339129e-07, "lr_fusion": 9.885656485339129e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 73.78357377987413, "velocity_mse_step1": 0.014195617562758603, "velocity_mse_step2": 0.01753798149625904, "velocity_mse_step3": 0.02817451559738921} +{"batch_in_epoch": 34, "epoch": 3, "global_step": 184, "grad_norm_max": 0.0439453125, "hidden_mse_step1": 0.01889127299987844, "hidden_mse_step2": 0.021581979468464848, "hidden_mse_step3": 0.02517524644333337, "latent_grad_l1_min": 0.0002678989458217984, "loss": 0.020689840021077543, "lr_blocks": 9.883937278009467e-07, "lr_fusion": 9.883937278009468e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 72.45336372637757, "velocity_mse_step1": 0.013370532848057335, "velocity_mse_step2": 0.01696276751629609, "velocity_mse_step3": 0.025171367393340915} +{"batch_in_epoch": 35, "epoch": 3, "global_step": 185, "grad_norm_max": 0.041259765625, "hidden_mse_step1": 0.021162605295622985, "hidden_mse_step2": 0.024152231003556932, "hidden_mse_step3": 0.027768577681854364, "latent_grad_l1_min": 0.00031276815934688784, "loss": 0.026458048112025217, "lr_blocks": 9.88220539414636e-07, "lr_fusion": 9.882205394146362e-06, "peak_memory_gib": 48.316489696502686, "trajectory_time_s": 74.59806434512575, "velocity_mse_step1": 0.019379272366807396, "velocity_mse_step2": 0.022338801238220185, "velocity_mse_step3": 0.030347727106085843} +{"batch_in_epoch": 36, "epoch": 3, "global_step": 186, "grad_norm_max": 0.03515625, "hidden_mse_step1": 0.015486492561259573, "hidden_mse_step2": 0.017281423401852538, "hidden_mse_step3": 0.02051860378456435, "latent_grad_l1_min": 0.0001712814046186395, "loss": 0.016137581079549688, "lr_blocks": 9.88046083824501e-07, "lr_fusion": 9.88046083824501e-06, "peak_memory_gib": 48.31737470626831, "trajectory_time_s": 74.5922869578826, "velocity_mse_step1": 0.010236258147675209, "velocity_mse_step2": 0.013032919576549569, "velocity_mse_step3": 0.01981491262891463} +{"batch_in_epoch": 37, "epoch": 3, "global_step": 187, "grad_norm_max": 0.03466796875, "hidden_mse_step1": 0.01928795406794441, "hidden_mse_step2": 0.020664511780653684, "hidden_mse_step3": 0.023328994450691552, "latent_grad_l1_min": 0.0001714231984806247, "loss": 0.019161478774289465, "lr_blocks": 9.878703614833507e-07, "lr_fusion": 9.878703614833507e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 74.5743226702507, "velocity_mse_step1": 0.012921931446596448, "velocity_mse_step2": 0.015672203548352366, "velocity_mse_step3": 0.0225621531384864} +{"batch_in_epoch": 38, "epoch": 3, "global_step": 188, "grad_norm_max": 0.0419921875, "hidden_mse_step1": 0.02028414582101894, "hidden_mse_step2": 0.022304104848964404, "hidden_mse_step3": 0.02583620654019926, "latent_grad_l1_min": 0.00026544532920524944, "loss": 0.023926582062683468, "lr_blocks": 9.876933728472826e-07, "lr_fusion": 9.876933728472826e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 73.00854191199687, "velocity_mse_step1": 0.016700009025433765, "velocity_mse_step2": 0.020364829638440697, "velocity_mse_step3": 0.027872459771710312} +{"batch_in_epoch": 39, "epoch": 3, "global_step": 189, "grad_norm_max": 0.040283203125, "hidden_mse_step1": 0.015601621275501592, "hidden_mse_step2": 0.018491919060969458, "hidden_mse_step3": 0.02193029540857034, "latent_grad_l1_min": 0.00021374716925492976, "loss": 0.0172998086075365, "lr_blocks": 9.875151183756806e-07, "lr_fusion": 9.875151183756806e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 74.1318911518756, "velocity_mse_step1": 0.011639332623287502, "velocity_mse_step2": 0.014203333125416454, "velocity_mse_step3": 0.02045437498184453} +{"batch_in_epoch": 40, "epoch": 3, "global_step": 190, "grad_norm_max": 0.037353515625, "hidden_mse_step1": 0.01750978239579126, "hidden_mse_step2": 0.020129261987416873, "hidden_mse_step3": 0.02291486967754151, "latent_grad_l1_min": 0.00026554854775895365, "loss": 0.019567673102886016, "lr_blocks": 9.87335598531214e-07, "lr_fusion": 9.873355985312141e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 71.72441401211836, "velocity_mse_step1": 0.014038235431403987, "velocity_mse_step2": 0.016231961411124624, "velocity_mse_step3": 0.022377429388663064} +{"batch_in_epoch": 41, "epoch": 3, "global_step": 191, "grad_norm_max": 0.03955078125, "hidden_mse_step1": 0.01999972404363299, "hidden_mse_step2": 0.02165158818076764, "hidden_mse_step3": 0.024572647658974996, "latent_grad_l1_min": 0.00019642316692625172, "loss": 0.02091417914821899, "lr_blocks": 9.871548137798367e-07, "lr_fusion": 9.871548137798367e-06, "peak_memory_gib": 48.320029735565186, "trajectory_time_s": 72.8223010983711, "velocity_mse_step1": 0.014663191456098242, "velocity_mse_step2": 0.016893498876015656, "velocity_mse_step3": 0.024563447904906097} +{"batch_in_epoch": 42, "epoch": 3, "global_step": 192, "grad_norm_max": 0.037353515625, "hidden_mse_step1": 0.011140174885180645, "hidden_mse_step2": 0.01443465202880491, "hidden_mse_step3": 0.018305740019838722, "latent_grad_l1_min": 0.0001943336483236635, "loss": 0.011573008781332255, "lr_blocks": 9.869727645907857e-07, "lr_fusion": 9.86972764590786e-06, "peak_memory_gib": 48.31737470626831, "trajectory_time_s": 71.50173364925286, "velocity_mse_step1": 0.0069185724111905855, "velocity_mse_step2": 0.008993364910046302, "velocity_mse_step3": 0.014419031404291413} +{"batch_in_epoch": 43, "epoch": 3, "global_step": 193, "grad_norm_max": 0.03125, "hidden_mse_step1": 0.01264625137472259, "hidden_mse_step2": 0.015797566614180268, "hidden_mse_step3": 0.019626919562662284, "latent_grad_l1_min": 0.0002151937205780996, "loss": 0.014094632962951437, "lr_blocks": 9.8678945143658e-07, "lr_fusion": 9.867894514365802e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 71.91157248837044, "velocity_mse_step1": 0.009123198357493882, "velocity_mse_step2": 0.01154438426809585, "velocity_mse_step3": 0.016809241039611927} +{"batch_in_epoch": 44, "epoch": 3, "global_step": 194, "grad_norm_max": 0.033203125, "hidden_mse_step1": 0.014474054794326159, "hidden_mse_step2": 0.01679506023979879, "hidden_mse_step3": 0.020322411321103573, "latent_grad_l1_min": 0.0001910152059281245, "loss": 0.014546040836389041, "lr_blocks": 9.866048747930192e-07, "lr_fusion": 9.866048747930194e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 73.39434499112394, "velocity_mse_step1": 0.010170841870214125, "velocity_mse_step2": 0.011294945267893907, "velocity_mse_step3": 0.017013181474924616} +{"batch_in_epoch": 45, "epoch": 3, "global_step": 195, "grad_norm_max": 0.03515625, "hidden_mse_step1": 0.0181159281886981, "hidden_mse_step2": 0.01992582690270085, "hidden_mse_step3": 0.023168451402203315, "latent_grad_l1_min": 0.00020276993836887414, "loss": 0.018002713099122044, "lr_blocks": 9.86419035139182e-07, "lr_fusion": 9.864190351391822e-06, "peak_memory_gib": 48.320029735565186, "trajectory_time_s": 71.81698475912344, "velocity_mse_step1": 0.012945138699641185, "velocity_mse_step2": 0.014390059372609745, "velocity_mse_step3": 0.020551919064018872} +{"batch_in_epoch": 46, "epoch": 3, "global_step": 196, "grad_norm_max": 0.033203125, "hidden_mse_step1": 0.016629957901646515, "hidden_mse_step2": 0.019247399909155707, "hidden_mse_step3": 0.023191210198482227, "latent_grad_l1_min": 0.00025323744193883613, "loss": 0.018361796575066237, "lr_blocks": 9.862319329574263e-07, "lr_fusion": 9.862319329574264e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 71.99675237449628, "velocity_mse_step1": 0.01241041862106483, "velocity_mse_step2": 0.014524561319766299, "velocity_mse_step3": 0.02224355112827782} +{"batch_in_epoch": 47, "epoch": 3, "global_step": 197, "grad_norm_max": 0.03662109375, "hidden_mse_step1": 0.01945128832760799, "hidden_mse_step2": 0.020581990214330805, "hidden_mse_step3": 0.023287450046544627, "latent_grad_l1_min": 0.00019730422627617372, "loss": 0.01971995485447613, "lr_blocks": 9.860435687333857e-07, "lr_fusion": 9.860435687333857e-06, "peak_memory_gib": 48.31737470626831, "trajectory_time_s": 74.32129192224966, "velocity_mse_step1": 0.014829726984316949, "velocity_mse_step2": 0.015676209480651387, "velocity_mse_step3": 0.022321852489507625} +{"batch_in_epoch": 48, "epoch": 3, "global_step": 198, "grad_norm_max": 0.034912109375, "hidden_mse_step1": 0.015002771168448295, "hidden_mse_step2": 0.018202398180229857, "hidden_mse_step3": 0.022358642891049385, "latent_grad_l1_min": 0.00023228171176015167, "loss": 0.018165228975703936, "lr_blocks": 9.858539429559704e-07, "lr_fusion": 9.858539429559705e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 71.03292989187321, "velocity_mse_step1": 0.01157527252299977, "velocity_mse_step2": 0.014777286967728287, "velocity_mse_step3": 0.022586745093576607} +{"batch_in_epoch": 49, "epoch": 3, "global_step": 199, "grad_norm_max": 0.032470703125, "hidden_mse_step1": 0.01958468283659645, "hidden_mse_step2": 0.021951640690011636, "hidden_mse_step3": 0.02488925595701273, "latent_grad_l1_min": 0.0002512120991013944, "loss": 0.020269797574396113, "lr_blocks": 9.856630561173646e-07, "lr_fusion": 9.856630561173648e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 72.77487706075044, "velocity_mse_step1": 0.014212623221932778, "velocity_mse_step2": 0.01644856993308557, "velocity_mse_step3": 0.023505639817033495} +{"batch_in_epoch": 50, "epoch": 3, "global_step": 200, "grad_norm_max": 0.04345703125, "hidden_mse_step1": 0.01746175710078595, "hidden_mse_step2": 0.020429548657765344, "hidden_mse_step3": 0.023741167288140522, "latent_grad_l1_min": 0.00019648957459139638, "loss": 0.019941647744937133, "lr_blocks": 9.85470908713026e-07, "lr_fusion": 9.854709087130261e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 74.95465241775673, "velocity_mse_step1": 0.01359171654829489, "velocity_mse_step2": 0.016625614786919738, "velocity_mse_step3": 0.023444362408814153} +{"batch_in_epoch": 1, "epoch": 4, "global_step": 201, "grad_norm_max": 0.036376953125, "hidden_mse_step1": 0.016087870683155155, "hidden_mse_step2": 0.01875614251808396, "hidden_mse_step3": 0.021904564911632663, "latent_grad_l1_min": 0.0001735080340949935, "loss": 0.015354114775878511, "lr_blocks": 9.852775012416838e-07, "lr_fusion": 9.85277501241684e-06, "peak_memory_gib": 48.32135725021362, "trajectory_time_s": 74.00157648037566, "velocity_mse_step1": 0.009979153691217236, "velocity_mse_step2": 0.011887409069458954, "velocity_mse_step3": 0.018520922308588133} +{"batch_in_epoch": 2, "epoch": 4, "global_step": 202, "grad_norm_max": 0.0284423828125, "hidden_mse_step1": 0.01914064277246195, "hidden_mse_step2": 0.02194440487905272, "hidden_mse_step3": 0.025352082015680413, "latent_grad_l1_min": 0.000284598459074914, "loss": 0.02134411910914683, "lr_blocks": 9.85082834205338e-07, "lr_fusion": 9.850828342053382e-06, "peak_memory_gib": 48.320029735565186, "trajectory_time_s": 72.10369293887197, "velocity_mse_step1": 0.01416659537686168, "velocity_mse_step2": 0.017780713620595634, "velocity_mse_step3": 0.02544133392595021} +{"batch_in_epoch": 3, "epoch": 4, "global_step": 203, "grad_norm_max": 0.03076171875, "hidden_mse_step1": 0.020554618336193795, "hidden_mse_step2": 0.024141105557126657, "hidden_mse_step3": 0.02825460266987128, "latent_grad_l1_min": 0.00025041820845217444, "loss": 0.026190828702445802, "lr_blocks": 9.84886908109258e-07, "lr_fusion": 9.848869081092581e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 74.47608105236577, "velocity_mse_step1": 0.01802898373820686, "velocity_mse_step2": 0.0217970320455996, "velocity_mse_step3": 0.03145143528568691} +{"batch_in_epoch": 4, "epoch": 4, "global_step": 204, "grad_norm_max": 0.031494140625, "hidden_mse_step1": 0.019816937976949184, "hidden_mse_step2": 0.021349981976007775, "hidden_mse_step3": 0.023800490008267974, "latent_grad_l1_min": 0.0001596251404407667, "loss": 0.020052472973475233, "lr_blocks": 9.846897234619808e-07, "lr_fusion": 9.84689723461981e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 74.04787831962676, "velocity_mse_step1": 0.014862973509090287, "velocity_mse_step2": 0.016172273080363602, "velocity_mse_step3": 0.022625429547458352} +{"batch_in_epoch": 5, "epoch": 4, "global_step": 205, "grad_norm_max": 0.047607421875, "hidden_mse_step1": 0.022384561240739585, "hidden_mse_step2": 0.024744881533219347, "hidden_mse_step3": 0.02736351436137089, "latent_grad_l1_min": 0.0003133121699647745, "loss": 0.028796452687986727, "lr_blocks": 9.844912807753102e-07, "lr_fusion": 9.844912807753105e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 74.88686204737314, "velocity_mse_step1": 0.023867492867533913, "velocity_mse_step2": 0.024938516668044027, "velocity_mse_step3": 0.03013404942716338} +{"batch_in_epoch": 6, "epoch": 4, "global_step": 206, "grad_norm_max": 0.031982421875, "hidden_mse_step1": 0.0165907290938776, "hidden_mse_step2": 0.01917490378920255, "hidden_mse_step3": 0.02347236763619418, "latent_grad_l1_min": 0.0002276194718433544, "loss": 0.0178604314070461, "lr_blocks": 9.842915805643156e-07, "lr_fusion": 9.842915805643156e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 72.84359660437622, "velocity_mse_step1": 0.01147244450112339, "velocity_mse_step2": 0.014170613152340852, "velocity_mse_step3": 0.022014434690520696} +{"batch_in_epoch": 7, "epoch": 4, "global_step": 207, "grad_norm_max": 0.033203125, "hidden_mse_step1": 0.021148196966221024, "hidden_mse_step2": 0.023319087872680805, "hidden_mse_step3": 0.025841019382434233, "latent_grad_l1_min": 0.0001921207003761083, "loss": 0.02114239187877891, "lr_blocks": 9.840906233473297e-07, "lr_fusion": 9.840906233473298e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 72.08407687550061, "velocity_mse_step1": 0.014513073773871706, "velocity_mse_step2": 0.017005959850004207, "velocity_mse_step3": 0.024877309250379245} +{"batch_in_epoch": 8, "epoch": 4, "global_step": 208, "grad_norm_max": 0.0281982421875, "hidden_mse_step1": 0.016421290156098882, "hidden_mse_step2": 0.018564312519239526, "hidden_mse_step3": 0.021509926849310954, "latent_grad_l1_min": 0.0001708886575215729, "loss": 0.01683249616845777, "lr_blocks": 9.838884096459486e-07, "lr_fusion": 9.838884096459486e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 71.19595479137934, "velocity_mse_step1": 0.011278468477290256, "velocity_mse_step2": 0.013318981414028843, "velocity_mse_step3": 0.020250484214297364} +{"batch_in_epoch": 9, "epoch": 4, "global_step": 209, "grad_norm_max": 0.03564453125, "hidden_mse_step1": 0.01427999375820426, "hidden_mse_step2": 0.0172207842863697, "hidden_mse_step3": 0.020893413107842203, "latent_grad_l1_min": 0.0001816123576645623, "loss": 0.013462292279915085, "lr_blocks": 9.83684939985029e-07, "lr_fusion": 9.836849399850291e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 74.49284568037547, "velocity_mse_step1": 0.008308440769074617, "velocity_mse_step2": 0.010407456269604154, "velocity_mse_step3": 0.01643155997901756} +{"batch_in_epoch": 10, "epoch": 4, "global_step": 210, "grad_norm_max": 0.039794921875, "hidden_mse_step1": 0.012372784877827923, "hidden_mse_step2": 0.015013692393300256, "hidden_mse_step3": 0.01855677259819848, "latent_grad_l1_min": 0.000157704203047615, "loss": 0.011781315917947462, "lr_blocks": 9.834802148926882e-07, "lr_fusion": 9.834802148926883e-06, "peak_memory_gib": 48.320029735565186, "trajectory_time_s": 70.76657633624927, "velocity_mse_step1": 0.0070602675976780506, "velocity_mse_step2": 0.0092566416819214, "velocity_mse_step3": 0.014432712528754821} +{"batch_in_epoch": 11, "epoch": 4, "global_step": 211, "grad_norm_max": 0.036376953125, "hidden_mse_step1": 0.013299637216342877, "hidden_mse_step2": 0.016103074070997536, "hidden_mse_step3": 0.019861516409686635, "latent_grad_l1_min": 0.00019618527312559308, "loss": 0.014702087708948444, "lr_blocks": 9.832742349003013e-07, "lr_fusion": 9.832742349003014e-06, "peak_memory_gib": 48.320029735565186, "trajectory_time_s": 71.57895948825171, "velocity_mse_step1": 0.009753407218210799, "velocity_mse_step2": 0.011851968696906363, "velocity_mse_step3": 0.017574463139421174} +{"batch_in_epoch": 12, "epoch": 4, "global_step": 212, "grad_norm_max": 0.032958984375, "hidden_mse_step1": 0.021206430492124388, "hidden_mse_step2": 0.0232927079611857, "hidden_mse_step3": 0.02691125201194414, "latent_grad_l1_min": 0.00032665881553839426, "loss": 0.025883880086309673, "lr_blocks": 9.830670005425012e-07, "lr_fusion": 9.830670005425012e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 71.25039996186388, "velocity_mse_step1": 0.019345457341322408, "velocity_mse_step2": 0.02144397735329611, "velocity_mse_step3": 0.02972116402816027} +{"batch_in_epoch": 13, "epoch": 4, "global_step": 213, "grad_norm_max": 0.032958984375, "hidden_mse_step1": 0.021184544037428817, "hidden_mse_step2": 0.023831303480879536, "hidden_mse_step3": 0.027085663102168058, "latent_grad_l1_min": 0.00023144297665567137, "loss": 0.022165742877405137, "lr_blocks": 9.828585123571762e-07, "lr_fusion": 9.828585123571763e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 71.32809778412775, "velocity_mse_step1": 0.014818808988950747, "velocity_mse_step2": 0.01817442345366414, "velocity_mse_step3": 0.02629384318632739} +{"batch_in_epoch": 14, "epoch": 4, "global_step": 214, "grad_norm_max": 0.03369140625, "hidden_mse_step1": 0.015718894033592994, "hidden_mse_step2": 0.018725300457195514, "hidden_mse_step3": 0.022422036006381464, "latent_grad_l1_min": 0.00021486452624230878, "loss": 0.018910788797906468, "lr_blocks": 9.826487708854692e-07, "lr_fusion": 9.826487708854693e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 71.47085484537092, "velocity_mse_step1": 0.014034545488747035, "velocity_mse_step2": 0.015601657393355187, "velocity_mse_step3": 0.021409538404051482} +{"batch_in_epoch": 15, "epoch": 4, "global_step": 215, "grad_norm_max": 0.034912109375, "hidden_mse_step1": 0.014019608189950562, "hidden_mse_step2": 0.01603904332400166, "hidden_mse_step3": 0.0193193371939872, "latent_grad_l1_min": 0.00022587595685763517, "loss": 0.014889261938930885, "lr_blocks": 9.824377766717758e-07, "lr_fusion": 9.824377766717758e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 74.78871742975025, "velocity_mse_step1": 0.010658863904869316, "velocity_mse_step2": 0.012181694224377027, "velocity_mse_step3": 0.016889427605617256} +{"batch_in_epoch": 16, "epoch": 4, "global_step": 216, "grad_norm_max": 0.034912109375, "hidden_mse_step1": 0.022400661779101935, "hidden_mse_step2": 0.02461531425693205, "hidden_mse_step3": 0.0277343105824132, "latent_grad_l1_min": 0.000259926568105584, "loss": 0.026010753362373046, "lr_blocks": 9.822255302637434e-07, "lr_fusion": 9.822255302637434e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 71.32368567549929, "velocity_mse_step1": 0.018718120754264024, "velocity_mse_step2": 0.021935388221338923, "velocity_mse_step3": 0.029903720906336924} +{"batch_in_epoch": 17, "epoch": 4, "global_step": 217, "grad_norm_max": 0.038818359375, "hidden_mse_step1": 0.016857590958742157, "hidden_mse_step2": 0.019116823960627825, "hidden_mse_step3": 0.022213084473540735, "latent_grad_l1_min": 0.00018626591554493643, "loss": 0.01796049510760765, "lr_blocks": 9.820120322122693e-07, "lr_fusion": 9.820120322122695e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 71.9848058984935, "velocity_mse_step1": 0.01266380289286774, "velocity_mse_step2": 0.014469083481734353, "velocity_mse_step3": 0.02092984820150637} +2026-07-19 18:23:39.140 | ERROR | +swanlab.sdk.internal.pkg.client.session:send:131 - [HTTP] POST +https://api.swanlab.cn/api/house/metrics -> 525 (5253ms) trace:unknown | [ERR] +code=unknown code message=unknown error +2026-07-19 18:23:39.191 | WARNING | +swanlab.sdk.internal.core_python.transport.thread:warn:187 - Upload failed due +to network or server issues, retrying automatically. +2026-07-19 18:23:44.369 | ERROR | +swanlab.sdk.internal.pkg.client.session:send:131 - [HTTP] POST +https://api.swanlab.cn/api/house/metrics -> 525 (5170ms) trace:unknown | [ERR] +code=unknown code message=unknown error +swanlab: Upload recovered, resuming normally. +2026-07-19 18:23:50.675 | ERROR | +swanlab.sdk.internal.pkg.client.session:send:131 - [HTTP] POST +https://api.swanlab.cn/api/house/metrics -> 522 (5185ms) trace:unknown | [ERR] +code=unknown code message=unknown error +2026-07-19 18:23:50.687 | WARNING | +swanlab.sdk.internal.core_python.transport.thread:warn:187 - Upload failed due +to network or server issues, retrying automatically. +swanlab: Upload recovered, resuming normally. +{"batch_in_epoch": 18, "epoch": 4, "global_step": 218, "grad_norm_max": 0.033447265625, "hidden_mse_step1": 0.023395928342194696, "hidden_mse_step2": 0.0259585607065154, "hidden_mse_step3": 0.029141727419170944, "latent_grad_l1_min": 0.0003319166789879091, "loss": 0.031240510928910222, "lr_blocks": 9.817972830715002e-07, "lr_fusion": 9.817972830715003e-06, "peak_memory_gib": 48.31737470626831, "trajectory_time_s": 72.53379796125228, "velocity_mse_step1": 0.024672360729060268, "velocity_mse_step2": 0.026866778465253965, "velocity_mse_step3": 0.0343327700143813} +{"batch_in_epoch": 19, "epoch": 4, "global_step": 219, "grad_norm_max": 0.049560546875, "hidden_mse_step1": 0.01897217599201082, "hidden_mse_step2": 0.0211875475735204, "hidden_mse_step3": 0.02419759057894615, "latent_grad_l1_min": 0.00020974003564333543, "loss": 0.019492877424844273, "lr_blocks": 9.81581283398829e-07, "lr_fusion": 9.815812833988292e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 71.70994135874571, "velocity_mse_step1": 0.013462468403304229, "velocity_mse_step2": 0.015659520563661183, "velocity_mse_step3": 0.022920909495691633} +{"batch_in_epoch": 20, "epoch": 4, "global_step": 220, "grad_norm_max": 0.048583984375, "hidden_mse_step1": 0.01663657828300659, "hidden_mse_step2": 0.019256434602929015, "hidden_mse_step3": 0.022063783203650798, "latent_grad_l1_min": 0.00019989852262369823, "loss": 0.017030598836884438, "lr_blocks": 9.813640337548954e-07, "lr_fusion": 9.813640337548955e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 71.54757896911178, "velocity_mse_step1": 0.011276944725458243, "velocity_mse_step2": 0.01394223335332104, "velocity_mse_step3": 0.020076936767769175} +{"batch_in_epoch": 21, "epoch": 4, "global_step": 221, "grad_norm_max": 0.036865234375, "hidden_mse_step1": 0.019397113713369305, "hidden_mse_step2": 0.022013362946121818, "hidden_mse_step3": 0.02495434404616909, "latent_grad_l1_min": 0.0002159953710361151, "loss": 0.021908202025640224, "lr_blocks": 9.811455347035826e-07, "lr_fusion": 9.811455347035827e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 71.60648026150011, "velocity_mse_step1": 0.015452594575305867, "velocity_mse_step2": 0.0181508758354799, "velocity_mse_step3": 0.025484652491286397} +{"batch_in_epoch": 22, "epoch": 4, "global_step": 222, "grad_norm_max": 0.041748046875, "hidden_mse_step1": 0.012307757761196365, "hidden_mse_step2": 0.014962767483666537, "hidden_mse_step3": 0.017931507817203443, "latent_grad_l1_min": 0.0001956723954208428, "loss": 0.012130800809245555, "lr_blocks": 9.809257868120176e-07, "lr_fusion": 9.809257868120176e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 71.55931060562216, "velocity_mse_step1": 0.008419908186104814, "velocity_mse_step2": 0.009724350475672897, "velocity_mse_step3": 0.013727939388315593} +{"batch_in_epoch": 23, "epoch": 4, "global_step": 223, "grad_norm_max": 0.032958984375, "hidden_mse_step1": 0.017971640914246172, "hidden_mse_step2": 0.019328217812082063, "hidden_mse_step3": 0.022069016637812765, "latent_grad_l1_min": 0.00021795978045702213, "loss": 0.017898193743479035, "lr_blocks": 9.80704790650568e-07, "lr_fusion": 9.807047906505683e-06, "peak_memory_gib": 48.3169322013855, "trajectory_time_s": 71.85643779000384, "velocity_mse_step1": 0.013576353636121245, "velocity_mse_step2": 0.014076166474426693, "velocity_mse_step3": 0.020105171557848474} +{"batch_in_epoch": 24, "epoch": 4, "global_step": 224, "grad_norm_max": 0.033935546875, "hidden_mse_step1": 0.02026992453361994, "hidden_mse_step2": 0.021935471965532215, "hidden_mse_step3": 0.025030387948001075, "latent_grad_l1_min": 0.00020937031513312832, "loss": 0.020884284200812023, "lr_blocks": 9.80482546792842e-07, "lr_fusion": 9.804825467928423e-06, "peak_memory_gib": 48.31958723068237, "trajectory_time_s": 71.44021140337645, "velocity_mse_step1": 0.015179097681539131, "velocity_mse_step2": 0.016851448636901165, "velocity_mse_step3": 0.0238987263375228} +{"batch_in_epoch": 25, "epoch": 4, "global_step": 225, "grad_norm_max": 0.035400390625, "hidden_mse_step1": 0.021024405099784156, "hidden_mse_step2": 0.023531914164777845, "hidden_mse_step3": 0.026900543365627524, "latent_grad_l1_min": 0.00021362577444961062, "loss": 0.02282512654450589, "lr_blocks": 9.802590558156861e-07, "lr_fusion": 9.802590558156863e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 71.48392805350522, "velocity_mse_step1": 0.016510618395321735, "velocity_mse_step2": 0.018578855738659122, "velocity_mse_step3": 0.026240216834204536} +{"batch_in_epoch": 26, "epoch": 4, "global_step": 226, "grad_norm_max": 0.03173828125, "hidden_mse_step1": 0.018385164547778134, "hidden_mse_step2": 0.02025087477938671, "hidden_mse_step3": 0.023526293441786295, "latent_grad_l1_min": 0.00021057009871583432, "loss": 0.018724103840733215, "lr_blocks": 9.800343182991834e-07, "lr_fusion": 9.800343182991837e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 71.79014477249802, "velocity_mse_step1": 0.013386453780445402, "velocity_mse_step2": 0.014991007470858415, "velocity_mse_step3": 0.02157861539827926} +{"batch_in_epoch": 27, "epoch": 4, "global_step": 227, "grad_norm_max": 0.031494140625, "hidden_mse_step1": 0.01901848134418417, "hidden_mse_step2": 0.020927966314567517, "hidden_mse_step3": 0.024207734220129038, "latent_grad_l1_min": 0.00020436725935724098, "loss": 0.019159430642113354, "lr_blocks": 9.79808334826653e-07, "lr_fusion": 9.798083348266531e-06, "peak_memory_gib": 48.320029735565186, "trajectory_time_s": 73.07757994162603, "velocity_mse_step1": 0.012718345115510081, "velocity_mse_step2": 0.015414386892059286, "velocity_mse_step3": 0.02293013931817508} +{"batch_in_epoch": 28, "epoch": 4, "global_step": 228, "grad_norm_max": 0.03271484375, "hidden_mse_step1": 0.014879835511757325, "hidden_mse_step2": 0.017793586635629514, "hidden_mse_step3": 0.02173359657172113, "latent_grad_l1_min": 0.00020319720351835713, "loss": 0.0164025253366812, "lr_blocks": 9.795811059846475e-07, "lr_fusion": 9.795811059846476e-06, "peak_memory_gib": 48.32135725021362, "trajectory_time_s": 71.67388911312082, "velocity_mse_step1": 0.010117661871065917, "velocity_mse_step2": 0.013256158542519967, "velocity_mse_step3": 0.0203930525375264} +{"batch_in_epoch": 29, "epoch": 4, "global_step": 229, "grad_norm_max": 0.029541015625, "hidden_mse_step1": 0.016672205421075757, "hidden_mse_step2": 0.019150108747583414, "hidden_mse_step3": 0.02246504197163241, "latent_grad_l1_min": 0.00027706106448022183, "loss": 0.02006289767034884, "lr_blocks": 9.793526323629521e-07, "lr_fusion": 9.793526323629522e-06, "peak_memory_gib": 48.31958723068237, "trajectory_time_s": 71.29878835024283, "velocity_mse_step1": 0.014924248181549568, "velocity_mse_step2": 0.016365153598599136, "velocity_mse_step3": 0.023070554082681022} +{"batch_in_epoch": 30, "epoch": 4, "global_step": 230, "grad_norm_max": 0.04345703125, "hidden_mse_step1": 0.01893966548128187, "hidden_mse_step2": 0.02246027498456117, "hidden_mse_step3": 0.025823301303067377, "latent_grad_l1_min": 0.00035281638884043787, "loss": 0.027631720935460177, "lr_blocks": 9.791229145545832e-07, "lr_fusion": 9.791229145545832e-06, "peak_memory_gib": 48.320029735565186, "trajectory_time_s": 71.70430265487084, "velocity_mse_step1": 0.02177222433497913, "velocity_mse_step2": 0.024669323111343792, "velocity_mse_step3": 0.029731289051207047} +{"batch_in_epoch": 31, "epoch": 4, "global_step": 231, "grad_norm_max": 0.030029296875, "hidden_mse_step1": 0.018122250513572778, "hidden_mse_step2": 0.020156256763064966, "hidden_mse_step3": 0.023058770540436462, "latent_grad_l1_min": 0.00021677853419532767, "loss": 0.019679397485950694, "lr_blocks": 9.788919531557859e-07, "lr_fusion": 9.78891953155786e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 71.67935819299964, "velocity_mse_step1": 0.014290297254254776, "velocity_mse_step2": 0.016254098927934786, "velocity_mse_step3": 0.022360066575596905} +{"batch_in_epoch": 32, "epoch": 4, "global_step": 232, "grad_norm_max": 0.035400390625, "hidden_mse_step1": 0.016633869996959608, "hidden_mse_step2": 0.018899899292072014, "hidden_mse_step3": 0.022044356072521105, "latent_grad_l1_min": 0.00019566763421607902, "loss": 0.018345512466372123, "lr_blocks": 9.786597487660335e-07, "lr_fusion": 9.786597487660336e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 71.91445306875175, "velocity_mse_step1": 0.012671479883595434, "velocity_mse_step2": 0.015119331477243186, "velocity_mse_step3": 0.021487912146507627} +{"batch_in_epoch": 33, "epoch": 4, "global_step": 233, "grad_norm_max": 0.035400390625, "hidden_mse_step1": 0.01549414768981348, "hidden_mse_step2": 0.01786514152939032, "hidden_mse_step3": 0.021004591652724357, "latent_grad_l1_min": 0.00020491351097007282, "loss": 0.01761227985012478, "lr_blocks": 9.784263019880259e-07, "lr_fusion": 9.78426301988026e-06, "peak_memory_gib": 48.31958723068237, "trajectory_time_s": 71.71755086199482, "velocity_mse_step1": 0.013209084595603467, "velocity_mse_step2": 0.014562307786296252, "velocity_mse_step3": 0.019629057388686175} +{"batch_in_epoch": 34, "epoch": 4, "global_step": 234, "grad_norm_max": 0.0294189453125, "hidden_mse_step1": 0.016133686203310002, "hidden_mse_step2": 0.018216262147429267, "hidden_mse_step3": 0.021118923040506027, "latent_grad_l1_min": 0.00018760093462333316, "loss": 0.01727657563917871, "lr_blocks": 9.78191613427687e-07, "lr_fusion": 9.781916134276873e-06, "peak_memory_gib": 48.320029735565186, "trajectory_time_s": 72.59837502337177, "velocity_mse_step1": 0.012587197070908067, "velocity_mse_step2": 0.014006686089227772, "velocity_mse_step3": 0.01968895559964169} +{"batch_in_epoch": 35, "epoch": 4, "global_step": 235, "grad_norm_max": 0.03955078125, "hidden_mse_step1": 0.014404942348067249, "hidden_mse_step2": 0.017747435535836433, "hidden_mse_step3": 0.02172148996032774, "latent_grad_l1_min": 0.00021666699649358634, "loss": 0.014776254426189033, "lr_blocks": 9.779556836941645e-07, "lr_fusion": 9.779556836941646e-06, "peak_memory_gib": 48.3169322013855, "trajectory_time_s": 74.1855883916287, "velocity_mse_step1": 0.00918950410310312, "velocity_mse_step2": 0.011487629497423768, "velocity_mse_step3": 0.01826424180762842} +{"batch_in_epoch": 36, "epoch": 4, "global_step": 236, "grad_norm_max": 0.040771484375, "hidden_mse_step1": 0.018058964525282915, "hidden_mse_step2": 0.019384521575245475, "hidden_mse_step3": 0.02178767275264752, "latent_grad_l1_min": 0.00017620920152694453, "loss": 0.017263055687570673, "lr_blocks": 9.777185133998268e-07, "lr_fusion": 9.777185133998268e-06, "peak_memory_gib": 48.31737470626831, "trajectory_time_s": 70.78898718900018, "velocity_mse_step1": 0.013689216750208288, "velocity_mse_step2": 0.013729359996172464, "velocity_mse_step3": 0.01844747271388769} +{"batch_in_epoch": 37, "epoch": 4, "global_step": 237, "grad_norm_max": 0.036376953125, "hidden_mse_step1": 0.013801921815944033, "hidden_mse_step2": 0.016576954421387718, "hidden_mse_step3": 0.019907385825977793, "latent_grad_l1_min": 0.0002777912486635614, "loss": 0.022223977563304024, "lr_blocks": 9.774801031602628e-07, "lr_fusion": 9.77480103160263e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 72.9303314047429, "velocity_mse_step1": 0.01970433198792827, "velocity_mse_step2": 0.01957766640823268, "velocity_mse_step3": 0.022361306598343483} +{"batch_in_epoch": 38, "epoch": 4, "global_step": 238, "grad_norm_max": 0.035400390625, "hidden_mse_step1": 0.02054881631296926, "hidden_mse_step2": 0.023189207064985697, "hidden_mse_step3": 0.02610837317271424, "latent_grad_l1_min": 0.00022384834937838605, "loss": 0.02276425153831951, "lr_blocks": 9.7724045359428e-07, "lr_fusion": 9.772404535942802e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 73.14582389887437, "velocity_mse_step1": 0.016959319828726747, "velocity_mse_step2": 0.018513886378579106, "velocity_mse_step3": 0.02583490702090785} +{"batch_in_epoch": 39, "epoch": 4, "global_step": 239, "grad_norm_max": 0.047607421875, "hidden_mse_step1": 0.015055037868608321, "hidden_mse_step2": 0.018123062743273164, "hidden_mse_step3": 0.022129740250030797, "latent_grad_l1_min": 0.00024031421526160557, "loss": 0.01720675804452704, "lr_blocks": 9.769995653239022e-07, "lr_fusion": 9.769995653239023e-06, "peak_memory_gib": 48.31737470626831, "trajectory_time_s": 74.40121631474904, "velocity_mse_step1": 0.01118277873112155, "velocity_mse_step2": 0.013894717196568047, "velocity_mse_step3": 0.021011992415878922} +{"batch_in_epoch": 40, "epoch": 4, "global_step": 240, "grad_norm_max": 0.032470703125, "hidden_mse_step1": 0.013702744125371931, "hidden_mse_step2": 0.015936750897838334, "hidden_mse_step3": 0.019165735691785812, "latent_grad_l1_min": 0.0002238335455331253, "loss": 0.015576518814279029, "lr_blocks": 9.767574389743681e-07, "lr_fusion": 9.767574389743683e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 71.48065702500026, "velocity_mse_step1": 0.011590600176922244, "velocity_mse_step2": 0.012573797677760012, "velocity_mse_step3": 0.01768463383112768} +{"batch_in_epoch": 41, "epoch": 4, "global_step": 241, "grad_norm_max": 0.03271484375, "hidden_mse_step1": 0.019670418122716776, "hidden_mse_step2": 0.022562709478994032, "hidden_mse_step3": 0.026640535964231406, "latent_grad_l1_min": 0.00031282613417715766, "loss": 0.023819826726269505, "lr_blocks": 9.765140751741305e-07, "lr_fusion": 9.765140751741307e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 72.13766944874442, "velocity_mse_step1": 0.01652728120929428, "velocity_mse_step2": 0.01971525020365204, "velocity_mse_step3": 0.028329579916317012} +{"batch_in_epoch": 42, "epoch": 4, "global_step": 242, "grad_norm_max": 0.035888671875, "hidden_mse_step1": 0.01389152681388493, "hidden_mse_step2": 0.01571096314416666, "hidden_mse_step3": 0.019384973973501474, "latent_grad_l1_min": 0.00015700065159762744, "loss": 0.014342206239234654, "lr_blocks": 9.76269474554854e-07, "lr_fusion": 9.76269474554854e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 72.32857522812265, "velocity_mse_step1": 0.010122512129393206, "velocity_mse_step2": 0.011021967615566348, "velocity_mse_step3": 0.016983391359514956} +{"batch_in_epoch": 43, "epoch": 4, "global_step": 243, "grad_norm_max": 0.038330078125, "hidden_mse_step1": 0.023196409231916598, "hidden_mse_step2": 0.026022038247901943, "hidden_mse_step3": 0.029262622336058745, "latent_grad_l1_min": 0.0003000889228133019, "loss": 0.028713478083123585, "lr_blocks": 9.760236377514128e-07, "lr_fusion": 9.760236377514128e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 71.81028312650233, "velocity_mse_step1": 0.02086604886738184, "velocity_mse_step2": 0.024900311604142192, "velocity_mse_step3": 0.03252596409791814} +{"batch_in_epoch": 44, "epoch": 4, "global_step": 244, "grad_norm_max": 0.047607421875, "hidden_mse_step1": 0.019968519041347985, "hidden_mse_step2": 0.02186252165951633, "hidden_mse_step3": 0.024999922257848084, "latent_grad_l1_min": 0.00022906714548298623, "loss": 0.021856563174099262, "lr_blocks": 9.757765654018903e-07, "lr_fusion": 9.757765654018905e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 72.0660080728776, "velocity_mse_step1": 0.016133295889760903, "velocity_mse_step2": 0.01799231696973688, "velocity_mse_step3": 0.02476097927761397} +{"batch_in_epoch": 45, "epoch": 4, "global_step": 245, "grad_norm_max": 0.049072265625, "hidden_mse_step1": 0.016546294446535673, "hidden_mse_step2": 0.019468042708467692, "hidden_mse_step3": 0.02319474543245243, "latent_grad_l1_min": 0.0002573144238340319, "loss": 0.019308523478684947, "lr_blocks": 9.755282581475767e-07, "lr_fusion": 9.755282581475769e-06, "peak_memory_gib": 48.320029735565186, "trajectory_time_s": 73.97123653862946, "velocity_mse_step1": 0.012597588989592623, "velocity_mse_step2": 0.016325305862535187, "velocity_mse_step3": 0.023081766420675973} +{"batch_in_epoch": 46, "epoch": 4, "global_step": 246, "grad_norm_max": 0.026123046875, "hidden_mse_step1": 0.0166409662467361, "hidden_mse_step2": 0.019323358628233624, "hidden_mse_step3": 0.022935183286400776, "latent_grad_l1_min": 0.0002600304869702086, "loss": 0.018841002302776487, "lr_blocks": 9.752787166329675e-07, "lr_fusion": 9.752787166329677e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 71.37229075475261, "velocity_mse_step1": 0.012611311430061634, "velocity_mse_step2": 0.015246397882167782, "velocity_mse_step3": 0.02277534560666286} +{"batch_in_epoch": 47, "epoch": 4, "global_step": 247, "grad_norm_max": 0.037353515625, "hidden_mse_step1": 0.016286818763806617, "hidden_mse_step2": 0.018362562355053208, "hidden_mse_step3": 0.022175405241016827, "latent_grad_l1_min": 0.00020340503670013277, "loss": 0.018994826025196483, "lr_blocks": 9.750279415057617e-07, "lr_fusion": 9.750279415057618e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 73.80029995661971, "velocity_mse_step1": 0.013151954777380785, "velocity_mse_step2": 0.015493119748758282, "velocity_mse_step3": 0.022656923608987457} +{"batch_in_epoch": 48, "epoch": 4, "global_step": 248, "grad_norm_max": 0.035888671875, "hidden_mse_step1": 0.019674921204568818, "hidden_mse_step2": 0.02190529995797468, "hidden_mse_step3": 0.024876085974808247, "latent_grad_l1_min": 0.00022866695144330151, "loss": 0.01993446884443983, "lr_blocks": 9.7477593341686e-07, "lr_fusion": 9.747759334168602e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 73.75987660161991, "velocity_mse_step1": 0.014328621780025839, "velocity_mse_step2": 0.015874300305897898, "velocity_mse_step3": 0.02295485188369639} +{"batch_in_epoch": 49, "epoch": 4, "global_step": 249, "grad_norm_max": 0.036865234375, "hidden_mse_step1": 0.014475587372934178, "hidden_mse_step2": 0.016893266126446958, "hidden_mse_step3": 0.020371558444042287, "latent_grad_l1_min": 0.00018280635413248092, "loss": 0.014078735157714357, "lr_blocks": 9.745226930203637e-07, "lr_fusion": 9.74522693020364e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 74.64999987750343, "velocity_mse_step1": 0.008816834208638673, "velocity_mse_step2": 0.010941749895989363, "velocity_mse_step3": 0.017303579279022027} +{"batch_in_epoch": 50, "epoch": 4, "global_step": 250, "grad_norm_max": 0.044921875, "hidden_mse_step1": 0.021556399845784266, "hidden_mse_step2": 0.02340295481761651, "hidden_mse_step3": 0.02641196738529418, "latent_grad_l1_min": 0.0002550216449890286, "loss": 0.02455793332774192, "lr_blocks": 9.742682209735727e-07, "lr_fusion": 9.742682209735727e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 74.53211107299285, "velocity_mse_step1": 0.018918947616059865, "velocity_mse_step2": 0.0204366279316933, "velocity_mse_step3": 0.02718108949816919} +{"batch_in_epoch": 1, "epoch": 5, "global_step": 251, "grad_norm_max": 0.03125, "hidden_mse_step1": 0.01762774698935183, "hidden_mse_step2": 0.019655656061201753, "hidden_mse_step3": 0.02299584719418947, "latent_grad_l1_min": 0.00023334830257226713, "loss": 0.020878697703925093, "lr_blocks": 9.74012517936983e-07, "lr_fusion": 9.740125179369833e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 74.45554254138187, "velocity_mse_step1": 0.015791064174014276, "velocity_mse_step2": 0.017028842807381546, "velocity_mse_step3": 0.023788259805379703} +{"batch_in_epoch": 2, "epoch": 5, "global_step": 252, "grad_norm_max": 0.0361328125, "hidden_mse_step1": 0.020224003449714343, "hidden_mse_step2": 0.022494422454786084, "hidden_mse_step3": 0.025277582812122997, "latent_grad_l1_min": 0.00020588503866747487, "loss": 0.021126384522566304, "lr_blocks": 9.737555845742868e-07, "lr_fusion": 9.737555845742869e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 74.6174283099972, "velocity_mse_step1": 0.015167228474248466, "velocity_mse_step2": 0.017343759091870328, "velocity_mse_step3": 0.024068563843944242} +{"batch_in_epoch": 3, "epoch": 5, "global_step": 253, "grad_norm_max": 0.03564453125, "hidden_mse_step1": 0.021698187571018934, "hidden_mse_step2": 0.02449759537453896, "hidden_mse_step3": 0.028270451867553805, "latent_grad_l1_min": 0.00028365822618070524, "loss": 0.025642396699238037, "lr_blocks": 9.734974215523683e-07, "lr_fusion": 9.734974215523684e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 73.76114694337593, "velocity_mse_step1": 0.01741379578015767, "velocity_mse_step2": 0.021176725858822465, "velocity_mse_step3": 0.03089004253602719} +{"batch_in_epoch": 4, "epoch": 5, "global_step": 254, "grad_norm_max": 0.0289306640625, "hidden_mse_step1": 0.01696581137782362, "hidden_mse_step2": 0.01937205272094746, "hidden_mse_step3": 0.022488101734779775, "latent_grad_l1_min": 0.00018407549760013353, "loss": 0.01961766414128111, "lr_blocks": 9.73238029541305e-07, "lr_fusion": 9.73238029541305e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 74.62307354949371, "velocity_mse_step1": 0.013716657891304097, "velocity_mse_step2": 0.016554922343597615, "velocity_mse_step3": 0.022698814011943926} +{"batch_in_epoch": 5, "epoch": 5, "global_step": 255, "grad_norm_max": 0.035400390625, "hidden_mse_step1": 0.01754582534028616, "hidden_mse_step2": 0.02030500838632828, "hidden_mse_step3": 0.023534050360987227, "latent_grad_l1_min": 0.0002783772697512177, "loss": 0.02499078064491706, "lr_blocks": 9.729774092143626e-07, "lr_fusion": 9.729774092143627e-06, "peak_memory_gib": 48.320029735565186, "trajectory_time_s": 73.79449725687664, "velocity_mse_step1": 0.020140850868691423, "velocity_mse_step2": 0.021987614088824815, "velocity_mse_step3": 0.026705386840538786} +{"batch_in_epoch": 6, "epoch": 5, "global_step": 256, "grad_norm_max": 0.0308837890625, "hidden_mse_step1": 0.017640581944890852, "hidden_mse_step2": 0.020292786349143298, "hidden_mse_step3": 0.02387525345797517, "latent_grad_l1_min": 0.0002566512848716229, "loss": 0.019479065543107157, "lr_blocks": 9.727155612479962e-07, "lr_fusion": 9.727155612479964e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 71.5051917901219, "velocity_mse_step1": 0.012814606496249324, "velocity_mse_step2": 0.01617281962535344, "velocity_mse_step3": 0.023268907064838067} +{"batch_in_epoch": 7, "epoch": 5, "global_step": 257, "grad_norm_max": 0.03662109375, "hidden_mse_step1": 0.014597455987573733, "hidden_mse_step2": 0.017141389311291277, "hidden_mse_step3": 0.020799652440473434, "latent_grad_l1_min": 0.00020756598587468034, "loss": 0.017087747218153836, "lr_blocks": 9.72452486321847e-07, "lr_fusion": 9.72452486321847e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 72.80502778500886, "velocity_mse_step1": 0.012289775146720265, "velocity_mse_step2": 0.01421654991905338, "velocity_mse_step3": 0.019503065626070435} +{"batch_in_epoch": 8, "epoch": 5, "global_step": 258, "grad_norm_max": 0.044189453125, "hidden_mse_step1": 0.016112076879445728, "hidden_mse_step2": 0.01942926412448287, "hidden_mse_step3": 0.022698962495529225, "latent_grad_l1_min": 0.0002824979228535085, "loss": 0.020578072385563115, "lr_blocks": 9.721881851187405e-07, "lr_fusion": 9.721881851187406e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 72.96667927413364, "velocity_mse_step1": 0.01473923037701752, "velocity_mse_step2": 0.017775222168503593, "velocity_mse_step3": 0.023395731995281364} +{"batch_in_epoch": 9, "epoch": 5, "global_step": 259, "grad_norm_max": 0.035888671875, "hidden_mse_step1": 0.013763907440339349, "hidden_mse_step2": 0.01610786111060796, "hidden_mse_step3": 0.019355458573305177, "latent_grad_l1_min": 0.00017032145842676982, "loss": 0.01386472800680037, "lr_blocks": 9.719226583246855e-07, "lr_fusion": 9.719226583246856e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 71.3666427116259, "velocity_mse_step1": 0.009716996665312243, "velocity_mse_step2": 0.011158876083624949, "velocity_mse_step3": 0.015795587291774736} +{"batch_in_epoch": 10, "epoch": 5, "global_step": 260, "grad_norm_max": 0.034423828125, "hidden_mse_step1": 0.01980099160573445, "hidden_mse_step2": 0.022110095738233727, "hidden_mse_step3": 0.025622531987859732, "latent_grad_l1_min": 0.0003368978723301552, "loss": 0.026853501339376504, "lr_blocks": 9.716559066288714e-07, "lr_fusion": 9.716559066288716e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 73.72475897262848, "velocity_mse_step1": 0.020889362438798083, "velocity_mse_step2": 0.02326350608408185, "velocity_mse_step3": 0.02965427145162331} +{"batch_in_epoch": 11, "epoch": 5, "global_step": 261, "grad_norm_max": 0.0296630859375, "hidden_mse_step1": 0.014774416976641596, "hidden_mse_step2": 0.016912926512304693, "hidden_mse_step3": 0.020078884329580302, "latent_grad_l1_min": 0.00016339575540769147, "loss": 0.01551853008485133, "lr_blocks": 9.713879307236677e-07, "lr_fusion": 9.713879307236678e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 71.0927476682482, "velocity_mse_step1": 0.011282841761255569, "velocity_mse_step2": 0.01247171769916479, "velocity_mse_step3": 0.017624406572265015} +{"batch_in_epoch": 12, "epoch": 5, "global_step": 262, "grad_norm_max": 0.039306640625, "hidden_mse_step1": 0.017036907374858856, "hidden_mse_step2": 0.019484474689566662, "hidden_mse_step3": 0.023456823380131803, "latent_grad_l1_min": 0.00023094404787116218, "loss": 0.018091731638248475, "lr_blocks": 9.711187313046206e-07, "lr_fusion": 9.711187313046207e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 71.2136165185002, "velocity_mse_step1": 0.012666302654127189, "velocity_mse_step2": 0.014400538115296513, "velocity_mse_step3": 0.02121053185380463} +{"batch_in_epoch": 13, "epoch": 5, "global_step": 263, "grad_norm_max": 0.035400390625, "hidden_mse_step1": 0.013471403329666438, "hidden_mse_step2": 0.016917000599538108, "hidden_mse_step3": 0.021290536662231067, "latent_grad_l1_min": 0.00021637458303302992, "loss": 0.015534736116283705, "lr_blocks": 9.708483090704523e-07, "lr_fusion": 9.708483090704524e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 71.47277334787213, "velocity_mse_step1": 0.009945727688318584, "velocity_mse_step2": 0.012346248348226903, "velocity_mse_step3": 0.019144336464315913} +{"batch_in_epoch": 14, "epoch": 5, "global_step": 264, "grad_norm_max": 0.033447265625, "hidden_mse_step1": 0.013550329017951817, "hidden_mse_step2": 0.01620158371016649, "hidden_mse_step3": 0.01955515305910792, "latent_grad_l1_min": 0.00016661005793139338, "loss": 0.013704693934414534, "lr_blocks": 9.705766647230588e-07, "lr_fusion": 9.70576664723059e-06, "peak_memory_gib": 48.31958723068237, "trajectory_time_s": 71.6969724738774, "velocity_mse_step1": 0.009070187825792735, "velocity_mse_step2": 0.011043257772273914, "velocity_mse_step3": 0.0160699288743282} +{"batch_in_epoch": 15, "epoch": 5, "global_step": 265, "grad_norm_max": 0.03662109375, "hidden_mse_step1": 0.018821303824162375, "hidden_mse_step2": 0.02159288552190576, "hidden_mse_step3": 0.02511450297398759, "latent_grad_l1_min": 0.00021402002494141925, "loss": 0.02059225125205038, "lr_blocks": 9.703037989675086e-07, "lr_fusion": 9.703037989675088e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 71.58108811886996, "velocity_mse_step1": 0.014599557778897827, "velocity_mse_step2": 0.016918113056038107, "velocity_mse_step3": 0.023706211592070755} +{"batch_in_epoch": 16, "epoch": 5, "global_step": 266, "grad_norm_max": 0.0311279296875, "hidden_mse_step1": 0.012548751858828056, "hidden_mse_step2": 0.015026170086847349, "hidden_mse_step3": 0.018300425361043638, "latent_grad_l1_min": 0.00015643720053049037, "loss": 0.01304872121129717, "lr_blocks": 9.700297125120397e-07, "lr_fusion": 9.7002971251204e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 71.2515441417454, "velocity_mse_step1": 0.008944561876497965, "velocity_mse_step2": 0.010238630002796914, "velocity_mse_step3": 0.015375435689098334} +{"batch_in_epoch": 17, "epoch": 5, "global_step": 267, "grad_norm_max": 0.0306396484375, "hidden_mse_step1": 0.020471475494559854, "hidden_mse_step2": 0.022641927469521757, "hidden_mse_step3": 0.02589064276045454, "latent_grad_l1_min": 0.00027234392655373085, "loss": 0.020699639804661274, "lr_blocks": 9.697544060680593e-07, "lr_fusion": 9.697544060680594e-06, "peak_memory_gib": 48.320029735565186, "trajectory_time_s": 71.04405349824447, "velocity_mse_step1": 0.014189860733625078, "velocity_mse_step2": 0.01642280876902597, "velocity_mse_step3": 0.024585843917780688} +{"batch_in_epoch": 18, "epoch": 5, "global_step": 268, "grad_norm_max": 0.03173828125, "hidden_mse_step1": 0.01955650814176936, "hidden_mse_step2": 0.022634170384013223, "hidden_mse_step3": 0.02611097132986677, "latent_grad_l1_min": 0.0002277532112202607, "loss": 0.021553250819644224, "lr_blocks": 9.694778803501401e-07, "lr_fusion": 9.694778803501404e-06, "peak_memory_gib": 48.31958723068237, "trajectory_time_s": 73.56048717974772, "velocity_mse_step1": 0.014882959257062922, "velocity_mse_step2": 0.017410181961687546, "velocity_mse_step3": 0.02553644474496001} +{"batch_in_epoch": 19, "epoch": 5, "global_step": 269, "grad_norm_max": 0.0277099609375, "hidden_mse_step1": 0.021056220775270567, "hidden_mse_step2": 0.023383286889709, "hidden_mse_step3": 0.026580027188174423, "latent_grad_l1_min": 0.0002110293225996429, "loss": 0.022874580970632708, "lr_blocks": 9.69200136076021e-07, "lr_fusion": 9.692001360760212e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 72.64085838975006, "velocity_mse_step1": 0.01567976378386707, "velocity_mse_step2": 0.018863550042234625, "velocity_mse_step3": 0.02697847391079579} +{"batch_in_epoch": 20, "epoch": 5, "global_step": 270, "grad_norm_max": 0.0255126953125, "hidden_mse_step1": 0.014771174672724945, "hidden_mse_step2": 0.017093668400775638, "hidden_mse_step3": 0.020052740167427273, "latent_grad_l1_min": 0.00014577631282008952, "loss": 0.014646209556043946, "lr_blocks": 9.689211739666022e-07, "lr_fusion": 9.689211739666023e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 71.55004671537972, "velocity_mse_step1": 0.009931565890188461, "velocity_mse_step2": 0.01167636559278305, "velocity_mse_step3": 0.017138937617086673} +{"batch_in_epoch": 21, "epoch": 5, "global_step": 271, "grad_norm_max": 0.03271484375, "hidden_mse_step1": 0.020884492667391893, "hidden_mse_step2": 0.02308064404808517, "hidden_mse_step3": 0.025631632023890103, "latent_grad_l1_min": 0.0002173604407289531, "loss": 0.023110062424426096, "lr_blocks": 9.686409947459457e-07, "lr_fusion": 9.68640994745946e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 74.32845027762596, "velocity_mse_step1": 0.017559484990280386, "velocity_mse_step2": 0.019349698858734752, "velocity_mse_step3": 0.025461323858637894} +{"batch_in_epoch": 22, "epoch": 5, "global_step": 272, "grad_norm_max": 0.0419921875, "hidden_mse_step1": 0.02035077245506857, "hidden_mse_step2": 0.02293955982895568, "hidden_mse_step3": 0.02619241994606065, "latent_grad_l1_min": 0.00025789463143155444, "loss": 0.02296775530391772, "lr_blocks": 9.683595991412724e-07, "lr_fusion": 9.683595991412725e-06, "peak_memory_gib": 48.31737470626831, "trajectory_time_s": 73.94567894362626, "velocity_mse_step1": 0.017017276851610013, "velocity_mse_step2": 0.018846972620979483, "velocity_mse_step3": 0.026090738579763895} +{"batch_in_epoch": 23, "epoch": 5, "global_step": 273, "grad_norm_max": 0.028564453125, "hidden_mse_step1": 0.015527866805704045, "hidden_mse_step2": 0.01733310203001435, "hidden_mse_step3": 0.020559366202048422, "latent_grad_l1_min": 0.0001583353332534898, "loss": 0.016381185285614004, "lr_blocks": 9.680769878829604e-07, "lr_fusion": 9.680769878829606e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 71.91539726962583, "velocity_mse_step1": 0.011945560316754771, "velocity_mse_step2": 0.013070577462453261, "velocity_mse_step3": 0.01878538292865934} +{"batch_in_epoch": 24, "epoch": 5, "global_step": 274, "grad_norm_max": 0.0289306640625, "hidden_mse_step1": 0.01889457063433448, "hidden_mse_step2": 0.020873594397146783, "hidden_mse_step3": 0.02408728575600045, "latent_grad_l1_min": 0.00018579006973595824, "loss": 0.01861064987938984, "lr_blocks": 9.67793161704543e-07, "lr_fusion": 9.677931617045432e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 72.21360462462326, "velocity_mse_step1": 0.012528605842297631, "velocity_mse_step2": 0.01494479113275052, "velocity_mse_step3": 0.021973005670588464} +{"batch_in_epoch": 25, "epoch": 5, "global_step": 275, "grad_norm_max": 0.0247802734375, "hidden_mse_step1": 0.014037399723227801, "hidden_mse_step2": 0.016241130635275373, "hidden_mse_step3": 0.019716414573070194, "latent_grad_l1_min": 0.0002049778850050643, "loss": 0.015059782957125987, "lr_blocks": 9.675081213427074e-07, "lr_fusion": 9.675081213427076e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 71.41234761436499, "velocity_mse_step1": 0.010389791321358642, "velocity_mse_step2": 0.012001488694555258, "velocity_mse_step3": 0.017788573218110417} +{"batch_in_epoch": 26, "epoch": 5, "global_step": 276, "grad_norm_max": 0.028076171875, "hidden_mse_step1": 0.019882616720029285, "hidden_mse_step2": 0.022258833083989363, "hidden_mse_step3": 0.02549452274771673, "latent_grad_l1_min": 0.000300440973660443, "loss": 0.024608161383574557, "lr_blocks": 9.672218675372913e-07, "lr_fusion": 9.672218675372914e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 73.45651559037287, "velocity_mse_step1": 0.020124915860443644, "velocity_mse_step2": 0.020700754497998526, "velocity_mse_step3": 0.02623521430151803} +{"batch_in_epoch": 27, "epoch": 5, "global_step": 277, "grad_norm_max": 0.032958984375, "hidden_mse_step1": 0.01820400016939467, "hidden_mse_step2": 0.02059511958837642, "hidden_mse_step3": 0.024591279132956903, "latent_grad_l1_min": 0.0002856419441741309, "loss": 0.021762750169727948, "lr_blocks": 9.669344010312828e-07, "lr_fusion": 9.669344010312829e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 73.83582807388302, "velocity_mse_step1": 0.015567112938047753, "velocity_mse_step2": 0.017991873658112514, "velocity_mse_step3": 0.02539022179137516} +{"batch_in_epoch": 28, "epoch": 5, "global_step": 278, "grad_norm_max": 0.037841796875, "hidden_mse_step1": 0.015405542998840763, "hidden_mse_step2": 0.016947695910598016, "hidden_mse_step3": 0.02071788117089974, "latent_grad_l1_min": 0.0001917325844260631, "loss": 0.016396300218600248, "lr_blocks": 9.666457225708172e-07, "lr_fusion": 9.666457225708175e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 70.9724210639979, "velocity_mse_step1": 0.012225285731671778, "velocity_mse_step2": 0.012555281236667984, "velocity_mse_step3": 0.019101220524005057} +{"batch_in_epoch": 29, "epoch": 5, "global_step": 279, "grad_norm_max": 0.034423828125, "hidden_mse_step1": 0.013784217777096533, "hidden_mse_step2": 0.01579702483390325, "hidden_mse_step3": 0.018826434704741195, "latent_grad_l1_min": 0.00015379277010652004, "loss": 0.013256262933802125, "lr_blocks": 9.66355832905176e-07, "lr_fusion": 9.663558329051761e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 72.7764287309983, "velocity_mse_step1": 0.00896400162829585, "velocity_mse_step2": 0.010397048492450265, "velocity_mse_step3": 0.015566969506575594} +{"batch_in_epoch": 30, "epoch": 5, "global_step": 280, "grad_norm_max": 0.0289306640625, "hidden_mse_step1": 0.01650786431855522, "hidden_mse_step2": 0.019143646300238154, "hidden_mse_step3": 0.022814003817204914, "latent_grad_l1_min": 0.0002791380466078408, "loss": 0.019465094270084853, "lr_blocks": 9.660647327867838e-07, "lr_fusion": 9.66064732786784e-06, "peak_memory_gib": 48.31958723068237, "trajectory_time_s": 72.06973712487888, "velocity_mse_step1": 0.013476539223380053, "velocity_mse_step2": 0.015931084315525368, "velocity_mse_step3": 0.0231411065573671} +{"batch_in_epoch": 31, "epoch": 5, "global_step": 281, "grad_norm_max": 0.0263671875, "hidden_mse_step1": 0.02040150591139017, "hidden_mse_step2": 0.022251369639499374, "hidden_mse_step3": 0.02529247412790677, "latent_grad_l1_min": 0.0002438424253341509, "loss": 0.023481937208479, "lr_blocks": 9.657724229712074e-07, "lr_fusion": 9.657724229712076e-06, "peak_memory_gib": 48.320029735565186, "trajectory_time_s": 74.04512726975372, "velocity_mse_step1": 0.018559774974294535, "velocity_mse_step2": 0.019554248004819134, "velocity_mse_step3": 0.025537252284785997} +{"batch_in_epoch": 32, "epoch": 5, "global_step": 282, "grad_norm_max": 0.048583984375, "hidden_mse_step1": 0.02321098192727991, "hidden_mse_step2": 0.02568132512637281, "hidden_mse_step3": 0.028758988998431177, "latent_grad_l1_min": 0.0002659694782778388, "loss": 0.025659738025361933, "lr_blocks": 9.654789042171534e-07, "lr_fusion": 9.654789042171535e-06, "peak_memory_gib": 48.31737470626831, "trajectory_time_s": 73.99706316050288, "velocity_mse_step1": 0.01856678903693267, "velocity_mse_step2": 0.021130338331983824, "velocity_mse_step3": 0.029516954315892817} +{"batch_in_epoch": 33, "epoch": 5, "global_step": 283, "grad_norm_max": 0.02685546875, "hidden_mse_step1": 0.019046427806772823, "hidden_mse_step2": 0.02150687455598797, "hidden_mse_step3": 0.024551866343244907, "latent_grad_l1_min": 0.0002555132978159236, "loss": 0.020950254560115615, "lr_blocks": 9.651841772864659e-07, "lr_fusion": 9.65184177286466e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 73.35503288137625, "velocity_mse_step1": 0.014700766137268926, "velocity_mse_step2": 0.01751506283679711, "velocity_mse_step3": 0.024124415707774457} +{"batch_in_epoch": 34, "epoch": 5, "global_step": 284, "grad_norm_max": 0.038818359375, "hidden_mse_step1": 0.022386437198812406, "hidden_mse_step2": 0.02491186713866357, "hidden_mse_step3": 0.02839466660017414, "latent_grad_l1_min": 0.0003043546257686103, "loss": 0.02539097040425986, "lr_blocks": 9.648882429441256e-07, "lr_fusion": 9.648882429441258e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 71.96623401387478, "velocity_mse_step1": 0.018036423658486456, "velocity_mse_step2": 0.02088780277907582, "velocity_mse_step3": 0.029679385080401383} +{"batch_in_epoch": 35, "epoch": 5, "global_step": 285, "grad_norm_max": 0.03857421875, "hidden_mse_step1": 0.018958730548287606, "hidden_mse_step2": 0.02123994457568707, "hidden_mse_step3": 0.023896729141207676, "latent_grad_l1_min": 0.0002470620402164059, "loss": 0.021408054445471083, "lr_blocks": 9.645911019582465e-07, "lr_fusion": 9.645911019582467e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 71.51160403250469, "velocity_mse_step1": 0.016792973848558695, "velocity_mse_step2": 0.017645490143747465, "velocity_mse_step3": 0.023376157183000547} +{"batch_in_epoch": 36, "epoch": 5, "global_step": 286, "grad_norm_max": 0.029541015625, "hidden_mse_step1": 0.01691167757131292, "hidden_mse_step2": 0.02046136383432895, "hidden_mse_step3": 0.02417745876924268, "latent_grad_l1_min": 0.00025533163989166496, "loss": 0.019873233707455387, "lr_blocks": 9.642927551000747e-07, "lr_fusion": 9.642927551000749e-06, "peak_memory_gib": 48.320029735565186, "trajectory_time_s": 71.37057428124899, "velocity_mse_step1": 0.01328005535885625, "velocity_mse_step2": 0.016725353784360256, "velocity_mse_step3": 0.02345924020378983} +{"batch_in_epoch": 37, "epoch": 5, "global_step": 287, "grad_norm_max": 0.037841796875, "hidden_mse_step1": 0.01522966060604501, "hidden_mse_step2": 0.01724720002052241, "hidden_mse_step3": 0.020613183749706616, "latent_grad_l1_min": 0.00017304102038906422, "loss": 0.015872431565575034, "lr_blocks": 9.639932031439865e-07, "lr_fusion": 9.639932031439866e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 74.4570080997546, "velocity_mse_step1": 0.011112571172166748, "velocity_mse_step2": 0.012477809033173668, "velocity_mse_step3": 0.01871790928998962} +{"batch_in_epoch": 38, "epoch": 5, "global_step": 288, "grad_norm_max": 0.054443359375, "hidden_mse_step1": 0.01690135397283094, "hidden_mse_step2": 0.019562352274078876, "hidden_mse_step3": 0.02268051615517054, "latent_grad_l1_min": 0.00022202470790944062, "loss": 0.01830257719432536, "lr_blocks": 9.636924468674855e-07, "lr_fusion": 9.636924468674856e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 75.08579154637482, "velocity_mse_step1": 0.012636067100434698, "velocity_mse_step2": 0.01508299599559645, "velocity_mse_step3": 0.021274244628979692} +{"batch_in_epoch": 39, "epoch": 5, "global_step": 289, "grad_norm_max": 0.037353515625, "hidden_mse_step1": 0.025440852265871526, "hidden_mse_step2": 0.027293687436862717, "hidden_mse_step3": 0.030422851127306267, "latent_grad_l1_min": 0.00024314719303220045, "loss": 0.02851773052576131, "lr_blocks": 9.633904870512013e-07, "lr_fusion": 9.633904870512015e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 73.5836709336254, "velocity_mse_step1": 0.021016516236289, "velocity_mse_step2": 0.023002811599456306, "velocity_mse_step3": 0.03321812127251178} +{"batch_in_epoch": 40, "epoch": 5, "global_step": 290, "grad_norm_max": 0.042724609375, "hidden_mse_step1": 0.01698260405100882, "hidden_mse_step2": 0.019342216754531752, "hidden_mse_step3": 0.02217096753884107, "latent_grad_l1_min": 0.00019290036834718194, "loss": 0.01777129789115861, "lr_blocks": 9.630873244788882e-07, "lr_fusion": 9.630873244788884e-06, "peak_memory_gib": 48.31958723068237, "trajectory_time_s": 72.1503421961279, "velocity_mse_step1": 0.01183487128998552, "velocity_mse_step2": 0.014509802243472744, "velocity_mse_step3": 0.02111963970985796} +{"batch_in_epoch": 41, "epoch": 5, "global_step": 291, "grad_norm_max": 0.035888671875, "hidden_mse_step1": 0.019352591966578202, "hidden_mse_step2": 0.021773058663321927, "hidden_mse_step3": 0.024992294193777655, "latent_grad_l1_min": 0.00028290445334278047, "loss": 0.021848250105644444, "lr_blocks": 9.627829599374214e-07, "lr_fusion": 9.627829599374214e-06, "peak_memory_gib": 48.32135725021362, "trajectory_time_s": 71.77780744262418, "velocity_mse_step1": 0.015782159447553568, "velocity_mse_step2": 0.018559530107138143, "velocity_mse_step3": 0.024591264863764596} +{"batch_in_epoch": 42, "epoch": 5, "global_step": 292, "grad_norm_max": 0.04443359375, "hidden_mse_step1": 0.020103673350864226, "hidden_mse_step2": 0.023210882126087587, "hidden_mse_step3": 0.02695577388762363, "latent_grad_l1_min": 0.00031520087031822186, "loss": 0.024555340475801907, "lr_blocks": 9.624773942167958e-07, "lr_fusion": 9.624773942167958e-06, "peak_memory_gib": 48.31958723068237, "trajectory_time_s": 72.86388560762134, "velocity_mse_step1": 0.01696340916428848, "velocity_mse_step2": 0.0207943102245086, "velocity_mse_step3": 0.028881266064542745} +{"batch_in_epoch": 43, "epoch": 5, "global_step": 293, "grad_norm_max": 0.0419921875, "hidden_mse_step1": 0.013827970396960154, "hidden_mse_step2": 0.016124866803043654, "hidden_mse_step3": 0.019624544182858826, "latent_grad_l1_min": 0.00024667210345796775, "loss": 0.017038219834960597, "lr_blocks": 9.621706281101247e-07, "lr_fusion": 9.62170628110125e-06, "peak_memory_gib": 48.320029735565186, "trajectory_time_s": 73.59759799124731, "velocity_mse_step1": 0.012358888135038017, "velocity_mse_step2": 0.014514863063855694, "velocity_mse_step3": 0.019283168770406128} +{"batch_in_epoch": 44, "epoch": 5, "global_step": 294, "grad_norm_max": 0.03857421875, "hidden_mse_step1": 0.02468799949357552, "hidden_mse_step2": 0.026005240855738517, "hidden_mse_step3": 0.028518275191475234, "latent_grad_l1_min": 0.00019875486759701744, "loss": 0.023443070922179943, "lr_blocks": 9.618626624136368e-07, "lr_fusion": 9.61862662413637e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 73.7136898115059, "velocity_mse_step1": 0.016952271469303275, "velocity_mse_step2": 0.01889019941778055, "velocity_mse_step3": 0.02656558779666998} +{"batch_in_epoch": 45, "epoch": 5, "global_step": 295, "grad_norm_max": 0.047119140625, "hidden_mse_step1": 0.011220104996547369, "hidden_mse_step2": 0.013621083069925329, "hidden_mse_step3": 0.01700519140078021, "latent_grad_l1_min": 0.0001262368632524158, "loss": 0.009941610673974668, "lr_blocks": 9.615534979266744e-07, "lr_fusion": 9.615534979266745e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 72.53552652549843, "velocity_mse_step1": 0.00623564850580546, "velocity_mse_step2": 0.007498405894563931, "velocity_mse_step3": 0.011906138704424456} +{"batch_in_epoch": 46, "epoch": 5, "global_step": 296, "grad_norm_max": 0.037841796875, "hidden_mse_step1": 0.01756083463884092, "hidden_mse_step2": 0.01987917499250865, "hidden_mse_step3": 0.022857979006533108, "latent_grad_l1_min": 0.00024451663011859637, "loss": 0.021239870165507976, "lr_blocks": 9.612431354516912e-07, "lr_fusion": 9.612431354516912e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 71.61023986211876, "velocity_mse_step1": 0.016448032373903385, "velocity_mse_step2": 0.017979953787289556, "velocity_mse_step3": 0.02326182376210844} +{"batch_in_epoch": 47, "epoch": 5, "global_step": 297, "grad_norm_max": 0.039794921875, "hidden_mse_step1": 0.018767827489812455, "hidden_mse_step2": 0.02118555637675204, "hidden_mse_step3": 0.025122627200159644, "latent_grad_l1_min": 0.00021556986757786945, "loss": 0.020816773051462532, "lr_blocks": 9.609315757942502e-07, "lr_fusion": 9.609315757942504e-06, "peak_memory_gib": 48.31737470626831, "trajectory_time_s": 71.97362208912091, "velocity_mse_step1": 0.014677367414281304, "velocity_mse_step2": 0.016917099095215753, "velocity_mse_step3": 0.02434824916001941} +{"batch_in_epoch": 48, "epoch": 5, "global_step": 298, "grad_norm_max": 0.039794921875, "hidden_mse_step1": 0.01825482294329309, "hidden_mse_step2": 0.019900036780329956, "hidden_mse_step3": 0.022959888663275963, "latent_grad_l1_min": 0.00023334942943620263, "loss": 0.018199596211031475, "lr_blocks": 9.606188197630222e-07, "lr_fusion": 9.606188197630224e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 71.88585420837626, "velocity_mse_step1": 0.012792689947153641, "velocity_mse_step2": 0.014546701896636347, "velocity_mse_step3": 0.02114792022621259} +{"batch_in_epoch": 49, "epoch": 5, "global_step": 299, "grad_norm_max": 0.033935546875, "hidden_mse_step1": 0.017891154275275767, "hidden_mse_step2": 0.02044886488667024, "hidden_mse_step3": 0.024022988392971453, "latent_grad_l1_min": 0.0001855107702795067, "loss": 0.01811627371768866, "lr_blocks": 9.603048681697833e-07, "lr_fusion": 9.603048681697834e-06, "peak_memory_gib": 48.3169322013855, "trajectory_time_s": 71.95312292924427, "velocity_mse_step1": 0.011881123321862624, "velocity_mse_step2": 0.014318311894645115, "velocity_mse_step3": 0.02191308401026098} +{"batch_in_epoch": 50, "epoch": 5, "global_step": 300, "grad_norm_max": 0.03857421875, "hidden_mse_step1": 0.011530450595143649, "hidden_mse_step2": 0.014367855952254363, "hidden_mse_step3": 0.018192639225162566, "latent_grad_l1_min": 0.0001986442948691547, "loss": 0.013505163402961833, "lr_blocks": 9.599897218294121e-07, "lr_fusion": 9.599897218294122e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 72.34439832200587, "velocity_mse_step1": 0.008173513673162753, "velocity_mse_step2": 0.01088172085915825, "velocity_mse_step3": 0.017051159894825623} +{"batch_in_epoch": 1, "epoch": 6, "global_step": 301, "grad_norm_max": 0.0322265625, "hidden_mse_step1": 0.01846481265134311, "hidden_mse_step2": 0.020913835889327208, "hidden_mse_step3": 0.023842563669729443, "latent_grad_l1_min": 0.00021469732200785074, "loss": 0.019259223024294312, "lr_blocks": 9.596733815598888e-07, "lr_fusion": 9.596733815598888e-06, "peak_memory_gib": 48.32135725021362, "trajectory_time_s": 72.13266710137395, "velocity_mse_step1": 0.015168706341812919, "velocity_mse_step2": 0.015093435599867784, "velocity_mse_step3": 0.021193404432519204} +{"batch_in_epoch": 2, "epoch": 6, "global_step": 302, "grad_norm_max": 0.04638671875, "hidden_mse_step1": 0.021684364074774615, "hidden_mse_step2": 0.02403474673129884, "hidden_mse_step3": 0.027228881255723533, "latent_grad_l1_min": 0.00019921122020605253, "loss": 0.025222611858875355, "lr_blocks": 9.59355848182292e-07, "lr_fusion": 9.593558481822923e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 71.65403848449932, "velocity_mse_step1": 0.018718931474722922, "velocity_mse_step2": 0.02100707753145668, "velocity_mse_step3": 0.02864702517399564} +{"batch_in_epoch": 3, "epoch": 6, "global_step": 303, "grad_norm_max": 0.05078125, "hidden_mse_step1": 0.016934738569294235, "hidden_mse_step2": 0.019461240236913518, "hidden_mse_step3": 0.02327450659192566, "latent_grad_l1_min": 0.00022108018856670242, "loss": 0.01901286549400538, "lr_blocks": 9.59037122520798e-07, "lr_fusion": 9.590371225207981e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 71.55140901825507, "velocity_mse_step1": 0.013195197759029855, "velocity_mse_step2": 0.015368606408758619, "velocity_mse_step3": 0.022507742255194377} +{"batch_in_epoch": 4, "epoch": 6, "global_step": 304, "grad_norm_max": 0.047607421875, "hidden_mse_step1": 0.023557730110561743, "hidden_mse_step2": 0.025867339863907542, "hidden_mse_step3": 0.027659435082958744, "latent_grad_l1_min": 0.00042662648229452316, "loss": 0.0387799304361189, "lr_blocks": 9.587172054026768e-07, "lr_fusion": 9.587172054026769e-06, "peak_memory_gib": 48.31737470626831, "trajectory_time_s": 71.82222317424748, "velocity_mse_step1": 0.03508910017886332, "velocity_mse_step2": 0.035887017808688276, "velocity_mse_step3": 0.037655220761994956} +{"batch_in_epoch": 5, "epoch": 6, "global_step": 305, "grad_norm_max": 0.03759765625, "hidden_mse_step1": 0.019862369197653607, "hidden_mse_step2": 0.022363811340515634, "hidden_mse_step3": 0.026201672835408578, "latent_grad_l1_min": 0.0002832114505508798, "loss": 0.021986194478813552, "lr_blocks": 9.583960976582913e-07, "lr_fusion": 9.583960976582914e-06, "peak_memory_gib": 48.316489696502686, "trajectory_time_s": 71.31214336463745, "velocity_mse_step1": 0.015345956918151517, "velocity_mse_step2": 0.017993998200316646, "velocity_mse_step3": 0.025775841387387893} +{"batch_in_epoch": 6, "epoch": 6, "global_step": 306, "grad_norm_max": 0.0279541015625, "hidden_mse_step1": 0.013901754477826347, "hidden_mse_step2": 0.016343936082973545, "hidden_mse_step3": 0.01968164218124002, "latent_grad_l1_min": 0.00018532349440647522, "loss": 0.013059208211156406, "lr_blocks": 9.580738001210944e-07, "lr_fusion": 9.580738001210946e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 71.73180856874387, "velocity_mse_step1": 0.008706662652132631, "velocity_mse_step2": 0.010155742684479004, "velocity_mse_step3": 0.0153224845791036} +{"batch_in_epoch": 7, "epoch": 6, "global_step": 307, "grad_norm_max": 0.041259765625, "hidden_mse_step1": 0.01677875535096973, "hidden_mse_step2": 0.018391151596525948, "hidden_mse_step3": 0.021423398683379804, "latent_grad_l1_min": 0.0001831092176871607, "loss": 0.015496982960030433, "lr_blocks": 9.57750313627628e-07, "lr_fusion": 9.57750313627628e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 71.66028624249884, "velocity_mse_step1": 0.010599750070209015, "velocity_mse_step2": 0.011978400318184868, "velocity_mse_step3": 0.018253466039563397} +{"batch_in_epoch": 8, "epoch": 6, "global_step": 308, "grad_norm_max": 0.044189453125, "hidden_mse_step1": 0.0166764787032402, "hidden_mse_step2": 0.019662146897254776, "hidden_mse_step3": 0.024025091053252776, "latent_grad_l1_min": 0.00022635211462329607, "loss": 0.01862101372431165, "lr_blocks": 9.57425639017519e-07, "lr_fusion": 9.574256390175192e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 71.06260661837587, "velocity_mse_step1": 0.012132499512517823, "velocity_mse_step2": 0.014699283621406978, "velocity_mse_step3": 0.02299488436463954} +{"batch_in_epoch": 9, "epoch": 6, "global_step": 309, "grad_norm_max": 0.034423828125, "hidden_mse_step1": 0.022773613011980026, "hidden_mse_step2": 0.024718410065231313, "hidden_mse_step3": 0.027604805438646247, "latent_grad_l1_min": 0.00020783185755135491, "loss": 0.024736751282554387, "lr_blocks": 9.57099777133479e-07, "lr_fusion": 9.57099777133479e-06, "peak_memory_gib": 48.320472240448, "trajectory_time_s": 71.64363252725889, "velocity_mse_step1": 0.017899682624244764, "velocity_mse_step2": 0.02061907275284674, "velocity_mse_step3": 0.02818181227693068} +{"batch_in_epoch": 10, "epoch": 6, "global_step": 310, "grad_norm_max": 0.0301513671875, "hidden_mse_step1": 0.017312705313088372, "hidden_mse_step2": 0.019883471233437634, "hidden_mse_step3": 0.023711136084914743, "latent_grad_l1_min": 0.00022190366507857107, "loss": 0.02081025654583105, "lr_blocks": 9.567727288213004e-07, "lr_fusion": 9.567727288213005e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 71.36149797161488, "velocity_mse_step1": 0.015167487544074123, "velocity_mse_step2": 0.017295524394804876, "velocity_mse_step3": 0.023877024334589284} +{"batch_in_epoch": 11, "epoch": 6, "global_step": 311, "grad_norm_max": 0.036376953125, "hidden_mse_step1": 0.01754388610632824, "hidden_mse_step2": 0.020276417265579633, "hidden_mse_step3": 0.02320717692574752, "latent_grad_l1_min": 0.00021485948855115566, "loss": 0.018193538200908473, "lr_blocks": 9.564444949298557e-07, "lr_fusion": 9.564444949298559e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 71.83995126287482, "velocity_mse_step1": 0.012429779087792017, "velocity_mse_step2": 0.014667089799851443, "velocity_mse_step3": 0.0213809957162344} +{"batch_in_epoch": 12, "epoch": 6, "global_step": 312, "grad_norm_max": 0.039794921875, "hidden_mse_step1": 0.01610028772016189, "hidden_mse_step2": 0.018535867142158425, "hidden_mse_step3": 0.0220628605789638, "latent_grad_l1_min": 0.00019518257067829836, "loss": 0.01734285880645205, "lr_blocks": 9.561150763110943e-07, "lr_fusion": 9.561150763110945e-06, "peak_memory_gib": 48.31737470626831, "trajectory_time_s": 71.55433733874816, "velocity_mse_step1": 0.011754535128212801, "velocity_mse_step2": 0.01421672072943433, "velocity_mse_step3": 0.020387417304196527} +{"batch_in_epoch": 13, "epoch": 6, "global_step": 313, "grad_norm_max": 0.03662109375, "hidden_mse_step1": 0.013349188990624887, "hidden_mse_step2": 0.016182765181708546, "hidden_mse_step3": 0.020653973316906816, "latent_grad_l1_min": 0.00022875490685692057, "loss": 0.016059820511145517, "lr_blocks": 9.557844738200409e-07, "lr_fusion": 9.557844738200408e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 71.52544096499332, "velocity_mse_step1": 0.010087196087364905, "velocity_mse_step2": 0.01267883192589839, "velocity_mse_step3": 0.020394839321462705} +{"batch_in_epoch": 14, "epoch": 6, "global_step": 314, "grad_norm_max": 0.027099609375, "hidden_mse_step1": 0.02034671971757364, "hidden_mse_step2": 0.02250556453197662, "hidden_mse_step3": 0.026180242953289832, "latent_grad_l1_min": 0.0002701067978705396, "loss": 0.023304246440862438, "lr_blocks": 9.554526883147925e-07, "lr_fusion": 9.554526883147926e-06, "peak_memory_gib": 48.320029735565186, "trajectory_time_s": 72.13282166735007, "velocity_mse_step1": 0.01678807560633036, "velocity_mse_step2": 0.019265433112325675, "velocity_mse_step3": 0.0269559761218261} +{"batch_in_epoch": 15, "epoch": 6, "global_step": 315, "grad_norm_max": 0.03662109375, "hidden_mse_step1": 0.020133972608683895, "hidden_mse_step2": 0.022612887384769107, "hidden_mse_step3": 0.02616480057726481, "latent_grad_l1_min": 0.00020533498900476843, "loss": 0.020914143375453668, "lr_blocks": 9.551197206565172e-07, "lr_fusion": 9.551197206565174e-06, "peak_memory_gib": 48.31958723068237, "trajectory_time_s": 71.86218558737164, "velocity_mse_step1": 0.01434098938729481, "velocity_mse_step2": 0.016761408507591113, "velocity_mse_step3": 0.024748863693925417} +{"batch_in_epoch": 16, "epoch": 6, "global_step": 316, "grad_norm_max": 0.04150390625, "hidden_mse_step1": 0.012775364292403016, "hidden_mse_step2": 0.015101948233288049, "hidden_mse_step3": 0.018750290527740222, "latent_grad_l1_min": 0.00016094153215817641, "loss": 0.01216530348548466, "lr_blocks": 9.547855717094513e-07, "lr_fusion": 9.547855717094516e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 71.65129500412877, "velocity_mse_step1": 0.008160252654176605, "velocity_mse_step2": 0.009405761817885962, "velocity_mse_step3": 0.014267134326343824} +{"batch_in_epoch": 17, "epoch": 6, "global_step": 317, "grad_norm_max": 0.031494140625, "hidden_mse_step1": 0.014850881125312299, "hidden_mse_step2": 0.01737367034158004, "hidden_mse_step3": 0.020452024431766143, "latent_grad_l1_min": 0.0001896113681141287, "loss": 0.01471462910662272, "lr_blocks": 9.544502423408971e-07, "lr_fusion": 9.544502423408974e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 71.60432590876007, "velocity_mse_step1": 0.010077556759435018, "velocity_mse_step2": 0.011712709292104202, "velocity_mse_step3": 0.01708596261284713} +{"batch_in_epoch": 18, "epoch": 6, "global_step": 318, "grad_norm_max": 0.0263671875, "hidden_mse_step1": 0.021232184504957074, "hidden_mse_step2": 0.02391724007403744, "hidden_mse_step3": 0.027401424212647334, "latent_grad_l1_min": 0.0002918654881796101, "loss": 0.025271311601889983, "lr_blocks": 9.54113733421221e-07, "lr_fusion": 9.541137334212212e-06, "peak_memory_gib": 48.316489696502686, "trajectory_time_s": 71.52033152576041, "velocity_mse_step1": 0.017881282010681128, "velocity_mse_step2": 0.021305826309669228, "velocity_mse_step3": 0.029371740063652393} +{"batch_in_epoch": 19, "epoch": 6, "global_step": 319, "grad_norm_max": 0.03955078125, "hidden_mse_step1": 0.01955664249336613, "hidden_mse_step2": 0.021946157960753353, "hidden_mse_step3": 0.02557813946623355, "latent_grad_l1_min": 0.0002732112607191084, "loss": 0.023490093349079998, "lr_blocks": 9.537760458238504e-07, "lr_fusion": 9.537760458238505e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 71.57837298700179, "velocity_mse_step1": 0.017946561449207366, "velocity_mse_step2": 0.019292677237120057, "velocity_mse_step3": 0.026522944997330863} +{"batch_in_epoch": 20, "epoch": 6, "global_step": 320, "grad_norm_max": 0.0341796875, "hidden_mse_step1": 0.01719680647199441, "hidden_mse_step2": 0.019603353259818893, "hidden_mse_step3": 0.023197639766814448, "latent_grad_l1_min": 0.0002408243472018512, "loss": 0.019326684539139802, "lr_blocks": 9.534371804252726e-07, "lr_fusion": 9.534371804252727e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 73.71832768724562, "velocity_mse_step1": 0.013202790686461543, "velocity_mse_step2": 0.015662553395876393, "velocity_mse_step3": 0.02311492816079408} +{"batch_in_epoch": 21, "epoch": 6, "global_step": 321, "grad_norm_max": 0.03271484375, "hidden_mse_step1": 0.01495786221182373, "hidden_mse_step2": 0.017303393247337744, "hidden_mse_step3": 0.020411407026196163, "latent_grad_l1_min": 0.00020373970073706005, "loss": 0.017088100176936544, "lr_blocks": 9.530971381050319e-07, "lr_fusion": 9.53097138105032e-06, "peak_memory_gib": 48.316489696502686, "trajectory_time_s": 70.65309933474782, "velocity_mse_step1": 0.01291114406610307, "velocity_mse_step2": 0.014487004294226476, "velocity_mse_step3": 0.01859888484302376} +{"batch_in_epoch": 22, "epoch": 6, "global_step": 322, "grad_norm_max": 0.0277099609375, "hidden_mse_step1": 0.017189045877395462, "hidden_mse_step2": 0.018968651603375162, "hidden_mse_step3": 0.022982996289751357, "latent_grad_l1_min": 0.00020065215358044952, "loss": 0.018566797213030176, "lr_blocks": 9.527559197457272e-07, "lr_fusion": 9.527559197457273e-06, "peak_memory_gib": 48.31737470626831, "trajectory_time_s": 71.41077139425033, "velocity_mse_step1": 0.013035408038246845, "velocity_mse_step2": 0.014339396621965402, "velocity_mse_step3": 0.022411516127509197} +{"batch_in_epoch": 23, "epoch": 6, "global_step": 323, "grad_norm_max": 0.037109375, "hidden_mse_step1": 0.015575178296005885, "hidden_mse_step2": 0.01854515034938231, "hidden_mse_step3": 0.021967571262004118, "latent_grad_l1_min": 0.0002195380857301643, "loss": 0.017201596056111157, "lr_blocks": 9.524135262330098e-07, "lr_fusion": 9.524135262330098e-06, "peak_memory_gib": 48.31958723068237, "trajectory_time_s": 71.65356075073942, "velocity_mse_step1": 0.012253388111795566, "velocity_mse_step2": 0.01362482289135057, "velocity_mse_step3": 0.02011778586477573} +{"batch_in_epoch": 24, "epoch": 6, "global_step": 324, "grad_norm_max": 0.03173828125, "hidden_mse_step1": 0.016125035759094835, "hidden_mse_step2": 0.01794460922246799, "hidden_mse_step3": 0.020613495651299933, "latent_grad_l1_min": 0.00019223410163249355, "loss": 0.015754057427069968, "lr_blocks": 9.520699584555812e-07, "lr_fusion": 9.520699584555813e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 71.56936904426402, "velocity_mse_step1": 0.01082793269805344, "velocity_mse_step2": 0.012589840575986144, "velocity_mse_step3": 0.018376083237983818} +{"batch_in_epoch": 25, "epoch": 6, "global_step": 325, "grad_norm_max": 0.036376953125, "hidden_mse_step1": 0.016644384670403918, "hidden_mse_step2": 0.019287172255904542, "hidden_mse_step3": 0.021994480613752136, "latent_grad_l1_min": 0.00017302857577305986, "loss": 0.015728178633643047, "lr_blocks": 9.517252173051911e-07, "lr_fusion": 9.517252173051912e-06, "peak_memory_gib": 48.316489696502686, "trajectory_time_s": 71.00223151562386, "velocity_mse_step1": 0.010506988380387026, "velocity_mse_step2": 0.012677405827811785, "velocity_mse_step3": 0.018207536486443132} +{"batch_in_epoch": 26, "epoch": 6, "global_step": 326, "grad_norm_max": 0.032470703125, "hidden_mse_step1": 0.019180247057062973, "hidden_mse_step2": 0.021682884194888175, "hidden_mse_step3": 0.025144319765136706, "latent_grad_l1_min": 0.00023198780399980024, "loss": 0.021463407003985982, "lr_blocks": 9.513793036766345e-07, "lr_fusion": 9.513793036766345e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 71.56200026586885, "velocity_mse_step1": 0.015232297664624637, "velocity_mse_step2": 0.017519314087361893, "velocity_mse_step3": 0.025037862352162068} +{"batch_in_epoch": 27, "epoch": 6, "global_step": 327, "grad_norm_max": 0.032958984375, "hidden_mse_step1": 0.018296291378127143, "hidden_mse_step2": 0.02022745494364894, "hidden_mse_step3": 0.02375562489864283, "latent_grad_l1_min": 0.000203994113689987, "loss": 0.02220518358600592, "lr_blocks": 9.510322184677492e-07, "lr_fusion": 9.510322184677494e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 71.46690939149994, "velocity_mse_step1": 0.016440738552773837, "velocity_mse_step2": 0.01840654845416014, "velocity_mse_step3": 0.025540324589071264} +{"batch_in_epoch": 28, "epoch": 6, "global_step": 328, "grad_norm_max": 0.031494140625, "hidden_mse_step1": 0.020417124332327924, "hidden_mse_step2": 0.02221914660185575, "hidden_mse_step3": 0.02478264308800655, "latent_grad_l1_min": 0.00022070763407100458, "loss": 0.02039523429370352, "lr_blocks": 9.506839625794151e-07, "lr_fusion": 9.506839625794152e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 71.60244598712598, "velocity_mse_step1": 0.01473478859822665, "velocity_mse_step2": 0.016444112157582173, "velocity_mse_step3": 0.0232649083731563} +{"batch_in_epoch": 29, "epoch": 6, "global_step": 329, "grad_norm_max": 0.033203125, "hidden_mse_step1": 0.01787014611597572, "hidden_mse_step2": 0.0197579681490814, "hidden_mse_step3": 0.022264508330928426, "latent_grad_l1_min": 0.0001842775509430794, "loss": 0.01658404168639598, "lr_blocks": 9.503345369155493e-07, "lr_fusion": 9.503345369155494e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 71.45157188086887, "velocity_mse_step1": 0.011903035778751861, "velocity_mse_step2": 0.013146911980584264, "velocity_mse_step3": 0.01871291371727628} +{"batch_in_epoch": 30, "epoch": 6, "global_step": 330, "grad_norm_max": 0.029541015625, "hidden_mse_step1": 0.023870176132602085, "hidden_mse_step2": 0.025700196191402413, "hidden_mse_step3": 0.028295500237228616, "latent_grad_l1_min": 0.00023370724284177413, "loss": 0.027608888219609586, "lr_blocks": 9.49983942383106e-07, "lr_fusion": 9.499839423831062e-06, "peak_memory_gib": 48.320029735565186, "trajectory_time_s": 71.97545723275107, "velocity_mse_step1": 0.021502040199785757, "velocity_mse_step2": 0.023778903813633533, "velocity_mse_step3": 0.029759131118355847} +{"batch_in_epoch": 31, "epoch": 6, "global_step": 331, "grad_norm_max": 0.043212890625, "hidden_mse_step1": 0.017876249804560623, "hidden_mse_step2": 0.020002740925909684, "hidden_mse_step3": 0.02285463622372065, "latent_grad_l1_min": 0.0002494676773494575, "loss": 0.01994816746030535, "lr_blocks": 9.49632179892073e-07, "lr_fusion": 9.49632179892073e-06, "peak_memory_gib": 48.31737470626831, "trajectory_time_s": 72.5377668697547, "velocity_mse_step1": 0.015507722082215228, "velocity_mse_step2": 0.016128147785950984, "velocity_mse_step3": 0.022135267516465062} +{"batch_in_epoch": 32, "epoch": 6, "global_step": 332, "grad_norm_max": 0.035888671875, "hidden_mse_step1": 0.014388210792926006, "hidden_mse_step2": 0.016728545322881212, "hidden_mse_step3": 0.020064192907219485, "latent_grad_l1_min": 0.00018247789284941973, "loss": 0.015293189166446348, "lr_blocks": 9.492792503554694e-07, "lr_fusion": 9.492792503554695e-06, "peak_memory_gib": 48.320029735565186, "trajectory_time_s": 71.33219649337843, "velocity_mse_step1": 0.010521800384075113, "velocity_mse_step2": 0.012583320573737313, "velocity_mse_step3": 0.017656350134140145} +{"batch_in_epoch": 33, "epoch": 6, "global_step": 333, "grad_norm_max": 0.0303955078125, "hidden_mse_step1": 0.01460627158771136, "hidden_mse_step2": 0.017422668635845184, "hidden_mse_step3": 0.02177950979343482, "latent_grad_l1_min": 0.00020520288853731472, "loss": 0.01573172700591385, "lr_blocks": 9.48925154689344e-07, "lr_fusion": 9.489251546893441e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 73.15779259199189, "velocity_mse_step1": 0.010546399091253988, "velocity_mse_step2": 0.012369598058285192, "velocity_mse_step3": 0.018898337181391462} +{"batch_in_epoch": 34, "epoch": 6, "global_step": 334, "grad_norm_max": 0.03173828125, "hidden_mse_step1": 0.01750945742657807, "hidden_mse_step2": 0.020670676670436348, "hidden_mse_step3": 0.02449769185789462, "latent_grad_l1_min": 0.0003103159342572326, "loss": 0.02091859681032864, "lr_blocks": 9.485698938127715e-07, "lr_fusion": 9.485698938127716e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 73.40498033713084, "velocity_mse_step1": 0.013664298382147015, "velocity_mse_step2": 0.017525253775862178, "velocity_mse_step3": 0.025298453734389374} +{"batch_in_epoch": 35, "epoch": 6, "global_step": 335, "grad_norm_max": 0.032958984375, "hidden_mse_step1": 0.017028289198476286, "hidden_mse_step2": 0.01939452839516369, "hidden_mse_step3": 0.022964063799008727, "latent_grad_l1_min": 0.0002249754097647383, "loss": 0.017768226805076535, "lr_blocks": 9.482134686478518e-07, "lr_fusion": 9.48213468647852e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 72.24095671386749, "velocity_mse_step1": 0.012391076834839101, "velocity_mse_step2": 0.014119211621748815, "velocity_mse_step3": 0.020855702850635028} +{"batch_in_epoch": 36, "epoch": 6, "global_step": 336, "grad_norm_max": 0.04052734375, "hidden_mse_step1": 0.013515857401736346, "hidden_mse_step2": 0.016083780933903263, "hidden_mse_step3": 0.02024346907689635, "latent_grad_l1_min": 0.0001995977972910623, "loss": 0.015215844414862137, "lr_blocks": 9.478558801197064e-07, "lr_fusion": 9.478558801197065e-06, "peak_memory_gib": 48.316489696502686, "trajectory_time_s": 71.80139748949296, "velocity_mse_step1": 0.00960856797714119, "velocity_mse_step2": 0.011738551547750832, "velocity_mse_step3": 0.019316101895778304} +{"batch_in_epoch": 37, "epoch": 6, "global_step": 337, "grad_norm_max": 0.033203125, "hidden_mse_step1": 0.017295766193586002, "hidden_mse_step2": 0.019768366997595876, "hidden_mse_step3": 0.023383947970744754, "latent_grad_l1_min": 0.00024064523495326284, "loss": 0.019799711748159355, "lr_blocks": 9.474971291564763e-07, "lr_fusion": 9.474971291564764e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 71.8330724016123, "velocity_mse_step1": 0.013751515291265344, "velocity_mse_step2": 0.016269329465493292, "velocity_mse_step3": 0.02333348074379111} +{"batch_in_epoch": 38, "epoch": 6, "global_step": 338, "grad_norm_max": 0.03955078125, "hidden_mse_step1": 0.01826773743544306, "hidden_mse_step2": 0.020320675901270336, "hidden_mse_step3": 0.02342521866583931, "latent_grad_l1_min": 0.00022293973233900033, "loss": 0.020127128152775443, "lr_blocks": 9.471372166893198e-07, "lr_fusion": 9.4713721668932e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 71.61220200574462, "velocity_mse_step1": 0.014940039649705535, "velocity_mse_step2": 0.016355520272294855, "velocity_mse_step3": 0.022884459317927913} +{"batch_in_epoch": 39, "epoch": 6, "global_step": 339, "grad_norm_max": 0.031982421875, "hidden_mse_step1": 0.01723744211735071, "hidden_mse_step2": 0.019190935309909812, "hidden_mse_step3": 0.02221781988295593, "latent_grad_l1_min": 0.00020176807447569445, "loss": 0.01809864430937783, "lr_blocks": 9.467761436524099e-07, "lr_fusion": 9.4677614365241e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 71.60388121326832, "velocity_mse_step1": 0.012744248831170677, "velocity_mse_step2": 0.014756611604492978, "velocity_mse_step3": 0.020930450696531416} +{"batch_in_epoch": 40, "epoch": 6, "global_step": 340, "grad_norm_max": 0.041748046875, "hidden_mse_step1": 0.015409106438580367, "hidden_mse_step2": 0.018148913314299922, "hidden_mse_step3": 0.021278396465017328, "latent_grad_l1_min": 0.00019280347805761266, "loss": 0.017430587912843163, "lr_blocks": 9.46413910982932e-07, "lr_fusion": 9.46413910982932e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 72.19726498849923, "velocity_mse_step1": 0.012671676497640356, "velocity_mse_step2": 0.014494906092295423, "velocity_mse_step3": 0.019641538325231522} +{"batch_in_epoch": 41, "epoch": 6, "global_step": 341, "grad_norm_max": 0.0517578125, "hidden_mse_step1": 0.014468434445526718, "hidden_mse_step2": 0.016583070142327676, "hidden_mse_step3": 0.01940625949230577, "latent_grad_l1_min": 0.00018538800213718787, "loss": 0.013316216924327556, "lr_blocks": 9.460505196210813e-07, "lr_fusion": 9.460505196210814e-06, "peak_memory_gib": 48.320472240448, "trajectory_time_s": 72.05355738249636, "velocity_mse_step1": 0.009278481767978517, "velocity_mse_step2": 0.010639233693447228, "velocity_mse_step3": 0.014985157900290298} +{"batch_in_epoch": 42, "epoch": 6, "global_step": 342, "grad_norm_max": 0.033935546875, "hidden_mse_step1": 0.01614189761208503, "hidden_mse_step2": 0.019547535397578027, "hidden_mse_step3": 0.023858859636155624, "latent_grad_l1_min": 0.0002602578642836306, "loss": 0.01983957056654617, "lr_blocks": 9.456859705100605e-07, "lr_fusion": 9.456859705100605e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 72.39991543724318, "velocity_mse_step1": 0.01304263772492829, "velocity_mse_step2": 0.016475020753984736, "velocity_mse_step3": 0.024046222542113225} +{"batch_in_epoch": 43, "epoch": 6, "global_step": 343, "grad_norm_max": 0.0303955078125, "hidden_mse_step1": 0.020592651504557576, "hidden_mse_step2": 0.022340628767519126, "hidden_mse_step3": 0.02547092381532171, "latent_grad_l1_min": 0.00022183661712915637, "loss": 0.022873201781684265, "lr_blocks": 9.453202645960773e-07, "lr_fusion": 9.453202645960775e-06, "peak_memory_gib": 48.31737470626831, "trajectory_time_s": 71.75407249286945, "velocity_mse_step1": 0.01658482969339405, "velocity_mse_step2": 0.018980114325781224, "velocity_mse_step3": 0.02621423952846921} +{"batch_in_epoch": 44, "epoch": 6, "global_step": 344, "grad_norm_max": 0.0341796875, "hidden_mse_step1": 0.016729116431503953, "hidden_mse_step2": 0.01873248426376709, "hidden_mse_step3": 0.02235585467756859, "latent_grad_l1_min": 0.00017685733655525837, "loss": 0.017822122163904296, "lr_blocks": 9.449534028283419e-07, "lr_fusion": 9.44953402828342e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 72.02485172576417, "velocity_mse_step1": 0.012828696637630595, "velocity_mse_step2": 0.014100227970629929, "velocity_mse_step3": 0.02075569482154346} +{"batch_in_epoch": 45, "epoch": 6, "global_step": 345, "grad_norm_max": 0.039306640625, "hidden_mse_step1": 0.022624939846407087, "hidden_mse_step2": 0.025238614429586696, "hidden_mse_step3": 0.02805764755300645, "latent_grad_l1_min": 0.00025420755991945043, "loss": 0.02462552506144025, "lr_blocks": 9.445853861590646e-07, "lr_fusion": 9.445853861590647e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 71.82659736674395, "velocity_mse_step1": 0.017581459130659432, "velocity_mse_step2": 0.02056659750607131, "velocity_mse_step3": 0.028136396813871603} +{"batch_in_epoch": 46, "epoch": 6, "global_step": 346, "grad_norm_max": 0.037353515625, "hidden_mse_step1": 0.018949106622520567, "hidden_mse_step2": 0.021714128345982835, "hidden_mse_step3": 0.0254895097210205, "latent_grad_l1_min": 0.0004152312485530274, "loss": 0.030017525855717916, "lr_blocks": 9.442162155434533e-07, "lr_fusion": 9.442162155434534e-06, "peak_memory_gib": 48.31737470626831, "trajectory_time_s": 71.91340989588934, "velocity_mse_step1": 0.024527721526932767, "velocity_mse_step2": 0.02661663912710667, "velocity_mse_step3": 0.032292940034364746} +{"batch_in_epoch": 47, "epoch": 6, "global_step": 347, "grad_norm_max": 0.029052734375, "hidden_mse_step1": 0.015539561373381212, "hidden_mse_step2": 0.018474329163187315, "hidden_mse_step3": 0.022017109357485812, "latent_grad_l1_min": 0.000170328889907978, "loss": 0.015405667920796465, "lr_blocks": 9.438458919397112e-07, "lr_fusion": 9.438458919397113e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 71.33521499489143, "velocity_mse_step1": 0.009347457474047714, "velocity_mse_step2": 0.012081360720912926, "velocity_mse_step3": 0.019185084271677103} +{"batch_in_epoch": 48, "epoch": 6, "global_step": 348, "grad_norm_max": 0.0400390625, "hidden_mse_step1": 0.01683029335358047, "hidden_mse_step2": 0.018883224551765512, "hidden_mse_step3": 0.02184287052867668, "latent_grad_l1_min": 0.00019826031712000258, "loss": 0.016757058578410318, "lr_blocks": 9.434744163090339e-07, "lr_fusion": 9.434744163090341e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 71.6711205738684, "velocity_mse_step1": 0.011458082793978974, "velocity_mse_step2": 0.013645941948717724, "velocity_mse_step3": 0.019411510363819873} +{"batch_in_epoch": 49, "epoch": 6, "global_step": 349, "grad_norm_max": 0.04638671875, "hidden_mse_step1": 0.018650086521769742, "hidden_mse_step2": 0.02094809105619788, "hidden_mse_step3": 0.024125341991228715, "latent_grad_l1_min": 0.0002288368486915715, "loss": 0.020604141929652542, "lr_blocks": 9.431017896156073e-07, "lr_fusion": 9.431017896156074e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 71.84747452562442, "velocity_mse_step1": 0.014939104256752347, "velocity_mse_step2": 0.017259916648202173, "velocity_mse_step3": 0.023241050674446987} +{"batch_in_epoch": 50, "epoch": 6, "global_step": 350, "grad_norm_max": 0.037841796875, "hidden_mse_step1": 0.019193825403427973, "hidden_mse_step2": 0.022565517291825796, "hidden_mse_step3": 0.025809198850765824, "latent_grad_l1_min": 0.00026264195548719727, "loss": 0.021295136102708057, "lr_blocks": 9.427280128266049e-07, "lr_fusion": 9.427280128266049e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 71.535456658763, "velocity_mse_step1": 0.014201668691904548, "velocity_mse_step2": 0.01754960784039992, "velocity_mse_step3": 0.025377276079130488} +{"batch_in_epoch": 1, "epoch": 7, "global_step": 351, "grad_norm_max": 0.03125, "hidden_mse_step1": 0.014686522772535682, "hidden_mse_step2": 0.01633919555959957, "hidden_mse_step3": 0.01956599302190755, "latent_grad_l1_min": 0.0001720943337204517, "loss": 0.01352569826745561, "lr_blocks": 9.423530869121854e-07, "lr_fusion": 9.423530869121855e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 71.36690171388182, "velocity_mse_step1": 0.009662105045988155, "velocity_mse_step2": 0.010379525261149478, "velocity_mse_step3": 0.015476292556351315} +{"batch_in_epoch": 2, "epoch": 7, "global_step": 352, "grad_norm_max": 0.0478515625, "hidden_mse_step1": 0.016005892867854397, "hidden_mse_step2": 0.018329795815848877, "hidden_mse_step3": 0.021386678264077218, "latent_grad_l1_min": 0.00016425146259280154, "loss": 0.014774893580969155, "lr_blocks": 9.419770128454898e-07, "lr_fusion": 9.4197701284549e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 73.05882965712226, "velocity_mse_step1": 0.010526810061751997, "velocity_mse_step2": 0.011469665744308648, "velocity_mse_step3": 0.016755966710791523} +{"batch_in_epoch": 3, "epoch": 7, "global_step": 353, "grad_norm_max": 0.044677734375, "hidden_mse_step1": 0.019227491591924, "hidden_mse_step2": 0.021182027612147585, "hidden_mse_step3": 0.02436858561954328, "latent_grad_l1_min": 0.00020819485143874772, "loss": 0.019787292977395867, "lr_blocks": 9.415997916026399e-07, "lr_fusion": 9.415997916026401e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 72.0327566726337, "velocity_mse_step1": 0.014619238653852205, "velocity_mse_step2": 0.015523629907485365, "velocity_mse_step3": 0.02274119833184938} +{"batch_in_epoch": 4, "epoch": 7, "global_step": 354, "grad_norm_max": 0.02978515625, "hidden_mse_step1": 0.017214954256944894, "hidden_mse_step2": 0.020108727339122976, "hidden_mse_step3": 0.024394224930022444, "latent_grad_l1_min": 0.0003100028643530095, "loss": 0.02150254462945408, "lr_blocks": 9.412214241627342e-07, "lr_fusion": 9.412214241627344e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 72.58461189513037, "velocity_mse_step1": 0.014261648133729717, "velocity_mse_step2": 0.01769320132111066, "velocity_mse_step3": 0.02638099222843136} +{"batch_in_epoch": 5, "epoch": 7, "global_step": 355, "grad_norm_max": 0.03466796875, "hidden_mse_step1": 0.019456448424274898, "hidden_mse_step2": 0.02180343645159155, "hidden_mse_step3": 0.024969061337677494, "latent_grad_l1_min": 0.00022737254766980186, "loss": 0.02155664828439642, "lr_blocks": 9.40841911507847e-07, "lr_fusion": 9.40841911507847e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 71.02645660164126, "velocity_mse_step1": 0.01567504676599388, "velocity_mse_step2": 0.017387736983697062, "velocity_mse_step3": 0.02498426426401628} +{"batch_in_epoch": 6, "epoch": 7, "global_step": 356, "grad_norm_max": 0.045654296875, "hidden_mse_step1": 0.020084767207403535, "hidden_mse_step2": 0.021918691303913614, "hidden_mse_step3": 0.025560079283812744, "latent_grad_l1_min": 0.00026122738108824706, "loss": 0.026419995111479824, "lr_blocks": 9.404612546230243e-07, "lr_fusion": 9.404612546230244e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 74.0335777186192, "velocity_mse_step1": 0.019863669160778433, "velocity_mse_step2": 0.022381936476449482, "velocity_mse_step3": 0.0302580224025795} +{"batch_in_epoch": 7, "epoch": 7, "global_step": 357, "grad_norm_max": 0.032958984375, "hidden_mse_step1": 0.01723729019951341, "hidden_mse_step2": 0.019175614654419144, "hidden_mse_step3": 0.022128169275155023, "latent_grad_l1_min": 0.0001870793530542869, "loss": 0.020142714243515263, "lr_blocks": 9.400794544962827e-07, "lr_fusion": 9.400794544962828e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 71.10670173986, "velocity_mse_step1": 0.015935965293985128, "velocity_mse_step2": 0.016906981356441975, "velocity_mse_step3": 0.021731086998313134} +{"batch_in_epoch": 8, "epoch": 7, "global_step": 358, "grad_norm_max": 0.027099609375, "hidden_mse_step1": 0.015649687290923402, "hidden_mse_step2": 0.018161368805782067, "hidden_mse_step3": 0.021499867950166972, "latent_grad_l1_min": 0.00018222680955659598, "loss": 0.015041573197647398, "lr_blocks": 9.396965121186058e-07, "lr_fusion": 9.396965121186058e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 72.67309230937099, "velocity_mse_step1": 0.009731317620857485, "velocity_mse_step2": 0.01189545464668689, "velocity_mse_step3": 0.01796685334660911} +{"batch_in_epoch": 9, "epoch": 7, "global_step": 359, "grad_norm_max": 0.0341796875, "hidden_mse_step1": 0.017134243950879733, "hidden_mse_step2": 0.019010332138610204, "hidden_mse_step3": 0.02231846546887287, "latent_grad_l1_min": 0.0002394559433014365, "loss": 0.018841667350248566, "lr_blocks": 9.393124284839422e-07, "lr_fusion": 9.393124284839424e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 72.54774128298595, "velocity_mse_step1": 0.013944518550748136, "velocity_mse_step2": 0.014818512647512501, "velocity_mse_step3": 0.021915664713430615} +{"batch_in_epoch": 10, "epoch": 7, "global_step": 360, "grad_norm_max": 0.04248046875, "hidden_mse_step1": 0.02046326279274321, "hidden_mse_step2": 0.023468747601977414, "hidden_mse_step3": 0.027039648748801222, "latent_grad_l1_min": 0.00039341523006442003, "loss": 0.0280602912784421, "lr_blocks": 9.389272045892023e-07, "lr_fusion": 9.389272045892023e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 75.72425777223543, "velocity_mse_step1": 0.02101576711200843, "velocity_mse_step2": 0.024734917595716457, "velocity_mse_step3": 0.03133302127077643} +{"batch_in_epoch": 11, "epoch": 7, "global_step": 361, "grad_norm_max": 0.03173828125, "hidden_mse_step1": 0.01951305846367696, "hidden_mse_step2": 0.021419560179180865, "hidden_mse_step3": 0.023999311246110922, "latent_grad_l1_min": 0.0001759172419042443, "loss": 0.019339935633719763, "lr_blocks": 9.385408414342563e-07, "lr_fusion": 9.385408414342565e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 72.78020943375304, "velocity_mse_step1": 0.013457108304594706, "velocity_mse_step2": 0.01568232923247186, "velocity_mse_step3": 0.0223871754341027} +{"batch_in_epoch": 12, "epoch": 7, "global_step": 362, "grad_norm_max": 0.031005859375, "hidden_mse_step1": 0.018095999515415833, "hidden_mse_step2": 0.020449416046696048, "hidden_mse_step3": 0.023856657058266655, "latent_grad_l1_min": 0.0002664766871021129, "loss": 0.01951244663879541, "lr_blocks": 9.381533400219317e-07, "lr_fusion": 9.381533400219319e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 72.31061886087264, "velocity_mse_step1": 0.013222070024182487, "velocity_mse_step2": 0.016006939595432153, "velocity_mse_step3": 0.023068121750839055} +{"batch_in_epoch": 13, "epoch": 7, "global_step": 363, "grad_norm_max": 0.046142578125, "hidden_mse_step1": 0.01525748098668243, "hidden_mse_step2": 0.018113257263654044, "hidden_mse_step3": 0.02176299241338191, "latent_grad_l1_min": 0.0002142256626029848, "loss": 0.017398856422265192, "lr_blocks": 9.377647013580101e-07, "lr_fusion": 9.377647013580102e-06, "peak_memory_gib": 48.31958723068237, "trajectory_time_s": 71.37522357038688, "velocity_mse_step1": 0.011861095547958807, "velocity_mse_step2": 0.01428262750830202, "velocity_mse_step3": 0.02053947156361703} +{"batch_in_epoch": 14, "epoch": 7, "global_step": 364, "grad_norm_max": 0.036376953125, "hidden_mse_step1": 0.02384063379473186, "hidden_mse_step2": 0.026660497483265187, "hidden_mse_step3": 0.03009380496639226, "latent_grad_l1_min": 0.00032325891152140684, "loss": 0.026933486174259863, "lr_blocks": 9.373749264512249e-07, "lr_fusion": 9.37374926451225e-06, "peak_memory_gib": 48.316489696502686, "trajectory_time_s": 71.67749868749524, "velocity_mse_step1": 0.019660117944502935, "velocity_mse_step2": 0.022186933733922026, "velocity_mse_step3": 0.030893911790501855} +{"batch_in_epoch": 15, "epoch": 7, "global_step": 365, "grad_norm_max": 0.030517578125, "hidden_mse_step1": 0.014859295595670117, "hidden_mse_step2": 0.017076853122229552, "hidden_mse_step3": 0.019837563474928693, "latent_grad_l1_min": 0.0001642655024625128, "loss": 0.014078080388052121, "lr_blocks": 9.36984016313259e-07, "lr_fusion": 9.36984016313259e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 71.02896810224047, "velocity_mse_step1": 0.009706106130449501, "velocity_mse_step2": 0.011408928428344163, "velocity_mse_step3": 0.015941834642684886} +{"batch_in_epoch": 16, "epoch": 7, "global_step": 366, "grad_norm_max": 0.040283203125, "hidden_mse_step1": 0.016779093429379697, "hidden_mse_step2": 0.01960411652856107, "hidden_mse_step3": 0.023651466322397542, "latent_grad_l1_min": 0.0001738509199640248, "loss": 0.01865762161157493, "lr_blocks": 9.365919719587412e-07, "lr_fusion": 9.365919719587413e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 71.840816849377, "velocity_mse_step1": 0.012030417627621708, "velocity_mse_step2": 0.014886387374385126, "velocity_mse_step3": 0.023052590342038975} +{"batch_in_epoch": 17, "epoch": 7, "global_step": 367, "grad_norm_max": 0.039306640625, "hidden_mse_step1": 0.01942677077853919, "hidden_mse_step2": 0.021767814103181335, "hidden_mse_step3": 0.024837795329014103, "latent_grad_l1_min": 0.0002919724975072313, "loss": 0.026790162573368956, "lr_blocks": 9.361987944052449e-07, "lr_fusion": 9.36198794405245e-06, "peak_memory_gib": 48.320029735565186, "trajectory_time_s": 71.16091280024557, "velocity_mse_step1": 0.022379570796536942, "velocity_mse_step2": 0.023511841893196102, "velocity_mse_step3": 0.02787583593245862} +{"batch_in_epoch": 18, "epoch": 7, "global_step": 368, "grad_norm_max": 0.0400390625, "hidden_mse_step1": 0.01855578611139208, "hidden_mse_step2": 0.0217364796720046, "hidden_mse_step3": 0.02508748001751623, "latent_grad_l1_min": 0.00026101330240635434, "loss": 0.022470643793764927, "lr_blocks": 9.358044846732846e-07, "lr_fusion": 9.358044846732848e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 73.84834947426134, "velocity_mse_step1": 0.015395555337558367, "velocity_mse_step2": 0.019416438843888627, "velocity_mse_step3": 0.026061961086400385} +{"batch_in_epoch": 19, "epoch": 7, "global_step": 369, "grad_norm_max": 0.02978515625, "hidden_mse_step1": 0.0191547856541417, "hidden_mse_step2": 0.021630425679696988, "hidden_mse_step3": 0.02458337977129434, "latent_grad_l1_min": 0.00018692826415644959, "loss": 0.01811504512027438, "lr_blocks": 9.354090437863131e-07, "lr_fusion": 9.354090437863133e-06, "peak_memory_gib": 48.31737470626831, "trajectory_time_s": 72.93967639988841, "velocity_mse_step1": 0.012654723025791879, "velocity_mse_step2": 0.01434918711934837, "velocity_mse_step3": 0.020804365340154618} +{"batch_in_epoch": 20, "epoch": 7, "global_step": 370, "grad_norm_max": 0.034912109375, "hidden_mse_step1": 0.017094147441509576, "hidden_mse_step2": 0.020507529910121645, "hidden_mse_step3": 0.023756230399677794, "latent_grad_l1_min": 0.000232468975809752, "loss": 0.019169776484237185, "lr_blocks": 9.350124727707196e-07, "lr_fusion": 9.350124727707197e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 72.10569195476273, "velocity_mse_step1": 0.013068144097425308, "velocity_mse_step2": 0.016088024929298887, "velocity_mse_step3": 0.02221736843265327} +{"batch_in_epoch": 21, "epoch": 7, "global_step": 371, "grad_norm_max": 0.034912109375, "hidden_mse_step1": 0.019977159041445702, "hidden_mse_step2": 0.022299409877242785, "hidden_mse_step3": 0.02528721125729914, "latent_grad_l1_min": 0.0002590903932286892, "loss": 0.022040195363972864, "lr_blocks": 9.346147726558264e-07, "lr_fusion": 9.346147726558266e-06, "peak_memory_gib": 48.320029735565186, "trajectory_time_s": 71.34874197112367, "velocity_mse_step1": 0.015961055481706614, "velocity_mse_step2": 0.017791201767977327, "velocity_mse_step3": 0.025611947972460514} +{"batch_in_epoch": 22, "epoch": 7, "global_step": 372, "grad_norm_max": 0.02880859375, "hidden_mse_step1": 0.020477974009866423, "hidden_mse_step2": 0.02270235493779182, "hidden_mse_step3": 0.026297602968822633, "latent_grad_l1_min": 0.0002858526750060264, "loss": 0.023066229910390187, "lr_blocks": 9.342159444738864e-07, "lr_fusion": 9.342159444738865e-06, "peak_memory_gib": 48.31737470626831, "trajectory_time_s": 71.55407796362124, "velocity_mse_step1": 0.015516955909593627, "velocity_mse_step2": 0.019384881494001353, "velocity_mse_step3": 0.027349057075168402} +{"batch_in_epoch": 23, "epoch": 7, "global_step": 373, "grad_norm_max": 0.036376953125, "hidden_mse_step1": 0.02004813732180212, "hidden_mse_step2": 0.022180156599331112, "hidden_mse_step3": 0.025827209764559357, "latent_grad_l1_min": 0.0002797110491883359, "loss": 0.021405456255057027, "lr_blocks": 9.338159892600807e-07, "lr_fusion": 9.338159892600809e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 73.3497024957469, "velocity_mse_step1": 0.015553956279680797, "velocity_mse_step2": 0.016671783411376442, "velocity_mse_step3": 0.025185077035400488} +{"batch_in_epoch": 24, "epoch": 7, "global_step": 374, "grad_norm_max": 0.039794921875, "hidden_mse_step1": 0.021538534417881495, "hidden_mse_step2": 0.023953114719396193, "hidden_mse_step3": 0.026874765587438426, "latent_grad_l1_min": 0.00020892155771434773, "loss": 0.02431224314828536, "lr_blocks": 9.334149080525153e-07, "lr_fusion": 9.334149080525154e-06, "peak_memory_gib": 48.320029735565186, "trajectory_time_s": 74.05548284575343, "velocity_mse_step1": 0.01756613840448803, "velocity_mse_step2": 0.020562204058348597, "velocity_mse_step3": 0.02757174350387816} +{"batch_in_epoch": 25, "epoch": 7, "global_step": 375, "grad_norm_max": 0.033203125, "hidden_mse_step1": 0.016338175237511417, "hidden_mse_step2": 0.01911060235995267, "hidden_mse_step3": 0.023105968014403643, "latent_grad_l1_min": 0.0002933100522568566, "loss": 0.021286563739912317, "lr_blocks": 9.330127018922193e-07, "lr_fusion": 9.330127018922195e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 72.18815864648786, "velocity_mse_step1": 0.015869130859625456, "velocity_mse_step2": 0.01816523922025226, "velocity_mse_step3": 0.0239698448235036} +{"batch_in_epoch": 26, "epoch": 7, "global_step": 376, "grad_norm_max": 0.0286865234375, "hidden_mse_step1": 0.016310625297981978, "hidden_mse_step2": 0.017751331590781256, "hidden_mse_step3": 0.020722372257815938, "latent_grad_l1_min": 0.00022823645667813253, "loss": 0.016774483415896872, "lr_blocks": 9.326093718231411e-07, "lr_fusion": 9.326093718231413e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 74.6508570696169, "velocity_mse_step1": 0.012197142592672856, "velocity_mse_step2": 0.013673337712784166, "velocity_mse_step3": 0.018974535670297752} +{"batch_in_epoch": 27, "epoch": 7, "global_step": 377, "grad_norm_max": 0.0380859375, "hidden_mse_step1": 0.022175983039362888, "hidden_mse_step2": 0.02439533032676471, "hidden_mse_step3": 0.028072166040406694, "latent_grad_l1_min": 0.0002926416336777038, "loss": 0.027424239794657163, "lr_blocks": 9.322049188921466e-07, "lr_fusion": 9.322049188921467e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 73.43401012137474, "velocity_mse_step1": 0.01953468558869125, "velocity_mse_step2": 0.023146800950468914, "velocity_mse_step3": 0.03212688267896218} +{"batch_in_epoch": 28, "epoch": 7, "global_step": 378, "grad_norm_max": 0.0400390625, "hidden_mse_step1": 0.020638344717943777, "hidden_mse_step2": 0.02347927664439859, "hidden_mse_step3": 0.02679323128956769, "latent_grad_l1_min": 0.0002661794624145841, "loss": 0.025663105658687915, "lr_blocks": 9.317993441490161e-07, "lr_fusion": 9.317993441490163e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 72.17333567775495, "velocity_mse_step1": 0.018852861908921374, "velocity_mse_step2": 0.021866552506773065, "velocity_mse_step3": 0.02917881447189886} +{"batch_in_epoch": 29, "epoch": 7, "global_step": 379, "grad_norm_max": 0.03466796875, "hidden_mse_step1": 0.019650680719808276, "hidden_mse_step2": 0.02175726888734581, "hidden_mse_step3": 0.024935715898339237, "latent_grad_l1_min": 0.0001777811921783723, "loss": 0.020181944610417953, "lr_blocks": 9.313926486464419e-07, "lr_fusion": 9.31392648646442e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 71.69867491486366, "velocity_mse_step1": 0.014451945951025532, "velocity_mse_step2": 0.01611444913266626, "velocity_mse_step3": 0.023345070874451524} +{"batch_in_epoch": 30, "epoch": 7, "global_step": 380, "grad_norm_max": 0.027099609375, "hidden_mse_step1": 0.017719922620537027, "hidden_mse_step2": 0.0203496630419977, "hidden_mse_step3": 0.023593899477938454, "latent_grad_l1_min": 0.0002226071474069613, "loss": 0.019786522000296305, "lr_blocks": 9.309848334400245e-07, "lr_fusion": 9.309848334400247e-06, "peak_memory_gib": 48.31958723068237, "trajectory_time_s": 73.81856486989273, "velocity_mse_step1": 0.01454233370480194, "velocity_mse_step2": 0.016157495398407003, "velocity_mse_step3": 0.022493386017491238} +{"batch_in_epoch": 31, "epoch": 7, "global_step": 381, "grad_norm_max": 0.0390625, "hidden_mse_step1": 0.016610788792604577, "hidden_mse_step2": 0.019244954746682197, "hidden_mse_step3": 0.022978252581586794, "latent_grad_l1_min": 0.00020657879576901905, "loss": 0.018496810234085257, "lr_blocks": 9.305758995882715e-07, "lr_fusion": 9.305758995882716e-06, "peak_memory_gib": 48.31737470626831, "trajectory_time_s": 73.10321280776407, "velocity_mse_step1": 0.01187893028587236, "velocity_mse_step2": 0.014996662794146687, "velocity_mse_step3": 0.02273143589263782} +{"batch_in_epoch": 32, "epoch": 7, "global_step": 382, "grad_norm_max": 0.0546875, "hidden_mse_step1": 0.01862922602283236, "hidden_mse_step2": 0.020579073012673428, "hidden_mse_step3": 0.023616316894601495, "latent_grad_l1_min": 0.00025413091952941613, "loss": 0.02046204760803708, "lr_blocks": 9.301658481525938e-07, "lr_fusion": 9.30165848152594e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 72.46156703363522, "velocity_mse_step1": 0.015396815414922977, "velocity_mse_step2": 0.016599024284265136, "velocity_mse_step3": 0.02310783986467868} +{"batch_in_epoch": 33, "epoch": 7, "global_step": 383, "grad_norm_max": 0.0400390625, "hidden_mse_step1": 0.019877092147778185, "hidden_mse_step2": 0.02245924690006567, "hidden_mse_step3": 0.025678946370525018, "latent_grad_l1_min": 0.0002495453336450737, "loss": 0.022552178390989344, "lr_blocks": 9.297546801973026e-07, "lr_fusion": 9.297546801973027e-06, "peak_memory_gib": 48.31958723068237, "trajectory_time_s": 73.53084746862442, "velocity_mse_step1": 0.015871961925378333, "velocity_mse_step2": 0.01867777179826849, "velocity_mse_step3": 0.02630527096334844} +{"batch_in_epoch": 34, "epoch": 7, "global_step": 384, "grad_norm_max": 0.033203125, "hidden_mse_step1": 0.013252918812213466, "hidden_mse_step2": 0.014964730088000318, "hidden_mse_step3": 0.01846125168127141, "latent_grad_l1_min": 0.00016935619532887358, "loss": 0.012628286050000626, "lr_blocks": 9.293423967896075e-07, "lr_fusion": 9.293423967896076e-06, "peak_memory_gib": 48.31958723068237, "trajectory_time_s": 74.12555843438167, "velocity_mse_step1": 0.00940000191206179, "velocity_mse_step2": 0.009497493777806603, "velocity_mse_step3": 0.014319470975481505} +{"batch_in_epoch": 35, "epoch": 7, "global_step": 385, "grad_norm_max": 0.038818359375, "hidden_mse_step1": 0.013558031026540058, "hidden_mse_step2": 0.016242521698586643, "hidden_mse_step3": 0.02040205811083849, "latent_grad_l1_min": 0.000206751550649642, "loss": 0.014269201567263474, "lr_blocks": 9.289289989996132e-07, "lr_fusion": 9.289289989996133e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 72.63166834574804, "velocity_mse_step1": 0.009358781599855448, "velocity_mse_step2": 0.011049914007474269, "velocity_mse_step3": 0.017378646996803578} +{"batch_in_epoch": 36, "epoch": 7, "global_step": 386, "grad_norm_max": 0.033935546875, "hidden_mse_step1": 0.012429432286548294, "hidden_mse_step2": 0.014789577623430106, "hidden_mse_step3": 0.01812577423905688, "latent_grad_l1_min": 0.0001479221418776433, "loss": 0.011243969863114347, "lr_blocks": 9.285144879003172e-07, "lr_fusion": 9.285144879003173e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 71.65269136386632, "velocity_mse_step1": 0.007000040913616041, "velocity_mse_step2": 0.008766430983087046, "velocity_mse_step3": 0.01343095811483051} +{"batch_in_epoch": 37, "epoch": 7, "global_step": 387, "grad_norm_max": 0.0302734375, "hidden_mse_step1": 0.02194548276020214, "hidden_mse_step2": 0.024164305800305947, "hidden_mse_step3": 0.027276742904047878, "latent_grad_l1_min": 0.0002336059096705867, "loss": 0.02409768947732768, "lr_blocks": 9.280988645676058e-07, "lr_fusion": 9.28098864567606e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 71.51344159249857, "velocity_mse_step1": 0.018620406793031306, "velocity_mse_step2": 0.019919286889489737, "velocity_mse_step3": 0.02641471899447164} +{"batch_in_epoch": 38, "epoch": 7, "global_step": 388, "grad_norm_max": 0.027587890625, "hidden_mse_step1": 0.016441991199306876, "hidden_mse_step2": 0.019005461936883093, "hidden_mse_step3": 0.022369949147105217, "latent_grad_l1_min": 0.00024566171123296954, "loss": 0.01881861814763397, "lr_blocks": 9.276821300802533e-07, "lr_fusion": 9.276821300802535e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 73.90760109238909, "velocity_mse_step1": 0.013814661781777558, "velocity_mse_step2": 0.0158143147688991, "velocity_mse_step3": 0.021045135479653254} +{"batch_in_epoch": 39, "epoch": 7, "global_step": 389, "grad_norm_max": 0.03125, "hidden_mse_step1": 0.019233894102009277, "hidden_mse_step2": 0.02207769338773297, "hidden_mse_step3": 0.026039692107588053, "latent_grad_l1_min": 0.00021160522373975255, "loss": 0.021964785252099057, "lr_blocks": 9.27264285519917e-07, "lr_fusion": 9.272642855199171e-06, "peak_memory_gib": 48.32135725021362, "trajectory_time_s": 71.30946256875905, "velocity_mse_step1": 0.014533095219771244, "velocity_mse_step2": 0.017565284057387286, "velocity_mse_step3": 0.027060846814752685} +{"batch_in_epoch": 40, "epoch": 7, "global_step": 390, "grad_norm_max": 0.034912109375, "hidden_mse_step1": 0.014112208113406917, "hidden_mse_step2": 0.016766792246406634, "hidden_mse_step3": 0.020612556878144717, "latent_grad_l1_min": 0.00027675605633703526, "loss": 0.019740435103553215, "lr_blocks": 9.268453319711362e-07, "lr_fusion": 9.268453319711362e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 72.55263808237214, "velocity_mse_step1": 0.01572742021484633, "velocity_mse_step2": 0.017008161423811022, "velocity_mse_step3": 0.021336565979124446} +{"batch_in_epoch": 41, "epoch": 7, "global_step": 391, "grad_norm_max": 0.0361328125, "hidden_mse_step1": 0.014902048652792082, "hidden_mse_step2": 0.01785949522828949, "hidden_mse_step3": 0.02144515956752002, "latent_grad_l1_min": 0.00020561143355735112, "loss": 0.019217673094577288, "lr_blocks": 9.26425270521328e-07, "lr_fusion": 9.264252705213281e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 73.72567700923537, "velocity_mse_step1": 0.014050410169667365, "velocity_mse_step2": 0.016535528991620856, "velocity_mse_step3": 0.021646407823677043} +{"batch_in_epoch": 42, "epoch": 7, "global_step": 392, "grad_norm_max": 0.0294189453125, "hidden_mse_step1": 0.01929113468421357, "hidden_mse_step2": 0.02184583357302472, "hidden_mse_step3": 0.025890728841269653, "latent_grad_l1_min": 0.00023444061662303284, "loss": 0.02056431527515607, "lr_blocks": 9.260041022607858e-07, "lr_fusion": 9.26004102260786e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 72.54743557648908, "velocity_mse_step1": 0.014035773652722128, "velocity_mse_step2": 0.016580521553156098, "velocity_mse_step3": 0.024373878566880842} +{"batch_in_epoch": 43, "epoch": 7, "global_step": 393, "grad_norm_max": 0.0294189453125, "hidden_mse_step1": 0.01387523732721872, "hidden_mse_step2": 0.016010530405245454, "hidden_mse_step3": 0.019426647132994336, "latent_grad_l1_min": 0.00018854820154956542, "loss": 0.01388715396127996, "lr_blocks": 9.255818282826755e-07, "lr_fusion": 9.255818282826755e-06, "peak_memory_gib": 48.31737470626831, "trajectory_time_s": 74.00943716576148, "velocity_mse_step1": 0.010003377142961003, "velocity_mse_step2": 0.010832609388411843, "velocity_mse_step3": 0.01589423305787412} +{"batch_in_epoch": 44, "epoch": 7, "global_step": 394, "grad_norm_max": 0.0301513671875, "hidden_mse_step1": 0.014293840762028205, "hidden_mse_step2": 0.016486150062909086, "hidden_mse_step3": 0.019743641023524106, "latent_grad_l1_min": 0.0001664732471908792, "loss": 0.014312953833723439, "lr_blocks": 9.251584496830326e-07, "lr_fusion": 9.251584496830329e-06, "peak_memory_gib": 48.31737470626831, "trajectory_time_s": 73.28002983688202, "velocity_mse_step1": 0.009328075986039557, "velocity_mse_step2": 0.011537212321335184, "velocity_mse_step3": 0.017021208851864294} +{"batch_in_epoch": 45, "epoch": 7, "global_step": 395, "grad_norm_max": 0.0390625, "hidden_mse_step1": 0.022399384561660032, "hidden_mse_step2": 0.02459197090606072, "hidden_mse_step3": 0.027662257497598014, "latent_grad_l1_min": 0.00022076769710110966, "loss": 0.023358565505727062, "lr_blocks": 9.247339675607605e-07, "lr_fusion": 9.247339675607606e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 72.83424095161172, "velocity_mse_step1": 0.016321342675447728, "velocity_mse_step2": 0.01916367816323015, "velocity_mse_step3": 0.02712531275548307} +{"batch_in_epoch": 46, "epoch": 7, "global_step": 396, "grad_norm_max": 0.0308837890625, "hidden_mse_step1": 0.01550569083857616, "hidden_mse_step2": 0.018100135244562162, "hidden_mse_step3": 0.021802780492829958, "latent_grad_l1_min": 0.00017384134207532043, "loss": 0.015825917225031714, "lr_blocks": 9.243083830176259e-07, "lr_fusion": 9.243083830176261e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 72.12025365562295, "velocity_mse_step1": 0.010500125460696706, "velocity_mse_step2": 0.01265598020420709, "velocity_mse_step3": 0.018780783949685943} +{"batch_in_epoch": 47, "epoch": 7, "global_step": 397, "grad_norm_max": 0.0242919921875, "hidden_mse_step1": 0.012547643867687187, "hidden_mse_step2": 0.015107361840948994, "hidden_mse_step3": 0.018795538918181722, "latent_grad_l1_min": 0.00017743440093909157, "loss": 0.013126053011676828, "lr_blocks": 9.238816971582577e-07, "lr_fusion": 9.238816971582578e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 71.51193283537577, "velocity_mse_step1": 0.008739600650317567, "velocity_mse_step2": 0.010463428005875486, "velocity_mse_step3": 0.015530074701278604} +{"batch_in_epoch": 48, "epoch": 7, "global_step": 398, "grad_norm_max": 0.032470703125, "hidden_mse_step1": 0.01337162621036571, "hidden_mse_step2": 0.0159973029928681, "hidden_mse_step3": 0.019248830803137804, "latent_grad_l1_min": 0.00018994109177583596, "loss": 0.014535268282218437, "lr_blocks": 9.23453911090143e-07, "lr_fusion": 9.23453911090143e-06, "peak_memory_gib": 48.320029735565186, "trajectory_time_s": 72.96255954637309, "velocity_mse_step1": 0.010119463302544316, "velocity_mse_step2": 0.011552382855110666, "velocity_mse_step3": 0.01707218163314142} +{"batch_in_epoch": 49, "epoch": 7, "global_step": 399, "grad_norm_max": 0.03076171875, "hidden_mse_step1": 0.02059969226164477, "hidden_mse_step2": 0.022183444766726877, "hidden_mse_step3": 0.024980321792619564, "latent_grad_l1_min": 0.00022059289767639711, "loss": 0.021876692347827232, "lr_blocks": 9.230250259236243e-07, "lr_fusion": 9.230250259236245e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 71.70352086262574, "velocity_mse_step1": 0.016413401221923, "velocity_mse_step2": 0.017837381580777998, "velocity_mse_step3": 0.024602946675648645} +{"batch_in_epoch": 50, "epoch": 7, "global_step": 400, "grad_norm_max": 0.035400390625, "hidden_mse_step1": 0.015898725529301116, "hidden_mse_step2": 0.018652611545154026, "hidden_mse_step3": 0.02174749476502516, "latent_grad_l1_min": 0.0002054925625998294, "loss": 0.018302162169545357, "lr_blocks": 9.225950427718974e-07, "lr_fusion": 9.225950427718974e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 73.80173354222643, "velocity_mse_step1": 0.013969824909249187, "velocity_mse_step2": 0.015326237393310292, "velocity_mse_step3": 0.019980539684183896} +{"batch_in_epoch": 1, "epoch": 8, "global_step": 401, "grad_norm_max": 0.0341796875, "hidden_mse_step1": 0.01712960553621607, "hidden_mse_step2": 0.019240655487270227, "hidden_mse_step3": 0.0221236204462392, "latent_grad_l1_min": 0.00018456191173754632, "loss": 0.019131011778621802, "lr_blocks": 9.221639627510075e-07, "lr_fusion": 9.221639627510076e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 75.70179285186896, "velocity_mse_step1": 0.013743339626151799, "velocity_mse_step2": 0.01592058395700795, "velocity_mse_step3": 0.021879722447400645} +{"batch_in_epoch": 2, "epoch": 8, "global_step": 402, "grad_norm_max": 0.0284423828125, "hidden_mse_step1": 0.017898282792884853, "hidden_mse_step2": 0.020295020259384598, "hidden_mse_step3": 0.023749925063124722, "latent_grad_l1_min": 0.00019775900000240654, "loss": 0.01896876235592312, "lr_blocks": 9.217317869798471e-07, "lr_fusion": 9.217317869798473e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 75.2664991436177, "velocity_mse_step1": 0.013349577663965254, "velocity_mse_step2": 0.015026952164979386, "velocity_mse_step3": 0.022335433201598268} +{"batch_in_epoch": 3, "epoch": 8, "global_step": 403, "grad_norm_max": 0.0341796875, "hidden_mse_step1": 0.013666272907617636, "hidden_mse_step2": 0.01673656982269936, "hidden_mse_step3": 0.02054085055299635, "latent_grad_l1_min": 0.00025215580353687983, "loss": 0.016726362498177744, "lr_blocks": 9.212985165801528e-07, "lr_fusion": 9.21298516580153e-06, "peak_memory_gib": 48.320029735565186, "trajectory_time_s": 73.60649376813672, "velocity_mse_step1": 0.011358303806835984, "velocity_mse_step2": 0.013890383542665015, "velocity_mse_step3": 0.01983602899209862} +{"batch_in_epoch": 4, "epoch": 8, "global_step": 404, "grad_norm_max": 0.036376953125, "hidden_mse_step1": 0.013318460971017233, "hidden_mse_step2": 0.01570644697806399, "hidden_mse_step3": 0.01813211293691503, "latent_grad_l1_min": 0.0002026047823164845, "loss": 0.015812767145689577, "lr_blocks": 9.208641526765022e-07, "lr_fusion": 9.208641526765024e-06, "peak_memory_gib": 48.32135725021362, "trajectory_time_s": 74.06598697911977, "velocity_mse_step1": 0.012799981139583645, "velocity_mse_step2": 0.013684748865281498, "velocity_mse_step3": 0.016237867746635205} +{"batch_in_epoch": 5, "epoch": 8, "global_step": 405, "grad_norm_max": 0.040283203125, "hidden_mse_step1": 0.017290453725893582, "hidden_mse_step2": 0.01974380714818835, "hidden_mse_step3": 0.02393178322485515, "latent_grad_l1_min": 0.00023829276142350864, "loss": 0.018759395187121948, "lr_blocks": 9.204286963963111e-07, "lr_fusion": 9.204286963963112e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 74.46364352175442, "velocity_mse_step1": 0.012547393969725817, "velocity_mse_step2": 0.015090031125250139, "velocity_mse_step3": 0.022544155080270554} +{"batch_in_epoch": 6, "epoch": 8, "global_step": 406, "grad_norm_max": 0.033203125, "hidden_mse_step1": 0.013132066790214074, "hidden_mse_step2": 0.01518024443482448, "hidden_mse_step3": 0.01830819122759359, "latent_grad_l1_min": 0.00015455893390026176, "loss": 0.013172944916212664, "lr_blocks": 9.199921488698307e-07, "lr_fusion": 9.199921488698307e-06, "peak_memory_gib": 48.31737470626831, "trajectory_time_s": 73.4547188421202, "velocity_mse_step1": 0.009020632702491378, "velocity_mse_step2": 0.010362954970332794, "velocity_mse_step3": 0.015473195688433146} +{"batch_in_epoch": 7, "epoch": 8, "global_step": 407, "grad_norm_max": 0.038818359375, "hidden_mse_step1": 0.01974995741121737, "hidden_mse_step2": 0.022288771728718917, "hidden_mse_step3": 0.025370868438455672, "latent_grad_l1_min": 0.00021185663536016364, "loss": 0.020972454471380582, "lr_blocks": 9.195545112301446e-07, "lr_fusion": 9.195545112301448e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 72.84285219724552, "velocity_mse_step1": 0.014088743220781904, "velocity_mse_step2": 0.017281525273574516, "velocity_mse_step3": 0.02480613353795239} +{"batch_in_epoch": 8, "epoch": 8, "global_step": 408, "grad_norm_max": 0.035400390625, "hidden_mse_step1": 0.015781945528163175, "hidden_mse_step2": 0.01871018569051687, "hidden_mse_step3": 0.02186236610370023, "latent_grad_l1_min": 0.00016571347623539623, "loss": 0.01689472431150664, "lr_blocks": 9.191157846131661e-07, "lr_fusion": 9.191157846131662e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 73.85220316537016, "velocity_mse_step1": 0.01231949304513234, "velocity_mse_step2": 0.013723693401386428, "velocity_mse_step3": 0.019005535137174383} +{"batch_in_epoch": 9, "epoch": 8, "global_step": 409, "grad_norm_max": 0.036865234375, "hidden_mse_step1": 0.01567085528846032, "hidden_mse_step2": 0.018049141292327216, "hidden_mse_step3": 0.021882548346184194, "latent_grad_l1_min": 0.00021322473821783205, "loss": 0.0158424357858686, "lr_blocks": 9.186759701576348e-07, "lr_fusion": 9.18675970157635e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 72.75518546013336, "velocity_mse_step1": 0.010061210252128408, "velocity_mse_step2": 0.012492460489738733, "velocity_mse_step3": 0.01941338075058801} +{"batch_in_epoch": 10, "epoch": 8, "global_step": 410, "grad_norm_max": 0.031982421875, "hidden_mse_step1": 0.012175722026898124, "hidden_mse_step2": 0.014702049499776746, "hidden_mse_step3": 0.017724612927330394, "latent_grad_l1_min": 0.00014563129752787063, "loss": 0.010784238601835179, "lr_blocks": 9.182350690051132e-07, "lr_fusion": 9.182350690051134e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 70.92419551224157, "velocity_mse_step1": 0.006297273260965344, "velocity_mse_step2": 0.008533465143825325, "velocity_mse_step3": 0.013061737866207426} +{"batch_in_epoch": 11, "epoch": 8, "global_step": 411, "grad_norm_max": 0.031005859375, "hidden_mse_step1": 0.018657013266679963, "hidden_mse_step2": 0.02092374426346006, "hidden_mse_step3": 0.023750856186130213, "latent_grad_l1_min": 0.0002291088376296102, "loss": 0.02009563673969491, "lr_blocks": 9.177930822999858e-07, "lr_fusion": 9.17793082299986e-06, "peak_memory_gib": 48.320029735565186, "trajectory_time_s": 73.47128472575423, "velocity_mse_step1": 0.015167644916800782, "velocity_mse_step2": 0.01638849745462981, "velocity_mse_step3": 0.022397604879058362} +{"batch_in_epoch": 12, "epoch": 8, "global_step": 412, "grad_norm_max": 0.03369140625, "hidden_mse_step1": 0.01641290023066436, "hidden_mse_step2": 0.018558528261824643, "hidden_mse_step3": 0.021250154249303575, "latent_grad_l1_min": 0.00016800280172901694, "loss": 0.016594505908350193, "lr_blocks": 9.173500111894535e-07, "lr_fusion": 9.173500111894536e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 72.97753591237415, "velocity_mse_step1": 0.011783644039886503, "velocity_mse_step2": 0.01335088544874452, "velocity_mse_step3": 0.01902682890483577} +{"batch_in_epoch": 13, "epoch": 8, "global_step": 413, "grad_norm_max": 0.030029296875, "hidden_mse_step1": 0.017726877072293843, "hidden_mse_step2": 0.02011230156391061, "hidden_mse_step3": 0.023994003771804273, "latent_grad_l1_min": 0.00022295160852081608, "loss": 0.019313447717909833, "lr_blocks": 9.169058568235323e-07, "lr_fusion": 9.169058568235324e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 73.4297329250112, "velocity_mse_step1": 0.01358285910516445, "velocity_mse_step2": 0.01514364151600083, "velocity_mse_step3": 0.023030522695210363} +{"batch_in_epoch": 14, "epoch": 8, "global_step": 414, "grad_norm_max": 0.03759765625, "hidden_mse_step1": 0.018096613165523325, "hidden_mse_step2": 0.02045657523974244, "hidden_mse_step3": 0.024021458313135163, "latent_grad_l1_min": 0.00023768282517266925, "loss": 0.01843108536143388, "lr_blocks": 9.164606203550497e-07, "lr_fusion": 9.164606203550498e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 74.00762985886831, "velocity_mse_step1": 0.012276848063005932, "velocity_mse_step2": 0.014616256496602933, "velocity_mse_step3": 0.02214268480227994} +{"batch_in_epoch": 15, "epoch": 8, "global_step": 415, "grad_norm_max": 0.033935546875, "hidden_mse_step1": 0.017512009491578544, "hidden_mse_step2": 0.019450642559344745, "hidden_mse_step3": 0.023239659877227883, "latent_grad_l1_min": 0.00023191152286017314, "loss": 0.020263938910959817, "lr_blocks": 9.160143029396421e-07, "lr_fusion": 9.160143029396422e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 74.52133327836782, "velocity_mse_step1": 0.01455387402009884, "velocity_mse_step2": 0.016642577996078343, "velocity_mse_step3": 0.02357513190612995} +{"batch_in_epoch": 16, "epoch": 8, "global_step": 416, "grad_norm_max": 0.032470703125, "hidden_mse_step1": 0.01580811252850773, "hidden_mse_step2": 0.01888382774944018, "hidden_mse_step3": 0.02242957005676414, "latent_grad_l1_min": 0.00021290157019393519, "loss": 0.01759827401838265, "lr_blocks": 9.155669057357513e-07, "lr_fusion": 9.155669057357515e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 73.46411207049823, "velocity_mse_step1": 0.0117225121440632, "velocity_mse_step2": 0.014218509546481073, "velocity_mse_step3": 0.02114164737784969} +{"batch_in_epoch": 17, "epoch": 8, "global_step": 417, "grad_norm_max": 0.02880859375, "hidden_mse_step1": 0.019674236649214954, "hidden_mse_step2": 0.0218003708370296, "hidden_mse_step3": 0.025260783657098985, "latent_grad_l1_min": 0.00020194833632558584, "loss": 0.020906594544482817, "lr_blocks": 9.15118429904622e-07, "lr_fusion": 9.151184299046221e-06, "peak_memory_gib": 48.31737470626831, "trajectory_time_s": 73.80393338136491, "velocity_mse_step1": 0.014271432422641998, "velocity_mse_step2": 0.016696215355685644, "velocity_mse_step3": 0.025078595333200483} +{"batch_in_epoch": 18, "epoch": 8, "global_step": 418, "grad_norm_max": 0.03466796875, "hidden_mse_step1": 0.020880710903189277, "hidden_mse_step2": 0.0233293543403436, "hidden_mse_step3": 0.026580889293524833, "latent_grad_l1_min": 0.00022903141598362708, "loss": 0.023038379224349877, "lr_blocks": 9.146688766102983e-07, "lr_fusion": 9.146688766102985e-06, "peak_memory_gib": 48.3169322013855, "trajectory_time_s": 73.09270475136873, "velocity_mse_step1": 0.01621532722908471, "velocity_mse_step2": 0.019309205389747924, "velocity_mse_step3": 0.026511507857191773} +{"batch_in_epoch": 19, "epoch": 8, "global_step": 419, "grad_norm_max": 0.038818359375, "hidden_mse_step1": 0.01976122082227708, "hidden_mse_step2": 0.02141709565018703, "hidden_mse_step3": 0.024560641480742818, "latent_grad_l1_min": 0.00020713225876534125, "loss": 0.01929137771367095, "lr_blocks": 9.142182470196212e-07, "lr_fusion": 9.142182470196213e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 72.55428618911537, "velocity_mse_step1": 0.01355977882173777, "velocity_mse_step2": 0.015046613870903716, "velocity_mse_step3": 0.022693842911394313} +{"batch_in_epoch": 20, "epoch": 8, "global_step": 420, "grad_norm_max": 0.0380859375, "hidden_mse_step1": 0.016626544147064642, "hidden_mse_step2": 0.01972076125509505, "hidden_mse_step3": 0.02350438544194081, "latent_grad_l1_min": 0.00023681080529058818, "loss": 0.02049964500058974, "lr_blocks": 9.13766542302225e-07, "lr_fusion": 9.13766542302225e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 72.62841477636539, "velocity_mse_step1": 0.014007786216097884, "velocity_mse_step2": 0.01721995701830435, "velocity_mse_step3": 0.024286021270589635} +{"batch_in_epoch": 21, "epoch": 8, "global_step": 421, "grad_norm_max": 0.028564453125, "hidden_mse_step1": 0.013993596982410443, "hidden_mse_step2": 0.016326814856646315, "hidden_mse_step3": 0.019723303973608247, "latent_grad_l1_min": 0.00019168797007296234, "loss": 0.015141791431233287, "lr_blocks": 9.133137636305344e-07, "lr_fusion": 9.133137636305345e-06, "peak_memory_gib": 48.31958723068237, "trajectory_time_s": 75.42989338636835, "velocity_mse_step1": 0.01099787610239998, "velocity_mse_step2": 0.012390975013009406, "velocity_mse_step3": 0.01703215044108219} +{"batch_in_epoch": 22, "epoch": 8, "global_step": 422, "grad_norm_max": 0.03515625, "hidden_mse_step1": 0.0138228691607115, "hidden_mse_step2": 0.016339110110753348, "hidden_mse_step3": 0.01963420636353216, "latent_grad_l1_min": 0.00026054377667605877, "loss": 0.0183223236194213, "lr_blocks": 9.12859912179762e-07, "lr_fusion": 9.12859912179762e-06, "peak_memory_gib": 48.320472240448, "trajectory_time_s": 74.26302061248862, "velocity_mse_step1": 0.014507902444165666, "velocity_mse_step2": 0.015871070755695525, "velocity_mse_step3": 0.01960837707156315} +{"batch_in_epoch": 23, "epoch": 8, "global_step": 423, "grad_norm_max": 0.038818359375, "hidden_mse_step1": 0.020437830410498594, "hidden_mse_step2": 0.022613195902002708, "hidden_mse_step3": 0.026193908631934652, "latent_grad_l1_min": 0.0002346827859582845, "loss": 0.021555694560187733, "lr_blocks": 9.12404989127905e-07, "lr_fusion": 9.124049891279052e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 71.63696898412309, "velocity_mse_step1": 0.014829151839616576, "velocity_mse_step2": 0.01782584888860583, "velocity_mse_step3": 0.025087586895097047} +{"batch_in_epoch": 24, "epoch": 8, "global_step": 424, "grad_norm_max": 0.03369140625, "hidden_mse_step1": 0.020622216464419446, "hidden_mse_step2": 0.0237433946791238, "hidden_mse_step3": 0.027208516839891672, "latent_grad_l1_min": 0.00034495357976993546, "loss": 0.028412176479053284, "lr_blocks": 9.119489956557414e-07, "lr_fusion": 9.119489956557416e-06, "peak_memory_gib": 48.320472240448, "trajectory_time_s": 72.23994302823849, "velocity_mse_step1": 0.022262541776789085, "velocity_mse_step2": 0.025220532835062058, "velocity_mse_step3": 0.030596039041743746} +{"batch_in_epoch": 25, "epoch": 8, "global_step": 425, "grad_norm_max": 0.031982421875, "hidden_mse_step1": 0.019356352060900205, "hidden_mse_step2": 0.0218181900480496, "hidden_mse_step3": 0.02504884262037064, "latent_grad_l1_min": 0.0002365291384194279, "loss": 0.021588548876544725, "lr_blocks": 9.114919329468282e-07, "lr_fusion": 9.114919329468283e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 75.42075007561652, "velocity_mse_step1": 0.014984682543269756, "velocity_mse_step2": 0.017945512081496418, "velocity_mse_step3": 0.02521311119198799} +{"batch_in_epoch": 26, "epoch": 8, "global_step": 426, "grad_norm_max": 0.040771484375, "hidden_mse_step1": 0.019408129820866243, "hidden_mse_step2": 0.02148267243423366, "hidden_mse_step3": 0.024761718215553886, "latent_grad_l1_min": 0.00019841984067170415, "loss": 0.02095859806286171, "lr_blocks": 9.110338021874969e-07, "lr_fusion": 9.11033802187497e-06, "peak_memory_gib": 48.320029735565186, "trajectory_time_s": 73.81926707961975, "velocity_mse_step1": 0.015998752457172874, "velocity_mse_step2": 0.01708125162986107, "velocity_mse_step3": 0.0232305362837256} +{"batch_in_epoch": 27, "epoch": 8, "global_step": 427, "grad_norm_max": 0.036376953125, "hidden_mse_step1": 0.015633818976181955, "hidden_mse_step2": 0.01823491123338629, "hidden_mse_step3": 0.02097403268063707, "latent_grad_l1_min": 0.0001515300027676858, "loss": 0.014648295265422866, "lr_blocks": 9.10574604566852e-07, "lr_fusion": 9.10574604566852e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 75.60250204850308, "velocity_mse_step1": 0.009592610846863994, "velocity_mse_step2": 0.011837354960984418, "velocity_mse_step3": 0.01703064238868787} +{"batch_in_epoch": 28, "epoch": 8, "global_step": 428, "grad_norm_max": 0.031982421875, "hidden_mse_step1": 0.017385485257753835, "hidden_mse_step2": 0.020549021054258834, "hidden_mse_step3": 0.024428056852359856, "latent_grad_l1_min": 0.0003273969123256393, "loss": 0.023641310781905695, "lr_blocks": 9.101143412767664e-07, "lr_fusion": 9.101143412767665e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 73.36030026099615, "velocity_mse_step1": 0.017388049259482483, "velocity_mse_step2": 0.020253129336716875, "velocity_mse_step3": 0.027046495965415876} +{"batch_in_epoch": 29, "epoch": 8, "global_step": 429, "grad_norm_max": 0.0322265625, "hidden_mse_step1": 0.016602649831578935, "hidden_mse_step2": 0.019904853667997355, "hidden_mse_step3": 0.023769271227398088, "latent_grad_l1_min": 0.00027945817328145495, "loss": 0.022652024386583692, "lr_blocks": 9.096530135118797e-07, "lr_fusion": 9.096530135118798e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 72.54221109662467, "velocity_mse_step1": 0.01735228950045504, "velocity_mse_step2": 0.019480871758007973, "velocity_mse_step3": 0.025095232907915488} +{"batch_in_epoch": 30, "epoch": 8, "global_step": 430, "grad_norm_max": 0.03662109375, "hidden_mse_step1": 0.017171542467882057, "hidden_mse_step2": 0.019827814165702354, "hidden_mse_step3": 0.023437471145630946, "latent_grad_l1_min": 0.00024077510897768661, "loss": 0.019627772142744755, "lr_blocks": 9.091906224695935e-07, "lr_fusion": 9.091906224695935e-06, "peak_memory_gib": 48.31737470626831, "trajectory_time_s": 72.87243660237436, "velocity_mse_step1": 0.013931259394407139, "velocity_mse_step2": 0.016054873271579187, "velocity_mse_step3": 0.022853499968602715} +{"batch_in_epoch": 31, "epoch": 8, "global_step": 431, "grad_norm_max": 0.045166015625, "hidden_mse_step1": 0.01742094862441133, "hidden_mse_step2": 0.02018592978960701, "hidden_mse_step3": 0.02344655568179275, "latent_grad_l1_min": 0.0002610669271234656, "loss": 0.02063343798675175, "lr_blocks": 9.0872716935007e-07, "lr_fusion": 9.087271693500701e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 74.19852633436676, "velocity_mse_step1": 0.014691775556587213, "velocity_mse_step2": 0.01738881001282217, "velocity_mse_step3": 0.02371438343210944} +{"batch_in_epoch": 32, "epoch": 8, "global_step": 432, "grad_norm_max": 0.032470703125, "hidden_mse_step1": 0.019872894809980477, "hidden_mse_step2": 0.022203768852965107, "hidden_mse_step3": 0.026389548205770552, "latent_grad_l1_min": 0.0003352677704242524, "loss": 0.02330926727570061, "lr_blocks": 9.08262655356228e-07, "lr_fusion": 9.08262655356228e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 72.86129483136756, "velocity_mse_step1": 0.016136467159542787, "velocity_mse_step2": 0.019418278372281092, "velocity_mse_step3": 0.02752643408686188} +{"batch_in_epoch": 33, "epoch": 8, "global_step": 433, "grad_norm_max": 0.03515625, "hidden_mse_step1": 0.016011348863165558, "hidden_mse_step2": 0.017967025075839565, "hidden_mse_step3": 0.021264023280569484, "latent_grad_l1_min": 0.00018927031669591088, "loss": 0.016263758834351653, "lr_blocks": 9.077970816937393e-07, "lr_fusion": 9.077970816937394e-06, "peak_memory_gib": 48.31958723068237, "trajectory_time_s": 71.33532358261436, "velocity_mse_step1": 0.011279637813069192, "velocity_mse_step2": 0.013160474598407744, "velocity_mse_step3": 0.018826922784293335} +{"batch_in_epoch": 34, "epoch": 8, "global_step": 434, "grad_norm_max": 0.04638671875, "hidden_mse_step1": 0.022395218564530035, "hidden_mse_step2": 0.02441522045826007, "hidden_mse_step3": 0.027715046623987806, "latent_grad_l1_min": 0.0003157191640639212, "loss": 0.02650909773988782, "lr_blocks": 9.073304495710265e-07, "lr_fusion": 9.073304495710267e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 72.82490939125273, "velocity_mse_step1": 0.019340975760964545, "velocity_mse_step2": 0.02242885393206961, "velocity_mse_step3": 0.030304912028701177} +{"batch_in_epoch": 35, "epoch": 8, "global_step": 435, "grad_norm_max": 0.0458984375, "hidden_mse_step1": 0.015448909348508871, "hidden_mse_step2": 0.017491756738828763, "hidden_mse_step3": 0.02051056420896202, "latent_grad_l1_min": 0.00018924084542959463, "loss": 0.015198796147680174, "lr_blocks": 9.068627601992598e-07, "lr_fusion": 9.0686276019926e-06, "peak_memory_gib": 48.31737470626831, "trajectory_time_s": 71.89028791961755, "velocity_mse_step1": 0.010316922956137984, "velocity_mse_step2": 0.012222516695536405, "velocity_mse_step3": 0.017711824496343197} +{"batch_in_epoch": 36, "epoch": 8, "global_step": 436, "grad_norm_max": 0.0296630859375, "hidden_mse_step1": 0.025349182741982595, "hidden_mse_step2": 0.02693585815307285, "hidden_mse_step3": 0.029634191288745823, "latent_grad_l1_min": 0.000248918506258633, "loss": 0.02887965219893626, "lr_blocks": 9.063940147923527e-07, "lr_fusion": 9.063940147923529e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 71.62091820112255, "velocity_mse_step1": 0.021890744383979054, "velocity_mse_step2": 0.024024470235287608, "velocity_mse_step3": 0.032531815547762175} +{"batch_in_epoch": 37, "epoch": 8, "global_step": 437, "grad_norm_max": 0.040771484375, "hidden_mse_step1": 0.0180769464973959, "hidden_mse_step2": 0.02021295035956427, "hidden_mse_step3": 0.023471758268507465, "latent_grad_l1_min": 0.00022577160052605905, "loss": 0.019696284559488828, "lr_blocks": 9.059242145669609e-07, "lr_fusion": 9.05924214566961e-06, "peak_memory_gib": 48.31737470626831, "trajectory_time_s": 71.83344286213105, "velocity_mse_step1": 0.014547794165472232, "velocity_mse_step2": 0.015824207316784716, "velocity_mse_step3": 0.0225406851552959} +{"batch_in_epoch": 38, "epoch": 8, "global_step": 438, "grad_norm_max": 0.035888671875, "hidden_mse_step1": 0.015994732467723743, "hidden_mse_step2": 0.018693044149715985, "hidden_mse_step3": 0.022592620774438337, "latent_grad_l1_min": 0.00025135016039712355, "loss": 0.017436613054347357, "lr_blocks": 9.054533607424769e-07, "lr_fusion": 9.05453360742477e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 71.30629464812228, "velocity_mse_step1": 0.011660219563054852, "velocity_mse_step2": 0.013763206925692169, "velocity_mse_step3": 0.021158371752660186} +{"batch_in_epoch": 39, "epoch": 8, "global_step": 439, "grad_norm_max": 0.052490234375, "hidden_mse_step1": 0.015814816354707415, "hidden_mse_step2": 0.018126795026806315, "hidden_mse_step3": 0.022303836029355546, "latent_grad_l1_min": 0.000198374370484089, "loss": 0.017404550254078846, "lr_blocks": 9.049814545410281e-07, "lr_fusion": 9.049814545410282e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 73.59356215524895, "velocity_mse_step1": 0.011892221064564574, "velocity_mse_step2": 0.01357168925044659, "velocity_mse_step3": 0.021125194338050535} +{"batch_in_epoch": 40, "epoch": 8, "global_step": 440, "grad_norm_max": 0.0341796875, "hidden_mse_step1": 0.017232389031310698, "hidden_mse_step2": 0.01980131600118641, "hidden_mse_step3": 0.023635471472516652, "latent_grad_l1_min": 0.00021017427570768632, "loss": 0.019000263793194395, "lr_blocks": 9.045084971874737e-07, "lr_fusion": 9.045084971874738e-06, "peak_memory_gib": 48.31958723068237, "trajectory_time_s": 71.42827605900675, "velocity_mse_step1": 0.013262234191643072, "velocity_mse_step2": 0.015403157659290755, "velocity_mse_step3": 0.02226847949870197} +{"batch_in_epoch": 41, "epoch": 8, "global_step": 441, "grad_norm_max": 0.038818359375, "hidden_mse_step1": 0.022366872223626287, "hidden_mse_step2": 0.025120022149556984, "hidden_mse_step3": 0.027675905813729123, "latent_grad_l1_min": 0.00024107210265356116, "loss": 0.025748333336585865, "lr_blocks": 9.04034489909401e-07, "lr_fusion": 9.04034489909401e-06, "peak_memory_gib": 48.31737470626831, "trajectory_time_s": 70.7527475481329, "velocity_mse_step1": 0.018233078319878717, "velocity_mse_step2": 0.021924458943041306, "velocity_mse_step3": 0.029571180963622674} +{"batch_in_epoch": 42, "epoch": 8, "global_step": 442, "grad_norm_max": 0.05810546875, "hidden_mse_step1": 0.018559874600863883, "hidden_mse_step2": 0.02065760612354747, "hidden_mse_step3": 0.023417270709095255, "latent_grad_l1_min": 0.00019218258603359573, "loss": 0.018233139608387967, "lr_blocks": 9.035594339371228e-07, "lr_fusion": 9.035594339371229e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 71.36284359099955, "velocity_mse_step1": 0.013265665649669243, "velocity_mse_step2": 0.014598022547683545, "velocity_mse_step3": 0.020572253951935893} +{"batch_in_epoch": 43, "epoch": 8, "global_step": 443, "grad_norm_max": 0.033447265625, "hidden_mse_step1": 0.01624419049559427, "hidden_mse_step2": 0.018096251637741388, "hidden_mse_step3": 0.02163062449212053, "latent_grad_l1_min": 0.00019045263252337463, "loss": 0.01803855983806508, "lr_blocks": 9.030833305036732e-07, "lr_fusion": 9.030833305036732e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 71.31857309438055, "velocity_mse_step1": 0.014738856307563504, "velocity_mse_step2": 0.014610529964556918, "velocity_mse_step3": 0.01916918551017131} +{"batch_in_epoch": 44, "epoch": 8, "global_step": 444, "grad_norm_max": 0.0308837890625, "hidden_mse_step1": 0.02398517402720505, "hidden_mse_step2": 0.025435333888578625, "hidden_mse_step3": 0.028057886719969765, "latent_grad_l1_min": 0.00020238499200786464, "loss": 0.024480181671346406, "lr_blocks": 9.026061808448055e-07, "lr_fusion": 9.026061808448055e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 73.23078163362516, "velocity_mse_step1": 0.018701984085575014, "velocity_mse_step2": 0.019988647861672298, "velocity_mse_step3": 0.02700207129653011} +{"batch_in_epoch": 45, "epoch": 8, "global_step": 445, "grad_norm_max": 0.02783203125, "hidden_mse_step1": 0.016093935434972596, "hidden_mse_step2": 0.0180891410314611, "hidden_mse_step3": 0.021243711218370924, "latent_grad_l1_min": 0.00018706760965869762, "loss": 0.015619567179653262, "lr_blocks": 9.021279861989884e-07, "lr_fusion": 9.021279861989884e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 73.61545252086944, "velocity_mse_step1": 0.01045149947250528, "velocity_mse_step2": 0.012455818652857227, "velocity_mse_step3": 0.018408702802844346} +{"batch_in_epoch": 46, "epoch": 8, "global_step": 446, "grad_norm_max": 0.03515625, "hidden_mse_step1": 0.013991667575152989, "hidden_mse_step2": 0.01629765863929476, "hidden_mse_step3": 0.020129446904840215, "latent_grad_l1_min": 0.00019661190708575305, "loss": 0.01549817566202754, "lr_blocks": 9.016487478074031e-07, "lr_fusion": 9.016487478074032e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 72.4702666237572, "velocity_mse_step1": 0.010968961239996133, "velocity_mse_step2": 0.012229515210492535, "velocity_mse_step3": 0.01825417181160966} +{"batch_in_epoch": 47, "epoch": 8, "global_step": 447, "grad_norm_max": 0.04052734375, "hidden_mse_step1": 0.023088723899231157, "hidden_mse_step2": 0.026187375853104253, "hidden_mse_step3": 0.03009601845405996, "latent_grad_l1_min": 0.0003660525235318346, "loss": 0.02877265264812325, "lr_blocks": 9.011684669139397e-07, "lr_fusion": 9.011684669139398e-06, "peak_memory_gib": 48.316489696502686, "trajectory_time_s": 74.57124800150632, "velocity_mse_step1": 0.020217621067006673, "velocity_mse_step2": 0.0244654247903132, "velocity_mse_step3": 0.03369769808237574} +{"batch_in_epoch": 48, "epoch": 8, "global_step": 448, "grad_norm_max": 0.0302734375, "hidden_mse_step1": 0.02001695436358984, "hidden_mse_step2": 0.021595770127273033, "hidden_mse_step3": 0.024388588948308358, "latent_grad_l1_min": 0.00022155819351610262, "loss": 0.02065828946485583, "lr_blocks": 9.006871447651941e-07, "lr_fusion": 9.006871447651941e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 71.05564885075, "velocity_mse_step1": 0.016119502730102143, "velocity_mse_step2": 0.016408381767437925, "velocity_mse_step3": 0.022846850965704234} +{"batch_in_epoch": 49, "epoch": 8, "global_step": 449, "grad_norm_max": 0.02880859375, "hidden_mse_step1": 0.01309909820390333, "hidden_mse_step2": 0.016036585389104272, "hidden_mse_step3": 0.0200267817958125, "latent_grad_l1_min": 0.0001932812410814222, "loss": 0.015189448255114257, "lr_blocks": 9.002047826104651e-07, "lr_fusion": 9.002047826104652e-06, "peak_memory_gib": 48.320029735565186, "trajectory_time_s": 72.22543895775016, "velocity_mse_step1": 0.011073334743351941, "velocity_mse_step2": 0.012054587714374065, "velocity_mse_step3": 0.017524174081959894} +{"batch_in_epoch": 50, "epoch": 8, "global_step": 450, "grad_norm_max": 0.0341796875, "hidden_mse_step1": 0.02026515281509741, "hidden_mse_step2": 0.023005022542617683, "hidden_mse_step3": 0.026447100348637568, "latent_grad_l1_min": 0.00024629505787743255, "loss": 0.02472843386099807, "lr_blocks": 8.997213817017506e-07, "lr_fusion": 8.997213817017508e-06, "peak_memory_gib": 48.316489696502686, "trajectory_time_s": 75.42385096826183, "velocity_mse_step1": 0.0179947733226332, "velocity_mse_step2": 0.021028330952893675, "velocity_mse_step3": 0.028190467057616582} +{"batch_in_epoch": 1, "epoch": 9, "global_step": 451, "grad_norm_max": 0.04736328125, "hidden_mse_step1": 0.017727442759288742, "hidden_mse_step2": 0.020100194969147975, "hidden_mse_step3": 0.022880024276673794, "latent_grad_l1_min": 0.00022410247584048193, "loss": 0.019780850407966812, "lr_blocks": 8.99236943293745e-07, "lr_fusion": 8.992369432937451e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 74.2420138905145, "velocity_mse_step1": 0.015273053709409264, "velocity_mse_step2": 0.01639154512252259, "velocity_mse_step3": 0.021607184919828014} +{"batch_in_epoch": 2, "epoch": 9, "global_step": 452, "grad_norm_max": 0.0225830078125, "hidden_mse_step1": 0.015969308795839816, "hidden_mse_step2": 0.018128855769256394, "hidden_mse_step3": 0.021119413847502853, "latent_grad_l1_min": 0.00017083790407923516, "loss": 0.01648912108586436, "lr_blocks": 8.987514686438353e-07, "lr_fusion": 8.987514686438353e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 74.15545531536918, "velocity_mse_step1": 0.01159032482973998, "velocity_mse_step2": 0.013403244705321933, "velocity_mse_step3": 0.018952034135249308} +{"batch_in_epoch": 3, "epoch": 9, "global_step": 453, "grad_norm_max": 0.03173828125, "hidden_mse_step1": 0.014157280540010626, "hidden_mse_step2": 0.01647986044658215, "hidden_mse_step3": 0.019472404416384443, "latent_grad_l1_min": 0.00016035561930038966, "loss": 0.015028735013662038, "lr_blocks": 8.982649590120981e-07, "lr_fusion": 8.982649590120982e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 72.78201742986857, "velocity_mse_step1": 0.009909595115459524, "velocity_mse_step2": 0.012415067650311227, "velocity_mse_step3": 0.017750586541036943} +{"batch_in_epoch": 4, "epoch": 9, "global_step": 454, "grad_norm_max": 0.034423828125, "hidden_mse_step1": 0.01524797317272584, "hidden_mse_step2": 0.017225934982499372, "hidden_mse_step3": 0.02037986974963652, "latent_grad_l1_min": 0.00018575330432213377, "loss": 0.016583618795266375, "lr_blocks": 8.977774156612968e-07, "lr_fusion": 8.977774156612968e-06, "peak_memory_gib": 48.31737470626831, "trajectory_time_s": 70.45499144562928, "velocity_mse_step1": 0.012307662352603594, "velocity_mse_step2": 0.013449544883249993, "velocity_mse_step3": 0.01870827060026516} +{"batch_in_epoch": 5, "epoch": 9, "global_step": 455, "grad_norm_max": 0.029052734375, "hidden_mse_step1": 0.011995274814710553, "hidden_mse_step2": 0.014940389365489993, "hidden_mse_step3": 0.018610275516818674, "latent_grad_l1_min": 0.0001780828279152047, "loss": 0.011984517020339678, "lr_blocks": 8.972888398568771e-07, "lr_fusion": 8.972888398568772e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 72.81529097686871, "velocity_mse_step1": 0.0072228700623132415, "velocity_mse_step2": 0.009455960844726568, "velocity_mse_step3": 0.014720125207012253} +{"batch_in_epoch": 6, "epoch": 9, "global_step": 456, "grad_norm_max": 0.032470703125, "hidden_mse_step1": 0.02016624929716012, "hidden_mse_step2": 0.02128027337935886, "hidden_mse_step3": 0.023536320027363083, "latent_grad_l1_min": 0.00019587741189752705, "loss": 0.021519708521996757, "lr_blocks": 8.967992328669653e-07, "lr_fusion": 8.967992328669655e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 71.83908892713225, "velocity_mse_step1": 0.017859603253392767, "velocity_mse_step2": 0.017717208255947164, "velocity_mse_step3": 0.022484028167257617} +{"batch_in_epoch": 7, "epoch": 9, "global_step": 457, "grad_norm_max": 0.0255126953125, "hidden_mse_step1": 0.017476426271189532, "hidden_mse_step2": 0.019828685842055292, "hidden_mse_step3": 0.023255115075569065, "latent_grad_l1_min": 0.00021819411449541803, "loss": 0.018256527113927792, "lr_blocks": 8.963085959623636e-07, "lr_fusion": 8.963085959623638e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 74.3364295173742, "velocity_mse_step1": 0.012497913865705153, "velocity_mse_step2": 0.014696996804559603, "velocity_mse_step3": 0.021518646323654262} +{"batch_in_epoch": 8, "epoch": 9, "global_step": 458, "grad_norm_max": 0.023681640625, "hidden_mse_step1": 0.013337039779539085, "hidden_mse_step2": 0.016248161746521612, "hidden_mse_step3": 0.020139325210558515, "latent_grad_l1_min": 0.00017996933183894726, "loss": 0.014914838572232319, "lr_blocks": 8.958169304165478e-07, "lr_fusion": 8.95816930416548e-06, "peak_memory_gib": 48.31737470626831, "trajectory_time_s": 74.08030234162288, "velocity_mse_step1": 0.009912229969423996, "velocity_mse_step2": 0.01206507473917944, "velocity_mse_step3": 0.01779475705032902} +{"batch_in_epoch": 9, "epoch": 9, "global_step": 459, "grad_norm_max": 0.0228271484375, "hidden_mse_step1": 0.01848619314841926, "hidden_mse_step2": 0.021292171798579923, "hidden_mse_step3": 0.0249421875923872, "latent_grad_l1_min": 0.00023043453074933495, "loss": 0.021179913252126426, "lr_blocks": 8.953242375056633e-07, "lr_fusion": 8.953242375056634e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 70.55540925263631, "velocity_mse_step1": 0.014789019411961946, "velocity_mse_step2": 0.017703033518046137, "velocity_mse_step3": 0.024575628872428618} +{"batch_in_epoch": 10, "epoch": 9, "global_step": 460, "grad_norm_max": 0.0311279296875, "hidden_mse_step1": 0.014669007043786611, "hidden_mse_step2": 0.017048542919967855, "hidden_mse_step3": 0.020926085117805214, "latent_grad_l1_min": 0.00019519493980624247, "loss": 0.014855276073961117, "lr_blocks": 8.948305185085224e-07, "lr_fusion": 8.948305185085226e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 71.70506626400311, "velocity_mse_step1": 0.009373282949257242, "velocity_mse_step2": 0.011466650558369498, "velocity_mse_step3": 0.018461529897259812} +{"batch_in_epoch": 11, "epoch": 9, "global_step": 461, "grad_norm_max": 0.030029296875, "hidden_mse_step1": 0.018900005496107042, "hidden_mse_step2": 0.021874629701155102, "hidden_mse_step3": 0.025671530681263124, "latent_grad_l1_min": 0.00033605817770876456, "loss": 0.024896916579954062, "lr_blocks": 8.943357747066003e-07, "lr_fusion": 8.943357747066004e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 71.42691191688937, "velocity_mse_step1": 0.018215308950207794, "velocity_mse_step2": 0.0214266795540295, "velocity_mse_step3": 0.028404142441494123} +{"batch_in_epoch": 12, "epoch": 9, "global_step": 462, "grad_norm_max": 0.02783203125, "hidden_mse_step1": 0.017423450826234848, "hidden_mse_step2": 0.019284328670307462, "hidden_mse_step3": 0.02248358496996973, "latent_grad_l1_min": 0.00020008458523079753, "loss": 0.019969226401631852, "lr_blocks": 8.938400073840324e-07, "lr_fusion": 8.938400073840325e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 74.25558203460969, "velocity_mse_step1": 0.015620141530234832, "velocity_mse_step2": 0.016501198564323465, "velocity_mse_step3": 0.021867201480615352} +{"batch_in_epoch": 13, "epoch": 9, "global_step": 463, "grad_norm_max": 0.031494140625, "hidden_mse_step1": 0.019625275603695105, "hidden_mse_step2": 0.021976532830324556, "hidden_mse_step3": 0.02533942663909069, "latent_grad_l1_min": 0.00022305696984403767, "loss": 0.020898540878468853, "lr_blocks": 8.933432178276105e-07, "lr_fusion": 8.933432178276108e-06, "peak_memory_gib": 48.31737470626831, "trajectory_time_s": 74.20215622399701, "velocity_mse_step1": 0.014319793366926853, "velocity_mse_step2": 0.017093426090598638, "velocity_mse_step3": 0.02458827839499073} +{"batch_in_epoch": 14, "epoch": 9, "global_step": 464, "grad_norm_max": 0.0301513671875, "hidden_mse_step1": 0.015408209458525692, "hidden_mse_step2": 0.01890855112911335, "hidden_mse_step3": 0.02363247056824288, "latent_grad_l1_min": 0.00023602247074450133, "loss": 0.018623371101317125, "lr_blocks": 8.9284540732678e-07, "lr_fusion": 8.928454073267801e-06, "peak_memory_gib": 48.3169322013855, "trajectory_time_s": 72.72084602862014, "velocity_mse_step1": 0.011701525619303408, "velocity_mse_step2": 0.014775805287562043, "velocity_mse_step3": 0.02359785758224981} +{"batch_in_epoch": 15, "epoch": 9, "global_step": 465, "grad_norm_max": 0.030029296875, "hidden_mse_step1": 0.020566765792734386, "hidden_mse_step2": 0.023389450257777104, "hidden_mse_step3": 0.027123414146314775, "latent_grad_l1_min": 0.00035538335214369, "loss": 0.02833518212095701, "lr_blocks": 8.923465771736359e-07, "lr_fusion": 8.923465771736361e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 74.70558109800913, "velocity_mse_step1": 0.0224977362543411, "velocity_mse_step2": 0.024635707967848117, "velocity_mse_step3": 0.03076413598111165} +{"batch_in_epoch": 16, "epoch": 9, "global_step": 466, "grad_norm_max": 0.0296630859375, "hidden_mse_step1": 0.012018879393248685, "hidden_mse_step2": 0.014944462762546858, "hidden_mse_step3": 0.01855109345966152, "latent_grad_l1_min": 0.0001925809665408451, "loss": 0.01289462757995352, "lr_blocks": 8.918467286629198e-07, "lr_fusion": 8.9184672866292e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 71.4173785842504, "velocity_mse_step1": 0.007820293943430963, "velocity_mse_step2": 0.010297669921003814, "velocity_mse_step3": 0.016014474103160734} +{"batch_in_epoch": 17, "epoch": 9, "global_step": 467, "grad_norm_max": 0.028076171875, "hidden_mse_step1": 0.01674355870012992, "hidden_mse_step2": 0.01937381376046687, "hidden_mse_step3": 0.022480949360345087, "latent_grad_l1_min": 0.00021241535614535678, "loss": 0.018806925567332655, "lr_blocks": 8.913458630920168e-07, "lr_fusion": 8.913458630920169e-06, "peak_memory_gib": 48.31737470626831, "trajectory_time_s": 72.71969852461189, "velocity_mse_step1": 0.013638481774250977, "velocity_mse_step2": 0.015390320369208763, "velocity_mse_step3": 0.021532140613999214} +{"batch_in_epoch": 18, "epoch": 9, "global_step": 468, "grad_norm_max": 0.0281982421875, "hidden_mse_step1": 0.021973101322406104, "hidden_mse_step2": 0.02376662301165717, "hidden_mse_step3": 0.027034198200064048, "latent_grad_l1_min": 0.00026765411166707054, "loss": 0.02324013663538997, "lr_blocks": 8.908439817609513e-07, "lr_fusion": 8.908439817609514e-06, "peak_memory_gib": 48.31958723068237, "trajectory_time_s": 71.38922606475535, "velocity_mse_step1": 0.016474272295973993, "velocity_mse_step2": 0.01890746986360422, "velocity_mse_step3": 0.027061273792891628} +{"batch_in_epoch": 19, "epoch": 9, "global_step": 469, "grad_norm_max": 0.040771484375, "hidden_mse_step1": 0.016641837835777547, "hidden_mse_step2": 0.0182951825803944, "hidden_mse_step3": 0.020886640147572116, "latent_grad_l1_min": 0.00016086511914181756, "loss": 0.01510118512669578, "lr_blocks": 8.903410859723846e-07, "lr_fusion": 8.903410859723848e-06, "peak_memory_gib": 48.31737470626831, "trajectory_time_s": 72.95094952100044, "velocity_mse_step1": 0.010401331534792136, "velocity_mse_step2": 0.012275478083340983, "velocity_mse_step3": 0.017044378644121543} +{"batch_in_epoch": 20, "epoch": 9, "global_step": 470, "grad_norm_max": 0.0262451171875, "hidden_mse_step1": 0.016582668043806082, "hidden_mse_step2": 0.018877429330521927, "hidden_mse_step3": 0.022565238111253296, "latent_grad_l1_min": 0.00017670331999397604, "loss": 0.016023185877462046, "lr_blocks": 8.898371770316111e-07, "lr_fusion": 8.898371770316113e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 72.11726379511674, "velocity_mse_step1": 0.010541927061110203, "velocity_mse_step2": 0.012705902955661128, "velocity_mse_step3": 0.019019192550331354} +{"batch_in_epoch": 21, "epoch": 9, "global_step": 471, "grad_norm_max": 0.029296875, "hidden_mse_step1": 0.012067387836785721, "hidden_mse_step2": 0.015050922237735772, "hidden_mse_step3": 0.018394522901092256, "latent_grad_l1_min": 0.00016589267215749715, "loss": 0.011852295611918503, "lr_blocks": 8.893322562465546e-07, "lr_fusion": 8.893322562465547e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 73.73448510099115, "velocity_mse_step1": 0.007793925567446941, "velocity_mse_step2": 0.009157760779739222, "velocity_mse_step3": 0.014053916117908164} +{"batch_in_epoch": 22, "epoch": 9, "global_step": 472, "grad_norm_max": 0.03076171875, "hidden_mse_step1": 0.019988736187640046, "hidden_mse_step2": 0.022636287014133165, "hidden_mse_step3": 0.026048913598060608, "latent_grad_l1_min": 0.00021766547070001252, "loss": 0.023898817033373886, "lr_blocks": 8.888263249277655e-07, "lr_fusion": 8.888263249277656e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 71.34039418587054, "velocity_mse_step1": 0.0178232402249705, "velocity_mse_step2": 0.019697487237863243, "velocity_mse_step3": 0.027308328276766197} +{"batch_in_epoch": 23, "epoch": 9, "global_step": 473, "grad_norm_max": 0.0257568359375, "hidden_mse_step1": 0.018020885905051336, "hidden_mse_step2": 0.02022848725651524, "hidden_mse_step3": 0.023814374032164257, "latent_grad_l1_min": 0.00023427293126587756, "loss": 0.01925789251358115, "lr_blocks": 8.883193843884168e-07, "lr_fusion": 8.883193843884169e-06, "peak_memory_gib": 48.31737470626831, "trajectory_time_s": 73.50451285425515, "velocity_mse_step1": 0.013675405810187967, "velocity_mse_step2": 0.015309572876763662, "velocity_mse_step3": 0.022582323223884614} +{"batch_in_epoch": 24, "epoch": 9, "global_step": 474, "grad_norm_max": 0.02880859375, "hidden_mse_step1": 0.016479245312179303, "hidden_mse_step2": 0.019346196919546595, "hidden_mse_step3": 0.02181822232835527, "latent_grad_l1_min": 0.0001493986874265829, "loss": 0.015275263302360793, "lr_blocks": 8.878114359443012e-07, "lr_fusion": 8.878114359443012e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 72.98563263237156, "velocity_mse_step1": 0.010272405390943666, "velocity_mse_step2": 0.012263536170524147, "velocity_mse_step3": 0.017525480478070676} +{"batch_in_epoch": 25, "epoch": 9, "global_step": 475, "grad_norm_max": 0.03125, "hidden_mse_step1": 0.02422025960238118, "hidden_mse_step2": 0.026233545138633674, "hidden_mse_step3": 0.03007039121751274, "latent_grad_l1_min": 0.0002911748888436705, "loss": 0.027572881729741178, "lr_blocks": 8.873024809138272e-07, "lr_fusion": 8.873024809138272e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 71.38143814712384, "velocity_mse_step1": 0.02010942640897286, "velocity_mse_step2": 0.022750080272089687, "velocity_mse_step3": 0.03180671632955117} +{"batch_in_epoch": 26, "epoch": 9, "global_step": 476, "grad_norm_max": 0.0361328125, "hidden_mse_step1": 0.015426212851057894, "hidden_mse_step2": 0.01771333794541923, "hidden_mse_step3": 0.021535749746752635, "latent_grad_l1_min": 0.0002476711861163494, "loss": 0.020019981935287687, "lr_blocks": 8.867925206180164e-07, "lr_fusion": 8.867925206180166e-06, "peak_memory_gib": 48.320029735565186, "trajectory_time_s": 73.66030476788728, "velocity_mse_step1": 0.015372205026713864, "velocity_mse_step2": 0.01680145487106139, "velocity_mse_step3": 0.02241875463265127} +{"batch_in_epoch": 27, "epoch": 9, "global_step": 477, "grad_norm_max": 0.0269775390625, "hidden_mse_step1": 0.01899934002514263, "hidden_mse_step2": 0.021308101223049946, "hidden_mse_step3": 0.024667854171379334, "latent_grad_l1_min": 0.00021990896129864268, "loss": 0.020316525333328173, "lr_blocks": 8.862815563804995e-07, "lr_fusion": 8.862815563804996e-06, "peak_memory_gib": 48.320029735565186, "trajectory_time_s": 72.48107149750285, "velocity_mse_step1": 0.013686711238863478, "velocity_mse_step2": 0.016769125564938543, "velocity_mse_step3": 0.02399620717291587} +{"batch_in_epoch": 28, "epoch": 9, "global_step": 478, "grad_norm_max": 0.026123046875, "hidden_mse_step1": 0.01667888070889083, "hidden_mse_step2": 0.018167023056386306, "hidden_mse_step3": 0.021183324221055955, "latent_grad_l1_min": 0.00019396631068957504, "loss": 0.0162546299564253, "lr_blocks": 8.857695895275126e-07, "lr_fusion": 8.857695895275127e-06, "peak_memory_gib": 48.31958723068237, "trajectory_time_s": 71.53448989800381, "velocity_mse_step1": 0.012379581991159024, "velocity_mse_step2": 0.012645721828448586, "velocity_mse_step3": 0.018135661359078115} +{"batch_in_epoch": 29, "epoch": 9, "global_step": 479, "grad_norm_max": 0.0400390625, "hidden_mse_step1": 0.016897669111910676, "hidden_mse_step2": 0.019613184442277998, "hidden_mse_step3": 0.023385805410466025, "latent_grad_l1_min": 0.0002159972118533915, "loss": 0.018318285126172534, "lr_blocks": 8.852566213878946e-07, "lr_fusion": 8.852566213878947e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 74.37999076786946, "velocity_mse_step1": 0.012390652835685093, "velocity_mse_step2": 0.014855209417873994, "velocity_mse_step3": 0.021719325932541063} +{"batch_in_epoch": 30, "epoch": 9, "global_step": 480, "grad_norm_max": 0.03125, "hidden_mse_step1": 0.01872219139893007, "hidden_mse_step2": 0.021395630477595004, "hidden_mse_step3": 0.024334363123801137, "latent_grad_l1_min": 0.00018996010294358712, "loss": 0.019802671819758998, "lr_blocks": 8.847426532930829e-07, "lr_fusion": 8.84742653293083e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 73.85247678648739, "velocity_mse_step1": 0.014270553595062144, "velocity_mse_step2": 0.016392270080229667, "velocity_mse_step3": 0.02229997163937826} +{"batch_in_epoch": 31, "epoch": 9, "global_step": 481, "grad_norm_max": 0.031005859375, "hidden_mse_step1": 0.01960550182099853, "hidden_mse_step2": 0.0224832667570029, "hidden_mse_step3": 0.026567485715661733, "latent_grad_l1_min": 0.000253487933150609, "loss": 0.024210035701149275, "lr_blocks": 8.842276865771108e-07, "lr_fusion": 8.84227686577111e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 72.07396686562424, "velocity_mse_step1": 0.016840032576250708, "velocity_mse_step2": 0.019928864494431764, "velocity_mse_step3": 0.028995582609370883} +{"batch_in_epoch": 32, "epoch": 9, "global_step": 482, "grad_norm_max": 0.03125, "hidden_mse_step1": 0.018228298926260322, "hidden_mse_step2": 0.02046828029311395, "hidden_mse_step3": 0.02344945118030799, "latent_grad_l1_min": 0.00018866096434067003, "loss": 0.017782657282493473, "lr_blocks": 8.837117225766031e-07, "lr_fusion": 8.837117225766033e-06, "peak_memory_gib": 48.31958723068237, "trajectory_time_s": 72.02957186998538, "velocity_mse_step1": 0.01203400081638912, "velocity_mse_step2": 0.014278728498278985, "velocity_mse_step3": 0.020820638464231576} +{"batch_in_epoch": 33, "epoch": 9, "global_step": 483, "grad_norm_max": 0.0299072265625, "hidden_mse_step1": 0.021781438271448544, "hidden_mse_step2": 0.024954959267883426, "hidden_mse_step3": 0.028984437979358645, "latent_grad_l1_min": 0.0003093328978138743, "loss": 0.027723736088124236, "lr_blocks": 8.831947626307734e-07, "lr_fusion": 8.831947626307735e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 73.87377388699679, "velocity_mse_step1": 0.021315234605156413, "velocity_mse_step2": 0.022958598428106467, "velocity_mse_step3": 0.03132528818345496} +{"batch_in_epoch": 34, "epoch": 9, "global_step": 484, "grad_norm_max": 0.033203125, "hidden_mse_step1": 0.015513400260325787, "hidden_mse_step2": 0.01829176894222785, "hidden_mse_step3": 0.021385067774515067, "latent_grad_l1_min": 0.00022408114818972535, "loss": 0.01799926666093857, "lr_blocks": 8.826768080814204e-07, "lr_fusion": 8.826768080814205e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 73.95804193588265, "velocity_mse_step1": 0.01252600915696738, "velocity_mse_step2": 0.014871299973622496, "velocity_mse_step3": 0.02108146564153555} +{"batch_in_epoch": 35, "epoch": 9, "global_step": 485, "grad_norm_max": 0.03466796875, "hidden_mse_step1": 0.017910456754699614, "hidden_mse_step2": 0.021360838594513813, "hidden_mse_step3": 0.02522176664621968, "latent_grad_l1_min": 0.00031556302747048903, "loss": 0.024201528115164753, "lr_blocks": 8.821578602729241e-07, "lr_fusion": 8.821578602729242e-06, "peak_memory_gib": 48.31958723068237, "trajectory_time_s": 72.73749110886274, "velocity_mse_step1": 0.01680623567517614, "velocity_mse_step2": 0.021240206356326646, "velocity_mse_step3": 0.028108834850302498} +{"batch_in_epoch": 36, "epoch": 9, "global_step": 486, "grad_norm_max": 0.025390625, "hidden_mse_step1": 0.0197094303979871, "hidden_mse_step2": 0.02189739480049216, "hidden_mse_step3": 0.024792583160368458, "latent_grad_l1_min": 0.00020521287387964549, "loss": 0.02113232045667246, "lr_blocks": 8.816379205522427e-07, "lr_fusion": 8.816379205522428e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 71.32605337525456, "velocity_mse_step1": 0.015323197856819858, "velocity_mse_step2": 0.01746310901736641, "velocity_mse_step3": 0.023970711336005476} +{"batch_in_epoch": 37, "epoch": 9, "global_step": 487, "grad_norm_max": 0.02734375, "hidden_mse_step1": 0.015311622268719863, "hidden_mse_step2": 0.017334438752316467, "hidden_mse_step3": 0.02026499618243958, "latent_grad_l1_min": 0.00016652312569931382, "loss": 0.016708513222900883, "lr_blocks": 8.81116990268909e-07, "lr_fusion": 8.81116990268909e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 73.57641418287676, "velocity_mse_step1": 0.01198536585018571, "velocity_mse_step2": 0.013886570705965691, "velocity_mse_step3": 0.018962496068395143} +{"batch_in_epoch": 38, "epoch": 9, "global_step": 488, "grad_norm_max": 0.032958984375, "hidden_mse_step1": 0.018421996825574233, "hidden_mse_step2": 0.02016796772867175, "hidden_mse_step3": 0.023811431485228237, "latent_grad_l1_min": 0.00023046797196002444, "loss": 0.019651343750800673, "lr_blocks": 8.805950707750267e-07, "lr_fusion": 8.805950707750268e-06, "peak_memory_gib": 48.320029735565186, "trajectory_time_s": 72.72678924225329, "velocity_mse_step1": 0.014528245896404508, "velocity_mse_step2": 0.016121535806243106, "velocity_mse_step3": 0.02206410782361802} +{"batch_in_epoch": 39, "epoch": 9, "global_step": 489, "grad_norm_max": 0.036865234375, "hidden_mse_step1": 0.02204802695944506, "hidden_mse_step2": 0.02367888733611575, "hidden_mse_step3": 0.02656208910048008, "latent_grad_l1_min": 0.0002361558854317991, "loss": 0.024049065416745306, "lr_blocks": 8.800721634252671e-07, "lr_fusion": 8.800721634252671e-06, "peak_memory_gib": 48.320029735565186, "trajectory_time_s": 71.57183792986325, "velocity_mse_step1": 0.017962635825304978, "velocity_mse_step2": 0.020165388767574247, "velocity_mse_step3": 0.026790269750303453} +{"batch_in_epoch": 40, "epoch": 9, "global_step": 490, "grad_norm_max": 0.0341796875, "hidden_mse_step1": 0.0228003230295144, "hidden_mse_step2": 0.025151308625936505, "hidden_mse_step3": 0.028819984324010357, "latent_grad_l1_min": 0.0003773651951632928, "loss": 0.031158687846202935, "lr_blocks": 8.795482695768656e-07, "lr_fusion": 8.795482695768658e-06, "peak_memory_gib": 48.31737470626831, "trajectory_time_s": 73.4699701767604, "velocity_mse_step1": 0.024922985726984086, "velocity_mse_step2": 0.027161130324072604, "velocity_mse_step3": 0.03371478284576109} +{"batch_in_epoch": 41, "epoch": 9, "global_step": 491, "grad_norm_max": 0.036865234375, "hidden_mse_step1": 0.019314956591447947, "hidden_mse_step2": 0.021677931417278678, "hidden_mse_step3": 0.025216132992812036, "latent_grad_l1_min": 0.00027361141201254213, "loss": 0.024358994693362283, "lr_blocks": 8.790233905896185e-07, "lr_fusion": 8.790233905896187e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 76.22463416362734, "velocity_mse_step1": 0.018722112818068126, "velocity_mse_step2": 0.02066240676608036, "velocity_mse_step3": 0.02707155994305919} +{"batch_in_epoch": 42, "epoch": 9, "global_step": 492, "grad_norm_max": 0.036376953125, "hidden_mse_step1": 0.022756555724689466, "hidden_mse_step2": 0.02500316257854657, "hidden_mse_step3": 0.028607847002734024, "latent_grad_l1_min": 0.0003012247379956534, "loss": 0.025724916344708096, "lr_blocks": 8.784975278258782e-07, "lr_fusion": 8.784975278258783e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 72.30369113600318, "velocity_mse_step1": 0.01827851665439084, "velocity_mse_step2": 0.0212226701551117, "velocity_mse_step3": 0.03003680417480479} +{"batch_in_epoch": 43, "epoch": 9, "global_step": 493, "grad_norm_max": 0.03271484375, "hidden_mse_step1": 0.013343761816421256, "hidden_mse_step2": 0.015895835789186612, "hidden_mse_step3": 0.01931663484512163, "latent_grad_l1_min": 0.00016021551709854975, "loss": 0.013661035586015452, "lr_blocks": 8.779706826505512e-07, "lr_fusion": 8.779706826505513e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 71.45353507913387, "velocity_mse_step1": 0.008953010999123632, "velocity_mse_step2": 0.010871027028770186, "velocity_mse_step3": 0.01630344466372792} +{"batch_in_epoch": 44, "epoch": 9, "global_step": 494, "grad_norm_max": 0.0291748046875, "hidden_mse_step1": 0.01898815805491592, "hidden_mse_step2": 0.021419038480546857, "hidden_mse_step3": 0.02482404742789055, "latent_grad_l1_min": 0.00021136329996807035, "loss": 0.019141902831116955, "lr_blocks": 8.774428564310937e-07, "lr_fusion": 8.774428564310939e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 72.50523574950785, "velocity_mse_step1": 0.01328110057095598, "velocity_mse_step2": 0.01536597348915945, "velocity_mse_step3": 0.02225550890268226} +{"batch_in_epoch": 45, "epoch": 9, "global_step": 495, "grad_norm_max": 0.028564453125, "hidden_mse_step1": 0.016280631443286047, "hidden_mse_step2": 0.018520770295123966, "hidden_mse_step3": 0.02194604176163141, "latent_grad_l1_min": 0.0002295053254783852, "loss": 0.016299564291590025, "lr_blocks": 8.769140505375083e-07, "lr_fusion": 8.769140505375084e-06, "peak_memory_gib": 48.320029735565186, "trajectory_time_s": 73.78778705660807, "velocity_mse_step1": 0.011284861013077066, "velocity_mse_step2": 0.012822892392120722, "velocity_mse_step3": 0.019116193779544637} +{"batch_in_epoch": 46, "epoch": 9, "global_step": 496, "grad_norm_max": 0.035400390625, "hidden_mse_step1": 0.016068649594672024, "hidden_mse_step2": 0.017623890415831865, "hidden_mse_step3": 0.020858997811696357, "latent_grad_l1_min": 0.0002334608425371698, "loss": 0.01677745795625794, "lr_blocks": 8.763842663423406e-07, "lr_fusion": 8.763842663423408e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 72.6487558450026, "velocity_mse_step1": 0.012484163571830971, "velocity_mse_step2": 0.01337938845764646, "velocity_mse_step3": 0.01901366667256557} +{"batch_in_epoch": 47, "epoch": 9, "global_step": 497, "grad_norm_max": 0.0390625, "hidden_mse_step1": 0.015052903871816983, "hidden_mse_step2": 0.017955165971735756, "hidden_mse_step3": 0.021990328079222566, "latent_grad_l1_min": 0.00021568058855336858, "loss": 0.017117239260447344, "lr_blocks": 8.758535052206748e-07, "lr_fusion": 8.758535052206749e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 70.93889616238448, "velocity_mse_step1": 0.011537067442467169, "velocity_mse_step2": 0.013780111629915024, "velocity_mse_step3": 0.020534696439946334} +{"batch_in_epoch": 48, "epoch": 9, "global_step": 498, "grad_norm_max": 0.033935546875, "hidden_mse_step1": 0.021165139582340738, "hidden_mse_step2": 0.02309251061108495, "hidden_mse_step3": 0.026113599023249532, "latent_grad_l1_min": 0.0002857435993064428, "loss": 0.024350001509966593, "lr_blocks": 8.753217685501316e-07, "lr_fusion": 8.753217685501317e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 71.54971188600757, "velocity_mse_step1": 0.018099150256603025, "velocity_mse_step2": 0.020465377452118055, "velocity_mse_step3": 0.027448350039776415} +{"batch_in_epoch": 49, "epoch": 9, "global_step": 499, "grad_norm_max": 0.0291748046875, "hidden_mse_step1": 0.014473920638140823, "hidden_mse_step2": 0.01664076550098668, "hidden_mse_step3": 0.020120648284708813, "latent_grad_l1_min": 0.0001679489132584422, "loss": 0.014286547021973612, "lr_blocks": 8.747890577108632e-07, "lr_fusion": 8.747890577108633e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 71.65795117362723, "velocity_mse_step1": 0.00973834110897899, "velocity_mse_step2": 0.011136693904908106, "velocity_mse_step3": 0.016861071868333962} +{"batch_in_epoch": 50, "epoch": 9, "global_step": 500, "grad_norm_max": 0.039306640625, "hidden_mse_step1": 0.02064468350727111, "hidden_mse_step2": 0.02312734576740435, "hidden_mse_step3": 0.026240975911995125, "latent_grad_l1_min": 0.00026355646150477696, "loss": 0.02240966109093279, "lr_blocks": 8.742553740855505e-07, "lr_fusion": 8.742553740855507e-06, "peak_memory_gib": 48.3169322013855, "trajectory_time_s": 74.61433909175685, "velocity_mse_step1": 0.015558873839576593, "velocity_mse_step2": 0.018476792270251147, "velocity_mse_step3": 0.02619201418045642} +{"batch_in_epoch": 1, "epoch": 10, "global_step": 501, "grad_norm_max": 0.038818359375, "hidden_mse_step1": 0.017657976381347647, "hidden_mse_step2": 0.020741760448020483, "hidden_mse_step3": 0.024061889969743785, "latent_grad_l1_min": 0.0002590463154774625, "loss": 0.01988944096124864, "lr_blocks": 8.737207190593993e-07, "lr_fusion": 8.737207190593995e-06, "peak_memory_gib": 48.31737470626831, "trajectory_time_s": 74.3417825483848, "velocity_mse_step1": 0.01397160322604967, "velocity_mse_step2": 0.01653165535286202, "velocity_mse_step3": 0.02291890075762889} +{"batch_in_epoch": 2, "epoch": 10, "global_step": 502, "grad_norm_max": 0.029541015625, "hidden_mse_step1": 0.021341206040233374, "hidden_mse_step2": 0.023232930888687924, "hidden_mse_step3": 0.02632279466238937, "latent_grad_l1_min": 0.0002623965010570828, "loss": 0.024878998918991, "lr_blocks": 8.731850940201368e-07, "lr_fusion": 8.731850940201369e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 72.33253560825688, "velocity_mse_step1": 0.018841865376868685, "velocity_mse_step2": 0.02041893878153392, "velocity_mse_step3": 0.02828649692569992} +{"batch_in_epoch": 3, "epoch": 10, "global_step": 503, "grad_norm_max": 0.03515625, "hidden_mse_step1": 0.01212697847430328, "hidden_mse_step2": 0.014652389983114387, "hidden_mse_step3": 0.018532594948607897, "latent_grad_l1_min": 0.00017343589206575416, "loss": 0.011836247976004544, "lr_blocks": 8.72648500358008e-07, "lr_fusion": 8.72648500358008e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 72.04502981000405, "velocity_mse_step1": 0.007708283222330334, "velocity_mse_step2": 0.008932288075032244, "velocity_mse_step3": 0.01433697513337912} +{"batch_in_epoch": 4, "epoch": 10, "global_step": 504, "grad_norm_max": 0.035400390625, "hidden_mse_step1": 0.021695671968960335, "hidden_mse_step2": 0.02361621963791549, "hidden_mse_step3": 0.0272435412708936, "latent_grad_l1_min": 0.00021718167590734083, "loss": 0.02194750854479415, "lr_blocks": 8.721109394657715e-07, "lr_fusion": 8.721109394657717e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 71.54058036500646, "velocity_mse_step1": 0.015651538394325013, "velocity_mse_step2": 0.017514987699022252, "velocity_mse_step3": 0.025420454630096043} +{"batch_in_epoch": 5, "epoch": 10, "global_step": 505, "grad_norm_max": 0.033447265625, "hidden_mse_step1": 0.01470817670009897, "hidden_mse_step2": 0.01721641472873411, "hidden_mse_step3": 0.02089973129997296, "latent_grad_l1_min": 0.00018272342458658386, "loss": 0.01601631595985964, "lr_blocks": 8.71572412738697e-07, "lr_fusion": 8.715724127386971e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 70.9789208843722, "velocity_mse_step1": 0.01156773136088824, "velocity_mse_step2": 0.012889237284460769, "velocity_mse_step3": 0.018309545520293926} +{"batch_in_epoch": 6, "epoch": 10, "global_step": 506, "grad_norm_max": 0.0311279296875, "hidden_mse_step1": 0.017016054488651986, "hidden_mse_step2": 0.019574819980854433, "hidden_mse_step3": 0.023368985259107182, "latent_grad_l1_min": 0.0001600801642780425, "loss": 0.0191116714060107, "lr_blocks": 8.710329215745611e-07, "lr_fusion": 8.710329215745612e-06, "peak_memory_gib": 48.320029735565186, "trajectory_time_s": 73.77922734225285, "velocity_mse_step1": 0.01303308835070181, "velocity_mse_step2": 0.015452860937719897, "velocity_mse_step3": 0.022853077780122737} +{"batch_in_epoch": 7, "epoch": 10, "global_step": 507, "grad_norm_max": 0.0419921875, "hidden_mse_step1": 0.01810127936602969, "hidden_mse_step2": 0.02017563416822148, "hidden_mse_step3": 0.02293390098826161, "latent_grad_l1_min": 0.000204719593966729, "loss": 0.01764639381352546, "lr_blocks": 8.704924673736429e-07, "lr_fusion": 8.70492467373643e-06, "peak_memory_gib": 48.31604719161987, "trajectory_time_s": 71.3257813329983, "velocity_mse_step1": 0.012733223170341392, "velocity_mse_step2": 0.014213363091195268, "velocity_mse_step3": 0.019871512128572376} +{"batch_in_epoch": 8, "epoch": 10, "global_step": 508, "grad_norm_max": 0.03759765625, "hidden_mse_step1": 0.02247991266943115, "hidden_mse_step2": 0.024271037446201908, "hidden_mse_step3": 0.027203166940515593, "latent_grad_l1_min": 0.00023053542099660262, "loss": 0.02467954548774287, "lr_blocks": 8.699510515387221e-07, "lr_fusion": 8.699510515387222e-06, "peak_memory_gib": 48.31737470626831, "trajectory_time_s": 72.31352912637522, "velocity_mse_step1": 0.017891848148013065, "velocity_mse_step2": 0.020531974105064625, "velocity_mse_step3": 0.028219399840704026} +{"batch_in_epoch": 9, "epoch": 10, "global_step": 509, "grad_norm_max": 0.0277099609375, "hidden_mse_step1": 0.015730780848701085, "hidden_mse_step2": 0.01776654846201252, "hidden_mse_step3": 0.021485147166198915, "latent_grad_l1_min": 0.00021271809328027302, "loss": 0.017813039203507026, "lr_blocks": 8.694086754750736e-07, "lr_fusion": 8.694086754750738e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 70.77246127949184, "velocity_mse_step1": 0.012882537692868417, "velocity_mse_step2": 0.014379818102627593, "velocity_mse_step3": 0.020678512897575274} +{"batch_in_epoch": 10, "epoch": 10, "global_step": 510, "grad_norm_max": 0.03857421875, "hidden_mse_step1": 0.017956297977694442, "hidden_mse_step2": 0.019900514993683567, "hidden_mse_step3": 0.023231669861291136, "latent_grad_l1_min": 0.00018759755585051607, "loss": 0.019237463266888098, "lr_blocks": 8.688653405904651e-07, "lr_fusion": 8.688653405904653e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 71.77310363700235, "velocity_mse_step1": 0.013942724425370605, "velocity_mse_step2": 0.01563844213628077, "velocity_mse_step3": 0.022022373483715843} +{"batch_in_epoch": 11, "epoch": 10, "global_step": 511, "grad_norm_max": 0.041748046875, "hidden_mse_step1": 0.01817416012636386, "hidden_mse_step2": 0.02038200770870649, "hidden_mse_step3": 0.02313984042432691, "latent_grad_l1_min": 0.0001949151137523586, "loss": 0.018591429231621857, "lr_blocks": 8.683210482951526e-07, "lr_fusion": 8.683210482951528e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 76.57488915100839, "velocity_mse_step1": 0.013399868365792957, "velocity_mse_step2": 0.015295066675337561, "velocity_mse_step3": 0.020909750210453885} +{"batch_in_epoch": 12, "epoch": 10, "global_step": 512, "grad_norm_max": 0.0252685546875, "hidden_mse_step1": 0.013868014725241143, "hidden_mse_step2": 0.016715513858278946, "hidden_mse_step3": 0.02049817090820787, "latent_grad_l1_min": 0.00021862145149498247, "loss": 0.014289923945658041, "lr_blocks": 8.677758000018775e-07, "lr_fusion": 8.677758000018777e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 71.5277737211145, "velocity_mse_step1": 0.009004235875701332, "velocity_mse_step2": 0.011430093142345348, "velocity_mse_step3": 0.017327271518297493} +{"batch_in_epoch": 13, "epoch": 10, "global_step": 513, "grad_norm_max": 0.044677734375, "hidden_mse_step1": 0.016125278376940902, "hidden_mse_step2": 0.019193534157238897, "hidden_mse_step3": 0.02341386062159602, "latent_grad_l1_min": 0.0003087661962126731, "loss": 0.021761141655068577, "lr_blocks": 8.672295971258625e-07, "lr_fusion": 8.672295971258624e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 72.00786908463488, "velocity_mse_step1": 0.01542506655823672, "velocity_mse_step2": 0.018265013252468112, "velocity_mse_step3": 0.02572007557111127} +{"batch_in_epoch": 14, "epoch": 10, "global_step": 514, "grad_norm_max": 0.034912109375, "hidden_mse_step1": 0.017693557888768342, "hidden_mse_step2": 0.0199660240655898, "hidden_mse_step3": 0.023579844489826685, "latent_grad_l1_min": 0.0001960060408237041, "loss": 0.02086495721180524, "lr_blocks": 8.666824410848074e-07, "lr_fusion": 8.666824410848075e-06, "peak_memory_gib": 48.316489696502686, "trajectory_time_s": 70.87740118975489, "velocity_mse_step1": 0.014931118471265237, "velocity_mse_step2": 0.017322021160258112, "velocity_mse_step3": 0.024217786383815106} +{"batch_in_epoch": 15, "epoch": 10, "global_step": 515, "grad_norm_max": 0.032958984375, "hidden_mse_step1": 0.02050227566256321, "hidden_mse_step2": 0.02290644704563809, "hidden_mse_step3": 0.02628661678837878, "latent_grad_l1_min": 0.000214367810258409, "loss": 0.023195783233469616, "lr_blocks": 8.661343332988868e-07, "lr_fusion": 8.661343332988869e-06, "peak_memory_gib": 48.320029735565186, "trajectory_time_s": 71.87379460049124, "velocity_mse_step1": 0.01710359113141229, "velocity_mse_step2": 0.01933035997873438, "velocity_mse_step3": 0.026183861839984144} +{"batch_in_epoch": 16, "epoch": 10, "global_step": 516, "grad_norm_max": 0.032470703125, "hidden_mse_step1": 0.012366864801151674, "hidden_mse_step2": 0.014373138488736004, "hidden_mse_step3": 0.01773470114650471, "latent_grad_l1_min": 0.00019987247833341826, "loss": 0.012427068739530762, "lr_blocks": 8.65585275190745e-07, "lr_fusion": 8.65585275190745e-06, "peak_memory_gib": 48.31958723068237, "trajectory_time_s": 73.8610457757386, "velocity_mse_step1": 0.008440301974977565, "velocity_mse_step2": 0.009549135681092075, "velocity_mse_step3": 0.014844296957432692} +{"batch_in_epoch": 17, "epoch": 10, "global_step": 517, "grad_norm_max": 0.029296875, "hidden_mse_step1": 0.018380954790960198, "hidden_mse_step2": 0.020310932196610208, "hidden_mse_step3": 0.023451843099402524, "latent_grad_l1_min": 0.00023691569549555425, "loss": 0.02051780542076033, "lr_blocks": 8.650352681854933e-07, "lr_fusion": 8.650352681854934e-06, "peak_memory_gib": 48.31737470626831, "trajectory_time_s": 76.98671638101223, "velocity_mse_step1": 0.014558404060413263, "velocity_mse_step2": 0.01680952311809441, "velocity_mse_step3": 0.02397111424943432} +{"batch_in_epoch": 18, "epoch": 10, "global_step": 518, "grad_norm_max": 0.0341796875, "hidden_mse_step1": 0.02032591146416962, "hidden_mse_step2": 0.022239235878389857, "hidden_mse_step3": 0.02502820615856243, "latent_grad_l1_min": 0.00024554126503062434, "loss": 0.022869799319388612, "lr_blocks": 8.644843137107057e-07, "lr_fusion": 8.644843137107058e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 72.90354007425049, "velocity_mse_step1": 0.01758763602369332, "velocity_mse_step2": 0.019255881684200307, "velocity_mse_step3": 0.02500654336264623} +{"batch_in_epoch": 19, "epoch": 10, "global_step": 519, "grad_norm_max": 0.04052734375, "hidden_mse_step1": 0.013602825343176455, "hidden_mse_step2": 0.016538781454853182, "hidden_mse_step3": 0.02041178158417876, "latent_grad_l1_min": 0.00019157280894432915, "loss": 0.015904373335485746, "lr_blocks": 8.639324131964156e-07, "lr_fusion": 8.639324131964156e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 72.48088289250882, "velocity_mse_step1": 0.010968465238485284, "velocity_mse_step2": 0.013019986953752645, "velocity_mse_step3": 0.018669327729315097} +{"batch_in_epoch": 20, "epoch": 10, "global_step": 520, "grad_norm_max": 0.0303955078125, "hidden_mse_step1": 0.01878045906778425, "hidden_mse_step2": 0.02119797862334443, "hidden_mse_step3": 0.02411732037684747, "latent_grad_l1_min": 0.00020880932970612776, "loss": 0.020525665342574936, "lr_blocks": 8.633795680751116e-07, "lr_fusion": 8.633795680751116e-06, "peak_memory_gib": 48.316489696502686, "trajectory_time_s": 73.37335033286945, "velocity_mse_step1": 0.015271149646391029, "velocity_mse_step2": 0.016824585732787712, "velocity_mse_step3": 0.02307168270428} +{"batch_in_epoch": 21, "epoch": 10, "global_step": 521, "grad_norm_max": 0.03173828125, "hidden_mse_step1": 0.020903720819790448, "hidden_mse_step2": 0.022339606125439913, "hidden_mse_step3": 0.025417107456762875, "latent_grad_l1_min": 0.00024453293917758856, "loss": 0.022647818043229302, "lr_blocks": 8.628257797817343e-07, "lr_fusion": 8.628257797817344e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 74.87889815388189, "velocity_mse_step1": 0.01716433263001298, "velocity_mse_step2": 0.018672289983702024, "velocity_mse_step3": 0.025240785924584737} +{"batch_in_epoch": 22, "epoch": 10, "global_step": 522, "grad_norm_max": 0.040283203125, "hidden_mse_step1": 0.023109415986774757, "hidden_mse_step2": 0.025396234829843573, "hidden_mse_step3": 0.028443886582473554, "latent_grad_l1_min": 0.00035851939173880965, "loss": 0.03156652213406882, "lr_blocks": 8.622710497536724e-07, "lr_fusion": 8.622710497536727e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 72.84490924125566, "velocity_mse_step1": 0.026528397820324505, "velocity_mse_step2": 0.02743508267615523, "velocity_mse_step3": 0.03304112939596442} +{"batch_in_epoch": 23, "epoch": 10, "global_step": 523, "grad_norm_max": 0.03173828125, "hidden_mse_step1": 0.020387603976165077, "hidden_mse_step2": 0.0225461655728785, "hidden_mse_step3": 0.024390134478120933, "latent_grad_l1_min": 0.0002133074031007709, "loss": 0.02030878286210022, "lr_blocks": 8.617153794307587e-07, "lr_fusion": 8.617153794307588e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 72.77075729235366, "velocity_mse_step1": 0.014739447223421718, "velocity_mse_step2": 0.017142109743352712, "velocity_mse_step3": 0.02231240039691329} +{"batch_in_epoch": 24, "epoch": 10, "global_step": 524, "grad_norm_max": 0.0302734375, "hidden_mse_step1": 0.018401334710818316, "hidden_mse_step2": 0.020988934581899748, "hidden_mse_step3": 0.024314462157365463, "latent_grad_l1_min": 0.00023944252825458534, "loss": 0.020013133313373795, "lr_blocks": 8.61158770255267e-07, "lr_fusion": 8.61158770255267e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 72.2590765088753, "velocity_mse_step1": 0.014223053492189918, "velocity_mse_step2": 0.016048786795831153, "velocity_mse_step3": 0.02339708515709} +{"batch_in_epoch": 25, "epoch": 10, "global_step": 525, "grad_norm_max": 0.02392578125, "hidden_mse_step1": 0.017304557974317243, "hidden_mse_step2": 0.01882609216928748, "hidden_mse_step3": 0.022210900887979994, "latent_grad_l1_min": 0.00017252611723961309, "loss": 0.017022662837657014, "lr_blocks": 8.606012236719073e-07, "lr_fusion": 8.606012236719073e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 73.08979277676553, "velocity_mse_step1": 0.011806647252732156, "velocity_mse_step2": 0.013316174711300326, "velocity_mse_step3": 0.02011100932057681} +{"batch_in_epoch": 26, "epoch": 10, "global_step": 526, "grad_norm_max": 0.0267333984375, "hidden_mse_step1": 0.013740874989058025, "hidden_mse_step2": 0.01603578353699829, "hidden_mse_step3": 0.01927243677034442, "latent_grad_l1_min": 0.00015508275100728497, "loss": 0.014361599061105934, "lr_blocks": 8.600427411278233e-07, "lr_fusion": 8.600427411278234e-06, "peak_memory_gib": 48.316489696502686, "trajectory_time_s": 72.47723468414915, "velocity_mse_step1": 0.0099420652758064, "velocity_mse_step2": 0.011161236973878528, "velocity_mse_step3": 0.01707658443566678} +{"batch_in_epoch": 27, "epoch": 10, "global_step": 527, "grad_norm_max": 0.0294189453125, "hidden_mse_step1": 0.018882694679112837, "hidden_mse_step2": 0.021417545437413664, "hidden_mse_step3": 0.02464860401648496, "latent_grad_l1_min": 0.0001963828299267334, "loss": 0.01939154608823758, "lr_blocks": 8.594833240725876e-07, "lr_fusion": 8.594833240725877e-06, "peak_memory_gib": 48.31737470626831, "trajectory_time_s": 71.41264068311284, "velocity_mse_step1": 0.013797127643816307, "velocity_mse_step2": 0.015413783432450145, "velocity_mse_step3": 0.022468840653475906} +{"batch_in_epoch": 28, "epoch": 10, "global_step": 528, "grad_norm_max": 0.03515625, "hidden_mse_step1": 0.01772316655011049, "hidden_mse_step2": 0.02074831988595958, "hidden_mse_step3": 0.02407451108696737, "latent_grad_l1_min": 0.00024632255372125655, "loss": 0.020764782542495853, "lr_blocks": 8.589229739581988e-07, "lr_fusion": 8.58922973958199e-06, "peak_memory_gib": 48.31737470626831, "trajectory_time_s": 71.85834002187767, "velocity_mse_step1": 0.014916248041637508, "velocity_mse_step2": 0.01736723677354998, "velocity_mse_step3": 0.02375626110006124} +{"batch_in_epoch": 29, "epoch": 10, "global_step": 529, "grad_norm_max": 0.034423828125, "hidden_mse_step1": 0.01909832276786411, "hidden_mse_step2": 0.021336819469330037, "hidden_mse_step3": 0.024661626700045805, "latent_grad_l1_min": 0.00027773247165896464, "loss": 0.020495543975682397, "lr_blocks": 8.58361692239077e-07, "lr_fusion": 8.583616922390772e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 72.11138117699738, "velocity_mse_step1": 0.014460743575390162, "velocity_mse_step2": 0.016714041262665496, "velocity_mse_step3": 0.023802167893986082} +{"batch_in_epoch": 30, "epoch": 10, "global_step": 530, "grad_norm_max": 0.0291748046875, "hidden_mse_step1": 0.01365186403771596, "hidden_mse_step2": 0.01653758416484509, "hidden_mse_step3": 0.020443241045411144, "latent_grad_l1_min": 0.00017855624446383445, "loss": 0.014068168421675051, "lr_blocks": 8.577994803720605e-07, "lr_fusion": 8.577994803720605e-06, "peak_memory_gib": 48.31737470626831, "trajectory_time_s": 75.04693460610724, "velocity_mse_step1": 0.008951519771861576, "velocity_mse_step2": 0.01104522607887962, "velocity_mse_step3": 0.017144489260057787} +{"batch_in_epoch": 31, "epoch": 10, "global_step": 531, "grad_norm_max": 0.03369140625, "hidden_mse_step1": 0.017140100053178942, "hidden_mse_step2": 0.018285501598646597, "hidden_mse_step3": 0.02130563527212611, "latent_grad_l1_min": 0.00019492577666824218, "loss": 0.016582711957328553, "lr_blocks": 8.572363398164016e-07, "lr_fusion": 8.572363398164017e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 75.72092664313095, "velocity_mse_step1": 0.013104076865212328, "velocity_mse_step2": 0.012737540864951108, "velocity_mse_step3": 0.01823339318590505} +{"batch_in_epoch": 32, "epoch": 10, "global_step": 532, "grad_norm_max": 0.029541015625, "hidden_mse_step1": 0.020199515149995153, "hidden_mse_step2": 0.022509073630707074, "hidden_mse_step3": 0.026328879042661617, "latent_grad_l1_min": 0.0002516113472665893, "loss": 0.023241274025557295, "lr_blocks": 8.566722720337634e-07, "lr_fusion": 8.566722720337634e-06, "peak_memory_gib": 48.31737470626831, "trajectory_time_s": 74.47244017900812, "velocity_mse_step1": 0.01624482052284293, "velocity_mse_step2": 0.019134577513406317, "velocity_mse_step3": 0.027440675916815444} +{"batch_in_epoch": 33, "epoch": 10, "global_step": 533, "grad_norm_max": 0.0341796875, "hidden_mse_step1": 0.013334453326284086, "hidden_mse_step2": 0.016039481112134775, "hidden_mse_step3": 0.019398743303359618, "latent_grad_l1_min": 0.00020011841297673527, "loss": 0.014996546021263515, "lr_blocks": 8.561072784882154e-07, "lr_fusion": 8.561072784882156e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 72.88515221486159, "velocity_mse_step1": 0.01017123726653933, "velocity_mse_step2": 0.01228893649697836, "velocity_mse_step3": 0.017652195129942677} +{"batch_in_epoch": 34, "epoch": 10, "global_step": 534, "grad_norm_max": 0.0299072265625, "hidden_mse_step1": 0.014423770676200678, "hidden_mse_step2": 0.017430326618653322, "hidden_mse_step3": 0.02194418984332255, "latent_grad_l1_min": 0.0002665973934199428, "loss": 0.017400650444739898, "lr_blocks": 8.5554136064623e-07, "lr_fusion": 8.555413606462302e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 72.53243481949903, "velocity_mse_step1": 0.011327797933647943, "velocity_mse_step2": 0.014090434951607935, "velocity_mse_step3": 0.0214038882321412} +{"batch_in_epoch": 35, "epoch": 10, "global_step": 535, "grad_norm_max": 0.033447265625, "hidden_mse_step1": 0.017980195149513226, "hidden_mse_step2": 0.020176583909363083, "hidden_mse_step3": 0.022723328870987252, "latent_grad_l1_min": 0.0001537492462375667, "loss": 0.017834016953461936, "lr_blocks": 8.549745199766791e-07, "lr_fusion": 8.549745199766792e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 73.67834290461178, "velocity_mse_step1": 0.01294433269608167, "velocity_mse_step2": 0.01458377727457056, "velocity_mse_step3": 0.01988592806238947} +{"batch_in_epoch": 36, "epoch": 10, "global_step": 536, "grad_norm_max": 0.0322265625, "hidden_mse_step1": 0.017114844701219614, "hidden_mse_step2": 0.019213333509729376, "hidden_mse_step3": 0.02261761639965698, "latent_grad_l1_min": 0.00019148779574607033, "loss": 0.018132552832997, "lr_blocks": 8.544067579508291e-07, "lr_fusion": 8.544067579508292e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 71.94388856463047, "velocity_mse_step1": 0.01264376219270551, "velocity_mse_step2": 0.014532414007200192, "velocity_mse_step3": 0.021326901204052513} +{"batch_in_epoch": 37, "epoch": 10, "global_step": 537, "grad_norm_max": 0.0263671875, "hidden_mse_step1": 0.01869382784103176, "hidden_mse_step2": 0.02159603154619357, "hidden_mse_step3": 0.02458576772395255, "latent_grad_l1_min": 0.00028042265785188647, "loss": 0.02450134834258019, "lr_blocks": 8.538380760423382e-07, "lr_fusion": 8.538380760423384e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 75.51762707588205, "velocity_mse_step1": 0.019040168180903753, "velocity_mse_step2": 0.02134483859117608, "velocity_mse_step3": 0.02663147271128504} +{"batch_in_epoch": 38, "epoch": 10, "global_step": 538, "grad_norm_max": 0.0262451171875, "hidden_mse_step1": 0.017510378088835363, "hidden_mse_step2": 0.020524548150465956, "hidden_mse_step3": 0.025198064744472504, "latent_grad_l1_min": 0.00029223430556157837, "loss": 0.020916915689927658, "lr_blocks": 8.532684757272526e-07, "lr_fusion": 8.532684757272527e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 75.5880311535002, "velocity_mse_step1": 0.01441524688769797, "velocity_mse_step2": 0.016832984135752276, "velocity_mse_step3": 0.02517921517470053} +{"batch_in_epoch": 39, "epoch": 10, "global_step": 539, "grad_norm_max": 0.029052734375, "hidden_mse_step1": 0.018969910352357795, "hidden_mse_step2": 0.02237472091136234, "hidden_mse_step3": 0.02621681880139346, "latent_grad_l1_min": 0.0002947029297502013, "loss": 0.02144339561345987, "lr_blocks": 8.526979584840015e-07, "lr_fusion": 8.526979584840016e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 73.77760098912404, "velocity_mse_step1": 0.014594938724517953, "velocity_mse_step2": 0.017527332733152434, "velocity_mse_step3": 0.025451768039991808} +{"batch_in_epoch": 40, "epoch": 10, "global_step": 540, "grad_norm_max": 0.025634765625, "hidden_mse_step1": 0.020755658849209012, "hidden_mse_step2": 0.022889511418595375, "hidden_mse_step3": 0.025570297464063124, "latent_grad_l1_min": 0.00022874211208545603, "loss": 0.02243767140732546, "lr_blocks": 8.521265257933947e-07, "lr_fusion": 8.521265257933948e-06, "peak_memory_gib": 48.31958723068237, "trajectory_time_s": 74.40554930337385, "velocity_mse_step1": 0.017057449367712252, "velocity_mse_step2": 0.01888425214981128, "velocity_mse_step3": 0.024449763079506477} +{"batch_in_epoch": 41, "epoch": 10, "global_step": 541, "grad_norm_max": 0.0277099609375, "hidden_mse_step1": 0.016125736774743667, "hidden_mse_step2": 0.01876852947420307, "hidden_mse_step3": 0.02193739083928189, "latent_grad_l1_min": 0.00018651901518751401, "loss": 0.015056769026809238, "lr_blocks": 8.515541791386176e-07, "lr_fusion": 8.515541791386176e-06, "peak_memory_gib": 48.32135725021362, "trajectory_time_s": 74.80974280186638, "velocity_mse_step1": 0.009604956133573848, "velocity_mse_step2": 0.011982434849155002, "velocity_mse_step3": 0.017899749542786073} +{"batch_in_epoch": 42, "epoch": 10, "global_step": 542, "grad_norm_max": 0.030517578125, "hidden_mse_step1": 0.015398548543868986, "hidden_mse_step2": 0.017734722687496936, "hidden_mse_step3": 0.021151542580420415, "latent_grad_l1_min": 0.00021915136494499166, "loss": 0.016484564785579484, "lr_blocks": 8.509809200052284e-07, "lr_fusion": 8.509809200052286e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 71.84286837561376, "velocity_mse_step1": 0.010534428479981475, "velocity_mse_step2": 0.013660881788902249, "velocity_mse_step3": 0.019829900947765315} +{"batch_in_epoch": 43, "epoch": 10, "global_step": 543, "grad_norm_max": 0.0224609375, "hidden_mse_step1": 0.015933563126184578, "hidden_mse_step2": 0.01916924739323024, "hidden_mse_step3": 0.02346829705805118, "latent_grad_l1_min": 0.00024998432672873605, "loss": 0.02138180224158402, "lr_blocks": 8.504067498811531e-07, "lr_fusion": 8.504067498811533e-06, "peak_memory_gib": 48.316489696502686, "trajectory_time_s": 74.15603651224956, "velocity_mse_step1": 0.015541124910149457, "velocity_mse_step2": 0.018211468854652982, "velocity_mse_step3": 0.024535700121694905} +{"batch_in_epoch": 44, "epoch": 10, "global_step": 544, "grad_norm_max": 0.033447265625, "hidden_mse_step1": 0.016370317961056053, "hidden_mse_step2": 0.017911152606497383, "hidden_mse_step3": 0.021163865865673866, "latent_grad_l1_min": 0.00015851767875574296, "loss": 0.015902217627236884, "lr_blocks": 8.498316702566826e-07, "lr_fusion": 8.498316702566828e-06, "peak_memory_gib": 48.320029735565186, "trajectory_time_s": 76.04405959312862, "velocity_mse_step1": 0.011101786598114163, "velocity_mse_step2": 0.012587694066626551, "velocity_mse_step3": 0.01847263697917307} +{"batch_in_epoch": 45, "epoch": 10, "global_step": 545, "grad_norm_max": 0.0301513671875, "hidden_mse_step1": 0.02266005119807752, "hidden_mse_step2": 0.02560476887239409, "hidden_mse_step3": 0.02872683071265263, "latent_grad_l1_min": 0.0002782826795737492, "loss": 0.02812965946005923, "lr_blocks": 8.492556826244686e-07, "lr_fusion": 8.492556826244687e-06, "peak_memory_gib": 48.320029735565186, "trajectory_time_s": 74.1959468808709, "velocity_mse_step1": 0.020416912764111266, "velocity_mse_step2": 0.024181600959439362, "velocity_mse_step3": 0.03209129792438553} +{"batch_in_epoch": 46, "epoch": 10, "global_step": 546, "grad_norm_max": 0.0286865234375, "hidden_mse_step1": 0.016515903898315237, "hidden_mse_step2": 0.019346204586327076, "hidden_mse_step3": 0.02341888059995004, "latent_grad_l1_min": 0.0002168410283047706, "loss": 0.018551331678671498, "lr_blocks": 8.486787884795187e-07, "lr_fusion": 8.48678788479519e-06, "peak_memory_gib": 48.32135725021362, "trajectory_time_s": 71.86367027123924, "velocity_mse_step1": 0.012876480715931395, "velocity_mse_step2": 0.014724477730591648, "velocity_mse_step3": 0.0221249359144297} +{"batch_in_epoch": 47, "epoch": 10, "global_step": 547, "grad_norm_max": 0.0267333984375, "hidden_mse_step1": 0.019350867972077263, "hidden_mse_step2": 0.02161907431270395, "hidden_mse_step3": 0.025436498657135025, "latent_grad_l1_min": 0.0003198604708813946, "loss": 0.023701288379795314, "lr_blocks": 8.481009893191946e-07, "lr_fusion": 8.481009893191947e-06, "peak_memory_gib": 48.3169322013855, "trajectory_time_s": 74.4563909472563, "velocity_mse_step1": 0.016673879646987185, "velocity_mse_step2": 0.02003396271071064, "velocity_mse_step3": 0.027755376683282002} +{"batch_in_epoch": 48, "epoch": 10, "global_step": 548, "grad_norm_max": 0.029296875, "hidden_mse_step1": 0.019553452443298217, "hidden_mse_step2": 0.021423271433117664, "hidden_mse_step3": 0.024403148854617026, "latent_grad_l1_min": 0.00019497274661262054, "loss": 0.019405830580840947, "lr_blocks": 8.475222866432063e-07, "lr_fusion": 8.475222866432065e-06, "peak_memory_gib": 48.3169322013855, "trajectory_time_s": 71.35524441100279, "velocity_mse_step1": 0.013961588105303234, "velocity_mse_step2": 0.01558045422591801, "velocity_mse_step3": 0.02213745965855196} +{"batch_in_epoch": 49, "epoch": 10, "global_step": 549, "grad_norm_max": 0.02978515625, "hidden_mse_step1": 0.019787772059706703, "hidden_mse_step2": 0.02179299369786999, "hidden_mse_step3": 0.025600170866320174, "latent_grad_l1_min": 0.00024357865731872153, "loss": 0.020564906806352413, "lr_blocks": 8.469426819536092e-07, "lr_fusion": 8.469426819536093e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 74.22012400261883, "velocity_mse_step1": 0.014105378450559718, "velocity_mse_step2": 0.016315363455630304, "velocity_mse_step3": 0.02455588295457086} +{"batch_in_epoch": 50, "epoch": 10, "global_step": 550, "grad_norm_max": 0.0400390625, "hidden_mse_step1": 0.012642059258983604, "hidden_mse_step2": 0.015397784258571585, "hidden_mse_step3": 0.018516130678887882, "latent_grad_l1_min": 0.0002493173005859717, "loss": 0.016853880270251205, "lr_blocks": 8.463621767547997e-07, "lr_fusion": 8.463621767547998e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 73.07266072326456, "velocity_mse_step1": 0.013416355716929371, "velocity_mse_step2": 0.014910862246844252, "velocity_mse_step3": 0.017578824256945937} +{"batch_in_epoch": 1, "epoch": 11, "global_step": 551, "grad_norm_max": 0.034912109375, "hidden_mse_step1": 0.01582627128975998, "hidden_mse_step2": 0.01796833419107965, "hidden_mse_step3": 0.021685902877444665, "latent_grad_l1_min": 0.0002234143885289086, "loss": 0.016744247221920104, "lr_blocks": 8.457807725535115e-07, "lr_fusion": 8.457807725535117e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 72.55575743089139, "velocity_mse_step1": 0.011906571536591009, "velocity_mse_step2": 0.013264051139620796, "velocity_mse_step3": 0.019514066582944776} +{"batch_in_epoch": 2, "epoch": 11, "global_step": 552, "grad_norm_max": 0.0299072265625, "hidden_mse_step1": 0.014740310401456163, "hidden_mse_step2": 0.017542948430803208, "hidden_mse_step3": 0.020534910610876974, "latent_grad_l1_min": 0.00014995323181210551, "loss": 0.014099859225098044, "lr_blocks": 8.451984708588121e-07, "lr_fusion": 8.451984708588122e-06, "peak_memory_gib": 48.31737470626831, "trajectory_time_s": 71.57078931651631, "velocity_mse_step1": 0.009498520504296589, "velocity_mse_step2": 0.011398428275632406, "velocity_mse_step3": 0.01612081045251606} +{"batch_in_epoch": 3, "epoch": 11, "global_step": 553, "grad_norm_max": 0.0260009765625, "hidden_mse_step1": 0.016483544700479667, "hidden_mse_step2": 0.018869071367329782, "hidden_mse_step3": 0.022413030765684586, "latent_grad_l1_min": 0.00030008043540874496, "loss": 0.02037988738240009, "lr_blocks": 8.446152731820983e-07, "lr_fusion": 8.446152731820984e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 73.94560206912865, "velocity_mse_step1": 0.01580732889747846, "velocity_mse_step2": 0.017449362373944104, "velocity_mse_step3": 0.022106403380998275} +{"batch_in_epoch": 4, "epoch": 11, "global_step": 554, "grad_norm_max": 0.031005859375, "hidden_mse_step1": 0.01953531377616205, "hidden_mse_step2": 0.02278487347731633, "hidden_mse_step3": 0.027796867453227084, "latent_grad_l1_min": 0.00028663755801972, "loss": 0.025052383708368455, "lr_blocks": 8.44031181037092e-07, "lr_fusion": 8.440311810370921e-06, "peak_memory_gib": 48.316489696502686, "trajectory_time_s": 73.13783187587251, "velocity_mse_step1": 0.016547468262225654, "velocity_mse_step2": 0.02102101921835648, "velocity_mse_step3": 0.030576955959466953} +{"batch_in_epoch": 5, "epoch": 11, "global_step": 555, "grad_norm_max": 0.03662109375, "hidden_mse_step1": 0.018296017955955382, "hidden_mse_step2": 0.021113925688301345, "hidden_mse_step3": 0.024497141945175827, "latent_grad_l1_min": 0.00021528295110329054, "loss": 0.01924302102997899, "lr_blocks": 8.434461959398376e-07, "lr_fusion": 8.434461959398377e-06, "peak_memory_gib": 48.316489696502686, "trajectory_time_s": 74.06322786810779, "velocity_mse_step1": 0.012683338797484922, "velocity_mse_step2": 0.015489102362023133, "velocity_mse_step3": 0.02316591200984216} +{"batch_in_epoch": 6, "epoch": 11, "global_step": 556, "grad_norm_max": 0.03125, "hidden_mse_step1": 0.01521636255451345, "hidden_mse_step2": 0.01734605827368796, "hidden_mse_step3": 0.02048649742001933, "latent_grad_l1_min": 0.00016442788273707265, "loss": 0.014891840592359326, "lr_blocks": 8.428603194086965e-07, "lr_fusion": 8.428603194086965e-06, "peak_memory_gib": 48.320029735565186, "trajectory_time_s": 71.12284718662704, "velocity_mse_step1": 0.010137531133035995, "velocity_mse_step2": 0.012057787767844275, "velocity_mse_step3": 0.0171753099919962} +{"batch_in_epoch": 7, "epoch": 11, "global_step": 557, "grad_norm_max": 0.028076171875, "hidden_mse_step1": 0.014309061990518655, "hidden_mse_step2": 0.016483673867436925, "hidden_mse_step3": 0.019769460320406197, "latent_grad_l1_min": 0.00017098383432312403, "loss": 0.014225500648275817, "lr_blocks": 8.422735529643443e-07, "lr_fusion": 8.422735529643445e-06, "peak_memory_gib": 48.31737470626831, "trajectory_time_s": 74.71804927225458, "velocity_mse_step1": 0.010014091667211946, "velocity_mse_step2": 0.011175916812914822, "velocity_mse_step3": 0.016430272596023445} +{"batch_in_epoch": 8, "epoch": 11, "global_step": 558, "grad_norm_max": 0.03369140625, "hidden_mse_step1": 0.016769400552480614, "hidden_mse_step2": 0.01893488047478188, "hidden_mse_step3": 0.022497583779373335, "latent_grad_l1_min": 0.00019137357594445348, "loss": 0.018194944818138277, "lr_blocks": 8.416858981297662e-07, "lr_fusion": 8.416858981297663e-06, "peak_memory_gib": 48.3169322013855, "trajectory_time_s": 73.34918100562209, "velocity_mse_step1": 0.012238132734117764, "velocity_mse_step2": 0.014723902871732466, "velocity_mse_step3": 0.021802610850759917} +{"batch_in_epoch": 9, "epoch": 11, "global_step": 559, "grad_norm_max": 0.037353515625, "hidden_mse_step1": 0.016022818753429292, "hidden_mse_step2": 0.018145931759915714, "hidden_mse_step3": 0.02143266324752143, "latent_grad_l1_min": 0.0002040948202193249, "loss": 0.017803647962864485, "lr_blocks": 8.410973564302533e-07, "lr_fusion": 8.410973564302534e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 71.75285497062578, "velocity_mse_step1": 0.012903157841979658, "velocity_mse_step2": 0.014515901815944482, "velocity_mse_step3": 0.020431740758275342} +{"batch_in_epoch": 10, "epoch": 11, "global_step": 560, "grad_norm_max": 0.0289306640625, "hidden_mse_step1": 0.01815730596925797, "hidden_mse_step2": 0.020116674655582756, "hidden_mse_step3": 0.02355195901223591, "latent_grad_l1_min": 0.00027167206644662656, "loss": 0.020730608051443205, "lr_blocks": 8.405079293933985e-07, "lr_fusion": 8.405079293933986e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 73.06951377038786, "velocity_mse_step1": 0.016011228184132573, "velocity_mse_step2": 0.01682217329341386, "velocity_mse_step3": 0.023175827277425145} +{"batch_in_epoch": 11, "epoch": 11, "global_step": 561, "grad_norm_max": 0.032958984375, "hidden_mse_step1": 0.020193059612730782, "hidden_mse_step2": 0.02252734509030623, "hidden_mse_step3": 0.025682490934351723, "latent_grad_l1_min": 0.0002092031754727941, "loss": 0.023613810672291683, "lr_blocks": 8.399176185490931e-07, "lr_fusion": 8.399176185490932e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 71.58520804673753, "velocity_mse_step1": 0.017302579105099927, "velocity_mse_step2": 0.019740695907135623, "velocity_mse_step3": 0.026957865247303352} +{"batch_in_epoch": 12, "epoch": 11, "global_step": 562, "grad_norm_max": 0.03271484375, "hidden_mse_step1": 0.021750775918397785, "hidden_mse_step2": 0.024715941806789488, "hidden_mse_step3": 0.028777131478169134, "latent_grad_l1_min": 0.00029282525065355003, "loss": 0.025383438682183623, "lr_blocks": 8.393264254295217e-07, "lr_fusion": 8.393264254295218e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 74.43731800486421, "velocity_mse_step1": 0.017326105421359116, "velocity_mse_step2": 0.020742420813933547, "velocity_mse_step3": 0.03055740236803623} +{"batch_in_epoch": 13, "epoch": 11, "global_step": 563, "grad_norm_max": 0.036865234375, "hidden_mse_step1": 0.02206702509595613, "hidden_mse_step2": 0.024795732219770016, "hidden_mse_step3": 0.02811023737636528, "latent_grad_l1_min": 0.0003819312096311478, "loss": 0.03128014995516943, "lr_blocks": 8.387343515691593e-07, "lr_fusion": 8.387343515691594e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 71.81132890837762, "velocity_mse_step1": 0.024939551433947466, "velocity_mse_step2": 0.027821429518683414, "velocity_mse_step3": 0.03358216711785644} +{"batch_in_epoch": 14, "epoch": 11, "global_step": 564, "grad_norm_max": 0.032470703125, "hidden_mse_step1": 0.016518876883700226, "hidden_mse_step2": 0.018206243253579096, "hidden_mse_step3": 0.021172530433562185, "latent_grad_l1_min": 0.00019077438082604203, "loss": 0.01695009008316057, "lr_blocks": 8.381413985047672e-07, "lr_fusion": 8.381413985047672e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 70.86844350011961, "velocity_mse_step1": 0.01142706709651975, "velocity_mse_step2": 0.013809038055894365, "velocity_mse_step3": 0.020024398499767164} +{"batch_in_epoch": 15, "epoch": 11, "global_step": 565, "grad_norm_max": 0.0306396484375, "hidden_mse_step1": 0.017427239591987536, "hidden_mse_step2": 0.019700439696732373, "hidden_mse_step3": 0.023649874542440684, "latent_grad_l1_min": 0.00025863119390123757, "loss": 0.019296273805334097, "lr_blocks": 8.37547567775388e-07, "lr_fusion": 8.375475677753882e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 74.85976515150833, "velocity_mse_step1": 0.012728748566587454, "velocity_mse_step2": 0.01583238390906315, "velocity_mse_step3": 0.023249931999349167} +{"batch_in_epoch": 16, "epoch": 11, "global_step": 566, "grad_norm_max": 0.03076171875, "hidden_mse_step1": 0.017893167798839774, "hidden_mse_step2": 0.019904998455396186, "hidden_mse_step3": 0.022818085812364837, "latent_grad_l1_min": 0.00018469001406629104, "loss": 0.01707637926613513, "lr_blocks": 8.369528609223428e-07, "lr_fusion": 8.36952860922343e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 71.47717333074979, "velocity_mse_step1": 0.011855566883501263, "velocity_mse_step2": 0.013897462768779534, "velocity_mse_step3": 0.019414481095736843} +{"batch_in_epoch": 17, "epoch": 11, "global_step": 567, "grad_norm_max": 0.042724609375, "hidden_mse_step1": 0.019888378579967787, "hidden_mse_step2": 0.02271402025196169, "hidden_mse_step3": 0.025421400787308812, "latent_grad_l1_min": 0.0001898157952382462, "loss": 0.01994943683634379, "lr_blocks": 8.363572794892266e-07, "lr_fusion": 8.363572794892268e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 72.85639861988602, "velocity_mse_step1": 0.013761385666189846, "velocity_mse_step2": 0.016228963300818574, "velocity_mse_step3": 0.02305557923890384} +{"batch_in_epoch": 18, "epoch": 11, "global_step": 568, "grad_norm_max": 0.031494140625, "hidden_mse_step1": 0.022316942798040276, "hidden_mse_step2": 0.023336560531918488, "hidden_mse_step3": 0.026165846618823707, "latent_grad_l1_min": 0.00023453143694496248, "loss": 0.024936889647506177, "lr_blocks": 8.357608250219046e-07, "lr_fusion": 8.357608250219046e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 73.42469829487527, "velocity_mse_step1": 0.02106898958196065, "velocity_mse_step2": 0.020458595003999238, "velocity_mse_step3": 0.026101147223796164} +{"batch_in_epoch": 19, "epoch": 11, "global_step": 569, "grad_norm_max": 0.032470703125, "hidden_mse_step1": 0.012431424264962385, "hidden_mse_step2": 0.015268119218360101, "hidden_mse_step3": 0.01898329430592379, "latent_grad_l1_min": 0.00019251738649472827, "loss": 0.014338584483734197, "lr_blocks": 8.351634990685078e-07, "lr_fusion": 8.35163499068508e-06, "peak_memory_gib": 48.320029735565186, "trajectory_time_s": 74.36693460174138, "velocity_mse_step1": 0.009724000287990617, "velocity_mse_step2": 0.011652410568785854, "velocity_mse_step3": 0.016971057156167393} +{"batch_in_epoch": 20, "epoch": 11, "global_step": 570, "grad_norm_max": 0.041748046875, "hidden_mse_step1": 0.015283294878567433, "hidden_mse_step2": 0.017677479831036177, "hidden_mse_step3": 0.020848658917072625, "latent_grad_l1_min": 0.0001594816021679435, "loss": 0.015215413445340734, "lr_blocks": 8.34565303179429e-07, "lr_fusion": 8.345653031794292e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 71.14350236213068, "velocity_mse_step1": 0.01077435924837898, "velocity_mse_step2": 0.011996212939266115, "velocity_mse_step3": 0.017494723940866864} +{"batch_in_epoch": 21, "epoch": 11, "global_step": 571, "grad_norm_max": 0.03173828125, "hidden_mse_step1": 0.015577969095569903, "hidden_mse_step2": 0.018428781959560832, "hidden_mse_step3": 0.021897945137295337, "latent_grad_l1_min": 0.00024706326621526387, "loss": 0.017429056153300083, "lr_blocks": 8.339662389073197e-07, "lr_fusion": 8.339662389073197e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 72.62472081537271, "velocity_mse_step1": 0.01153493121507511, "velocity_mse_step2": 0.01410824487434833, "velocity_mse_step3": 0.021053521461518745} +{"batch_in_epoch": 22, "epoch": 11, "global_step": 572, "grad_norm_max": 0.032470703125, "hidden_mse_step1": 0.020495596724296253, "hidden_mse_step2": 0.022691400272638666, "hidden_mse_step3": 0.02603619580622762, "latent_grad_l1_min": 0.00024308541742357193, "loss": 0.022273733846044967, "lr_blocks": 8.333663078070845e-07, "lr_fusion": 8.333663078070845e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 72.25736873636924, "velocity_mse_step1": 0.016517675294640606, "velocity_mse_step2": 0.018003337925100435, "velocity_mse_step3": 0.025377867244450104} +{"batch_in_epoch": 23, "epoch": 11, "global_step": 573, "grad_norm_max": 0.0272216796875, "hidden_mse_step1": 0.020941442065772465, "hidden_mse_step2": 0.02313195347752688, "hidden_mse_step3": 0.02597966803503888, "latent_grad_l1_min": 0.00024591083638370037, "loss": 0.023932598315046304, "lr_blocks": 8.327655114358781e-07, "lr_fusion": 8.327655114358782e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 71.88350794662256, "velocity_mse_step1": 0.018249122423834963, "velocity_mse_step2": 0.020215035395397405, "velocity_mse_step3": 0.026328328797327622} +{"batch_in_epoch": 24, "epoch": 11, "global_step": 574, "grad_norm_max": 0.0380859375, "hidden_mse_step1": 0.017444712716886503, "hidden_mse_step2": 0.018955338638209338, "hidden_mse_step3": 0.02156522515828588, "latent_grad_l1_min": 0.00018772784733300796, "loss": 0.01703515155324047, "lr_blocks": 8.321638513531018e-07, "lr_fusion": 8.321638513531018e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 71.52155701737502, "velocity_mse_step1": 0.013574229006605622, "velocity_mse_step2": 0.013542217192090382, "velocity_mse_step3": 0.018192479740329354} +{"batch_in_epoch": 25, "epoch": 11, "global_step": 575, "grad_norm_max": 0.026123046875, "hidden_mse_step1": 0.019850288068742623, "hidden_mse_step2": 0.021905664188255153, "hidden_mse_step3": 0.024788836848789027, "latent_grad_l1_min": 0.00021115801791893318, "loss": 0.020451951306313276, "lr_blocks": 8.315613291203976e-07, "lr_fusion": 8.315613291203977e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 74.00095170575514, "velocity_mse_step1": 0.015377326801951443, "velocity_mse_step2": 0.016287950823815275, "velocity_mse_step3": 0.02303609576275838} +{"batch_in_epoch": 26, "epoch": 11, "global_step": 576, "grad_norm_max": 0.029296875, "hidden_mse_step1": 0.019822798627761325, "hidden_mse_step2": 0.022207239351700988, "hidden_mse_step3": 0.025521824419099306, "latent_grad_l1_min": 0.00020690674864454195, "loss": 0.021483973009578352, "lr_blocks": 8.30957946301646e-07, "lr_fusion": 8.309579463016461e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 74.87455200287513, "velocity_mse_step1": 0.01542696879602902, "velocity_mse_step2": 0.017780652502551675, "velocity_mse_step3": 0.02448910942101585} +{"batch_in_epoch": 27, "epoch": 11, "global_step": 577, "grad_norm_max": 0.028076171875, "hidden_mse_step1": 0.016049075809340656, "hidden_mse_step2": 0.01943353210143479, "hidden_mse_step3": 0.023087480022305883, "latent_grad_l1_min": 0.000351647662682808, "loss": 0.024061593659488217, "lr_blocks": 8.30353704462961e-07, "lr_fusion": 8.303537044629611e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 74.97358206250647, "velocity_mse_step1": 0.019553353666976494, "velocity_mse_step2": 0.02137143603925194, "velocity_mse_step3": 0.02540298105200885} +{"batch_in_epoch": 28, "epoch": 11, "global_step": 578, "grad_norm_max": 0.03271484375, "hidden_mse_step1": 0.017763533770838485, "hidden_mse_step2": 0.019377422460820526, "hidden_mse_step3": 0.022234451291816577, "latent_grad_l1_min": 0.00019581899141485337, "loss": 0.017446636629756537, "lr_blocks": 8.297486051726862e-07, "lr_fusion": 8.297486051726864e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 73.50852805987233, "velocity_mse_step1": 0.013450319530550455, "velocity_mse_step2": 0.013783476764469274, "velocity_mse_step3": 0.019168571061787328} +{"batch_in_epoch": 29, "epoch": 11, "global_step": 579, "grad_norm_max": 0.0279541015625, "hidden_mse_step1": 0.016993566520146226, "hidden_mse_step2": 0.019814483721607497, "hidden_mse_step3": 0.023340280284173787, "latent_grad_l1_min": 0.00019360240185051225, "loss": 0.018175116149775152, "lr_blocks": 8.291426500013907e-07, "lr_fusion": 8.291426500013909e-06, "peak_memory_gib": 48.31737470626831, "trajectory_time_s": 75.00711433688411, "velocity_mse_step1": 0.012361335983899022, "velocity_mse_step2": 0.01453195756351176, "velocity_mse_step3": 0.02161722125518801} +{"batch_in_epoch": 30, "epoch": 11, "global_step": 580, "grad_norm_max": 0.0184326171875, "hidden_mse_step1": 0.01305795068453465, "hidden_mse_step2": 0.015787028848925338, "hidden_mse_step3": 0.019399680130715878, "latent_grad_l1_min": 0.00016306787438225, "loss": 0.013624323581877562, "lr_blocks": 8.285358405218654e-07, "lr_fusion": 8.285358405218655e-06, "peak_memory_gib": 48.31958723068237, "trajectory_time_s": 72.01826511949184, "velocity_mse_step1": 0.008944121260096185, "velocity_mse_step2": 0.010759629101292896, "velocity_mse_step3": 0.01634475325200973} +{"batch_in_epoch": 31, "epoch": 11, "global_step": 581, "grad_norm_max": 0.0279541015625, "hidden_mse_step1": 0.019755894717361246, "hidden_mse_step2": 0.021634285620945902, "hidden_mse_step3": 0.025302443470406746, "latent_grad_l1_min": 0.00024455750099150464, "loss": 0.02354295461139242, "lr_blocks": 8.279281783091181e-07, "lr_fusion": 8.279281783091182e-06, "peak_memory_gib": 48.31737470626831, "trajectory_time_s": 72.34345770686923, "velocity_mse_step1": 0.01771426249511673, "velocity_mse_step2": 0.01980998582439497, "velocity_mse_step3": 0.0264353505003133} +{"batch_in_epoch": 32, "epoch": 11, "global_step": 582, "grad_norm_max": 0.02685546875, "hidden_mse_step1": 0.01722820881488068, "hidden_mse_step2": 0.019765340190912993, "hidden_mse_step3": 0.02309786912519484, "latent_grad_l1_min": 0.00028289347574173007, "loss": 0.02139293705529001, "lr_blocks": 8.273196649403702e-07, "lr_fusion": 8.273196649403703e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 71.19266733086988, "velocity_mse_step1": 0.015360396090337805, "velocity_mse_step2": 0.018212644834420644, "velocity_mse_step3": 0.024596626354780576} +{"batch_in_epoch": 33, "epoch": 11, "global_step": 583, "grad_norm_max": 0.041259765625, "hidden_mse_step1": 0.0171778427528417, "hidden_mse_step2": 0.019811357288355272, "hidden_mse_step3": 0.0234867657335209, "latent_grad_l1_min": 0.00021123510305187665, "loss": 0.020101976585075523, "lr_blocks": 8.267103019950528e-07, "lr_fusion": 8.267103019950529e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 73.40196416475374, "velocity_mse_step1": 0.01391000819837375, "velocity_mse_step2": 0.01681128837766924, "velocity_mse_step3": 0.023537034958800565} +{"batch_in_epoch": 34, "epoch": 11, "global_step": 584, "grad_norm_max": 0.0257568359375, "hidden_mse_step1": 0.020617650256359147, "hidden_mse_step2": 0.022632389778404358, "hidden_mse_step3": 0.025747942438881313, "latent_grad_l1_min": 0.00026662002164812293, "loss": 0.02428092703589105, "lr_blocks": 8.261000910548009e-07, "lr_fusion": 8.26100091054801e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 71.35724297188426, "velocity_mse_step1": 0.018894592905001315, "velocity_mse_step2": 0.020321800448333042, "velocity_mse_step3": 0.02672658745099657} +{"batch_in_epoch": 35, "epoch": 11, "global_step": 585, "grad_norm_max": 0.02587890625, "hidden_mse_step1": 0.013216458152913087, "hidden_mse_step2": 0.015834995453977693, "hidden_mse_step3": 0.019493900937959552, "latent_grad_l1_min": 0.00017115903210651595, "loss": 0.013286051074309009, "lr_blocks": 8.254890337034519e-07, "lr_fusion": 8.25489033703452e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 71.63072283648216, "velocity_mse_step1": 0.008654925807994525, "velocity_mse_step2": 0.010117745139203698, "velocity_mse_step3": 0.01623094536729955} +{"batch_in_epoch": 36, "epoch": 11, "global_step": 586, "grad_norm_max": 0.033447265625, "hidden_mse_step1": 0.02186234632972628, "hidden_mse_step2": 0.024497043540967364, "hidden_mse_step3": 0.027832619263790548, "latent_grad_l1_min": 0.0002849182674253825, "loss": 0.02699131260825587, "lr_blocks": 8.248771315270391e-07, "lr_fusion": 8.248771315270391e-06, "peak_memory_gib": 48.320029735565186, "trajectory_time_s": 72.57267673575552, "velocity_mse_step1": 0.01983765442024118, "velocity_mse_step2": 0.022895931706963375, "velocity_mse_step3": 0.030821148389285163} +{"batch_in_epoch": 37, "epoch": 11, "global_step": 587, "grad_norm_max": 0.0361328125, "hidden_mse_step1": 0.019390616879848364, "hidden_mse_step2": 0.022083102475984818, "hidden_mse_step3": 0.025669333125863756, "latent_grad_l1_min": 0.00026874920149566606, "loss": 0.02365473404851011, "lr_blocks": 8.24264386113789e-07, "lr_fusion": 8.242643861137891e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 74.25209512362926, "velocity_mse_step1": 0.016828418216651438, "velocity_mse_step2": 0.020036404846385785, "velocity_mse_step3": 0.027385071381494132} +{"batch_in_epoch": 38, "epoch": 11, "global_step": 588, "grad_norm_max": 0.02880859375, "hidden_mse_step1": 0.01025379292384189, "hidden_mse_step2": 0.0127201843923623, "hidden_mse_step3": 0.016168367816135287, "latent_grad_l1_min": 0.0001711848144623218, "loss": 0.009708792174933477, "lr_blocks": 8.236507990541168e-07, "lr_fusion": 8.23650799054117e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 71.8225426247518, "velocity_mse_step1": 0.0059742169936985836, "velocity_mse_step2": 0.007479044874863965, "velocity_mse_step3": 0.011758879284440938} +{"batch_in_epoch": 39, "epoch": 11, "global_step": 589, "grad_norm_max": 0.031982421875, "hidden_mse_step1": 0.022040340941332812, "hidden_mse_step2": 0.02400854949207444, "hidden_mse_step3": 0.026647829966220473, "latent_grad_l1_min": 0.00021259392633510288, "loss": 0.023441572889818678, "lr_blocks": 8.230363719406223e-07, "lr_fusion": 8.230363719406223e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 73.81607678900036, "velocity_mse_step1": 0.017254488804610443, "velocity_mse_step2": 0.01952297479147091, "velocity_mse_step3": 0.02627758151252887} +{"batch_in_epoch": 40, "epoch": 11, "global_step": 590, "grad_norm_max": 0.02783203125, "hidden_mse_step1": 0.016548639903443733, "hidden_mse_step2": 0.019961207115557045, "hidden_mse_step3": 0.024087495478202717, "latent_grad_l1_min": 0.00020955481159035116, "loss": 0.019722927241153747, "lr_blocks": 8.224211063680852e-07, "lr_fusion": 8.224211063680854e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 72.41243408574519, "velocity_mse_step1": 0.013146349048059036, "velocity_mse_step2": 0.01638942418503575, "velocity_mse_step3": 0.02357327293637874} +{"batch_in_epoch": 41, "epoch": 11, "global_step": 591, "grad_norm_max": 0.02587890625, "hidden_mse_step1": 0.01461470273456403, "hidden_mse_step2": 0.016509299016823725, "hidden_mse_step3": 0.019999587259787534, "latent_grad_l1_min": 0.00019016239912161836, "loss": 0.013930630138409988, "lr_blocks": 8.218050039334623e-07, "lr_fusion": 8.218050039334624e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 75.70758437373297, "velocity_mse_step1": 0.009347440764291344, "velocity_mse_step2": 0.010709623727182458, "velocity_mse_step3": 0.016622465947875753} +{"batch_in_epoch": 42, "epoch": 11, "global_step": 592, "grad_norm_max": 0.03515625, "hidden_mse_step1": 0.01596156778811876, "hidden_mse_step2": 0.018925326926234578, "hidden_mse_step3": 0.023399058912348537, "latent_grad_l1_min": 0.00020331941414042376, "loss": 0.018732897604682615, "lr_blocks": 8.211880662358816e-07, "lr_fusion": 8.211880662358818e-06, "peak_memory_gib": 48.31958723068237, "trajectory_time_s": 75.43460914212483, "velocity_mse_step1": 0.012589205989830328, "velocity_mse_step2": 0.01497235317115805, "velocity_mse_step3": 0.022808536421507594} +{"batch_in_epoch": 43, "epoch": 11, "global_step": 593, "grad_norm_max": 0.02880859375, "hidden_mse_step1": 0.022302410241016847, "hidden_mse_step2": 0.02398164803162217, "hidden_mse_step3": 0.026873993155147345, "latent_grad_l1_min": 0.00022173066463437863, "loss": 0.02377771310109113, "lr_blocks": 8.2057029487664e-07, "lr_fusion": 8.2057029487664e-06, "peak_memory_gib": 48.316489696502686, "trajectory_time_s": 74.13711555700138, "velocity_mse_step1": 0.01747269438263694, "velocity_mse_step2": 0.01981815058804516, "velocity_mse_step3": 0.026726487067727636} +{"batch_in_epoch": 44, "epoch": 11, "global_step": 594, "grad_norm_max": 0.0279541015625, "hidden_mse_step1": 0.020656695215231072, "hidden_mse_step2": 0.023059665799207454, "hidden_mse_step3": 0.027275434046584578, "latent_grad_l1_min": 0.00024810284048726317, "loss": 0.02211028686308834, "lr_blocks": 8.199516914591975e-07, "lr_fusion": 8.199516914591975e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 72.07153504500457, "velocity_mse_step1": 0.015826334752741138, "velocity_mse_step2": 0.017216682172147557, "velocity_mse_step3": 0.026188662283987335} +{"batch_in_epoch": 45, "epoch": 11, "global_step": 595, "grad_norm_max": 0.024658203125, "hidden_mse_step1": 0.024477301148830777, "hidden_mse_step2": 0.025791444965372126, "hidden_mse_step3": 0.028725142141671052, "latent_grad_l1_min": 0.0002736520364123862, "loss": 0.02590802262836535, "lr_blocks": 8.193322575891739e-07, "lr_fusion": 8.19332257589174e-06, "peak_memory_gib": 48.31737470626831, "trajectory_time_s": 72.03151958301896, "velocity_mse_step1": 0.018840211110987833, "velocity_mse_step2": 0.021155939661964242, "velocity_mse_step3": 0.02982852599650089} +{"batch_in_epoch": 46, "epoch": 11, "global_step": 596, "grad_norm_max": 0.0255126953125, "hidden_mse_step1": 0.012005877681076527, "hidden_mse_step2": 0.01448102366079443, "hidden_mse_step3": 0.017708741976613443, "latent_grad_l1_min": 0.0001463199787394842, "loss": 0.012155154474645054, "lr_blocks": 8.187119948743449e-07, "lr_fusion": 8.18711994874345e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 71.61833172636398, "velocity_mse_step1": 0.007768043564283289, "velocity_mse_step2": 0.009659616480348632, "velocity_mse_step3": 0.014618238238784083} +{"batch_in_epoch": 47, "epoch": 11, "global_step": 597, "grad_norm_max": 0.0223388671875, "hidden_mse_step1": 0.014131101176774661, "hidden_mse_step2": 0.01673068235894399, "hidden_mse_step3": 0.020237423695756922, "latent_grad_l1_min": 0.00019778194109676406, "loss": 0.015013846714282408, "lr_blocks": 8.180909049246369e-07, "lr_fusion": 8.18090904924637e-06, "peak_memory_gib": 48.32135725021362, "trajectory_time_s": 71.37583412212553, "velocity_mse_step1": 0.00990377271955367, "velocity_mse_step2": 0.01202089110198098, "velocity_mse_step3": 0.01800695413424234} +{"batch_in_epoch": 48, "epoch": 11, "global_step": 598, "grad_norm_max": 0.035400390625, "hidden_mse_step1": 0.01804294637154921, "hidden_mse_step2": 0.021291177046285675, "hidden_mse_step3": 0.02455419222159045, "latent_grad_l1_min": 0.00024967355602711905, "loss": 0.021289969969075166, "lr_blocks": 8.174689893521238e-07, "lr_fusion": 8.174689893521239e-06, "peak_memory_gib": 48.31958723068237, "trajectory_time_s": 71.34560351675464, "velocity_mse_step1": 0.01500842729832844, "velocity_mse_step2": 0.017812141400229718, "velocity_mse_step3": 0.024660507092318894} +{"batch_in_epoch": 49, "epoch": 11, "global_step": 599, "grad_norm_max": 0.026611328125, "hidden_mse_step1": 0.016935231879220476, "hidden_mse_step2": 0.01963362661522946, "hidden_mse_step3": 0.023257778458563347, "latent_grad_l1_min": 0.0002719341791816987, "loss": 0.01871690689586103, "lr_blocks": 8.168462497710226e-07, "lr_fusion": 8.168462497710227e-06, "peak_memory_gib": 48.3169322013855, "trajectory_time_s": 71.16034198923444, "velocity_mse_step1": 0.012665428239935342, "velocity_mse_step2": 0.01530682765379814, "velocity_mse_step3": 0.022195799534009502} +{"batch_in_epoch": 50, "epoch": 11, "global_step": 600, "grad_norm_max": 0.0245361328125, "hidden_mse_step1": 0.014815086958281296, "hidden_mse_step2": 0.01752794038787085, "hidden_mse_step3": 0.021356444647868296, "latent_grad_l1_min": 0.00022732777051714947, "loss": 0.01651827023098511, "lr_blocks": 8.162226877976886e-07, "lr_fusion": 8.162226877976886e-06, "peak_memory_gib": 48.31737470626831, "trajectory_time_s": 73.50159668562264, "velocity_mse_step1": 0.010679505268594116, "velocity_mse_step2": 0.013410387181335992, "velocity_mse_step3": 0.020094969334812567} +{"batch_in_epoch": 1, "epoch": 12, "global_step": 601, "grad_norm_max": 0.0306396484375, "hidden_mse_step1": 0.015431440738861315, "hidden_mse_step2": 0.017241413455589542, "hidden_mse_step3": 0.020113902831716195, "latent_grad_l1_min": 0.00013854089593223762, "loss": 0.01497124331321434, "lr_blocks": 8.155983050506121e-07, "lr_fusion": 8.155983050506123e-06, "peak_memory_gib": 48.31737470626831, "trajectory_time_s": 71.5929059030168, "velocity_mse_step1": 0.010784418750388014, "velocity_mse_step2": 0.011935327906810145, "velocity_mse_step3": 0.016915306357467283} +{"batch_in_epoch": 2, "epoch": 12, "global_step": 602, "grad_norm_max": 0.023681640625, "hidden_mse_step1": 0.016440853027493825, "hidden_mse_step2": 0.01901164266746491, "hidden_mse_step3": 0.022888836633813168, "latent_grad_l1_min": 0.00024003815997275524, "loss": 0.018185061469141926, "lr_blocks": 8.149731031504134e-07, "lr_fusion": 8.149731031504136e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 72.5134376615024, "velocity_mse_step1": 0.011987607121617266, "velocity_mse_step2": 0.014696957580911527, "velocity_mse_step3": 0.022036485024727877} +{"batch_in_epoch": 3, "epoch": 12, "global_step": 603, "grad_norm_max": 0.032958984375, "hidden_mse_step1": 0.017491398720137243, "hidden_mse_step2": 0.020422714188628427, "hidden_mse_step3": 0.023808895635219023, "latent_grad_l1_min": 0.00027054322072217474, "loss": 0.02068447954038025, "lr_blocks": 8.143470837198393e-07, "lr_fusion": 8.143470837198394e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 71.86448730473785, "velocity_mse_step1": 0.014419057391933164, "velocity_mse_step2": 0.017323695767637606, "velocity_mse_step3": 0.024138382515437633} +{"batch_in_epoch": 4, "epoch": 12, "global_step": 604, "grad_norm_max": 0.0308837890625, "hidden_mse_step1": 0.017406122151961814, "hidden_mse_step2": 0.02054217019550768, "hidden_mse_step3": 0.02421548373864165, "latent_grad_l1_min": 0.0001777231736923568, "loss": 0.02024136503210424, "lr_blocks": 8.137202483837583e-07, "lr_fusion": 8.137202483837584e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 72.68906617562607, "velocity_mse_step1": 0.01457843516878451, "velocity_mse_step2": 0.016807988518849015, "velocity_mse_step3": 0.02312129295231508} +{"batch_in_epoch": 5, "epoch": 12, "global_step": 605, "grad_norm_max": 0.037353515625, "hidden_mse_step1": 0.014985255824285558, "hidden_mse_step2": 0.017037405249928788, "hidden_mse_step3": 0.020178178366872346, "latent_grad_l1_min": 0.0001566133269079728, "loss": 0.014529829805334366, "lr_blocks": 8.130925987691568e-07, "lr_fusion": 8.13092598769157e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 72.52430444662605, "velocity_mse_step1": 0.009985997802557125, "velocity_mse_step2": 0.011416214080652154, "velocity_mse_step3": 0.016967192067698176} +{"batch_in_epoch": 6, "epoch": 12, "global_step": 606, "grad_norm_max": 0.033447265625, "hidden_mse_step1": 0.01815122483198398, "hidden_mse_step2": 0.021183972754183093, "hidden_mse_step3": 0.025752079107665585, "latent_grad_l1_min": 0.00024214198219851824, "loss": 0.021756355176746314, "lr_blocks": 8.124641365051345e-07, "lr_fusion": 8.124641365051346e-06, "peak_memory_gib": 48.3169322013855, "trajectory_time_s": 70.93749535413372, "velocity_mse_step1": 0.014363365848631864, "velocity_mse_step2": 0.017683118093243265, "velocity_mse_step3": 0.026713852321596017} +{"batch_in_epoch": 7, "epoch": 12, "global_step": 607, "grad_norm_max": 0.0230712890625, "hidden_mse_step1": 0.013717775386924456, "hidden_mse_step2": 0.015940746358994926, "hidden_mse_step3": 0.019213083906964, "latent_grad_l1_min": 0.00020013321500300663, "loss": 0.013785726785759574, "lr_blocks": 8.118348632229006e-07, "lr_fusion": 8.118348632229006e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 73.03467115549574, "velocity_mse_step1": 0.009877133791243456, "velocity_mse_step2": 0.01067811764369253, "velocity_mse_step3": 0.01591476735692205} +{"batch_in_epoch": 8, "epoch": 12, "global_step": 608, "grad_norm_max": 0.0230712890625, "hidden_mse_step1": 0.019747853462052132, "hidden_mse_step2": 0.02246518729121557, "hidden_mse_step3": 0.026990466551589112, "latent_grad_l1_min": 0.0002555562350607943, "loss": 0.021404310088005977, "lr_blocks": 8.112047805557692e-07, "lr_fusion": 8.112047805557693e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 71.32385739999154, "velocity_mse_step1": 0.014764553661474824, "velocity_mse_step2": 0.01681455151577081, "velocity_mse_step3": 0.025713472361011163} +{"batch_in_epoch": 9, "epoch": 12, "global_step": 609, "grad_norm_max": 0.033447265625, "hidden_mse_step1": 0.018473359282195033, "hidden_mse_step2": 0.021092210663482547, "hidden_mse_step3": 0.024995008816144297, "latent_grad_l1_min": 0.00025676841414679075, "loss": 0.021215775761187876, "lr_blocks": 8.105738901391551e-07, "lr_fusion": 8.105738901391553e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 71.58156514050643, "velocity_mse_step1": 0.013814957303111441, "velocity_mse_step2": 0.017622963820550855, "velocity_mse_step3": 0.02575334679568186} +{"batch_in_epoch": 10, "epoch": 12, "global_step": 610, "grad_norm_max": 0.0458984375, "hidden_mse_step1": 0.021731326148645688, "hidden_mse_step2": 0.023613364211217097, "hidden_mse_step3": 0.02601033361029944, "latent_grad_l1_min": 0.00032713979089749046, "loss": 0.027539081472371305, "lr_blocks": 8.099421936105702e-07, "lr_fusion": 8.099421936105702e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 72.00458051924943, "velocity_mse_step1": 0.022935639976432345, "velocity_mse_step2": 0.02379654058105578, "velocity_mse_step3": 0.028749558821021176} +{"batch_in_epoch": 11, "epoch": 12, "global_step": 611, "grad_norm_max": 0.028076171875, "hidden_mse_step1": 0.015623592164567006, "hidden_mse_step2": 0.01783173075611038, "hidden_mse_step3": 0.02054937308587666, "latent_grad_l1_min": 0.00016885073910088977, "loss": 0.01559426903259009, "lr_blocks": 8.093096926096177e-07, "lr_fusion": 8.093096926096178e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 71.4131609383694, "velocity_mse_step1": 0.010209573804916414, "velocity_mse_step2": 0.012707671834505163, "velocity_mse_step3": 0.018465090825754614} +{"batch_in_epoch": 12, "epoch": 12, "global_step": 612, "grad_norm_max": 0.030517578125, "hidden_mse_step1": 0.01315205222428111, "hidden_mse_step2": 0.015458315269955034, "hidden_mse_step3": 0.01885571460505681, "latent_grad_l1_min": 0.0001843088421082939, "loss": 0.014886774642426251, "lr_blocks": 8.086763887779899e-07, "lr_fusion": 8.0867638877799e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 71.27483831650898, "velocity_mse_step1": 0.010522890586539036, "velocity_mse_step2": 0.012061886464445186, "velocity_mse_step3": 0.017328937063991492} +{"batch_in_epoch": 13, "epoch": 12, "global_step": 613, "grad_norm_max": 0.0306396484375, "hidden_mse_step1": 0.01448875103129207, "hidden_mse_step2": 0.016460036626085636, "hidden_mse_step3": 0.020281733994904374, "latent_grad_l1_min": 0.0001771249835655908, "loss": 0.013923924786338051, "lr_blocks": 8.080422837594625e-07, "lr_fusion": 8.080422837594627e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 71.09202203511086, "velocity_mse_step1": 0.00898709640646952, "velocity_mse_step2": 0.01059787396141993, "velocity_mse_step3": 0.0170637507191194} +{"batch_in_epoch": 14, "epoch": 12, "global_step": 614, "grad_norm_max": 0.03271484375, "hidden_mse_step1": 0.020107959230829562, "hidden_mse_step2": 0.02313387265894562, "hidden_mse_step3": 0.026093399135528928, "latent_grad_l1_min": 0.0002168773535231594, "loss": 0.02302285383588501, "lr_blocks": 8.074073791998905e-07, "lr_fusion": 8.074073791998907e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 72.01728216450283, "velocity_mse_step1": 0.01569617835671774, "velocity_mse_step2": 0.019618121133784628, "velocity_mse_step3": 0.026820737219947786} +{"batch_in_epoch": 15, "epoch": 12, "global_step": 615, "grad_norm_max": 0.033935546875, "hidden_mse_step1": 0.0157162194830432, "hidden_mse_step2": 0.018017799668346664, "hidden_mse_step3": 0.021037268036577315, "latent_grad_l1_min": 0.00019328223334014183, "loss": 0.015447037252930125, "lr_blocks": 8.067716767472044e-07, "lr_fusion": 8.067716767472045e-06, "peak_memory_gib": 48.31958723068237, "trajectory_time_s": 71.68823130337842, "velocity_mse_step1": 0.010342798840221283, "velocity_mse_step2": 0.012314381907344794, "velocity_mse_step3": 0.018206800596090034} +{"batch_in_epoch": 16, "epoch": 12, "global_step": 616, "grad_norm_max": 0.036376953125, "hidden_mse_step1": 0.021092528643618734, "hidden_mse_step2": 0.023478836602797464, "hidden_mse_step3": 0.025644317053125372, "latent_grad_l1_min": 0.00034202982897113543, "loss": 0.02699825329805857, "lr_blocks": 8.061351780514056e-07, "lr_fusion": 8.061351780514057e-06, "peak_memory_gib": 48.316489696502686, "trajectory_time_s": 74.37123070248344, "velocity_mse_step1": 0.02264054710394703, "velocity_mse_step2": 0.023709773002857606, "velocity_mse_step3": 0.02762286948771881} +{"batch_in_epoch": 17, "epoch": 12, "global_step": 617, "grad_norm_max": 0.0272216796875, "hidden_mse_step1": 0.011814400408184154, "hidden_mse_step2": 0.014864992366970649, "hidden_mse_step3": 0.01886907515082774, "latent_grad_l1_min": 0.0002510669564799173, "loss": 0.015000623068772255, "lr_blocks": 8.054978847645621e-07, "lr_fusion": 8.054978847645623e-06, "peak_memory_gib": 48.31737470626831, "trajectory_time_s": 71.32901159375615, "velocity_mse_step1": 0.010617820243559046, "velocity_mse_step2": 0.012724543909176385, "velocity_mse_step3": 0.017104657466656393} +{"batch_in_epoch": 18, "epoch": 12, "global_step": 618, "grad_norm_max": 0.0269775390625, "hidden_mse_step1": 0.013508871060496728, "hidden_mse_step2": 0.016225147859326428, "hidden_mse_step3": 0.020106674188614955, "latent_grad_l1_min": 0.00017762368679541396, "loss": 0.014526202742542538, "lr_blocks": 8.048597985408047e-07, "lr_fusion": 8.048597985408047e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 71.67341188550199, "velocity_mse_step1": 0.009700013686337373, "velocity_mse_step2": 0.011448471960777949, "velocity_mse_step3": 0.01744605182154503} +{"batch_in_epoch": 19, "epoch": 12, "global_step": 619, "grad_norm_max": 0.028564453125, "hidden_mse_step1": 0.019844458218098483, "hidden_mse_step2": 0.02133193499841062, "hidden_mse_step3": 0.02426137035113892, "latent_grad_l1_min": 0.00017471957653469872, "loss": 0.020556754939856803, "lr_blocks": 8.042209210363216e-07, "lr_fusion": 8.042209210363216e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 71.35035516099742, "velocity_mse_step1": 0.015470879135786422, "velocity_mse_step2": 0.01645222328287283, "velocity_mse_step3": 0.0232033841145624} +{"batch_in_epoch": 20, "epoch": 12, "global_step": 620, "grad_norm_max": 0.026123046875, "hidden_mse_step1": 0.019475925356216194, "hidden_mse_step2": 0.021150712256452868, "hidden_mse_step3": 0.024013479107192582, "latent_grad_l1_min": 0.0002273258924105903, "loss": 0.018671677165132544, "lr_blocks": 8.035812539093556e-07, "lr_fusion": 8.035812539093557e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 71.50400418351637, "velocity_mse_step1": 0.01363241486667123, "velocity_mse_step2": 0.014522946572729518, "velocity_mse_step3": 0.021395656978711482} +{"batch_in_epoch": 21, "epoch": 12, "global_step": 621, "grad_norm_max": 0.03515625, "hidden_mse_step1": 0.021035095892979628, "hidden_mse_step2": 0.022925782948732376, "hidden_mse_step3": 0.025745393741609793, "latent_grad_l1_min": 0.00027826500809169374, "loss": 0.02381916792364791, "lr_blocks": 8.029407988201985e-07, "lr_fusion": 8.029407988201986e-06, "peak_memory_gib": 48.320029735565186, "trajectory_time_s": 71.71917736963223, "velocity_mse_step1": 0.01818276484846137, "velocity_mse_step2": 0.02014987281290814, "velocity_mse_step3": 0.026154237682931125} +{"batch_in_epoch": 22, "epoch": 12, "global_step": 622, "grad_norm_max": 0.025634765625, "hidden_mse_step1": 0.01841226642552231, "hidden_mse_step2": 0.02085019766153501, "hidden_mse_step3": 0.02449926607576864, "latent_grad_l1_min": 0.000223183023990714, "loss": 0.019744886193490987, "lr_blocks": 8.022995574311875e-07, "lr_fusion": 8.022995574311876e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 71.84912585112033, "velocity_mse_step1": 0.013370325992582364, "velocity_mse_step2": 0.016075938071090992, "velocity_mse_step3": 0.023412220254873058} +{"batch_in_epoch": 23, "epoch": 12, "global_step": 623, "grad_norm_max": 0.028076171875, "hidden_mse_step1": 0.017360780845462744, "hidden_mse_step2": 0.019472004903946072, "hidden_mse_step3": 0.02229259442538023, "latent_grad_l1_min": 0.00021822926737513626, "loss": 0.018063016551812843, "lr_blocks": 8.016575314067004e-07, "lr_fusion": 8.016575314067005e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 71.65617431650026, "velocity_mse_step1": 0.013106917870962726, "velocity_mse_step2": 0.014702772846378918, "velocity_mse_step3": 0.020466819442974935} +{"batch_in_epoch": 24, "epoch": 12, "global_step": 624, "grad_norm_max": 0.0245361328125, "hidden_mse_step1": 0.015135454974370076, "hidden_mse_step2": 0.01712873227162553, "hidden_mse_step3": 0.020488157801862274, "latent_grad_l1_min": 0.00016852668159117457, "loss": 0.015521230645910171, "lr_blocks": 8.010147224131522e-07, "lr_fusion": 8.010147224131523e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 72.84834576299181, "velocity_mse_step1": 0.010964472186710086, "velocity_mse_step2": 0.012542780020989344, "velocity_mse_step3": 0.017781203891250437} +{"batch_in_epoch": 25, "epoch": 12, "global_step": 625, "grad_norm_max": 0.0303955078125, "hidden_mse_step1": 0.013820755341190046, "hidden_mse_step2": 0.01677638730533155, "hidden_mse_step3": 0.02074920472555927, "latent_grad_l1_min": 0.0002183568158216076, "loss": 0.015327434999302827, "lr_blocks": 8.003711321189895e-07, "lr_fusion": 8.003711321189895e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 74.23359881324723, "velocity_mse_step1": 0.00967372186797937, "velocity_mse_step2": 0.0122837336509422, "velocity_mse_step3": 0.018890213025900136} +{"batch_in_epoch": 26, "epoch": 12, "global_step": 626, "grad_norm_max": 0.037353515625, "hidden_mse_step1": 0.022688135163792006, "hidden_mse_step2": 0.025104349545602287, "hidden_mse_step3": 0.028705504722893238, "latent_grad_l1_min": 0.0002992380359501112, "loss": 0.02728643512818962, "lr_blocks": 7.99726762194687e-07, "lr_fusion": 7.997267621946871e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 73.16839320000145, "velocity_mse_step1": 0.019542568076368685, "velocity_mse_step2": 0.022823838018147004, "velocity_mse_step3": 0.031843097730805826} +{"batch_in_epoch": 27, "epoch": 12, "global_step": 627, "grad_norm_max": 0.03173828125, "hidden_mse_step1": 0.0199054004208717, "hidden_mse_step2": 0.021816931831251298, "hidden_mse_step3": 0.024599909083917734, "latent_grad_l1_min": 0.00020258407494111452, "loss": 0.020413062593434003, "lr_blocks": 7.990816143127431e-07, "lr_fusion": 7.990816143127431e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 73.9811981611274, "velocity_mse_step1": 0.014941983380205264, "velocity_mse_step2": 0.016574815497733653, "velocity_mse_step3": 0.02309016231447458} +{"batch_in_epoch": 28, "epoch": 12, "global_step": 628, "grad_norm_max": 0.0260009765625, "hidden_mse_step1": 0.01817025680793449, "hidden_mse_step2": 0.02057445029328976, "hidden_mse_step3": 0.024529763563935245, "latent_grad_l1_min": 0.0002483052376192063, "loss": 0.02001028896276174, "lr_blocks": 7.984356901476755e-07, "lr_fusion": 7.984356901476755e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 71.77503059199807, "velocity_mse_step1": 0.013732261311296108, "velocity_mse_step2": 0.016018244184254266, "velocity_mse_step3": 0.023952912214943868} +{"batch_in_epoch": 29, "epoch": 12, "global_step": 629, "grad_norm_max": 0.0245361328125, "hidden_mse_step1": 0.01621620531243804, "hidden_mse_step2": 0.019164254863946035, "hidden_mse_step3": 0.022708094695449935, "latent_grad_l1_min": 0.00025626783371990314, "loss": 0.02053545635759032, "lr_blocks": 7.977889913760163e-07, "lr_fusion": 7.977889913760163e-06, "peak_memory_gib": 48.3169322013855, "trajectory_time_s": 72.96231809238816, "velocity_mse_step1": 0.015054721740723052, "velocity_mse_step2": 0.017786119465849227, "velocity_mse_step3": 0.022956671958257045} +{"batch_in_epoch": 30, "epoch": 12, "global_step": 630, "grad_norm_max": 0.0322265625, "hidden_mse_step1": 0.01454664738100421, "hidden_mse_step2": 0.018504389927589467, "hidden_mse_step3": 0.023789652589974657, "latent_grad_l1_min": 0.0002963587594422279, "loss": 0.017832806126014993, "lr_blocks": 7.971415196763087e-07, "lr_fusion": 7.971415196763088e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 71.08874687576463, "velocity_mse_step1": 0.010835577081057375, "velocity_mse_step2": 0.014207363353177373, "velocity_mse_step3": 0.022771407700409845} +{"batch_in_epoch": 31, "epoch": 12, "global_step": 631, "grad_norm_max": 0.02783203125, "hidden_mse_step1": 0.017495385013587238, "hidden_mse_step2": 0.02048355548842145, "hidden_mse_step3": 0.02421363917112882, "latent_grad_l1_min": 0.000204945666155254, "loss": 0.018604424000451602, "lr_blocks": 7.964932767291018e-07, "lr_fusion": 7.964932767291019e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 73.4793171336205, "velocity_mse_step1": 0.01135032684708546, "velocity_mse_step2": 0.014725552327165912, "velocity_mse_step3": 0.02351813333890667} +{"batch_in_epoch": 32, "epoch": 12, "global_step": 632, "grad_norm_max": 0.026611328125, "hidden_mse_step1": 0.019180465094645376, "hidden_mse_step2": 0.021942321535399448, "hidden_mse_step3": 0.025746721608032073, "latent_grad_l1_min": 0.00024386261247855145, "loss": 0.022176539458866627, "lr_blocks": 7.958442642169468e-07, "lr_fusion": 7.958442642169469e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 73.03926512162434, "velocity_mse_step1": 0.016399639201283983, "velocity_mse_step2": 0.01833658221377326, "velocity_mse_step3": 0.02510644449752622} +{"batch_in_epoch": 33, "epoch": 12, "global_step": 633, "grad_norm_max": 0.0277099609375, "hidden_mse_step1": 0.01724685560787163, "hidden_mse_step2": 0.019361057397743155, "hidden_mse_step3": 0.022015457773315053, "latent_grad_l1_min": 0.00022979520690569188, "loss": 0.020532062719991827, "lr_blocks": 7.951944838243916e-07, "lr_fusion": 7.951944838243916e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 71.93529984464112, "velocity_mse_step1": 0.01594127277816629, "velocity_mse_step2": 0.017347197499475442, "velocity_mse_step3": 0.022445378396826397} +{"batch_in_epoch": 34, "epoch": 12, "global_step": 634, "grad_norm_max": 0.04443359375, "hidden_mse_step1": 0.023583713611255268, "hidden_mse_step2": 0.025588353763201406, "hidden_mse_step3": 0.028447662646483095, "latent_grad_l1_min": 0.00022593023459194228, "loss": 0.023901847016531974, "lr_blocks": 7.94543937237978e-07, "lr_fusion": 7.945439372379782e-06, "peak_memory_gib": 48.3169322013855, "trajectory_time_s": 71.73070360376005, "velocity_mse_step1": 0.016999986434322115, "velocity_mse_step2": 0.01951402671069705, "velocity_mse_step3": 0.027429552915106927} +{"batch_in_epoch": 35, "epoch": 12, "global_step": 635, "grad_norm_max": 0.032470703125, "hidden_mse_step1": 0.015424242035286235, "hidden_mse_step2": 0.017234355943011384, "hidden_mse_step3": 0.02036672900430858, "latent_grad_l1_min": 0.0001956726537173381, "loss": 0.016698134281406443, "lr_blocks": 7.938926261462365e-07, "lr_fusion": 7.938926261462366e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 74.46274889362394, "velocity_mse_step1": 0.012345906174492223, "velocity_mse_step2": 0.013622014388342256, "velocity_mse_step3": 0.018823948223143816} +{"batch_in_epoch": 36, "epoch": 12, "global_step": 636, "grad_norm_max": 0.0263671875, "hidden_mse_step1": 0.018365466227156242, "hidden_mse_step2": 0.02139330084901303, "hidden_mse_step3": 0.025057696753979793, "latent_grad_l1_min": 0.00027628563657344785, "loss": 0.022829332642556564, "lr_blocks": 7.932405522396812e-07, "lr_fusion": 7.932405522396814e-06, "peak_memory_gib": 48.31737470626831, "trajectory_time_s": 72.09716389450477, "velocity_mse_step1": 0.016519606100960767, "velocity_mse_step2": 0.01921546488301828, "velocity_mse_step3": 0.026271279655962383} +{"batch_in_epoch": 37, "epoch": 12, "global_step": 637, "grad_norm_max": 0.036376953125, "hidden_mse_step1": 0.027588910722572888, "hidden_mse_step2": 0.029909641573405154, "hidden_mse_step3": 0.03263705988813723, "latent_grad_l1_min": 0.0003557071995601291, "loss": 0.03615805337904021, "lr_blocks": 7.925877172108067e-07, "lr_fusion": 7.925877172108067e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 75.08827239237144, "velocity_mse_step1": 0.03046596826918955, "velocity_mse_step2": 0.031414463018466315, "velocity_mse_step3": 0.037580163725319186} +{"batch_in_epoch": 38, "epoch": 12, "global_step": 638, "grad_norm_max": 0.0262451171875, "hidden_mse_step1": 0.0159946882506899, "hidden_mse_step2": 0.01867836236488074, "hidden_mse_step3": 0.022540309715883007, "latent_grad_l1_min": 0.000217118355067214, "loss": 0.01903256835898251, "lr_blocks": 7.919341227540828e-07, "lr_fusion": 7.919341227540828e-06, "peak_memory_gib": 48.31958723068237, "trajectory_time_s": 73.0137789177461, "velocity_mse_step1": 0.01297659908803845, "velocity_mse_step2": 0.015874145336316096, "velocity_mse_step3": 0.022525622617519857} +{"batch_in_epoch": 39, "epoch": 12, "global_step": 639, "grad_norm_max": 0.0308837890625, "hidden_mse_step1": 0.016471830595816885, "hidden_mse_step2": 0.0188174619023422, "hidden_mse_step3": 0.022093619037020416, "latent_grad_l1_min": 0.00019185604287486058, "loss": 0.018676882593094236, "lr_blocks": 7.912797705659506e-07, "lr_fusion": 7.912797705659508e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 72.84278296036064, "velocity_mse_step1": 0.01328157416927362, "velocity_mse_step2": 0.01538370387113121, "velocity_mse_step3": 0.021627076884864697} +{"batch_in_epoch": 40, "epoch": 12, "global_step": 640, "grad_norm_max": 0.028076171875, "hidden_mse_step1": 0.01892618900663885, "hidden_mse_step2": 0.021630243165418502, "hidden_mse_step3": 0.02499442221596837, "latent_grad_l1_min": 0.00020516981385299005, "loss": 0.020583415595215874, "lr_blocks": 7.906246623448183e-07, "lr_fusion": 7.906246623448184e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 72.77327053512272, "velocity_mse_step1": 0.014201536577145037, "velocity_mse_step2": 0.01693470018549955, "velocity_mse_step3": 0.02405892256813656} +{"batch_in_epoch": 41, "epoch": 12, "global_step": 641, "grad_norm_max": 0.0244140625, "hidden_mse_step1": 0.018517394649929235, "hidden_mse_step2": 0.02009593353640022, "hidden_mse_step3": 0.022604273781845607, "latent_grad_l1_min": 0.00017719569677865366, "loss": 0.016437495227104852, "lr_blocks": 7.899687997910558e-07, "lr_fusion": 7.899687997910558e-06, "peak_memory_gib": 48.31737470626831, "trajectory_time_s": 70.61336709273746, "velocity_mse_step1": 0.011423340408198002, "velocity_mse_step2": 0.013196941858041098, "velocity_mse_step3": 0.018570441661203013} +{"batch_in_epoch": 42, "epoch": 12, "global_step": 642, "grad_norm_max": 0.0272216796875, "hidden_mse_step1": 0.019433789441661374, "hidden_mse_step2": 0.0218033654964529, "hidden_mse_step3": 0.025117168984641988, "latent_grad_l1_min": 0.00024985521849885117, "loss": 0.02255393615840668, "lr_blocks": 7.893121846069913e-07, "lr_fusion": 7.893121846069913e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 71.63651640624448, "velocity_mse_step1": 0.016297233444804854, "velocity_mse_step2": 0.01895242492485392, "velocity_mse_step3": 0.025776715500147217} +{"batch_in_epoch": 43, "epoch": 12, "global_step": 643, "grad_norm_max": 0.0245361328125, "hidden_mse_step1": 0.013925874917601633, "hidden_mse_step2": 0.016524278625313724, "hidden_mse_step3": 0.020116816008729593, "latent_grad_l1_min": 0.0002082207029161509, "loss": 0.014109487907262519, "lr_blocks": 7.886548184969062e-07, "lr_fusion": 7.886548184969063e-06, "peak_memory_gib": 48.31737470626831, "trajectory_time_s": 71.05930725674989, "velocity_mse_step1": 0.009466822768445127, "velocity_mse_step2": 0.011155235317086667, "velocity_mse_step3": 0.01664970740343311} +{"batch_in_epoch": 44, "epoch": 12, "global_step": 644, "grad_norm_max": 0.022705078125, "hidden_mse_step1": 0.017549767969675095, "hidden_mse_step2": 0.01944646014765437, "hidden_mse_step3": 0.022335560410283502, "latent_grad_l1_min": 0.0001731321253828355, "loss": 0.017052127414249947, "lr_blocks": 7.879967031670312e-07, "lr_fusion": 7.879967031670313e-06, "peak_memory_gib": 48.31958723068237, "trajectory_time_s": 74.60438109098322, "velocity_mse_step1": 0.011444220156200962, "velocity_mse_step2": 0.013890769906408551, "velocity_mse_step3": 0.019888211440827162} +{"batch_in_epoch": 45, "epoch": 12, "global_step": 645, "grad_norm_max": 0.025390625, "hidden_mse_step1": 0.017107596878694106, "hidden_mse_step2": 0.019673097446294766, "hidden_mse_step3": 0.02406742578438882, "latent_grad_l1_min": 0.00022232441733649466, "loss": 0.020443497882557234, "lr_blocks": 7.873378403255419e-07, "lr_fusion": 7.87337840325542e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 72.21440985787922, "velocity_mse_step1": 0.014150012737705503, "velocity_mse_step2": 0.016696773802063292, "velocity_mse_step3": 0.02439889369998127} +{"batch_in_epoch": 46, "epoch": 12, "global_step": 646, "grad_norm_max": 0.020751953125, "hidden_mse_step1": 0.017169804054511975, "hidden_mse_step2": 0.02003132731520704, "hidden_mse_step3": 0.023904488338822766, "latent_grad_l1_min": 0.00023400468762702076, "loss": 0.02007063593815214, "lr_blocks": 7.866782316825534e-07, "lr_fusion": 7.866782316825535e-06, "peak_memory_gib": 48.320029735565186, "trajectory_time_s": 73.02506692187308, "velocity_mse_step1": 0.014084796202951111, "velocity_mse_step2": 0.01708777233891721, "velocity_mse_step3": 0.022928776136333387} +{"batch_in_epoch": 47, "epoch": 12, "global_step": 647, "grad_norm_max": 0.0224609375, "hidden_mse_step1": 0.02057555880829958, "hidden_mse_step2": 0.023043309014091, "hidden_mse_step3": 0.025968127534724772, "latent_grad_l1_min": 0.00022541436192113906, "loss": 0.021909431055454273, "lr_blocks": 7.860178789501172e-07, "lr_fusion": 7.860178789501173e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 72.33067206726264, "velocity_mse_step1": 0.017056388560117087, "velocity_mse_step2": 0.017758031317498535, "velocity_mse_step3": 0.023955172210532637} +{"batch_in_epoch": 48, "epoch": 12, "global_step": 648, "grad_norm_max": 0.028076171875, "hidden_mse_step1": 0.014787918380794247, "hidden_mse_step2": 0.01736676914151758, "hidden_mse_step3": 0.020957841388215975, "latent_grad_l1_min": 0.00022185424495546613, "loss": 0.01617591662631769, "lr_blocks": 7.853567838422159e-07, "lr_fusion": 7.85356783842216e-06, "peak_memory_gib": 48.31737470626831, "trajectory_time_s": 71.69107934212661, "velocity_mse_step1": 0.011404946292584229, "velocity_mse_step2": 0.01302406283295048, "velocity_mse_step3": 0.018787485929871246} +{"batch_in_epoch": 49, "epoch": 12, "global_step": 649, "grad_norm_max": 0.022705078125, "hidden_mse_step1": 0.01619815089673336, "hidden_mse_step2": 0.01788403583590739, "hidden_mse_step3": 0.021405180766513307, "latent_grad_l1_min": 0.00020014167785120662, "loss": 0.016826064275976802, "lr_blocks": 7.846949480747588e-07, "lr_fusion": 7.846949480747588e-06, "peak_memory_gib": 48.31737470626831, "trajectory_time_s": 74.0370308101119, "velocity_mse_step1": 0.01325350163720681, "velocity_mse_step2": 0.012783725949702783, "velocity_mse_step3": 0.01889222679892555} +{"batch_in_epoch": 50, "epoch": 12, "global_step": 650, "grad_norm_max": 0.02734375, "hidden_mse_step1": 0.01927645849562915, "hidden_mse_step2": 0.020660426351241767, "hidden_mse_step3": 0.023958637495525178, "latent_grad_l1_min": 0.00021411163106677122, "loss": 0.020735050959046923, "lr_blocks": 7.840323733655778e-07, "lr_fusion": 7.84032373365578e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 71.24041597674659, "velocity_mse_step1": 0.015198398290002451, "velocity_mse_step2": 0.016758604120695964, "velocity_mse_step3": 0.02385859640448221} +{"batch_in_epoch": 1, "epoch": 13, "global_step": 651, "grad_norm_max": 0.03076171875, "hidden_mse_step1": 0.022827528350587403, "hidden_mse_step2": 0.02561125631577202, "hidden_mse_step3": 0.02970159577671438, "latent_grad_l1_min": 0.00033531987719470635, "loss": 0.026248597523330575, "lr_blocks": 7.833690614344231e-07, "lr_fusion": 7.833690614344232e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 71.560409465259, "velocity_mse_step1": 0.018526657143541212, "velocity_mse_step2": 0.02123247491129275, "velocity_mse_step3": 0.031172620165827013} +{"batch_in_epoch": 2, "epoch": 13, "global_step": 652, "grad_norm_max": 0.025146484375, "hidden_mse_step1": 0.020204108582609997, "hidden_mse_step2": 0.022732403578369746, "hidden_mse_step3": 0.026717090580080236, "latent_grad_l1_min": 0.0002580469154054299, "loss": 0.02253473795384967, "lr_blocks": 7.827050140029576e-07, "lr_fusion": 7.827050140029578e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 70.81445829749282, "velocity_mse_step1": 0.015970930024715405, "velocity_mse_step2": 0.018502797461613745, "velocity_mse_step3": 0.026165124328274808} +{"batch_in_epoch": 3, "epoch": 13, "global_step": 653, "grad_norm_max": 0.02685546875, "hidden_mse_step1": 0.01514266146945634, "hidden_mse_step2": 0.016983320861722207, "hidden_mse_step3": 0.019673755458955253, "latent_grad_l1_min": 0.00019365922526048962, "loss": 0.016987651852624755, "lr_blocks": 7.820402327947542e-07, "lr_fusion": 7.820402327947543e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 75.26617094012181, "velocity_mse_step1": 0.013665033662359096, "velocity_mse_step2": 0.013841288864828778, "velocity_mse_step3": 0.018276657584855065} +{"batch_in_epoch": 4, "epoch": 13, "global_step": 654, "grad_norm_max": 0.02587890625, "hidden_mse_step1": 0.020799840513583537, "hidden_mse_step2": 0.022829277116605744, "hidden_mse_step3": 0.025554448249749836, "latent_grad_l1_min": 0.00019481526578601915, "loss": 0.02081394067084017, "lr_blocks": 7.813747195352894e-07, "lr_fusion": 7.813747195352896e-06, "peak_memory_gib": 48.31737470626831, "trajectory_time_s": 70.86357304749981, "velocity_mse_step1": 0.01438951692086578, "velocity_mse_step2": 0.01698761950163836, "velocity_mse_step3": 0.02414632700882586} +{"batch_in_epoch": 5, "epoch": 13, "global_step": 655, "grad_norm_max": 0.02392578125, "hidden_mse_step1": 0.016132162796566263, "hidden_mse_step2": 0.018276721164251542, "hidden_mse_step3": 0.021418892545625567, "latent_grad_l1_min": 0.00016992882592603564, "loss": 0.015431994127409, "lr_blocks": 7.807084759519404e-07, "lr_fusion": 7.807084759519405e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 71.75841748824314, "velocity_mse_step1": 0.010529462112962, "velocity_mse_step2": 0.012046569646502445, "velocity_mse_step3": 0.01813717112444075} +{"batch_in_epoch": 6, "epoch": 13, "global_step": 656, "grad_norm_max": 0.028564453125, "hidden_mse_step1": 0.017541740060551092, "hidden_mse_step2": 0.020086309756152332, "hidden_mse_step3": 0.023730833133283464, "latent_grad_l1_min": 0.0002741281441558385, "loss": 0.020302233136525114, "lr_blocks": 7.8004150377398e-07, "lr_fusion": 7.800415037739802e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 72.16440852088272, "velocity_mse_step1": 0.014600836419309156, "velocity_mse_step2": 0.016798559551326822, "velocity_mse_step3": 0.023371412780501744} +{"batch_in_epoch": 7, "epoch": 13, "global_step": 657, "grad_norm_max": 0.027099609375, "hidden_mse_step1": 0.01767422059284789, "hidden_mse_step2": 0.01902562314145533, "hidden_mse_step3": 0.022471777363015068, "latent_grad_l1_min": 0.00021338474562071497, "loss": 0.020327572881277386, "lr_blocks": 7.793738047325716e-07, "lr_fusion": 7.793738047325718e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 73.84448161674663, "velocity_mse_step1": 0.015676826026590006, "velocity_mse_step2": 0.01660270071338995, "velocity_mse_step3": 0.022786028799600892} +{"batch_in_epoch": 8, "epoch": 13, "global_step": 658, "grad_norm_max": 0.0341796875, "hidden_mse_step1": 0.018275767053377683, "hidden_mse_step2": 0.02100001230636345, "hidden_mse_step3": 0.024034497311471826, "latent_grad_l1_min": 0.0002115636507369345, "loss": 0.018574925301696302, "lr_blocks": 7.787053805607659e-07, "lr_fusion": 7.78705380560766e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 74.22040398512763, "velocity_mse_step1": 0.012440465509176386, "velocity_mse_step2": 0.01524522847778696, "velocity_mse_step3": 0.021708052659440518} +{"batch_in_epoch": 9, "epoch": 13, "global_step": 659, "grad_norm_max": 0.025390625, "hidden_mse_step1": 0.015712833838603855, "hidden_mse_step2": 0.018353604611807635, "hidden_mse_step3": 0.021244383907677338, "latent_grad_l1_min": 0.00019449160026852041, "loss": 0.016532714572219578, "lr_blocks": 7.78036232993495e-07, "lr_fusion": 7.780362329934953e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 73.00307286611496, "velocity_mse_step1": 0.011318650676132943, "velocity_mse_step2": 0.01367788824323465, "velocity_mse_step3": 0.01907052073095526} +{"batch_in_epoch": 10, "epoch": 13, "global_step": 660, "grad_norm_max": 0.032958984375, "hidden_mse_step1": 0.011535130732227117, "hidden_mse_step2": 0.013530152542183975, "hidden_mse_step3": 0.01718127527939422, "latent_grad_l1_min": 0.00016702120319678215, "loss": 0.011759209163886095, "lr_blocks": 7.773663637675694e-07, "lr_fusion": 7.773663637675695e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 72.91497950474877, "velocity_mse_step1": 0.007942218495632654, "velocity_mse_step2": 0.009127938471335384, "velocity_mse_step3": 0.013982813946703181} +{"batch_in_epoch": 11, "epoch": 13, "global_step": 661, "grad_norm_max": 0.0277099609375, "hidden_mse_step1": 0.018347149781350578, "hidden_mse_step2": 0.020114376475768424, "hidden_mse_step3": 0.023625117560316405, "latent_grad_l1_min": 0.00020560151369863888, "loss": 0.01951092682845358, "lr_blocks": 7.76695774621672e-07, "lr_fusion": 7.76695774621672e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 74.54591092500777, "velocity_mse_step1": 0.013736337026265184, "velocity_mse_step2": 0.01575438755597653, "velocity_mse_step3": 0.022833389612579982} +{"batch_in_epoch": 12, "epoch": 13, "global_step": 662, "grad_norm_max": 0.0299072265625, "hidden_mse_step1": 0.017071582638891414, "hidden_mse_step2": 0.01905466114736295, "hidden_mse_step3": 0.022028565456691594, "latent_grad_l1_min": 0.00021046345682407264, "loss": 0.018929292123565183, "lr_blocks": 7.760244672963547e-07, "lr_fusion": 7.760244672963548e-06, "peak_memory_gib": 48.320029735565186, "trajectory_time_s": 72.19051354686235, "velocity_mse_step1": 0.013983771234052254, "velocity_mse_step2": 0.015922088899450106, "velocity_mse_step3": 0.02106653311888554} +{"batch_in_epoch": 13, "epoch": 13, "global_step": 663, "grad_norm_max": 0.033447265625, "hidden_mse_step1": 0.01836053389706649, "hidden_mse_step2": 0.020854131401782584, "hidden_mse_step3": 0.023754364045869027, "latent_grad_l1_min": 0.00019473103566269856, "loss": 0.018957099631994163, "lr_blocks": 7.753524435340333e-07, "lr_fusion": 7.753524435340334e-06, "peak_memory_gib": 48.316489696502686, "trajectory_time_s": 71.99271249486628, "velocity_mse_step1": 0.013151411979509118, "velocity_mse_step2": 0.015095996404332773, "velocity_mse_step3": 0.022326985947854285} +{"batch_in_epoch": 14, "epoch": 13, "global_step": 664, "grad_norm_max": 0.032470703125, "hidden_mse_step1": 0.02452322049897962, "hidden_mse_step2": 0.026664769426653426, "hidden_mse_step3": 0.02938367670867592, "latent_grad_l1_min": 0.0003465020090516191, "loss": 0.03249159679814641, "lr_blocks": 7.746797050789833e-07, "lr_fusion": 7.746797050789834e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 70.85712291576056, "velocity_mse_step1": 0.026130200139180354, "velocity_mse_step2": 0.028448536385050308, "velocity_mse_step3": 0.03483888442029378} +{"batch_in_epoch": 15, "epoch": 13, "global_step": 665, "grad_norm_max": 0.0252685546875, "hidden_mse_step1": 0.017026225572668148, "hidden_mse_step2": 0.01938891640332128, "hidden_mse_step3": 0.023325705776057606, "latent_grad_l1_min": 0.00020898329785268288, "loss": 0.019999222133524848, "lr_blocks": 7.74006253677335e-07, "lr_fusion": 7.740062536773352e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 74.09165635098907, "velocity_mse_step1": 0.014357111648028617, "velocity_mse_step2": 0.016289829646536545, "velocity_mse_step3": 0.023376638984440694} +{"batch_in_epoch": 16, "epoch": 13, "global_step": 666, "grad_norm_max": 0.032470703125, "hidden_mse_step1": 0.018094542220101824, "hidden_mse_step2": 0.021393069797860726, "hidden_mse_step3": 0.025398677896841294, "latent_grad_l1_min": 0.00026426622389408294, "loss": 0.021729341450346898, "lr_blocks": 7.733320910770693e-07, "lr_fusion": 7.733320910770695e-06, "peak_memory_gib": 48.320029735565186, "trajectory_time_s": 70.97563721162442, "velocity_mse_step1": 0.014569738680230718, "velocity_mse_step2": 0.018029594694131186, "velocity_mse_step3": 0.026100060486766906} +{"batch_in_epoch": 17, "epoch": 13, "global_step": 667, "grad_norm_max": 0.0255126953125, "hidden_mse_step1": 0.012796838709618893, "hidden_mse_step2": 0.015450322917396467, "hidden_mse_step3": 0.01917789179632174, "latent_grad_l1_min": 0.00018807194101100322, "loss": 0.014097799763216503, "lr_blocks": 7.726572190280133e-07, "lr_fusion": 7.726572190280134e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 72.71666933287634, "velocity_mse_step1": 0.009478973650922332, "velocity_mse_step2": 0.011211123841349036, "velocity_mse_step3": 0.016860795290475444} +{"batch_in_epoch": 18, "epoch": 13, "global_step": 668, "grad_norm_max": 0.034423828125, "hidden_mse_step1": 0.012319325353018941, "hidden_mse_step2": 0.014670804517144073, "hidden_mse_step3": 0.01786224536564467, "latent_grad_l1_min": 0.00015994999648683006, "loss": 0.012803823096744184, "lr_blocks": 7.719816392818353e-07, "lr_fusion": 7.719816392818354e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 71.54282704574143, "velocity_mse_step1": 0.008531027405946849, "velocity_mse_step2": 0.010315586621540465, "velocity_mse_step3": 0.015079616402674998} +{"batch_in_epoch": 19, "epoch": 13, "global_step": 669, "grad_norm_max": 0.03125, "hidden_mse_step1": 0.016945150851305307, "hidden_mse_step2": 0.019611898884509822, "hidden_mse_step3": 0.022470394922752993, "latent_grad_l1_min": 0.0001887434327727533, "loss": 0.01736684313592767, "lr_blocks": 7.713053535920402e-07, "lr_fusion": 7.713053535920403e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 72.20769297749939, "velocity_mse_step1": 0.012319251679790406, "velocity_mse_step2": 0.01386142022342288, "velocity_mse_step3": 0.020017111199974482} +{"batch_in_epoch": 20, "epoch": 13, "global_step": 670, "grad_norm_max": 0.0322265625, "hidden_mse_step1": 0.02475600633104997, "hidden_mse_step2": 0.026837194920517504, "hidden_mse_step3": 0.02974814547425402, "latent_grad_l1_min": 0.00025364193606947083, "loss": 0.02775179336978388, "lr_blocks": 7.706283637139657e-07, "lr_fusion": 7.706283637139658e-06, "peak_memory_gib": 48.320029735565186, "trajectory_time_s": 71.5974626572206, "velocity_mse_step1": 0.02053491595350871, "velocity_mse_step2": 0.023208841610799676, "velocity_mse_step3": 0.03137748488058735} +{"batch_in_epoch": 21, "epoch": 13, "global_step": 671, "grad_norm_max": 0.033935546875, "hidden_mse_step1": 0.01565211289978054, "hidden_mse_step2": 0.01818040076510182, "hidden_mse_step3": 0.02210834595773901, "latent_grad_l1_min": 0.00018568182531453203, "loss": 0.018477896328217214, "lr_blocks": 7.699506714047769e-07, "lr_fusion": 7.69950671404777e-06, "peak_memory_gib": 48.320029735565186, "trajectory_time_s": 71.14328930062766, "velocity_mse_step1": 0.012899029519758187, "velocity_mse_step2": 0.015335276635596527, "velocity_mse_step3": 0.02160529451378222} +{"batch_in_epoch": 22, "epoch": 13, "global_step": 672, "grad_norm_max": 0.02880859375, "hidden_mse_step1": 0.019079687385653545, "hidden_mse_step2": 0.022612924937025776, "hidden_mse_step3": 0.026846347892257785, "latent_grad_l1_min": 0.0003590516898839269, "loss": 0.026691846221053445, "lr_blocks": 7.692722784234624e-07, "lr_fusion": 7.692722784234624e-06, "peak_memory_gib": 48.320029735565186, "trajectory_time_s": 72.1825762506196, "velocity_mse_step1": 0.01902747965193287, "velocity_mse_step2": 0.023040168494584838, "velocity_mse_step3": 0.031153991934843358} +{"batch_in_epoch": 23, "epoch": 13, "global_step": 673, "grad_norm_max": 0.030517578125, "hidden_mse_step1": 0.012965842890220559, "hidden_mse_step2": 0.016052045759611895, "hidden_mse_step3": 0.019513482510644407, "latent_grad_l1_min": 0.00019138947664032457, "loss": 0.014285259917544731, "lr_blocks": 7.685931865308292e-07, "lr_fusion": 7.685931865308293e-06, "peak_memory_gib": 48.320472240448, "trajectory_time_s": 71.80581834923942, "velocity_mse_step1": 0.009262619182533984, "velocity_mse_step2": 0.011656807625383537, "velocity_mse_step3": 0.017083214820429148} +{"batch_in_epoch": 24, "epoch": 13, "global_step": 674, "grad_norm_max": 0.025390625, "hidden_mse_step1": 0.01820963164625157, "hidden_mse_step2": 0.020806762356577177, "hidden_mse_step3": 0.024593687998796146, "latent_grad_l1_min": 0.00021649802511092275, "loss": 0.02092575853956597, "lr_blocks": 7.679133974894982e-07, "lr_fusion": 7.679133974894984e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 71.55732003800222, "velocity_mse_step1": 0.013922178321601156, "velocity_mse_step2": 0.017145729403377375, "velocity_mse_step3": 0.02534835865455014} +{"batch_in_epoch": 25, "epoch": 13, "global_step": 675, "grad_norm_max": 0.02880859375, "hidden_mse_step1": 0.01568864330848945, "hidden_mse_step2": 0.018098644005866457, "hidden_mse_step3": 0.021977672569586756, "latent_grad_l1_min": 0.0001857995412137825, "loss": 0.016744444296429198, "lr_blocks": 7.672329130639005e-07, "lr_fusion": 7.672329130639007e-06, "peak_memory_gib": 48.320029735565186, "trajectory_time_s": 71.4975262588705, "velocity_mse_step1": 0.010661152386871566, "velocity_mse_step2": 0.01339272841661503, "velocity_mse_step3": 0.020602955058815757} +{"batch_in_epoch": 26, "epoch": 13, "global_step": 676, "grad_norm_max": 0.0234375, "hidden_mse_step1": 0.01623993892488735, "hidden_mse_step2": 0.01861858377898378, "hidden_mse_step3": 0.022685588844719207, "latent_grad_l1_min": 0.00019798985704255756, "loss": 0.01703740083030425, "lr_blocks": 7.665517350202714e-07, "lr_fusion": 7.665517350202715e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 71.60447257637861, "velocity_mse_step1": 0.01164384555678615, "velocity_mse_step2": 0.013227477981542635, "velocity_mse_step3": 0.020486466640639783} +{"batch_in_epoch": 27, "epoch": 13, "global_step": 677, "grad_norm_max": 0.031982421875, "hidden_mse_step1": 0.015551659935486635, "hidden_mse_step2": 0.017877279747543592, "hidden_mse_step3": 0.021622232244616107, "latent_grad_l1_min": 0.0002139849593731924, "loss": 0.01617533152291019, "lr_blocks": 7.658698651266467e-07, "lr_fusion": 7.658698651266467e-06, "peak_memory_gib": 48.32135725021362, "trajectory_time_s": 72.65851510450739, "velocity_mse_step1": 0.01074634461630402, "velocity_mse_step2": 0.012561519048176706, "velocity_mse_step3": 0.019713012252135998} +{"batch_in_epoch": 28, "epoch": 13, "global_step": 678, "grad_norm_max": 0.0220947265625, "hidden_mse_step1": 0.01634646514763257, "hidden_mse_step2": 0.01792498228938452, "hidden_mse_step3": 0.020419792987273207, "latent_grad_l1_min": 0.0002324343959116959, "loss": 0.017496826114698442, "lr_blocks": 7.65187305152858e-07, "lr_fusion": 7.651873051528582e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 71.96218745937222, "velocity_mse_step1": 0.013549472379963845, "velocity_mse_step2": 0.01461971179482394, "velocity_mse_step3": 0.01885216842804636} +{"batch_in_epoch": 29, "epoch": 13, "global_step": 679, "grad_norm_max": 0.033447265625, "hidden_mse_step1": 0.017093807475508323, "hidden_mse_step2": 0.019674708626033474, "hidden_mse_step3": 0.023041798683282516, "latent_grad_l1_min": 0.00020927208242937922, "loss": 0.01843829340733854, "lr_blocks": 7.645040568705281e-07, "lr_fusion": 7.645040568705281e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 71.43533947176911, "velocity_mse_step1": 0.01272890034098444, "velocity_mse_step2": 0.015308362772754791, "velocity_mse_step3": 0.02129658434673079} +{"batch_in_epoch": 30, "epoch": 13, "global_step": 680, "grad_norm_max": 0.0260009765625, "hidden_mse_step1": 0.020333123913068057, "hidden_mse_step2": 0.023771695011029284, "hidden_mse_step3": 0.02783910243306309, "latent_grad_l1_min": 0.00030428201807808364, "loss": 0.02610391767146731, "lr_blocks": 7.638201220530663e-07, "lr_fusion": 7.638201220530664e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 71.6080102107444, "velocity_mse_step1": 0.018211787260952406, "velocity_mse_step2": 0.022613938475842588, "velocity_mse_step3": 0.030291633755301257} +{"batch_in_epoch": 31, "epoch": 13, "global_step": 681, "grad_norm_max": 0.0274658203125, "hidden_mse_step1": 0.019630924939909682, "hidden_mse_step2": 0.022220507630013996, "hidden_mse_step3": 0.02520259388256818, "latent_grad_l1_min": 0.0002590438689367147, "loss": 0.022121950094255487, "lr_blocks": 7.631355024756638e-07, "lr_fusion": 7.63135502475664e-06, "peak_memory_gib": 48.3169322013855, "trajectory_time_s": 72.20558715025254, "velocity_mse_step1": 0.017332573840186733, "velocity_mse_step2": 0.018081304717010686, "velocity_mse_step3": 0.02424656772719962} +{"batch_in_epoch": 32, "epoch": 13, "global_step": 682, "grad_norm_max": 0.024658203125, "hidden_mse_step1": 0.0169832845817187, "hidden_mse_step2": 0.019481934091475393, "hidden_mse_step3": 0.022451700942058644, "latent_grad_l1_min": 0.00019769007667491678, "loss": 0.016000750640939386, "lr_blocks": 7.624501999152893e-07, "lr_fusion": 7.6245019991528934e-06, "peak_memory_gib": 48.31604719161987, "trajectory_time_s": 72.47502035273646, "velocity_mse_step1": 0.010222171508108398, "velocity_mse_step2": 0.012663575338332778, "velocity_mse_step3": 0.019224811852577007} +{"batch_in_epoch": 33, "epoch": 13, "global_step": 683, "grad_norm_max": 0.033203125, "hidden_mse_step1": 0.01846506819960528, "hidden_mse_step2": 0.019831662798034295, "hidden_mse_step3": 0.023755814098486944, "latent_grad_l1_min": 0.00021257348635117523, "loss": 0.018126907367591878, "lr_blocks": 7.617642161506836e-07, "lr_fusion": 7.617642161506837e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 70.8475756672342, "velocity_mse_step1": 0.013466681287224802, "velocity_mse_step2": 0.014033446554094555, "velocity_mse_step3": 0.020675338164437562} +{"batch_in_epoch": 34, "epoch": 13, "global_step": 684, "grad_norm_max": 0.029541015625, "hidden_mse_step1": 0.017832529851667847, "hidden_mse_step2": 0.020442549446930313, "hidden_mse_step3": 0.024295437831564674, "latent_grad_l1_min": 0.00021037564420112176, "loss": 0.01993229380293217, "lr_blocks": 7.610775529623568e-07, "lr_fusion": 7.610775529623568e-06, "peak_memory_gib": 48.31958723068237, "trajectory_time_s": 73.9609233978772, "velocity_mse_step1": 0.014085968470940964, "velocity_mse_step2": 0.016095420877848352, "velocity_mse_step3": 0.023358439178472117} +{"batch_in_epoch": 35, "epoch": 13, "global_step": 685, "grad_norm_max": 0.0267333984375, "hidden_mse_step1": 0.0231436577326219, "hidden_mse_step2": 0.026258092473394102, "hidden_mse_step3": 0.029913362032467763, "latent_grad_l1_min": 0.0003704300634126412, "loss": 0.028989511708329828, "lr_blocks": 7.603902121325811e-07, "lr_fusion": 7.603902121325813e-06, "peak_memory_gib": 48.31737470626831, "trajectory_time_s": 71.59374316398316, "velocity_mse_step1": 0.02190684473108766, "velocity_mse_step2": 0.02430061719080965, "velocity_mse_step3": 0.0328295593499206} +{"batch_in_epoch": 36, "epoch": 13, "global_step": 686, "grad_norm_max": 0.0341796875, "hidden_mse_step1": 0.013433205238210835, "hidden_mse_step2": 0.015531745548027434, "hidden_mse_step3": 0.019213148650513694, "latent_grad_l1_min": 0.0001712147713988088, "loss": 0.0132663657755724, "lr_blocks": 7.597021954453886e-07, "lr_fusion": 7.597021954453887e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 71.47547496412153, "velocity_mse_step1": 0.00882713053059498, "velocity_mse_step2": 0.010462138429699864, "velocity_mse_step3": 0.015692017099354416} +{"batch_in_epoch": 37, "epoch": 13, "global_step": 687, "grad_norm_max": 0.027587890625, "hidden_mse_step1": 0.016380310769558752, "hidden_mse_step2": 0.01843114297038742, "hidden_mse_step3": 0.021392669412307438, "latent_grad_l1_min": 0.00017437721180613153, "loss": 0.015146230089677762, "lr_blocks": 7.590135046865651e-07, "lr_fusion": 7.590135046865652e-06, "peak_memory_gib": 48.31958723068237, "trajectory_time_s": 73.56420048674772, "velocity_mse_step1": 0.011298538767732678, "velocity_mse_step2": 0.011672705686318554, "velocity_mse_step3": 0.016847032155575495} +{"batch_in_epoch": 38, "epoch": 13, "global_step": 688, "grad_norm_max": 0.0242919921875, "hidden_mse_step1": 0.02002580739125343, "hidden_mse_step2": 0.02229355121796418, "hidden_mse_step3": 0.025529122612039955, "latent_grad_l1_min": 0.00023386376597045455, "loss": 0.02150933723480973, "lr_blocks": 7.583241416436461e-07, "lr_fusion": 7.583241416436462e-06, "peak_memory_gib": 48.31958723068237, "trajectory_time_s": 70.90936186951149, "velocity_mse_step1": 0.014865018491816174, "velocity_mse_step2": 0.01779847369471099, "velocity_mse_step3": 0.025079670129343867} +{"batch_in_epoch": 39, "epoch": 13, "global_step": 689, "grad_norm_max": 0.0303955078125, "hidden_mse_step1": 0.015571653705722253, "hidden_mse_step2": 0.017935391532124152, "hidden_mse_step3": 0.020996997157843512, "latent_grad_l1_min": 0.00017301139996561687, "loss": 0.014681880707420141, "lr_blocks": 7.576341081059124e-07, "lr_fusion": 7.576341081059125e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 71.23287265450926, "velocity_mse_step1": 0.009642367802111298, "velocity_mse_step2": 0.011708664973931651, "velocity_mse_step3": 0.01724420378533458} +{"batch_in_epoch": 40, "epoch": 13, "global_step": 690, "grad_norm_max": 0.0230712890625, "hidden_mse_step1": 0.019737186168120906, "hidden_mse_step2": 0.021717567246274223, "hidden_mse_step3": 0.024751197330520624, "latent_grad_l1_min": 0.00019701933342730626, "loss": 0.0201501682500488, "lr_blocks": 7.569434058643843e-07, "lr_fusion": 7.5694340586438446e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 74.29880526062334, "velocity_mse_step1": 0.013912204569870872, "velocity_mse_step2": 0.01639692470364805, "velocity_mse_step3": 0.023520778548637673} +{"batch_in_epoch": 41, "epoch": 13, "global_step": 691, "grad_norm_max": 0.0233154296875, "hidden_mse_step1": 0.014183343666705435, "hidden_mse_step2": 0.016666256988953264, "hidden_mse_step3": 0.020503455947618928, "latent_grad_l1_min": 0.0001549712906125933, "loss": 0.014313737443964261, "lr_blocks": 7.562520367118186e-07, "lr_fusion": 7.562520367118187e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 73.73859328400431, "velocity_mse_step1": 0.00936878511960718, "velocity_mse_step2": 0.011647088085218065, "velocity_mse_step3": 0.016790032141476074} +{"batch_in_epoch": 42, "epoch": 13, "global_step": 692, "grad_norm_max": 0.0235595703125, "hidden_mse_step1": 0.02032750009259741, "hidden_mse_step2": 0.02256833795191986, "hidden_mse_step3": 0.025738188797341927, "latent_grad_l1_min": 0.00027042586043535266, "loss": 0.023401153548288027, "lr_blocks": 7.555600024427027e-07, "lr_fusion": 7.555600024427028e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 74.8855793529874, "velocity_mse_step1": 0.016971628131744052, "velocity_mse_step2": 0.019605268649424294, "velocity_mse_step3": 0.026763159149725522} +{"batch_in_epoch": 43, "epoch": 13, "global_step": 693, "grad_norm_max": 0.0299072265625, "hidden_mse_step1": 0.016253691071012457, "hidden_mse_step2": 0.01832701953493857, "hidden_mse_step3": 0.021714209462516006, "latent_grad_l1_min": 0.0001895015802801936, "loss": 0.016540300778745274, "lr_blocks": 7.548673048532503e-07, "lr_fusion": 7.548673048532504e-06, "peak_memory_gib": 48.3169322013855, "trajectory_time_s": 72.99049587663467, "velocity_mse_step1": 0.01141998055183129, "velocity_mse_step2": 0.013076823013502038, "velocity_mse_step3": 0.019494605525064147} +{"batch_in_epoch": 44, "epoch": 13, "global_step": 694, "grad_norm_max": 0.0289306640625, "hidden_mse_step1": 0.01755917920464916, "hidden_mse_step2": 0.01979979585822938, "hidden_mse_step3": 0.023062607356613238, "latent_grad_l1_min": 0.00021471507989190286, "loss": 0.01824624400718936, "lr_blocks": 7.541739457413969e-07, "lr_fusion": 7.54173945741397e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 71.85217520913284, "velocity_mse_step1": 0.012869873348141221, "velocity_mse_step2": 0.014687873618510951, "velocity_mse_step3": 0.02113882579474843} +{"batch_in_epoch": 45, "epoch": 13, "global_step": 695, "grad_norm_max": 0.0299072265625, "hidden_mse_step1": 0.01705226560547349, "hidden_mse_step2": 0.019705017500590265, "hidden_mse_step3": 0.02310812199721113, "latent_grad_l1_min": 0.0003130858749500476, "loss": 0.024226598716008345, "lr_blocks": 7.534799269067951e-07, "lr_fusion": 7.534799269067952e-06, "peak_memory_gib": 48.320029735565186, "trajectory_time_s": 76.24934590777411, "velocity_mse_step1": 0.01957282882514326, "velocity_mse_step2": 0.02147703222622762, "velocity_mse_step3": 0.02564339226228185} +{"batch_in_epoch": 46, "epoch": 13, "global_step": 696, "grad_norm_max": 0.0224609375, "hidden_mse_step1": 0.022133633644053977, "hidden_mse_step2": 0.024506347761156833, "hidden_mse_step3": 0.028417722305415994, "latent_grad_l1_min": 0.00031015042804938275, "loss": 0.025956735139646168, "lr_blocks": 7.527852501508099e-07, "lr_fusion": 7.527852501508099e-06, "peak_memory_gib": 48.31737470626831, "trajectory_time_s": 71.41333197887434, "velocity_mse_step1": 0.019472209963298938, "velocity_mse_step2": 0.021339917056528584, "velocity_mse_step3": 0.02955230576584914} +{"batch_in_epoch": 47, "epoch": 13, "global_step": 697, "grad_norm_max": 0.0238037109375, "hidden_mse_step1": 0.016374709912010336, "hidden_mse_step2": 0.01862980467766257, "hidden_mse_step3": 0.022021684787302678, "latent_grad_l1_min": 0.00020410185516084312, "loss": 0.01905694381899333, "lr_blocks": 7.520899172765135e-07, "lr_fusion": 7.520899172765136e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 71.94220745287748, "velocity_mse_step1": 0.01414079119006471, "velocity_mse_step2": 0.015747893150546588, "velocity_mse_step3": 0.021579525882511268} +{"batch_in_epoch": 48, "epoch": 13, "global_step": 698, "grad_norm_max": 0.03369140625, "hidden_mse_step1": 0.01603462346663166, "hidden_mse_step2": 0.017223524994083812, "hidden_mse_step3": 0.0192914246082572, "latent_grad_l1_min": 0.00019592511671362445, "loss": 0.01775830485192793, "lr_blocks": 7.513939300886816e-07, "lr_fusion": 7.513939300886816e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 71.56893828124885, "velocity_mse_step1": 0.01533822246398943, "velocity_mse_step2": 0.014797692883543535, "velocity_mse_step3": 0.017884039602774595} +{"batch_in_epoch": 49, "epoch": 13, "global_step": 699, "grad_norm_max": 0.0264892578125, "hidden_mse_step1": 0.018446469496536468, "hidden_mse_step2": 0.02103150102941851, "hidden_mse_step3": 0.02401217510591128, "latent_grad_l1_min": 0.00022348883430822752, "loss": 0.020028182288764844, "lr_blocks": 7.506972903937878e-07, "lr_fusion": 7.506972903937879e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 71.59991348174663, "velocity_mse_step1": 0.015401382525202017, "velocity_mse_step2": 0.01650861169244828, "velocity_mse_step3": 0.02182553639535659} +{"batch_in_epoch": 50, "epoch": 13, "global_step": 700, "grad_norm_max": 0.0262451171875, "hidden_mse_step1": 0.020156792356699173, "hidden_mse_step2": 0.02294562031082543, "hidden_mse_step3": 0.026417889998161366, "latent_grad_l1_min": 0.00032459802059747744, "loss": 0.026130061754624227, "lr_blocks": 7.5e-07, "lr_fusion": 7.500000000000001e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 72.35992205061484, "velocity_mse_step1": 0.020293837463084077, "velocity_mse_step2": 0.02234819224603208, "velocity_mse_step3": 0.02879612412237163} +{"batch_in_epoch": 1, "epoch": 14, "global_step": 701, "grad_norm_max": 0.028076171875, "hidden_mse_step1": 0.013353700491799307, "hidden_mse_step2": 0.01519496928501342, "hidden_mse_step3": 0.018084249425945536, "latent_grad_l1_min": 0.00016181117189262295, "loss": 0.012594127029712712, "lr_blocks": 7.493020607171742e-07, "lr_fusion": 7.4930206071717435e-06, "peak_memory_gib": 48.31958723068237, "trajectory_time_s": 72.16767017274105, "velocity_mse_step1": 0.008682842904818244, "velocity_mse_step2": 0.009876454506801175, "velocity_mse_step3": 0.014559791015926747} +{"batch_in_epoch": 2, "epoch": 14, "global_step": 702, "grad_norm_max": 0.0301513671875, "hidden_mse_step1": 0.015991671713501483, "hidden_mse_step2": 0.01878209552627855, "hidden_mse_step3": 0.022524599526410123, "latent_grad_l1_min": 0.00022363501193467528, "loss": 0.017687066982034594, "lr_blocks": 7.486034743568511e-07, "lr_fusion": 7.486034743568512e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 71.43600829113711, "velocity_mse_step1": 0.012394636860075739, "velocity_mse_step2": 0.01427360048858515, "velocity_mse_step3": 0.020663125743989697} +{"batch_in_epoch": 3, "epoch": 14, "global_step": 703, "grad_norm_max": 0.0238037109375, "hidden_mse_step1": 0.02143357903696596, "hidden_mse_step2": 0.023434687006686414, "hidden_mse_step3": 0.027053399510415535, "latent_grad_l1_min": 0.00022418150547309779, "loss": 0.023820338995262445, "lr_blocks": 7.479042427322509e-07, "lr_fusion": 7.479042427322509e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 72.24149611900066, "velocity_mse_step1": 0.017127117200288918, "velocity_mse_step2": 0.019467520369549414, "velocity_mse_step3": 0.027674210299405137} +{"batch_in_epoch": 4, "epoch": 14, "global_step": 704, "grad_norm_max": 0.0234375, "hidden_mse_step1": 0.013188152467981643, "hidden_mse_step2": 0.015577354006901648, "hidden_mse_step3": 0.01907576244723584, "latent_grad_l1_min": 0.00015491965223191073, "loss": 0.012091043405234814, "lr_blocks": 7.472043676582685e-07, "lr_fusion": 7.472043676582685e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 71.44265730286861, "velocity_mse_step1": 0.007239759995822428, "velocity_mse_step2": 0.009302944881450715, "velocity_mse_step3": 0.014946297343288149} +{"batch_in_epoch": 5, "epoch": 14, "global_step": 705, "grad_norm_max": 0.032470703125, "hidden_mse_step1": 0.02477484121586063, "hidden_mse_step2": 0.026759361548881446, "hidden_mse_step3": 0.03072525802001889, "latent_grad_l1_min": 0.00038987816515145823, "loss": 0.03050387249926903, "lr_blocks": 7.465038509514687e-07, "lr_fusion": 7.465038509514688e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 71.77609703936469, "velocity_mse_step1": 0.023049250802224766, "velocity_mse_step2": 0.025203982650834532, "velocity_mse_step3": 0.03503243472161037} +{"batch_in_epoch": 6, "epoch": 14, "global_step": 706, "grad_norm_max": 0.024169921875, "hidden_mse_step1": 0.020050288635372584, "hidden_mse_step2": 0.02260584330152986, "hidden_mse_step3": 0.02543078524260116, "latent_grad_l1_min": 0.00025236860710720066, "loss": 0.0228601893593025, "lr_blocks": 7.458026944300824e-07, "lr_fusion": 7.4580269443008245e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 71.42157888987276, "velocity_mse_step1": 0.016506606729568114, "velocity_mse_step2": 0.019160100109209967, "velocity_mse_step3": 0.02610516800944294} +{"batch_in_epoch": 7, "epoch": 14, "global_step": 707, "grad_norm_max": 0.0308837890625, "hidden_mse_step1": 0.01537631744784968, "hidden_mse_step2": 0.017792855031854873, "hidden_mse_step3": 0.02093673070027892, "latent_grad_l1_min": 0.00020679205135820666, "loss": 0.017397751449607313, "lr_blocks": 7.451008999140004e-07, "lr_fusion": 7.451008999140005e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 74.23963904651464, "velocity_mse_step1": 0.01230351631238591, "velocity_mse_step2": 0.014438126390034865, "velocity_mse_step3": 0.02004102027941761} +{"batch_in_epoch": 8, "epoch": 14, "global_step": 708, "grad_norm_max": 0.03125, "hidden_mse_step1": 0.015255714995354146, "hidden_mse_step2": 0.018348527897614982, "hidden_mse_step3": 0.0219884709721165, "latent_grad_l1_min": 0.00020721213604701916, "loss": 0.01756266877470937, "lr_blocks": 7.4439846922477e-07, "lr_fusion": 7.443984692247701e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 74.5879429025008, "velocity_mse_step1": 0.012377331424171903, "velocity_mse_step2": 0.014581570759349103, "velocity_mse_step3": 0.020169831072312912} +{"batch_in_epoch": 9, "epoch": 14, "global_step": 709, "grad_norm_max": 0.030029296875, "hidden_mse_step1": 0.015087696544859292, "hidden_mse_step2": 0.017082375183235854, "hidden_mse_step3": 0.020149366364681294, "latent_grad_l1_min": 0.00020551904162857682, "loss": 0.015525044199811027, "lr_blocks": 7.436954041855892e-07, "lr_fusion": 7.436954041855893e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 71.75477909012261, "velocity_mse_step1": 0.011509906974554594, "velocity_mse_step2": 0.012279118161781557, "velocity_mse_step3": 0.017554162494239533} +{"batch_in_epoch": 10, "epoch": 14, "global_step": 710, "grad_norm_max": 0.0269775390625, "hidden_mse_step1": 0.013210702180263718, "hidden_mse_step2": 0.015963448394489075, "hidden_mse_step3": 0.01923300711704152, "latent_grad_l1_min": 0.00018291254673385993, "loss": 0.013209181671429957, "lr_blocks": 7.429917066213029e-07, "lr_fusion": 7.42991706621303e-06, "peak_memory_gib": 48.31737470626831, "trajectory_time_s": 74.25975628363085, "velocity_mse_step1": 0.009053389845316165, "velocity_mse_step2": 0.010406989013842707, "velocity_mse_step3": 0.015326449281669086} +{"batch_in_epoch": 11, "epoch": 14, "global_step": 711, "grad_norm_max": 0.029541015625, "hidden_mse_step1": 0.01757303607882932, "hidden_mse_step2": 0.019825438253714567, "hidden_mse_step3": 0.02223767441630895, "latent_grad_l1_min": 0.00017733588356350083, "loss": 0.01697208152784567, "lr_blocks": 7.422873783583981e-07, "lr_fusion": 7.422873783583982e-06, "peak_memory_gib": 48.316489696502686, "trajectory_time_s": 73.84190574013337, "velocity_mse_step1": 0.011967195960875998, "velocity_mse_step2": 0.01408289058599621, "velocity_mse_step3": 0.018902541562316143} +{"batch_in_epoch": 12, "epoch": 14, "global_step": 712, "grad_norm_max": 0.025634765625, "hidden_mse_step1": 0.01892161276607242, "hidden_mse_step2": 0.02130685109711651, "hidden_mse_step3": 0.024186980960491513, "latent_grad_l1_min": 0.00024142258553183638, "loss": 0.019869877391361764, "lr_blocks": 7.415824212249976e-07, "lr_fusion": 7.415824212249977e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 72.74556808988564, "velocity_mse_step1": 0.014348445212817751, "velocity_mse_step2": 0.016256550884073864, "velocity_mse_step3": 0.022563089483550618} +{"batch_in_epoch": 13, "epoch": 14, "global_step": 713, "grad_norm_max": 0.0257568359375, "hidden_mse_step1": 0.014861127968677982, "hidden_mse_step2": 0.01620103253350992, "hidden_mse_step3": 0.019796308911671594, "latent_grad_l1_min": 0.0001726720147416927, "loss": 0.01565371185591045, "lr_blocks": 7.408768370508576e-07, "lr_fusion": 7.408768370508577e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 71.6722250362509, "velocity_mse_step1": 0.011974717577686533, "velocity_mse_step2": 0.0121943927702627, "velocity_mse_step3": 0.01770617704771991} +{"batch_in_epoch": 14, "epoch": 14, "global_step": 714, "grad_norm_max": 0.025634765625, "hidden_mse_step1": 0.02081583263186206, "hidden_mse_step2": 0.02387785499948742, "hidden_mse_step3": 0.02727147068695298, "latent_grad_l1_min": 0.00033726379297149833, "loss": 0.02675588981531161, "lr_blocks": 7.401706276673615e-07, "lr_fusion": 7.401706276673616e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 71.81495747761801, "velocity_mse_step1": 0.019904466099563117, "velocity_mse_step2": 0.02306142510081242, "velocity_mse_step3": 0.03010525997628324} +{"batch_in_epoch": 15, "epoch": 14, "global_step": 715, "grad_norm_max": 0.0274658203125, "hidden_mse_step1": 0.015609813305283231, "hidden_mse_step2": 0.01819654889238466, "hidden_mse_step3": 0.022071584909489107, "latent_grad_l1_min": 0.0002025497415161226, "loss": 0.0161376905743964, "lr_blocks": 7.394637949075154e-07, "lr_fusion": 7.3946379490751545e-06, "peak_memory_gib": 48.31958723068237, "trajectory_time_s": 72.09779859601258, "velocity_mse_step1": 0.010554929112134103, "velocity_mse_step2": 0.013016369260315384, "velocity_mse_step3": 0.01925397714200829} +{"batch_in_epoch": 16, "epoch": 14, "global_step": 716, "grad_norm_max": 0.0341796875, "hidden_mse_step1": 0.014287149674990878, "hidden_mse_step2": 0.015861412111137594, "hidden_mse_step3": 0.01864751370158046, "latent_grad_l1_min": 0.00020164589841442648, "loss": 0.013577282470318355, "lr_blocks": 7.387563406059432e-07, "lr_fusion": 7.387563406059433e-06, "peak_memory_gib": 48.31737470626831, "trajectory_time_s": 73.9392230327503, "velocity_mse_step1": 0.009257503462971986, "velocity_mse_step2": 0.010831609904666296, "velocity_mse_step3": 0.01576312498322555} +{"batch_in_epoch": 17, "epoch": 14, "global_step": 717, "grad_norm_max": 0.0220947265625, "hidden_mse_step1": 0.012812863004260829, "hidden_mse_step2": 0.015077426489109973, "hidden_mse_step3": 0.018863573371033584, "latent_grad_l1_min": 0.00015796082516317256, "loss": 0.013201505319947111, "lr_blocks": 7.380482665988825e-07, "lr_fusion": 7.380482665988826e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 71.80266723749082, "velocity_mse_step1": 0.008136546212231872, "velocity_mse_step2": 0.01037258001451846, "velocity_mse_step3": 0.01642000236149345} +{"batch_in_epoch": 18, "epoch": 14, "global_step": 718, "grad_norm_max": 0.02294921875, "hidden_mse_step1": 0.015827132480418577, "hidden_mse_step2": 0.01876716688275337, "hidden_mse_step3": 0.021622852688389163, "latent_grad_l1_min": 0.00020459702682273928, "loss": 0.015945360405437117, "lr_blocks": 7.373395747241791e-07, "lr_fusion": 7.373395747241792e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 73.39676477586909, "velocity_mse_step1": 0.01072137450475046, "velocity_mse_step2": 0.012961767426791733, "velocity_mse_step3": 0.018531223135401627} +{"batch_in_epoch": 19, "epoch": 14, "global_step": 719, "grad_norm_max": 0.03515625, "hidden_mse_step1": 0.01640999267276909, "hidden_mse_step2": 0.01847084200874503, "hidden_mse_step3": 0.021639420767314732, "latent_grad_l1_min": 0.00018911180450231768, "loss": 0.016006753779947754, "lr_blocks": 7.366302668212825e-07, "lr_fusion": 7.3663026682128255e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 72.81578876663116, "velocity_mse_step1": 0.011236877035116773, "velocity_mse_step2": 0.012856345923085297, "velocity_mse_step3": 0.018275010881812444} +{"batch_in_epoch": 20, "epoch": 14, "global_step": 720, "grad_norm_max": 0.02783203125, "hidden_mse_step1": 0.02004702857604051, "hidden_mse_step2": 0.021946420577088635, "hidden_mse_step3": 0.0250246410141699, "latent_grad_l1_min": 0.0002681978512555361, "loss": 0.0226324730631729, "lr_blocks": 7.35920344731241e-07, "lr_fusion": 7.35920344731241e-06, "peak_memory_gib": 48.31737470626831, "trajectory_time_s": 70.55284271487471, "velocity_mse_step1": 0.01817958983000218, "velocity_mse_step2": 0.01816158753977756, "velocity_mse_step3": 0.024854430521372706} +{"batch_in_epoch": 21, "epoch": 14, "global_step": 721, "grad_norm_max": 0.027099609375, "hidden_mse_step1": 0.019106798468523527, "hidden_mse_step2": 0.021596543349525224, "hidden_mse_step3": 0.025269709710430882, "latent_grad_l1_min": 0.00020316563313826919, "loss": 0.022221397275903394, "lr_blocks": 7.352098102966977e-07, "lr_fusion": 7.352098102966978e-06, "peak_memory_gib": 48.31737470626831, "trajectory_time_s": 73.20711980074702, "velocity_mse_step1": 0.01601525280732728, "velocity_mse_step2": 0.018116404456252764, "velocity_mse_step3": 0.02593522779144613} +{"batch_in_epoch": 22, "epoch": 14, "global_step": 722, "grad_norm_max": 0.02978515625, "hidden_mse_step1": 0.019009122375531918, "hidden_mse_step2": 0.021231732580677738, "hidden_mse_step3": 0.025146604465719844, "latent_grad_l1_min": 0.0002201406878157286, "loss": 0.02264059046033903, "lr_blocks": 7.344986653618843e-07, "lr_fusion": 7.344986653618844e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 71.68658371387573, "velocity_mse_step1": 0.016228712569240345, "velocity_mse_step2": 0.01863083004094993, "velocity_mse_step3": 0.026523480628384277} +{"batch_in_epoch": 23, "epoch": 14, "global_step": 723, "grad_norm_max": 0.035888671875, "hidden_mse_step1": 0.017177729019229964, "hidden_mse_step2": 0.019280090928077694, "hidden_mse_step3": 0.022240959556906352, "latent_grad_l1_min": 0.0001729913574308739, "loss": 0.01731115734686942, "lr_blocks": 7.337869117726175e-07, "lr_fusion": 7.337869117726176e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 74.059798122762, "velocity_mse_step1": 0.013056771128114113, "velocity_mse_step2": 0.01364917869796045, "velocity_mse_step3": 0.01935764296130011} +{"batch_in_epoch": 24, "epoch": 14, "global_step": 724, "grad_norm_max": 0.039306640625, "hidden_mse_step1": 0.01957927517027461, "hidden_mse_step2": 0.021738358615298888, "hidden_mse_step3": 0.025111028791538308, "latent_grad_l1_min": 0.00022942823397897882, "loss": 0.021072553861553645, "lr_blocks": 7.330745513762936e-07, "lr_fusion": 7.330745513762937e-06, "peak_memory_gib": 48.31958723068237, "trajectory_time_s": 72.06112703998224, "velocity_mse_step1": 0.01571379558715437, "velocity_mse_step2": 0.01710642631847544, "velocity_mse_step3": 0.02375457153954942} +{"batch_in_epoch": 25, "epoch": 14, "global_step": 725, "grad_norm_max": 0.02978515625, "hidden_mse_step1": 0.019468248701221973, "hidden_mse_step2": 0.021802925646105514, "hidden_mse_step3": 0.025651934789493677, "latent_grad_l1_min": 0.00022581267876375932, "loss": 0.022373607997516434, "lr_blocks": 7.323615860218842e-07, "lr_fusion": 7.323615860218844e-06, "peak_memory_gib": 48.31737470626831, "trajectory_time_s": 74.33607806374494, "velocity_mse_step1": 0.01612134938685423, "velocity_mse_step2": 0.018288232618942853, "velocity_mse_step3": 0.026018928732290593} +{"batch_in_epoch": 26, "epoch": 14, "global_step": 726, "grad_norm_max": 0.0267333984375, "hidden_mse_step1": 0.01321797852038539, "hidden_mse_step2": 0.015474386455025522, "hidden_mse_step3": 0.01905274121756, "latent_grad_l1_min": 0.00017118851974373683, "loss": 0.012703147309366612, "lr_blocks": 7.316480175599308e-07, "lr_fusion": 7.31648017559931e-06, "peak_memory_gib": 48.31958723068237, "trajectory_time_s": 71.39955767137144, "velocity_mse_step1": 0.008073257766747182, "velocity_mse_step2": 0.010006373638004462, "velocity_mse_step3": 0.01525529913072075} +{"batch_in_epoch": 27, "epoch": 14, "global_step": 727, "grad_norm_max": 0.0311279296875, "hidden_mse_step1": 0.021261756079703834, "hidden_mse_step2": 0.023363653470629027, "hidden_mse_step3": 0.026367460360883596, "latent_grad_l1_min": 0.00026077600341523066, "loss": 0.026048928224814256, "lr_blocks": 7.309338478425404e-07, "lr_fusion": 7.309338478425405e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 72.08505237835925, "velocity_mse_step1": 0.01942908406116268, "velocity_mse_step2": 0.022183980518353304, "velocity_mse_step3": 0.029434430995024737} +{"batch_in_epoch": 28, "epoch": 14, "global_step": 728, "grad_norm_max": 0.03173828125, "hidden_mse_step1": 0.022215118240897676, "hidden_mse_step2": 0.02462477591221354, "hidden_mse_step3": 0.027541425810860738, "latent_grad_l1_min": 0.00032644576822349336, "loss": 0.02755331261349576, "lr_blocks": 7.302190787233807e-07, "lr_fusion": 7.302190787233808e-06, "peak_memory_gib": 48.31737470626831, "trajectory_time_s": 72.33774175501458, "velocity_mse_step1": 0.02186529402492202, "velocity_mse_step2": 0.02371341421634757, "velocity_mse_step3": 0.029643095194062746} +{"batch_in_epoch": 29, "epoch": 14, "global_step": 729, "grad_norm_max": 0.034912109375, "hidden_mse_step1": 0.01535835783579387, "hidden_mse_step2": 0.018461003924520415, "hidden_mse_step3": 0.02200843032915145, "latent_grad_l1_min": 0.0002721509708862868, "loss": 0.01912340088996903, "lr_blocks": 7.295037120576749e-07, "lr_fusion": 7.29503712057675e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 74.36046599013207, "velocity_mse_step1": 0.013381161508732475, "velocity_mse_step2": 0.0163384717099169, "velocity_mse_step3": 0.022067788855305737} +{"batch_in_epoch": 30, "epoch": 14, "global_step": 730, "grad_norm_max": 0.025634765625, "hidden_mse_step1": 0.016992190129323195, "hidden_mse_step2": 0.019404421277743365, "hidden_mse_step3": 0.023184629872308248, "latent_grad_l1_min": 0.00018694728714763187, "loss": 0.017956604208198505, "lr_blocks": 7.287877497021977e-07, "lr_fusion": 7.287877497021978e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 74.63561719163408, "velocity_mse_step1": 0.012965248205416302, "velocity_mse_step2": 0.014361749207767257, "velocity_mse_step3": 0.020584688663283095} +{"batch_in_epoch": 31, "epoch": 14, "global_step": 731, "grad_norm_max": 0.03955078125, "hidden_mse_step1": 0.02314668784051069, "hidden_mse_step2": 0.02565931934597237, "hidden_mse_step3": 0.02927989891863295, "latent_grad_l1_min": 0.0002900834115280304, "loss": 0.027660426376054863, "lr_blocks": 7.280711935152691e-07, "lr_fusion": 7.2807119351526915e-06, "peak_memory_gib": 48.31737470626831, "trajectory_time_s": 73.39810955474968, "velocity_mse_step1": 0.019382917126806985, "velocity_mse_step2": 0.02344437912272821, "velocity_mse_step3": 0.03234538954815694} +{"batch_in_epoch": 32, "epoch": 14, "global_step": 732, "grad_norm_max": 0.03564453125, "hidden_mse_step1": 0.02134068062462445, "hidden_mse_step2": 0.02362759270805067, "hidden_mse_step3": 0.02682152616658381, "latent_grad_l1_min": 0.00032949683009064756, "loss": 0.02967001961743725, "lr_blocks": 7.273540453567512e-07, "lr_fusion": 7.273540453567512e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 72.56343797499721, "velocity_mse_step1": 0.02512772050769334, "velocity_mse_step2": 0.025922806575960876, "velocity_mse_step3": 0.030780549435543692} +{"batch_in_epoch": 33, "epoch": 14, "global_step": 733, "grad_norm_max": 0.0322265625, "hidden_mse_step1": 0.016805811419284767, "hidden_mse_step2": 0.019058887580675737, "hidden_mse_step3": 0.022698701242916283, "latent_grad_l1_min": 0.00020672759637818672, "loss": 0.016910304381911243, "lr_blocks": 7.266363070880423e-07, "lr_fusion": 7.266363070880424e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 71.38358685387357, "velocity_mse_step1": 0.011659243830113804, "velocity_mse_step2": 0.013324024197312871, "velocity_mse_step3": 0.019891303456721028} +{"batch_in_epoch": 34, "epoch": 14, "global_step": 734, "grad_norm_max": 0.0260009765625, "hidden_mse_step1": 0.017753172408057644, "hidden_mse_step2": 0.019877389705340774, "hidden_mse_step3": 0.023269947368784676, "latent_grad_l1_min": 0.00023228191821544897, "loss": 0.018147784006682093, "lr_blocks": 7.259179805720725e-07, "lr_fusion": 7.259179805720726e-06, "peak_memory_gib": 48.31958723068237, "trajectory_time_s": 71.80944307750906, "velocity_mse_step1": 0.012826507329009473, "velocity_mse_step2": 0.014275068034684016, "velocity_mse_step3": 0.021251724346906746} +{"batch_in_epoch": 35, "epoch": 14, "global_step": 735, "grad_norm_max": 0.031982421875, "hidden_mse_step1": 0.01547609589761123, "hidden_mse_step2": 0.018362498642610654, "hidden_mse_step3": 0.0218033797240683, "latent_grad_l1_min": 0.00017106454288295936, "loss": 0.015601170394802466, "lr_blocks": 7.251990676732984e-07, "lr_fusion": 7.251990676732985e-06, "peak_memory_gib": 48.31737470626831, "trajectory_time_s": 71.69680140074342, "velocity_mse_step1": 0.0098113105414086, "velocity_mse_step2": 0.0124413754258837, "velocity_mse_step3": 0.01898662672777261} +{"batch_in_epoch": 36, "epoch": 14, "global_step": 736, "grad_norm_max": 0.03369140625, "hidden_mse_step1": 0.017024192119216814, "hidden_mse_step2": 0.01963108663247632, "hidden_mse_step3": 0.022433631072219993, "latent_grad_l1_min": 0.00024032914552662987, "loss": 0.022220518485742753, "lr_blocks": 7.244795702576989e-07, "lr_fusion": 7.24479570257699e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 71.26396278387983, "velocity_mse_step1": 0.017044807948488075, "velocity_mse_step2": 0.019611704331639754, "velocity_mse_step3": 0.024096150358673185} +{"batch_in_epoch": 37, "epoch": 14, "global_step": 737, "grad_norm_max": 0.0247802734375, "hidden_mse_step1": 0.017640881015852625, "hidden_mse_step2": 0.019785724929533895, "hidden_mse_step3": 0.02296200730571789, "latent_grad_l1_min": 0.00025684665160952136, "loss": 0.01981344392074139, "lr_blocks": 7.237594901927698e-07, "lr_fusion": 7.237594901927699e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 71.27462839200598, "velocity_mse_step1": 0.014713303361661798, "velocity_mse_step2": 0.01637449646243892, "velocity_mse_step3": 0.022313668039194975} +{"batch_in_epoch": 38, "epoch": 14, "global_step": 738, "grad_norm_max": 0.03125, "hidden_mse_step1": 0.019698593840335628, "hidden_mse_step2": 0.021984744484403306, "hidden_mse_step3": 0.02543608222289809, "latent_grad_l1_min": 0.0002289573021698743, "loss": 0.021775952440553473, "lr_blocks": 7.230388293475195e-07, "lr_fusion": 7.2303882934751965e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 72.21667895687278, "velocity_mse_step1": 0.014966153740325743, "velocity_mse_step2": 0.01782917563105002, "velocity_mse_step3": 0.025820583795263828} +{"batch_in_epoch": 39, "epoch": 14, "global_step": 739, "grad_norm_max": 0.034912109375, "hidden_mse_step1": 0.018640288426208178, "hidden_mse_step2": 0.020736868094120703, "hidden_mse_step3": 0.02386810485872307, "latent_grad_l1_min": 0.00020548833708744496, "loss": 0.01981278041577233, "lr_blocks": 7.223175895924637e-07, "lr_fusion": 7.223175895924638e-06, "peak_memory_gib": 48.320472240448, "trajectory_time_s": 71.78391993362311, "velocity_mse_step1": 0.013724419608479366, "velocity_mse_step2": 0.016300717626498748, "velocity_mse_step3": 0.023088675665868707} +{"batch_in_epoch": 40, "epoch": 14, "global_step": 740, "grad_norm_max": 0.03564453125, "hidden_mse_step1": 0.02044434536946937, "hidden_mse_step2": 0.02230407973651641, "hidden_mse_step3": 0.026047606370411813, "latent_grad_l1_min": 0.00025900581749738194, "loss": 0.021463243124474368, "lr_blocks": 7.215957727996207e-07, "lr_fusion": 7.215957727996208e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 71.36966850863246, "velocity_mse_step1": 0.01589852504340732, "velocity_mse_step2": 0.017210821637750735, "velocity_mse_step3": 0.024400777657449774} +{"batch_in_epoch": 41, "epoch": 14, "global_step": 741, "grad_norm_max": 0.0294189453125, "hidden_mse_step1": 0.021188914044094936, "hidden_mse_step2": 0.024198071814940443, "hidden_mse_step3": 0.02769761926694108, "latent_grad_l1_min": 0.0003082774601352867, "loss": 0.02790221234317869, "lr_blocks": 7.208733808425062e-07, "lr_fusion": 7.208733808425064e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 73.55146878050436, "velocity_mse_step1": 0.020571734020320167, "velocity_mse_step2": 0.023722624880195196, "velocity_mse_step3": 0.03210381591426474} +{"batch_in_epoch": 42, "epoch": 14, "global_step": 742, "grad_norm_max": 0.030029296875, "hidden_mse_step1": 0.017561913351528347, "hidden_mse_step2": 0.020252641431787715, "hidden_mse_step3": 0.024315238414731405, "latent_grad_l1_min": 0.0002146275810446241, "loss": 0.019985887454822656, "lr_blocks": 7.201504155961295e-07, "lr_fusion": 7.201504155961296e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 73.26573871161236, "velocity_mse_step1": 0.013693162942737605, "velocity_mse_step2": 0.016212420779213838, "velocity_mse_step3": 0.0238390970459607} +{"batch_in_epoch": 43, "epoch": 14, "global_step": 743, "grad_norm_max": 0.0284423828125, "hidden_mse_step1": 0.018621037331675843, "hidden_mse_step2": 0.021876356697508266, "hidden_mse_step3": 0.02579185725854976, "latent_grad_l1_min": 0.00026697220346250106, "loss": 0.021369904446016462, "lr_blocks": 7.194268789369874e-07, "lr_fusion": 7.194268789369875e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 71.86084142849722, "velocity_mse_step1": 0.013924981571368074, "velocity_mse_step2": 0.01786641799013263, "velocity_mse_step3": 0.02568938712855535} +{"batch_in_epoch": 44, "epoch": 14, "global_step": 744, "grad_norm_max": 0.0279541015625, "hidden_mse_step1": 0.019973741773615723, "hidden_mse_step2": 0.022827318495339047, "hidden_mse_step3": 0.026847719963240837, "latent_grad_l1_min": 0.0003320114328744239, "loss": 0.02593365968121881, "lr_blocks": 7.187027727430599e-07, "lr_fusion": 7.1870277274306e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 72.51718302212248, "velocity_mse_step1": 0.018685307283054238, "velocity_mse_step2": 0.021439738363759325, "velocity_mse_step3": 0.030711053130549507} +{"batch_in_epoch": 45, "epoch": 14, "global_step": 745, "grad_norm_max": 0.031494140625, "hidden_mse_step1": 0.014287740058664764, "hidden_mse_step2": 0.016408351399669688, "hidden_mse_step3": 0.019890802762737234, "latent_grad_l1_min": 0.00019441928725427715, "loss": 0.0129649253961231, "lr_blocks": 7.179780988938051e-07, "lr_fusion": 7.179780988938051e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 71.68768338186783, "velocity_mse_step1": 0.008265375494374894, "velocity_mse_step2": 0.00995709909641716, "velocity_mse_step3": 0.015613611255373272} +{"batch_in_epoch": 46, "epoch": 14, "global_step": 746, "grad_norm_max": 0.035888671875, "hidden_mse_step1": 0.017064295596875514, "hidden_mse_step2": 0.01994112114022885, "hidden_mse_step3": 0.023718796703698378, "latent_grad_l1_min": 0.00022525707754539326, "loss": 0.019101030908391943, "lr_blocks": 7.172528592701549e-07, "lr_fusion": 7.17252859270155e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 71.95742209237505, "velocity_mse_step1": 0.013194557678486618, "velocity_mse_step2": 0.015595587298906009, "velocity_mse_step3": 0.02244052475517882} +{"batch_in_epoch": 47, "epoch": 14, "global_step": 747, "grad_norm_max": 0.03271484375, "hidden_mse_step1": 0.017022743334694366, "hidden_mse_step2": 0.019652433385739902, "hidden_mse_step3": 0.022995878642957126, "latent_grad_l1_min": 0.00018956464373331983, "loss": 0.016227991553023454, "lr_blocks": 7.165270557545094e-07, "lr_fusion": 7.165270557545094e-06, "peak_memory_gib": 48.31958723068237, "trajectory_time_s": 73.32080011087965, "velocity_mse_step1": 0.01003484074525269, "velocity_mse_step2": 0.013073498187752974, "velocity_mse_step3": 0.019608529010188898} +{"batch_in_epoch": 48, "epoch": 14, "global_step": 748, "grad_norm_max": 0.0257568359375, "hidden_mse_step1": 0.021108200395246968, "hidden_mse_step2": 0.023764128780125505, "hidden_mse_step3": 0.02699623638597716, "latent_grad_l1_min": 0.00023734517981210956, "loss": 0.02356833009149081, "lr_blocks": 7.15800690230732e-07, "lr_fusion": 7.158006902307322e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 71.86658000137686, "velocity_mse_step1": 0.017304841996519826, "velocity_mse_step2": 0.019975795073800588, "velocity_mse_step3": 0.026237495463906916} +{"batch_in_epoch": 49, "epoch": 14, "global_step": 749, "grad_norm_max": 0.0296630859375, "hidden_mse_step1": 0.013198996923165396, "hidden_mse_step2": 0.015294528574616249, "hidden_mse_step3": 0.019552198330140005, "latent_grad_l1_min": 0.00017917917739396216, "loss": 0.014881160193389016, "lr_blocks": 7.15073764584146e-07, "lr_fusion": 7.15073764584146e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 72.52488822310988, "velocity_mse_step1": 0.010250260581545132, "velocity_mse_step2": 0.011612867649611351, "velocity_mse_step3": 0.017975778497722267} +{"batch_in_epoch": 50, "epoch": 14, "global_step": 750, "grad_norm_max": 0.0400390625, "hidden_mse_step1": 0.015762812541132525, "hidden_mse_step2": 0.017601199248539548, "hidden_mse_step3": 0.02055415306572935, "latent_grad_l1_min": 0.00016306674842780922, "loss": 0.01493270169677479, "lr_blocks": 7.14346280701527e-07, "lr_fusion": 7.143462807015271e-06, "peak_memory_gib": 48.320029735565186, "trajectory_time_s": 73.85082211835834, "velocity_mse_step1": 0.01013264590957468, "velocity_mse_step2": 0.012054988993830711, "velocity_mse_step3": 0.017218651915235177} +{"batch_in_epoch": 1, "epoch": 15, "global_step": 751, "grad_norm_max": 0.0272216796875, "hidden_mse_step1": 0.013657295074413663, "hidden_mse_step2": 0.016315215034410357, "hidden_mse_step3": 0.019938160515656427, "latent_grad_l1_min": 0.00017521120025776327, "loss": 0.013468623560454163, "lr_blocks": 7.136182404711008e-07, "lr_fusion": 7.1361824047110084e-06, "peak_memory_gib": 48.31958723068237, "trajectory_time_s": 74.2337345081105, "velocity_mse_step1": 0.008135747459683833, "velocity_mse_step2": 0.010570004952439507, "velocity_mse_step3": 0.016709050022265207} +{"batch_in_epoch": 2, "epoch": 15, "global_step": 752, "grad_norm_max": 0.03515625, "hidden_mse_step1": 0.01691888488962182, "hidden_mse_step2": 0.019366507817591937, "hidden_mse_step3": 0.022404712608217126, "latent_grad_l1_min": 0.00019678312492033, "loss": 0.017382856184017977, "lr_blocks": 7.128896457825363e-07, "lr_fusion": 7.128896457825364e-06, "peak_memory_gib": 48.31737470626831, "trajectory_time_s": 74.93997963276342, "velocity_mse_step1": 0.011329470466340094, "velocity_mse_step2": 0.014251731190597638, "velocity_mse_step3": 0.020698354350835346} +{"batch_in_epoch": 3, "epoch": 15, "global_step": 753, "grad_norm_max": 0.028564453125, "hidden_mse_step1": 0.024182705017405427, "hidden_mse_step2": 0.026775863936303983, "hidden_mse_step3": 0.029853511418748112, "latent_grad_l1_min": 0.0002434251109662, "loss": 0.02747223835571536, "lr_blocks": 7.121604985269422e-07, "lr_fusion": 7.121604985269423e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 72.91780098661548, "velocity_mse_step1": 0.01938800796467279, "velocity_mse_step2": 0.023173781336351694, "velocity_mse_step3": 0.03177371505548113} +{"batch_in_epoch": 4, "epoch": 15, "global_step": 754, "grad_norm_max": 0.03173828125, "hidden_mse_step1": 0.017857511656724716, "hidden_mse_step2": 0.020153704631541456, "hidden_mse_step3": 0.02353133971337229, "latent_grad_l1_min": 0.00023364588469121372, "loss": 0.01976695353265053, "lr_blocks": 7.114308005968609e-07, "lr_fusion": 7.11430800596861e-06, "peak_memory_gib": 48.3169322013855, "trajectory_time_s": 71.74649057311763, "velocity_mse_step1": 0.0139059895975931, "velocity_mse_step2": 0.01606140926430401, "velocity_mse_step3": 0.02317920384562707} +{"batch_in_epoch": 5, "epoch": 15, "global_step": 755, "grad_norm_max": 0.030517578125, "hidden_mse_step1": 0.016031665494665504, "hidden_mse_step2": 0.018061076357428516, "hidden_mse_step3": 0.021152847829008743, "latent_grad_l1_min": 0.00018054243446385954, "loss": 0.016648381782163466, "lr_blocks": 7.107005538862646e-07, "lr_fusion": 7.107005538862647e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 71.86041119774745, "velocity_mse_step1": 0.011448275969347116, "velocity_mse_step2": 0.013525752810112732, "velocity_mse_step3": 0.01944655576621049} +{"batch_in_epoch": 6, "epoch": 15, "global_step": 756, "grad_norm_max": 0.032958984375, "hidden_mse_step1": 0.013681870984977907, "hidden_mse_step2": 0.01633808174769261, "hidden_mse_step3": 0.020732877144057832, "latent_grad_l1_min": 0.00022969213205215055, "loss": 0.014958455771972823, "lr_blocks": 7.099697602905493e-07, "lr_fusion": 7.099697602905494e-06, "peak_memory_gib": 48.31604719161987, "trajectory_time_s": 71.72055917412945, "velocity_mse_step1": 0.009875574738543946, "velocity_mse_step2": 0.011490961673968871, "velocity_mse_step3": 0.018433546996675428} +{"batch_in_epoch": 7, "epoch": 15, "global_step": 757, "grad_norm_max": 0.0311279296875, "hidden_mse_step1": 0.01197876555878403, "hidden_mse_step2": 0.014499413281945246, "hidden_mse_step3": 0.01805779163260013, "latent_grad_l1_min": 0.00017762034076440614, "loss": 0.012474665039917452, "lr_blocks": 7.092384217065314e-07, "lr_fusion": 7.092384217065315e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 71.66059094386583, "velocity_mse_step1": 0.008520743532114596, "velocity_mse_step2": 0.009758771889859678, "velocity_mse_step3": 0.014690881694800087} +{"batch_in_epoch": 8, "epoch": 15, "global_step": 758, "grad_norm_max": 0.0291748046875, "hidden_mse_step1": 0.017000003314543783, "hidden_mse_step2": 0.018530801287852228, "hidden_mse_step3": 0.02173657306203885, "latent_grad_l1_min": 0.00016674312701070448, "loss": 0.01853812300916096, "lr_blocks": 7.085065400324406e-07, "lr_fusion": 7.085065400324407e-06, "peak_memory_gib": 48.31958723068237, "trajectory_time_s": 75.3119168261328, "velocity_mse_step1": 0.01408326004665079, "velocity_mse_step2": 0.014968256804942421, "velocity_mse_step3": 0.02083611276300092} +{"batch_in_epoch": 9, "epoch": 15, "global_step": 759, "grad_norm_max": 0.0296630859375, "hidden_mse_step1": 0.015870426640114083, "hidden_mse_step2": 0.018194288489342264, "hidden_mse_step3": 0.021157639957631803, "latent_grad_l1_min": 0.00016952382975432556, "loss": 0.017211304599186402, "lr_blocks": 7.077741171679172e-07, "lr_fusion": 7.0777411716791734e-06, "peak_memory_gib": 48.31737470626831, "trajectory_time_s": 72.24574383461731, "velocity_mse_step1": 0.011992479321114451, "velocity_mse_step2": 0.014269108624180911, "velocity_mse_step3": 0.019850089331157506} +{"batch_in_epoch": 10, "epoch": 15, "global_step": 760, "grad_norm_max": 0.0269775390625, "hidden_mse_step1": 0.02325972513062879, "hidden_mse_step2": 0.026650155013027994, "hidden_mse_step3": 0.03018667806671666, "latent_grad_l1_min": 0.000377816118998453, "loss": 0.03077103674877435, "lr_blocks": 7.07041155014006e-07, "lr_fusion": 7.07041155014006e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 71.79649460237124, "velocity_mse_step1": 0.023534606888590912, "velocity_mse_step2": 0.02687779078093756, "velocity_mse_step3": 0.033891053787166524} +{"batch_in_epoch": 11, "epoch": 15, "global_step": 761, "grad_norm_max": 0.030517578125, "hidden_mse_step1": 0.024445099664652453, "hidden_mse_step2": 0.027198882889933884, "hidden_mse_step3": 0.030709978974690397, "latent_grad_l1_min": 0.00033063386763387825, "loss": 0.029952307597600984, "lr_blocks": 7.063076554731512e-07, "lr_fusion": 7.0630765547315125e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 72.36827560811798, "velocity_mse_step1": 0.02236307260214484, "velocity_mse_step2": 0.025581004239419204, "velocity_mse_step3": 0.03367744808617447} +{"batch_in_epoch": 12, "epoch": 15, "global_step": 762, "grad_norm_max": 0.037841796875, "hidden_mse_step1": 0.014521894411050848, "hidden_mse_step2": 0.016600133684862937, "hidden_mse_step3": 0.01960106687952897, "latent_grad_l1_min": 0.0001684491417108802, "loss": 0.013335390889551489, "lr_blocks": 7.055736204491921e-07, "lr_fusion": 7.055736204491922e-06, "peak_memory_gib": 48.31737470626831, "trajectory_time_s": 72.04009074912028, "velocity_mse_step1": 0.008680623328213446, "velocity_mse_step2": 0.010508148851970742, "velocity_mse_step3": 0.015745089846729697} +{"batch_in_epoch": 13, "epoch": 15, "global_step": 763, "grad_norm_max": 0.0242919921875, "hidden_mse_step1": 0.01647886082563283, "hidden_mse_step2": 0.018510604892591282, "hidden_mse_step3": 0.022045147729971046, "latent_grad_l1_min": 0.00018348849152971525, "loss": 0.01684292174676167, "lr_blocks": 7.048390518473578e-07, "lr_fusion": 7.048390518473579e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 72.06114694924327, "velocity_mse_step1": 0.011663318325840269, "velocity_mse_step2": 0.013452361354471316, "velocity_mse_step3": 0.01970962225875285} +{"batch_in_epoch": 14, "epoch": 15, "global_step": 764, "grad_norm_max": 0.03271484375, "hidden_mse_step1": 0.015628766808991452, "hidden_mse_step2": 0.018613825286073343, "hidden_mse_step3": 0.022620197984256914, "latent_grad_l1_min": 0.00021908017333771568, "loss": 0.017566900949792137, "lr_blocks": 7.041039515742625e-07, "lr_fusion": 7.041039515742626e-06, "peak_memory_gib": 48.31737470626831, "trajectory_time_s": 71.42781436873338, "velocity_mse_step1": 0.011063613145129887, "velocity_mse_step2": 0.014225731508174378, "velocity_mse_step3": 0.02172507799696177} +{"batch_in_epoch": 15, "epoch": 15, "global_step": 765, "grad_norm_max": 0.0283203125, "hidden_mse_step1": 0.014067509249019037, "hidden_mse_step2": 0.016072797198181173, "hidden_mse_step3": 0.019469494083231044, "latent_grad_l1_min": 0.00017675323397270404, "loss": 0.014664802997555982, "lr_blocks": 7.033683215379002e-07, "lr_fusion": 7.033683215379002e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 71.99019716475595, "velocity_mse_step1": 0.010585377793177031, "velocity_mse_step2": 0.01158517360766252, "velocity_mse_step3": 0.016862876322453042} +{"batch_in_epoch": 16, "epoch": 15, "global_step": 766, "grad_norm_max": 0.0308837890625, "hidden_mse_step1": 0.020247917564120144, "hidden_mse_step2": 0.022412412359179658, "hidden_mse_step3": 0.026151156668285172, "latent_grad_l1_min": 0.00023003096794127487, "loss": 0.02012281636208562, "lr_blocks": 7.026321636476396e-07, "lr_fusion": 7.026321636476398e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 71.75343242289091, "velocity_mse_step1": 0.014043393775604531, "velocity_mse_step2": 0.015708052549078793, "velocity_mse_step3": 0.023735852246837955} +{"batch_in_epoch": 17, "epoch": 15, "global_step": 767, "grad_norm_max": 0.0306396484375, "hidden_mse_step1": 0.01879842021818539, "hidden_mse_step2": 0.02110470777344224, "hidden_mse_step3": 0.023963951422566816, "latent_grad_l1_min": 0.0002653408746482455, "loss": 0.02238975836163653, "lr_blocks": 7.018954798142204e-07, "lr_fusion": 7.018954798142205e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 70.84462356525182, "velocity_mse_step1": 0.016526307035487013, "velocity_mse_step2": 0.019256728657637723, "velocity_mse_step3": 0.02499952992574046} +{"batch_in_epoch": 18, "epoch": 15, "global_step": 768, "grad_norm_max": 0.0260009765625, "hidden_mse_step1": 0.019540220116531212, "hidden_mse_step2": 0.021645556370328577, "hidden_mse_step3": 0.024459991304736053, "latent_grad_l1_min": 0.0002320282610526192, "loss": 0.02250844145393265, "lr_blocks": 7.011582719497465e-07, "lr_fusion": 7.011582719497466e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 72.15997354050342, "velocity_mse_step1": 0.016785887733151737, "velocity_mse_step2": 0.019058807237472917, "velocity_mse_step3": 0.02511605025834537} +{"batch_in_epoch": 19, "epoch": 15, "global_step": 769, "grad_norm_max": 0.0291748046875, "hidden_mse_step1": 0.01991992134052063, "hidden_mse_step2": 0.02180382752391909, "hidden_mse_step3": 0.024810049516548003, "latent_grad_l1_min": 0.00023222158051794395, "loss": 0.020776747081462023, "lr_blocks": 7.004205419676825e-07, "lr_fusion": 7.004205419676826e-06, "peak_memory_gib": 48.31958723068237, "trajectory_time_s": 70.8678495628701, "velocity_mse_step1": 0.014890312954126525, "velocity_mse_step2": 0.017040016778212572, "velocity_mse_step3": 0.023746529190767824} +{"batch_in_epoch": 20, "epoch": 15, "global_step": 770, "grad_norm_max": 0.031005859375, "hidden_mse_step1": 0.01677074866269582, "hidden_mse_step2": 0.01942033314013055, "hidden_mse_step3": 0.022708105372398025, "latent_grad_l1_min": 0.00020668980505433865, "loss": 0.018743122553652417, "lr_blocks": 6.996822917828476e-07, "lr_fusion": 6.9968229178284775e-06, "peak_memory_gib": 48.320029735565186, "trajectory_time_s": 73.80502887675539, "velocity_mse_step1": 0.013453040681529922, "velocity_mse_step2": 0.01564043094653503, "velocity_mse_step3": 0.021245975438172797} +{"batch_in_epoch": 21, "epoch": 15, "global_step": 771, "grad_norm_max": 0.03173828125, "hidden_mse_step1": 0.01847465836908668, "hidden_mse_step2": 0.02068079901593072, "hidden_mse_step3": 0.024477806823727275, "latent_grad_l1_min": 0.0002472192190907663, "loss": 0.02291554892768285, "lr_blocks": 6.989435233114123e-07, "lr_fusion": 6.989435233114124e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 73.97482202562969, "velocity_mse_step1": 0.017378275355440564, "velocity_mse_step2": 0.01911403927286821, "velocity_mse_step3": 0.02589100402214431} +{"batch_in_epoch": 22, "epoch": 15, "global_step": 772, "grad_norm_max": 0.0306396484375, "hidden_mse_step1": 0.018549937871284783, "hidden_mse_step2": 0.021680716599803414, "hidden_mse_step3": 0.026249165912824016, "latent_grad_l1_min": 0.00028022841797792353, "loss": 0.022683963661880364, "lr_blocks": 6.982042384708911e-07, "lr_fusion": 6.982042384708911e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 74.92436370213545, "velocity_mse_step1": 0.015632013561636477, "velocity_mse_step2": 0.01861888663344351, "velocity_mse_step3": 0.027153006488723413} +{"batch_in_epoch": 23, "epoch": 15, "global_step": 773, "grad_norm_max": 0.027587890625, "hidden_mse_step1": 0.019925687349833812, "hidden_mse_step2": 0.021550656254735907, "hidden_mse_step3": 0.024834426286231188, "latent_grad_l1_min": 0.00020444921028683893, "loss": 0.02005048143577629, "lr_blocks": 6.974644391801394e-07, "lr_fusion": 6.974644391801395e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 70.79990322925005, "velocity_mse_step1": 0.014362602091361101, "velocity_mse_step2": 0.016288162520920323, "velocity_mse_step3": 0.022869600798003372} +{"batch_in_epoch": 24, "epoch": 15, "global_step": 774, "grad_norm_max": 0.0341796875, "hidden_mse_step1": 0.02043451034530465, "hidden_mse_step2": 0.021394755358674696, "hidden_mse_step3": 0.024219670033614547, "latent_grad_l1_min": 0.00027142598810314666, "loss": 0.020807912473433783, "lr_blocks": 6.967241273593479e-07, "lr_fusion": 6.9672412735934805e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 71.99707334374398, "velocity_mse_step1": 0.01673291071035367, "velocity_mse_step2": 0.016449494944286665, "velocity_mse_step3": 0.022636436300152645} +{"batch_in_epoch": 25, "epoch": 15, "global_step": 775, "grad_norm_max": 0.029296875, "hidden_mse_step1": 0.014903765918071648, "hidden_mse_step2": 0.01763759006280452, "hidden_mse_step3": 0.02165416568251593, "latent_grad_l1_min": 0.000198997647203214, "loss": 0.016308308142470196, "lr_blocks": 6.959833049300375e-07, "lr_fusion": 6.959833049300376e-06, "peak_memory_gib": 48.320029735565186, "trajectory_time_s": 72.27990771012264, "velocity_mse_step1": 0.01058928841666784, "velocity_mse_step2": 0.012902019417586934, "velocity_mse_step3": 0.020014062910507033} +{"batch_in_epoch": 26, "epoch": 15, "global_step": 776, "grad_norm_max": 0.031494140625, "hidden_mse_step1": 0.022792158673317835, "hidden_mse_step2": 0.024321560955805968, "hidden_mse_step3": 0.02695022930856794, "latent_grad_l1_min": 0.00021423209909698926, "loss": 0.02423843166824164, "lr_blocks": 6.952419738150544e-07, "lr_fusion": 6.952419738150546e-06, "peak_memory_gib": 48.31958723068237, "trajectory_time_s": 71.33169690724753, "velocity_mse_step1": 0.017706154797841527, "velocity_mse_step2": 0.020216262059485804, "velocity_mse_step3": 0.02738648125003757} +{"batch_in_epoch": 27, "epoch": 15, "global_step": 777, "grad_norm_max": 0.0213623046875, "hidden_mse_step1": 0.022212585213960013, "hidden_mse_step2": 0.023412891788341637, "hidden_mse_step3": 0.026565879267374318, "latent_grad_l1_min": 0.00024019178999878932, "loss": 0.021992895220007212, "lr_blocks": 6.945001359385651e-07, "lr_fusion": 6.9450013593856515e-06, "peak_memory_gib": 48.31958723068237, "trajectory_time_s": 71.47594860097888, "velocity_mse_step1": 0.01605604218951027, "velocity_mse_step2": 0.0177541499558304, "velocity_mse_step3": 0.024949355857513313} +{"batch_in_epoch": 28, "epoch": 15, "global_step": 778, "grad_norm_max": 0.031005859375, "hidden_mse_step1": 0.019150653729281784, "hidden_mse_step2": 0.02278394512333242, "hidden_mse_step3": 0.026633415637271743, "latent_grad_l1_min": 0.00038795404543634504, "loss": 0.026846295376474566, "lr_blocks": 6.937577932260514e-07, "lr_fusion": 6.9375779322605154e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 71.87531624861003, "velocity_mse_step1": 0.021229361102866405, "velocity_mse_step2": 0.02339894459666019, "velocity_mse_step3": 0.02905377565184608} +{"batch_in_epoch": 29, "epoch": 15, "global_step": 779, "grad_norm_max": 0.028076171875, "hidden_mse_step1": 0.016616792409747307, "hidden_mse_step2": 0.01839314744574949, "hidden_mse_step3": 0.020818226521701684, "latent_grad_l1_min": 0.00017024210228555603, "loss": 0.015693355871397734, "lr_blocks": 6.930149476043058e-07, "lr_fusion": 6.930149476043059e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 71.24690510127402, "velocity_mse_step1": 0.011941036809210865, "velocity_mse_step2": 0.012512949451255344, "velocity_mse_step3": 0.017043263830211278} +{"batch_in_epoch": 30, "epoch": 15, "global_step": 780, "grad_norm_max": 0.03125, "hidden_mse_step1": 0.015796083108788092, "hidden_mse_step2": 0.01793516473013109, "hidden_mse_step3": 0.021674870114241327, "latent_grad_l1_min": 0.0002337999294468318, "loss": 0.01748206214480368, "lr_blocks": 6.922716010014255e-07, "lr_fusion": 6.922716010014256e-06, "peak_memory_gib": 48.316489696502686, "trajectory_time_s": 72.76501594673755, "velocity_mse_step1": 0.011994057407719083, "velocity_mse_step2": 0.01388101656006516, "velocity_mse_step3": 0.021030499258943434} +{"batch_in_epoch": 31, "epoch": 15, "global_step": 781, "grad_norm_max": 0.032958984375, "hidden_mse_step1": 0.018084884704356746, "hidden_mse_step2": 0.02057433555767472, "hidden_mse_step3": 0.02403756532086325, "latent_grad_l1_min": 0.00033183791038027266, "loss": 0.026236187292462475, "lr_blocks": 6.915277553468082e-07, "lr_fusion": 6.915277553468083e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 74.43679572613473, "velocity_mse_step1": 0.020956437552480826, "velocity_mse_step2": 0.023074753403696892, "velocity_mse_step3": 0.028407688805600628} +{"batch_in_epoch": 32, "epoch": 15, "global_step": 782, "grad_norm_max": 0.0380859375, "hidden_mse_step1": 0.021299406836208488, "hidden_mse_step2": 0.02478564405880336, "hidden_mse_step3": 0.02818465222870665, "latent_grad_l1_min": 0.0002622645133669721, "loss": 0.02499658378240253, "lr_blocks": 6.907834125711476e-07, "lr_fusion": 6.9078341257114765e-06, "peak_memory_gib": 48.320029735565186, "trajectory_time_s": 72.92963270538166, "velocity_mse_step1": 0.01700988380304937, "velocity_mse_step2": 0.021257288022232906, "velocity_mse_step3": 0.02929560765291431} +{"batch_in_epoch": 33, "epoch": 15, "global_step": 783, "grad_norm_max": 0.0274658203125, "hidden_mse_step1": 0.01672097273071163, "hidden_mse_step2": 0.01899060941234763, "hidden_mse_step3": 0.022241893956171613, "latent_grad_l1_min": 0.00023773244356561918, "loss": 0.018713105706930425, "lr_blocks": 6.900385746064268e-07, "lr_fusion": 6.900385746064268e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 74.08927963837777, "velocity_mse_step1": 0.013717462773326718, "velocity_mse_step2": 0.015467445846297778, "velocity_mse_step3": 0.02115905996262362} +{"batch_in_epoch": 34, "epoch": 15, "global_step": 784, "grad_norm_max": 0.0225830078125, "hidden_mse_step1": 0.019011374538032606, "hidden_mse_step2": 0.021025200341162936, "hidden_mse_step3": 0.024157453045648116, "latent_grad_l1_min": 0.00027739627148548607, "loss": 0.022727864228987264, "lr_blocks": 6.892932433859146e-07, "lr_fusion": 6.892932433859147e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 71.46148940363491, "velocity_mse_step1": 0.017303858546906015, "velocity_mse_step2": 0.01866635045735165, "velocity_mse_step3": 0.025793979692805027} +{"batch_in_epoch": 35, "epoch": 15, "global_step": 785, "grad_norm_max": 0.0308837890625, "hidden_mse_step1": 0.014973111076480042, "hidden_mse_step2": 0.01784889223719282, "hidden_mse_step3": 0.021071544233044342, "latent_grad_l1_min": 0.00020493464489845792, "loss": 0.01567301196545096, "lr_blocks": 6.885474208441601e-07, "lr_fusion": 6.885474208441602e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 71.50678820548637, "velocity_mse_step1": 0.01061180736620112, "velocity_mse_step2": 0.012736007824839492, "velocity_mse_step3": 0.018281864883777286} +{"batch_in_epoch": 36, "epoch": 15, "global_step": 786, "grad_norm_max": 0.03369140625, "hidden_mse_step1": 0.023454474143883478, "hidden_mse_step2": 0.02539580425114504, "hidden_mse_step3": 0.0280954438660826, "latent_grad_l1_min": 0.00027709155619959347, "loss": 0.027537005728975467, "lr_blocks": 6.878011089169877e-07, "lr_fusion": 6.878011089169878e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 71.77831295999931, "velocity_mse_step1": 0.022644988984601308, "velocity_mse_step2": 0.023250186757650226, "velocity_mse_step3": 0.02902126675637971} +{"batch_in_epoch": 37, "epoch": 15, "global_step": 787, "grad_norm_max": 0.026123046875, "hidden_mse_step1": 0.015826575379053662, "hidden_mse_step2": 0.017941445295166752, "hidden_mse_step3": 0.021005264982315048, "latent_grad_l1_min": 0.0001755169105308596, "loss": 0.01500060881621071, "lr_blocks": 6.870543095414918e-07, "lr_fusion": 6.870543095414919e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 71.39766633449472, "velocity_mse_step1": 0.0110732135633173, "velocity_mse_step2": 0.011758864058979919, "velocity_mse_step3": 0.016692419253688837} +{"batch_in_epoch": 38, "epoch": 15, "global_step": 788, "grad_norm_max": 0.0201416015625, "hidden_mse_step1": 0.017040079284925014, "hidden_mse_step2": 0.019554687181620723, "hidden_mse_step3": 0.023643475158938335, "latent_grad_l1_min": 0.00018768952941172756, "loss": 0.018719902652914504, "lr_blocks": 6.863070246560318e-07, "lr_fusion": 6.863070246560319e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 72.29291662300238, "velocity_mse_step1": 0.012367141324960227, "velocity_mse_step2": 0.015094925607887227, "velocity_mse_step3": 0.022673815455553786} +{"batch_in_epoch": 39, "epoch": 15, "global_step": 789, "grad_norm_max": 0.0281982421875, "hidden_mse_step1": 0.019867428046252043, "hidden_mse_step2": 0.022097743731657307, "hidden_mse_step3": 0.0250850442763684, "latent_grad_l1_min": 0.0002302993252669694, "loss": 0.023608120881752778, "lr_blocks": 6.855592562002281e-07, "lr_fusion": 6.855592562002281e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 74.6528691888816, "velocity_mse_step1": 0.018919380747580105, "velocity_mse_step2": 0.019351729482877996, "velocity_mse_step3": 0.025848228101884675} +{"batch_in_epoch": 40, "epoch": 15, "global_step": 790, "grad_norm_max": 0.03515625, "hidden_mse_step1": 0.012382907867764253, "hidden_mse_step2": 0.014712213064610423, "hidden_mse_step3": 0.01823181171701955, "latent_grad_l1_min": 0.00015492362035729457, "loss": 0.012997809967990697, "lr_blocks": 6.848110061149555e-07, "lr_fusion": 6.848110061149555e-06, "peak_memory_gib": 48.31958723068237, "trajectory_time_s": 74.15724087948911, "velocity_mse_step1": 0.0083237322084772, "velocity_mse_step2": 0.010216307090429057, "velocity_mse_step3": 0.015920696630408716} +{"batch_in_epoch": 41, "epoch": 15, "global_step": 791, "grad_norm_max": 0.0286865234375, "hidden_mse_step1": 0.015433820546604691, "hidden_mse_step2": 0.01833566792941253, "hidden_mse_step3": 0.02132544894370117, "latent_grad_l1_min": 0.00017276203379879007, "loss": 0.01570250431541353, "lr_blocks": 6.840622763423391e-07, "lr_fusion": 6.840622763423391e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 73.83798498912802, "velocity_mse_step1": 0.010176876048457675, "velocity_mse_step2": 0.01288037889545584, "velocity_mse_step3": 0.018540762847156396} +{"batch_in_epoch": 42, "epoch": 15, "global_step": 792, "grad_norm_max": 0.0252685546875, "hidden_mse_step1": 0.019102839407943457, "hidden_mse_step2": 0.021951830397093933, "hidden_mse_step3": 0.02606607093808374, "latent_grad_l1_min": 0.00027363337540009525, "loss": 0.022551078565551764, "lr_blocks": 6.833130688257488e-07, "lr_fusion": 6.833130688257489e-06, "peak_memory_gib": 48.31870222091675, "trajectory_time_s": 70.6613527567606, "velocity_mse_step1": 0.01589882448440351, "velocity_mse_step2": 0.018635258204345258, "velocity_mse_step3": 0.02640707787525441} +{"batch_in_epoch": 43, "epoch": 15, "global_step": 793, "grad_norm_max": 0.01953125, "hidden_mse_step1": 0.010033171025237868, "hidden_mse_step2": 0.012876305887142996, "hidden_mse_step3": 0.016996986232697964, "latent_grad_l1_min": 0.00013992690492159454, "loss": 0.009891987100542923, "lr_blocks": 6.825633855097953e-07, "lr_fusion": 6.825633855097954e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 72.7687126105011, "velocity_mse_step1": 0.00587234106413754, "velocity_mse_step2": 0.007530536718799599, "velocity_mse_step3": 0.01228243621465351} +{"batch_in_epoch": 44, "epoch": 15, "global_step": 794, "grad_norm_max": 0.0281982421875, "hidden_mse_step1": 0.016221325769687867, "hidden_mse_step2": 0.018424853773987185, "hidden_mse_step3": 0.022153450791457932, "latent_grad_l1_min": 0.00018815874591382453, "loss": 0.01741147195986871, "lr_blocks": 6.818132283403235e-07, "lr_fusion": 6.818132283403236e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 71.71759146299883, "velocity_mse_step1": 0.01262911585425692, "velocity_mse_step2": 0.013828852870834192, "velocity_mse_step3": 0.020096482817150117} +{"batch_in_epoch": 45, "epoch": 15, "global_step": 795, "grad_norm_max": 0.027099609375, "hidden_mse_step1": 0.02091852441662922, "hidden_mse_step2": 0.02320431500473725, "hidden_mse_step3": 0.026488442023816918, "latent_grad_l1_min": 0.000285794692899799, "loss": 0.024115877899540852, "lr_blocks": 6.810625992644083e-07, "lr_fusion": 6.810625992644085e-06, "peak_memory_gib": 48.31737470626831, "trajectory_time_s": 73.86659476451314, "velocity_mse_step1": 0.017054595073984404, "velocity_mse_step2": 0.019656423940822215, "velocity_mse_step3": 0.028575484574373273} +{"batch_in_epoch": 46, "epoch": 15, "global_step": 796, "grad_norm_max": 0.032470703125, "hidden_mse_step1": 0.018007588980253786, "hidden_mse_step2": 0.02041362611843007, "hidden_mse_step3": 0.023922842228785157, "latent_grad_l1_min": 0.0001697817879176, "loss": 0.017319983328759135, "lr_blocks": 6.803115002303498e-07, "lr_fusion": 6.8031150023035e-06, "peak_memory_gib": 48.31914472579956, "trajectory_time_s": 71.65782182337716, "velocity_mse_step1": 0.0115377735221825, "velocity_mse_step2": 0.013609376403370073, "velocity_mse_step3": 0.02057839334676308} +{"batch_in_epoch": 47, "epoch": 15, "global_step": 797, "grad_norm_max": 0.028076171875, "hidden_mse_step1": 0.014405702772949423, "hidden_mse_step2": 0.01692406491409721, "hidden_mse_step3": 0.020322806043882986, "latent_grad_l1_min": 0.0002052852596534649, "loss": 0.014681007471933426, "lr_blocks": 6.795599331876678e-07, "lr_fusion": 6.795599331876679e-06, "peak_memory_gib": 48.31604719161987, "trajectory_time_s": 72.83437883799343, "velocity_mse_step1": 0.010379408240883745, "velocity_mse_step2": 0.011490918315498025, "velocity_mse_step3": 0.01700743721864585} +{"batch_in_epoch": 48, "epoch": 15, "global_step": 798, "grad_norm_max": 0.0301513671875, "hidden_mse_step1": 0.012657188519369809, "hidden_mse_step2": 0.014970229622641842, "hidden_mse_step3": 0.01879019406624138, "latent_grad_l1_min": 0.00016280782074318267, "loss": 0.012297364916386347, "lr_blocks": 6.788079000870965e-07, "lr_fusion": 6.788079000870966e-06, "peak_memory_gib": 48.320472240448, "trajectory_time_s": 73.6150811116313, "velocity_mse_step1": 0.00753759619588631, "velocity_mse_step2": 0.009496138335504967, "velocity_mse_step3": 0.015216597879771145} +{"batch_in_epoch": 49, "epoch": 15, "global_step": 799, "grad_norm_max": 0.02392578125, "hidden_mse_step1": 0.017427555426755652, "hidden_mse_step2": 0.01949680806137621, "hidden_mse_step3": 0.02330722556715565, "latent_grad_l1_min": 0.00019521734520822065, "loss": 0.018926311824803373, "lr_blocks": 6.780554028805802e-07, "lr_fusion": 6.780554028805803e-06, "peak_memory_gib": 48.318259716033936, "trajectory_time_s": 72.44892987250932, "velocity_mse_step1": 0.013631362914955909, "velocity_mse_step2": 0.0150280746165663, "velocity_mse_step3": 0.022096337492777302} +{"batch_in_epoch": 50, "epoch": 15, "global_step": 800, "grad_norm_max": 0.0242919921875, "hidden_mse_step1": 0.0172010229019049, "hidden_mse_step2": 0.019855877783681666, "hidden_mse_step3": 0.02282433639213975, "latent_grad_l1_min": 0.0002100704905387829, "loss": 0.019813201805975816, "lr_blocks": 6.773024435212677e-07, "lr_fusion": 6.773024435212678e-06, "peak_memory_gib": 48.31781721115112, "trajectory_time_s": 71.73968932988646, "velocity_mse_step1": 0.014688789068064319, "velocity_mse_step2": 0.01652200617952206, "velocity_mse_step3": 0.02224068478348532} +{"batch_in_epoch": 1, "epoch": 16, "global_step": 801, "grad_norm_max": 0.023193359375, "hidden_mse_step1": 0.013776558917015791, "hidden_mse_step2": 0.0165282438351174, "hidden_mse_step3": 0.020478698840763948, "latent_grad_l1_min": 0.0002258516524307197, "loss": 0.014732466141660032, "lr_blocks": 6.765490239635075e-07, "lr_fusion": 6.765490239635075e-06, "peak_memory_gib": 48.316489696502686, "trajectory_time_s": 71.74956142462179, "velocity_mse_step1": 0.009137548882530868, "velocity_mse_step2": 0.01177482953581992, "velocity_mse_step3": 0.018206668855522624} +{"batch_in_epoch": 2, "epoch": 16, "global_step": 802, "grad_norm_max": 0.0252685546875, "hidden_mse_step1": 0.01701674675236323, "hidden_mse_step2": 0.018968069560027546, "hidden_mse_step3": 0.02128446045597749, "latent_grad_l1_min": 0.00019726605569303501, "loss": 0.01932941880243431, "lr_blocks": 6.757951461628415e-07, "lr_fusion": 6.757951461628416e-06, "peak_memory_gib": 48.320029735565186, "trajectory_time_s": 74.49828984899796, "velocity_mse_step1": 0.01524179445030833, "velocity_mse_step2": 0.01660974711246256, "velocity_mse_step3": 0.02040978577653212} +W0720 06:15:28.622000 4030588 /mnt/efs/zoubin/envs/hy_worldplay/lib/python3.10/site-packages/torch/distributed/elastic/agent/server/api.py:719] Received Signals.SIGTERM death signal, shutting down workers +W0720 06:15:28.628000 4030588 /mnt/efs/zoubin/envs/hy_worldplay/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/api.py:897] Sending process 4030635 closing signal SIGTERM +W0720 06:15:28.632000 4030588 /mnt/efs/zoubin/envs/hy_worldplay/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/api.py:897] Sending process 4030636 closing signal SIGTERM +W0720 06:15:28.634000 4030588 /mnt/efs/zoubin/envs/hy_worldplay/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/api.py:897] Sending process 4030637 closing signal SIGTERM +W0720 06:15:28.637000 4030588 /mnt/efs/zoubin/envs/hy_worldplay/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/api.py:897] Sending process 4030638 closing signal SIGTERM +W0720 06:15:28.639000 4030588 /mnt/efs/zoubin/envs/hy_worldplay/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/api.py:897] Sending process 4030639 closing signal SIGTERM +W0720 06:15:28.642000 4030588 /mnt/efs/zoubin/envs/hy_worldplay/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/api.py:897] Sending process 4030640 closing signal SIGTERM +W0720 06:15:28.644000 4030588 /mnt/efs/zoubin/envs/hy_worldplay/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/api.py:897] Sending process 4030641 closing signal SIGTERM +W0720 06:15:28.645000 4030588 /mnt/efs/zoubin/envs/hy_worldplay/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/api.py:897] Sending process 4030642 closing signal SIGTERM +Traceback (most recent call last): + File "/mnt/s3files/s3-us-west2-default/zoubin/cz/envs/hy_worldplay/lib/python3.10/runpy.py", line 196, in _run_module_as_main + return _run_code(code, main_globals, None, + File "/mnt/s3files/s3-us-west2-default/zoubin/cz/envs/hy_worldplay/lib/python3.10/runpy.py", line 86, in _run_code + exec(code, run_globals) + File "/mnt/s3files/s3-us-west2-default/zoubin/cz/envs/hy_worldplay/lib/python3.10/site-packages/torch/distributed/run.py", line 922, in + main() + File "/mnt/s3files/s3-us-west2-default/zoubin/cz/envs/hy_worldplay/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + File "/mnt/s3files/s3-us-west2-default/zoubin/cz/envs/hy_worldplay/lib/python3.10/site-packages/torch/distributed/run.py", line 918, in main + run(args) + File "/mnt/s3files/s3-us-west2-default/zoubin/cz/envs/hy_worldplay/lib/python3.10/site-packages/torch/distributed/run.py", line 909, in run + elastic_launch( + File "/mnt/s3files/s3-us-west2-default/zoubin/cz/envs/hy_worldplay/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 138, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + File "/mnt/s3files/s3-us-west2-default/zoubin/cz/envs/hy_worldplay/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 260, in launch_agent + result = agent.run() + File "/mnt/s3files/s3-us-west2-default/zoubin/cz/envs/hy_worldplay/lib/python3.10/site-packages/torch/distributed/elastic/metrics/api.py", line 137, in wrapper + result = f(*args, **kwargs) + File "/mnt/s3files/s3-us-west2-default/zoubin/cz/envs/hy_worldplay/lib/python3.10/site-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + File "/mnt/s3files/s3-us-west2-default/zoubin/cz/envs/hy_worldplay/lib/python3.10/site-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/mnt/s3files/s3-us-west2-default/zoubin/cz/envs/hy_worldplay/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 4030588 got signal: 15 diff --git a/logs/predictor_v4_stage1_previous_condition_ablation_seed0/both.log b/logs/predictor_v4_stage1_previous_condition_ablation_seed0/both.log new file mode 100644 index 0000000000000000000000000000000000000000..734b4705de966f8c70463ae59d73461a3002907d --- /dev/null +++ b/logs/predictor_v4_stage1_previous_condition_ablation_seed0/both.log @@ -0,0 +1,303 @@ +[start] mode=both 2026-08-23T14:51:32Z +{ + "manifest": "/mnt/local_nvme/cz/projects/HY-WorldPlay-DEV-Predictor/datasets/hyworldplay_predictor_prefeature_v4/train_eval_manifest.jsonl", + "teacher_checkpoint": "/mnt/local_nvme/cz/checkpoints/hy_worldplay/huggingface/hub/models--tencent--HY-WorldPlay/snapshots/f4c29235647707b571479a69b569e4166f9f5bf8/ar_distilled_action_model/diffusion_pytorch_model.safetensors", + "output_dir": "/mnt/local_nvme/cz/projects/HY-WorldPlay-DEV-Predictor/checkpoints/predictor_v4_stage1_previous_condition_ablation_seed0/both", + "source_blocks": [ + 1, + 52 + ], + "predictor_variant": "v4", + "previous_condition_mode": "both", + "predictor_fusion_type": "concat", + "gate_hidden_dim": 256, + "gate_initial_probability": 0.8, + "predictor_arch": "baseline", + "atc_variant": "full", + "atc_mlp_hidden_dim": 4096, + "atc_gate_hidden_dim": 512, + "atc_gate_initial_probability": 0.3, + "atc_transport_heads": 8, + "atc_transport_window": 3, + "atc_transport_query_chunk_size": 128, + "atc_transport_initial_scale": 0.1, + "max_steps": 2000, + "batch_size": 16, + "gradient_accumulation_steps": 1, + "block_warmup_steps": 100, + "block_warmup_mode": "delayed", + "scheduler_warmup_steps": 100, + "fusion_lr": 0.0001, + "blocks_lr": 1e-05, + "weight_decay": 0.01, + "hidden_weight": 0.1, + "velocity_weight": 1.0, + "grad_clip": 1.0, + "log_every": 10, + "save_every": 100, + "keep_snapshots": 20, + "seed": 0, + "num_workers": 0, + "max_records": null, + "bucket_by_context": true, + "gradient_checkpointing": true, + "fp32_trainable_params": true, + "train_delta_only": false, + "train_fusion_residual_only": false, + "save_optimizer_state": true, + "init_training_state": null, + "resume": null, + "swanlab_project": "HY-WorldPlay-Predictor-v4", + "swanlab_workspace": null, + "swanlab_experiment": "predictor-v4-stage1-previous-both-seed0-blocks1-52-fp32master-8gpu-torch26-noflash", + "swanlab_mode": "cloud", + "predictor_version": "v4_prefeature", + "schema_version": "predictor_context_prefeature_bf16_v1", + "supervision_pairs": [ + [ + 0, + 1 + ], + [ + 1, + 2 + ], + [ + 2, + 3 + ] + ], + "previous_condition_source": "offline_full_teacher_hidden", + "records": 2800, + "dataset_pairs": 8400, + "steps_per_epoch_per_rank": 64, + "world_size": 8, + "per_rank_batch_size": 16, + "global_batch_size": 128, + "trainable_parameters": 205611520, + "frozen_projector_parameters": 16785664, + "dtype": "bfloat16", + "activation_dtype": "bfloat16", + "trainable_parameter_dtype": "float32", + "optimizer_moment_dtype": "float32", + "inference_checkpoint_dtype": "bfloat16", + "gpu": "NVIDIA H100 80GB HBM3" +} +swanlab: Currently logged in as: ZhuoChen to https://api.swanlab.cn. Use +`swanlab login --relogin` to force relogin +swanlab: Tracking run with swanlab version 0.9.0 +swanlab: 💾 Run data saved at +/mnt/local_nvme/cz/projects/HY-WorldPlay-DEV-Predictor/checkpoints/predictor_v4_ +stage1_previous_condition_ablation_seed0/both/swanlab/run-20260823_145158-7zvpua +sj +swanlab: 📁 View project at +https://swanlab.cn/@ZhuoChen/HY-WorldPlay-Predictor-v4 +swanlab: 🚀 View run pre...cks1-52-fp32master-8gpu-torch26-noflash at +https://swanlab.cn/@ZhuoChen/HY-WorldPlay-Predictor-v4/runs/7zvpuasj +{"swanlab_run_id": "7zvpuasj"} +{"avg_context_frames": 4.0, "avg_micro_time_s": 5.678752202133182, "epoch": 1, "global_step": 1, "grad_norm_max": 0.045006927102804184, "hidden_mse": 0.019336470868438482, "loss": 0.018770750728435814, "lr_blocks": 0.0, "lr_fusion": 2.0000000000000003e-06, "peak_memory_gib": 40.80115556716919, "velocity_mse": 0.016837103641591966} +{"avg_context_frames": 14.666666666666666, "avg_micro_time_s": 4.9869485107451, "epoch": 1, "global_step": 10, "grad_norm_max": 0.047025665640830994, "hidden_mse": 0.015848804381676018, "loss": 0.012685119982860567, "lr_blocks": 0.0, "lr_fusion": 1.1000000000000001e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.011100239511708625} +{"avg_context_frames": 13.600000000000001, "avg_micro_time_s": 4.69876731086697, "epoch": 1, "global_step": 20, "grad_norm_max": 0.07692644000053406, "hidden_mse": 0.015845339559018613, "loss": 0.012918069114675745, "lr_blocks": 0.0, "lr_fusion": 2.1e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.011333535134326667} +{"avg_context_frames": 14.4, "avg_micro_time_s": 4.866441152976768, "epoch": 1, "global_step": 30, "grad_norm_max": 0.05289289727807045, "hidden_mse": 0.01638748454861343, "loss": 0.013678718352457509, "lr_blocks": 0.0, "lr_fusion": 3.1e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.012039969919715078} +{"avg_context_frames": 14.4, "avg_micro_time_s": 4.859211045448319, "epoch": 1, "global_step": 40, "grad_norm_max": 0.0343487411737442, "hidden_mse": 0.01808105275267735, "loss": 0.015317701228195802, "lr_blocks": 0.0, "lr_fusion": 4.1e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.013509595941286534} +{"avg_context_frames": 14.0, "avg_micro_time_s": 4.761487837895402, "epoch": 1, "global_step": 50, "grad_norm_max": 0.040569327771663666, "hidden_mse": 0.015547957411035896, "loss": 0.01253144278889522, "lr_blocks": 0.0, "lr_fusion": 5.1000000000000006e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.010976647044299171} +{"avg_context_frames": 16.0, "avg_micro_time_s": 5.183884204379865, "epoch": 1, "global_step": 60, "grad_norm_max": 0.06463341414928436, "hidden_mse": 0.014454990939702839, "loss": 0.012776940863113852, "lr_blocks": 0.0, "lr_fusion": 6.1e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.011331441858783365} +{"avg_context_frames": 14.0, "avg_micro_time_s": 4.7270118881409875, "epoch": 2, "global_step": 70, "grad_norm_max": 0.041777774691581726, "hidden_mse": 0.015296078671235593, "loss": 0.013207275868626312, "lr_blocks": 0.0, "lr_fusion": 7.1e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.01167766799335368} +{"avg_context_frames": 14.8, "avg_micro_time_s": 4.886555617186241, "epoch": 2, "global_step": 80, "grad_norm_max": 0.07195698469877243, "hidden_mse": 0.014872863295022399, "loss": 0.013008171453839168, "lr_blocks": 0.0, "lr_fusion": 8.1e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.011520885111531244} +{"avg_context_frames": 15.600000000000001, "avg_micro_time_s": 5.057493073465594, "epoch": 2, "global_step": 90, "grad_norm_max": 0.07349945604801178, "hidden_mse": 0.014462979719974102, "loss": 0.012593457580078394, "lr_blocks": 0.0, "lr_fusion": 9.1e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.011147159576648847} +{"avg_context_frames": 16.0, "avg_micro_time_s": 5.119254122606072, "epoch": 2, "global_step": 100, "grad_norm_max": 0.3667074739933014, "hidden_mse": 0.01388552238349803, "loss": 0.01244637044146657, "lr_blocks": 1e-05, "lr_fusion": 0.0001, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.011057818250264972} +{"avg_context_frames": 13.600000000000001, "avg_micro_time_s": 4.612508657136641, "epoch": 2, "global_step": 110, "grad_norm_max": 0.17710991203784943, "hidden_mse": 0.01444822830380872, "loss": 0.014019656670279802, "lr_blocks": 9.999316524962347e-06, "lr_fusion": 9.999316524962345e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.012574833765393124} +{"avg_context_frames": 12.4, "avg_micro_time_s": 4.386142864583235, "epoch": 2, "global_step": 120, "grad_norm_max": 0.09518104791641235, "hidden_mse": 0.017400684009771794, "loss": 0.015868137759389358, "lr_blocks": 9.99726628670463e-06, "lr_fusion": 9.997266286704631e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.014128069375874476} +{"avg_context_frames": 14.0, "avg_micro_time_s": 4.717114774644142, "epoch": 3, "global_step": 130, "grad_norm_max": 0.17353485524654388, "hidden_mse": 0.016381604538764805, "loss": 0.01480554402805865, "lr_blocks": 9.993849845741525e-06, "lr_fusion": 9.993849845741524e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.013167383591644467} +{"avg_context_frames": 17.2, "avg_micro_time_s": 5.44278566686553, "epoch": 3, "global_step": 140, "grad_norm_max": 0.08169068396091461, "hidden_mse": 0.014518131932709366, "loss": 0.013095536472974345, "lr_blocks": 9.989068136093873e-06, "lr_fusion": 9.989068136093873e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.011643723287852481} +{"avg_context_frames": 12.0, "avg_micro_time_s": 4.30352853211225, "epoch": 3, "global_step": 150, "grad_norm_max": 0.07080712914466858, "hidden_mse": 0.01599123899359256, "loss": 0.01418825417640619, "lr_blocks": 9.98292246503335e-06, "lr_fusion": 9.98292246503335e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.01258913031197153} +{"avg_context_frames": 13.600000000000001, "avg_micro_time_s": 4.652177372388542, "epoch": 3, "global_step": 160, "grad_norm_max": 0.054620757699012756, "hidden_mse": 0.015374813065864147, "loss": 0.013711851270636545, "lr_blocks": 9.975414512725058e-06, "lr_fusion": 9.975414512725057e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.012174370017601178} +{"avg_context_frames": 13.600000000000001, "avg_micro_time_s": 4.6500839769098095, "epoch": 3, "global_step": 170, "grad_norm_max": 0.025663215667009354, "hidden_mse": 0.015321133169345559, "loss": 0.013859270565444604, "lr_blocks": 9.966546331768192e-06, "lr_fusion": 9.966546331768191e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.012327157316030936} +{"avg_context_frames": 15.200000000000001, "avg_micro_time_s": 5.022405777298264, "epoch": 3, "global_step": 180, "grad_norm_max": 0.03851979970932007, "hidden_mse": 0.01405228766379878, "loss": 0.012440450303256512, "lr_blocks": 9.956320346634877e-06, "lr_fusion": 9.956320346634876e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.011035221541533247} +{"avg_context_frames": 15.600000000000001, "avg_micro_time_s": 5.080187334422954, "epoch": 3, "global_step": 190, "grad_norm_max": 0.058892782777547836, "hidden_mse": 0.013663378299679608, "loss": 0.011654425953747705, "lr_blocks": 9.944739353007344e-06, "lr_fusion": 9.944739353007344e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.010288088105153293} +{"avg_context_frames": 12.8, "avg_micro_time_s": 4.443207342831011, "epoch": 4, "global_step": 200, "grad_norm_max": 0.05625917762517929, "hidden_mse": 0.014428281475557015, "loss": 0.01309173521003686, "lr_blocks": 9.931806517013612e-06, "lr_fusion": 9.931806517013612e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.01164890707586892} +{"avg_context_frames": 14.0, "avg_micro_time_s": 4.734948478084698, "epoch": 4, "global_step": 210, "grad_norm_max": 0.04480276256799698, "hidden_mse": 0.014067418506601826, "loss": 0.012678628659341485, "lr_blocks": 9.917525374361913e-06, "lr_fusion": 9.917525374361912e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.011271886812755838} +{"avg_context_frames": 14.4, "avg_micro_time_s": 4.800792300279136, "epoch": 4, "global_step": 220, "grad_norm_max": 0.06990489363670349, "hidden_mse": 0.0138236973551102, "loss": 0.012755018839379774, "lr_blocks": 9.901899829374048e-06, "lr_fusion": 9.901899829374047e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.01137264907010831} +{"avg_context_frames": 13.600000000000001, "avg_micro_time_s": 4.65740185349423, "epoch": 4, "global_step": 230, "grad_norm_max": 0.04012608528137207, "hidden_mse": 0.015592286066384986, "loss": 0.013487128377892078, "lr_blocks": 9.884934153917998e-06, "lr_fusion": 9.884934153917997e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.011927899718284608} +{"avg_context_frames": 13.200000000000001, "avg_micro_time_s": 4.573810783892987, "epoch": 4, "global_step": 240, "grad_norm_max": 0.04310674965381622, "hidden_mse": 0.015632650582119823, "loss": 0.0144193907675799, "lr_blocks": 9.86663298624003e-06, "lr_fusion": 9.86663298624003e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.012856125709367916} +{"avg_context_frames": 16.0, "avg_micro_time_s": 5.183240557213139, "epoch": 4, "global_step": 250, "grad_norm_max": 0.038402408361434937, "hidden_mse": 0.01300415717996657, "loss": 0.01149783332948573, "lr_blocks": 9.847001329696653e-06, "lr_fusion": 9.847001329696653e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.010197417647577823} +{"avg_context_frames": 16.8, "avg_micro_time_s": 5.37530422180571, "epoch": 5, "global_step": 260, "grad_norm_max": 0.03225686401128769, "hidden_mse": 0.01396547164185904, "loss": 0.013050097081577406, "lr_blocks": 9.826044551386743e-06, "lr_fusion": 9.826044551386744e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.01165354996919632} +{"avg_context_frames": 10.4, "avg_micro_time_s": 3.9564984125652702, "epoch": 5, "global_step": 270, "grad_norm_max": 0.06465400010347366, "hidden_mse": 0.015836354822386055, "loss": 0.01489259697846137, "lr_blocks": 9.803768380684242e-06, "lr_fusion": 9.803768380684242e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.013308961514849216} +{"avg_context_frames": 14.0, "avg_micro_time_s": 4.754884787039191, "epoch": 5, "global_step": 280, "grad_norm_max": 0.05922531709074974, "hidden_mse": 0.014305408037034796, "loss": 0.012934351281728595, "lr_blocks": 9.780178907671788e-06, "lr_fusion": 9.780178907671789e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.011503810435533524} +{"avg_context_frames": 15.200000000000001, "avg_micro_time_s": 4.991409005787864, "epoch": 5, "global_step": 290, "grad_norm_max": 0.04620140418410301, "hidden_mse": 0.013154810410924256, "loss": 0.01230526170693338, "lr_blocks": 9.755282581475769e-06, "lr_fusion": 9.755282581475769e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.01098978063382674} +{"avg_context_frames": 14.8, "avg_micro_time_s": 4.906175856137998, "epoch": 5, "global_step": 300, "grad_norm_max": 0.04410897567868233, "hidden_mse": 0.012639705295441672, "loss": 0.012001178856007755, "lr_blocks": 9.729086208503174e-06, "lr_fusion": 9.729086208503174e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.010737208361388185} +{"avg_context_frames": 16.8, "avg_micro_time_s": 5.30887135763187, "epoch": 5, "global_step": 310, "grad_norm_max": 0.0367090180516243, "hidden_mse": 0.012110810185549782, "loss": 0.011056482343701646, "lr_blocks": 9.701596950580807e-06, "lr_fusion": 9.701596950580806e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.009845401349593885} +{"avg_context_frames": 14.4, "avg_micro_time_s": 4.8219852912880015, "epoch": 5, "global_step": 320, "grad_norm_max": 0.07931792736053467, "hidden_mse": 0.013647312251850964, "loss": 0.01241676102508791, "lr_blocks": 9.672822322997305e-06, "lr_fusion": 9.672822322997305e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.011052029783604667} +{"avg_context_frames": 15.600000000000001, "avg_micro_time_s": 5.087632401741576, "epoch": 6, "global_step": 330, "grad_norm_max": 0.04424087703227997, "hidden_mse": 0.013128775986842811, "loss": 0.011932604259345682, "lr_blocks": 9.642770192448537e-06, "lr_fusion": 9.642770192448536e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.010619726672302932} +{"avg_context_frames": 16.8, "avg_micro_time_s": 5.32751318676892, "epoch": 6, "global_step": 340, "grad_norm_max": 0.06357772648334503, "hidden_mse": 0.012644789292244241, "loss": 0.01165091579896398, "lr_blocks": 9.611448774886925e-06, "lr_fusion": 9.611448774886924e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.010386436863336714} +{"avg_context_frames": 16.0, "avg_micro_time_s": 5.18907181514951, "epoch": 6, "global_step": 350, "grad_norm_max": 0.026117408648133278, "hidden_mse": 0.012417624722002075, "loss": 0.011300427973037586, "lr_blocks": 9.578866633275289e-06, "lr_fusion": 9.578866633275288e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.010058665514225141} +{"avg_context_frames": 12.8, "avg_micro_time_s": 4.4675404071531375, "epoch": 6, "global_step": 360, "grad_norm_max": 0.062091853469610214, "hidden_mse": 0.014877772179897875, "loss": 0.01396142343292013, "lr_blocks": 9.545032675245814e-06, "lr_fusion": 9.545032675245813e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.012473646248690785} +{"avg_context_frames": 14.0, "avg_micro_time_s": 4.731026683301025, "epoch": 6, "global_step": 370, "grad_norm_max": 0.040123265236616135, "hidden_mse": 0.014111349743325265, "loss": 0.013519475952489302, "lr_blocks": 9.509956150664796e-06, "lr_fusion": 9.509956150664796e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.012108340975828469} +{"avg_context_frames": 10.8, "avg_micro_time_s": 4.030816273787059, "epoch": 6, "global_step": 380, "grad_norm_max": 0.020823348313570023, "hidden_mse": 0.014053907786728815, "loss": 0.013168109755497426, "lr_blocks": 9.473646649103819e-06, "lr_fusion": 9.473646649103818e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.011762718920363114} +{"avg_context_frames": 14.4, "avg_micro_time_s": 4.821141281501332, "epoch": 7, "global_step": 390, "grad_norm_max": 0.035574499517679214, "hidden_mse": 0.012839521083515139, "loss": 0.011941187264164911, "lr_blocks": 9.43611409721806e-06, "lr_fusion": 9.43611409721806e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.01065723512438126} +{"avg_context_frames": 15.200000000000001, "avg_micro_time_s": 4.986738722560403, "epoch": 7, "global_step": 400, "grad_norm_max": 0.028655298054218292, "hidden_mse": 0.012679027632111685, "loss": 0.011727167258504778, "lr_blocks": 9.397368756032445e-06, "lr_fusion": 9.397368756032445e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.010459264490054921} +{"avg_context_frames": 16.8, "avg_micro_time_s": 5.325210076765507, "epoch": 7, "global_step": 410, "grad_norm_max": 0.04797655716538429, "hidden_mse": 0.012437554087955506, "loss": 0.011876341153401882, "lr_blocks": 9.357421218136387e-06, "lr_fusion": 9.357421218136386e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.010632585716666655} +{"avg_context_frames": 15.200000000000001, "avg_micro_time_s": 4.972153821858228, "epoch": 7, "global_step": 420, "grad_norm_max": 0.028172533959150314, "hidden_mse": 0.0125812633021269, "loss": 0.011770283954683693, "lr_blocks": 9.31628240478787e-06, "lr_fusion": 9.316282404787871e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.010512157646007837} +{"avg_context_frames": 10.0, "avg_micro_time_s": 3.842109894884925, "epoch": 7, "global_step": 430, "grad_norm_max": 0.03509347513318062, "hidden_mse": 0.014261556713609026, "loss": 0.013347374030854554, "lr_blocks": 9.273963562927695e-06, "lr_fusion": 9.273963562927695e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.011921218392672017} +{"avg_context_frames": 14.4, "avg_micro_time_s": 4.813104902210762, "epoch": 7, "global_step": 440, "grad_norm_max": 0.04009347781538963, "hidden_mse": 0.013169020065106452, "loss": 0.012601784808794037, "lr_blocks": 9.230476262104678e-06, "lr_fusion": 9.230476262104677e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.011284882773179562} +{"avg_context_frames": 14.8, "avg_micro_time_s": 4.914495927050303, "epoch": 8, "global_step": 450, "grad_norm_max": 0.04004997760057449, "hidden_mse": 0.012759855610784144, "loss": 0.01196123966947198, "lr_blocks": 9.185832391312644e-06, "lr_fusion": 9.185832391312644e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.010685254109557719} +{"avg_context_frames": 12.4, "avg_micro_time_s": 4.408888483086776, "epoch": 8, "global_step": 460, "grad_norm_max": 0.03200272098183632, "hidden_mse": 0.014400396059500054, "loss": 0.013772651209728794, "lr_blocks": 9.140044155740102e-06, "lr_fusion": 9.140044155740101e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.012332611647434533} +{"avg_context_frames": 12.0, "avg_micro_time_s": 4.328126795114076, "epoch": 8, "global_step": 470, "grad_norm_max": 0.033700667321681976, "hidden_mse": 0.014296967152040452, "loss": 0.014066640456439928, "lr_blocks": 9.093124073433464e-06, "lr_fusion": 9.093124073433463e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.012636943726101892} +{"avg_context_frames": 13.600000000000001, "avg_micro_time_s": 4.64108264833776, "epoch": 8, "global_step": 480, "grad_norm_max": 0.040272440761327744, "hidden_mse": 0.012330204661702739, "loss": 0.011031136312521995, "lr_blocks": 9.045084971874738e-06, "lr_fusion": 9.045084971874738e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.009798115864396096} +{"avg_context_frames": 16.0, "avg_micro_time_s": 5.209970523514494, "epoch": 8, "global_step": 490, "grad_norm_max": 0.037569738924503326, "hidden_mse": 0.012711368879536167, "loss": 0.01210305403219536, "lr_blocks": 8.995939984474624e-06, "lr_fusion": 8.995939984474624e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.010831917109317147} +{"avg_context_frames": 17.6, "avg_micro_time_s": 5.527175450035429, "epoch": 8, "global_step": 500, "grad_norm_max": 0.03873500972986221, "hidden_mse": 0.011477849434595556, "loss": 0.010430093860486524, "lr_blocks": 8.94570254698197e-06, "lr_fusion": 8.945702546981969e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.009282308939145879} +{"avg_context_frames": 14.8, "avg_micro_time_s": 4.919707209969056, "epoch": 8, "global_step": 510, "grad_norm_max": 0.03909330070018768, "hidden_mse": 0.012474938726518304, "loss": 0.0117218864208553, "lr_blocks": 8.894386393810563e-06, "lr_fusion": 8.894386393810563e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.010474392530159094} +{"avg_context_frames": 14.0, "avg_micro_time_s": 4.726613732382248, "epoch": 9, "global_step": 520, "grad_norm_max": 0.05085160955786705, "hidden_mse": 0.011974741116864607, "loss": 0.011234481947030873, "lr_blocks": 8.842005554284296e-06, "lr_fusion": 8.842005554284296e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.010037007875507698} +{"avg_context_frames": 15.600000000000001, "avg_micro_time_s": 5.08309825466713, "epoch": 9, "global_step": 530, "grad_norm_max": 0.0448479987680912, "hidden_mse": 0.013470317429164425, "loss": 0.012341475888388232, "lr_blocks": 8.788574348801676e-06, "lr_fusion": 8.788574348801675e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.01099444412975572} +{"avg_context_frames": 17.2, "avg_micro_time_s": 5.42189069060114, "epoch": 9, "global_step": 540, "grad_norm_max": 0.044435203075408936, "hidden_mse": 0.011698086501564831, "loss": 0.01084382500848733, "lr_blocks": 8.734107384920771e-06, "lr_fusion": 8.73410738492077e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.00967401631642133} +{"avg_context_frames": 14.4, "avg_micro_time_s": 4.8204727924385224, "epoch": 9, "global_step": 550, "grad_norm_max": 0.02751782350242138, "hidden_mse": 0.012882103119045496, "loss": 0.01243859903770499, "lr_blocks": 8.67861955336566e-06, "lr_fusion": 8.678619553365659e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.011150388704845682} +{"avg_context_frames": 14.0, "avg_micro_time_s": 4.719422702860903, "epoch": 9, "global_step": 560, "grad_norm_max": 0.03273526951670647, "hidden_mse": 0.012030398956267164, "loss": 0.011344008398009465, "lr_blocks": 8.622126023955446e-06, "lr_fusion": 8.622126023955446e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.010140968504128978} +{"avg_context_frames": 9.600000000000001, "avg_micro_time_s": 3.771413992259477, "epoch": 9, "global_step": 570, "grad_norm_max": 0.03181091696023941, "hidden_mse": 0.013318515225546435, "loss": 0.013125775242224337, "lr_blocks": 8.564642241456986e-06, "lr_fusion": 8.564642241456986e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.011793923727236689} +{"avg_context_frames": 13.600000000000001, "avg_micro_time_s": 4.670920807879884, "epoch": 10, "global_step": 580, "grad_norm_max": 0.027685070410370827, "hidden_mse": 0.013084714865544812, "loss": 0.012447188462829217, "lr_blocks": 8.506183921362443e-06, "lr_fusion": 8.506183921362443e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.01113871701527387} +{"avg_context_frames": 16.8, "avg_micro_time_s": 5.347909609867202, "epoch": 10, "global_step": 590, "grad_norm_max": 0.025338510051369667, "hidden_mse": 0.010903313546441496, "loss": 0.010352345078717918, "lr_blocks": 8.446767045592829e-06, "lr_fusion": 8.44676704559283e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.009262013656552881} +{"avg_context_frames": 14.0, "avg_micro_time_s": 4.7357259601005355, "epoch": 10, "global_step": 600, "grad_norm_max": 0.03063010424375534, "hidden_mse": 0.013168954622233288, "loss": 0.0130295607552398, "lr_blocks": 8.386407858128707e-06, "lr_fusion": 8.386407858128706e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.01171266527962871} +{"avg_context_frames": 14.8, "avg_micro_time_s": 4.9100341239900445, "epoch": 10, "global_step": 610, "grad_norm_max": 0.038056351244449615, "hidden_mse": 0.012423367868177593, "loss": 0.011564345302758738, "lr_blocks": 8.325122860569241e-06, "lr_fusion": 8.32512286056924e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.010322008503135294} +{"avg_context_frames": 11.600000000000001, "avg_micro_time_s": 4.195297092778492, "epoch": 10, "global_step": 620, "grad_norm_max": 0.035617273300886154, "hidden_mse": 0.013125881046289579, "loss": 0.012470233044587076, "lr_blocks": 8.262928807620843e-06, "lr_fusion": 8.262928807620843e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.011157644935883582} +{"avg_context_frames": 15.600000000000001, "avg_micro_time_s": 5.085111705910823, "epoch": 10, "global_step": 630, "grad_norm_max": 0.025537189096212387, "hidden_mse": 0.011752407019957901, "loss": 0.010974270798033104, "lr_blocks": 8.199842702516584e-06, "lr_fusion": 8.199842702516583e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.009799030108842999} +{"avg_context_frames": 15.600000000000001, "avg_micro_time_s": 5.067129819851835, "epoch": 10, "global_step": 640, "grad_norm_max": 0.03250515088438988, "hidden_mse": 0.012694398389430718, "loss": 0.012180439330404625, "lr_blocks": 8.135881792367686e-06, "lr_fusion": 8.135881792367686e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.010910999478073792} +{"avg_context_frames": 13.200000000000001, "avg_micro_time_s": 4.5460795130959015, "epoch": 11, "global_step": 650, "grad_norm_max": 0.022728320211172104, "hidden_mse": 0.012705811113119126, "loss": 0.012037340016104282, "lr_blocks": 8.071063563448341e-06, "lr_fusion": 8.07106356344834e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.010766758921090514} +{"avg_context_frames": 17.6, "avg_micro_time_s": 5.513876575080213, "epoch": 11, "global_step": 660, "grad_norm_max": 0.021445617079734802, "hidden_mse": 0.012317167350556702, "loss": 0.011524672619998455, "lr_blocks": 8.005405736415127e-06, "lr_fusion": 8.005405736415126e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.01029295590124093} +{"avg_context_frames": 15.600000000000001, "avg_micro_time_s": 5.078429749362113, "epoch": 11, "global_step": 670, "grad_norm_max": 0.03843489661812782, "hidden_mse": 0.012524095620028675, "loss": 0.011860672035254538, "lr_blocks": 7.938926261462366e-06, "lr_fusion": 7.938926261462366e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.010608262510504575} +{"avg_context_frames": 14.0, "avg_micro_time_s": 4.732145177536586, "epoch": 11, "global_step": 680, "grad_norm_max": 0.03108818829059601, "hidden_mse": 0.01183104386436753, "loss": 0.011368469058652408, "lr_blocks": 7.871643313414718e-06, "lr_fusion": 7.871643313414718e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.010185364648350514} +{"avg_context_frames": 10.8, "avg_micro_time_s": 4.042355540317658, "epoch": 11, "global_step": 690, "grad_norm_max": 0.01818351075053215, "hidden_mse": 0.012648749747313559, "loss": 0.012277464260114357, "lr_blocks": 7.803575286758365e-06, "lr_fusion": 7.803575286758364e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.011012589291203767} +{"avg_context_frames": 14.8, "avg_micro_time_s": 4.891382265406719, "epoch": 11, "global_step": 700, "grad_norm_max": 0.024554915726184845, "hidden_mse": 0.011927576072048397, "loss": 0.011489623971283437, "lr_blocks": 7.734740790612137e-06, "lr_fusion": 7.734740790612136e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.010296866396674887} +{"avg_context_frames": 12.4, "avg_micro_time_s": 4.381526562676299, "epoch": 12, "global_step": 710, "grad_norm_max": 0.023981647565960884, "hidden_mse": 0.012630624312441796, "loss": 0.012124884355580436, "lr_blocks": 7.66515864363997e-06, "lr_fusion": 7.66515864363997e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.010861821938306093} +{"avg_context_frames": 16.400000000000002, "avg_micro_time_s": 5.229092814735487, "epoch": 12, "global_step": 720, "grad_norm_max": 0.03212783485651016, "hidden_mse": 0.011585104855475948, "loss": 0.010935989135759883, "lr_blocks": 7.594847868906076e-06, "lr_fusion": 7.594847868906076e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.009777478597243316} +{"avg_context_frames": 13.600000000000001, "avg_micro_time_s": 4.625633412835305, "epoch": 12, "global_step": 730, "grad_norm_max": 0.02539096772670746, "hidden_mse": 0.011723813292337583, "loss": 0.011688095919089392, "lr_blocks": 7.52382768867422e-06, "lr_fusion": 7.52382768867422e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.010515714617213234} +{"avg_context_frames": 16.8, "avg_micro_time_s": 5.334037301102945, "epoch": 12, "global_step": 740, "grad_norm_max": 0.029416512697935104, "hidden_mse": 0.011872701771790162, "loss": 0.011209124152082951, "lr_blocks": 7.452117519152542e-06, "lr_fusion": 7.452117519152542e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.010021853970829398} +{"avg_context_frames": 16.0, "avg_micro_time_s": 5.150024331327586, "epoch": 12, "global_step": 750, "grad_norm_max": 0.02455662004649639, "hidden_mse": 0.012682124902494252, "loss": 0.011773688689572737, "lr_blocks": 7.379736965185369e-06, "lr_fusion": 7.379736965185368e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.010505476238904521} +{"avg_context_frames": 13.600000000000001, "avg_micro_time_s": 4.617077546336804, "epoch": 12, "global_step": 760, "grad_norm_max": 0.03465154767036438, "hidden_mse": 0.011634590977337211, "loss": 0.01099877167143859, "lr_blocks": 7.30670581489344e-06, "lr_fusion": 7.30670581489344e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.009835312550421804} +{"avg_context_frames": 12.4, "avg_micro_time_s": 4.369915128742287, "epoch": 13, "global_step": 770, "grad_norm_max": 0.026537969708442688, "hidden_mse": 0.011837815929902719, "loss": 0.011557188781443984, "lr_blocks": 7.233044034264034e-06, "lr_fusion": 7.233044034264034e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.010373407212318853} +{"avg_context_frames": 15.600000000000001, "avg_micro_time_s": 5.077053551097925, "epoch": 13, "global_step": 780, "grad_norm_max": 0.027972284704446793, "hidden_mse": 0.013373839907580988, "loss": 0.012345575907966123, "lr_blocks": 7.158771761692464e-06, "lr_fusion": 7.158771761692464e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.011008191935252399} +{"avg_context_frames": 15.200000000000001, "avg_micro_time_s": 5.005364236234164, "epoch": 13, "global_step": 790, "grad_norm_max": 0.025723490864038467, "hidden_mse": 0.01172727735247463, "loss": 0.011146590390126221, "lr_blocks": 7.083909302476453e-06, "lr_fusion": 7.083909302476453e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.009973862644983456} +{"avg_context_frames": 11.200000000000001, "avg_micro_time_s": 4.114617601607461, "epoch": 13, "global_step": 800, "grad_norm_max": 0.019907310605049133, "hidden_mse": 0.012267228442942724, "loss": 0.011972358723869548, "lr_blocks": 7.008477123264849e-06, "lr_fusion": 7.008477123264848e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.0107456358615309} +{"avg_context_frames": 14.4, "avg_micro_time_s": 4.812116305157542, "epoch": 13, "global_step": 810, "grad_norm_max": 0.01769108697772026, "hidden_mse": 0.01103784803999588, "loss": 0.010753370594466106, "lr_blocks": 6.932495846462262e-06, "lr_fusion": 6.932495846462261e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.009649585757870227} +{"avg_context_frames": 14.4, "avg_micro_time_s": 4.806550437118858, "epoch": 13, "global_step": 820, "grad_norm_max": 0.020690351724624634, "hidden_mse": 0.01134402338648215, "loss": 0.010883403144543991, "lr_blocks": 6.855986244591104e-06, "lr_fusion": 6.855986244591104e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.00974900086876005} +{"avg_context_frames": 15.200000000000001, "avg_micro_time_s": 5.0021984103019355, "epoch": 13, "global_step": 830, "grad_norm_max": 0.02440754510462284, "hidden_mse": 0.01217756257392466, "loss": 0.011580503406003119, "lr_blocks": 6.778969234612583e-06, "lr_fusion": 6.778969234612584e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.01036274707876146} +{"avg_context_frames": 15.600000000000001, "avg_micro_time_s": 5.101568386130385, "epoch": 14, "global_step": 840, "grad_norm_max": 0.02297673001885414, "hidden_mse": 0.012946964445291087, "loss": 0.012155569938477129, "lr_blocks": 6.701465872208216e-06, "lr_fusion": 6.701465872208216e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.010860873467754573} +{"avg_context_frames": 14.0, "avg_micro_time_s": 4.714361679725698, "epoch": 14, "global_step": 850, "grad_norm_max": 0.03143686428666115, "hidden_mse": 0.011695909610716627, "loss": 0.011157575895776973, "lr_blocks": 6.6234973460234184e-06, "lr_fusion": 6.623497346023418e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.00998798487125896} +{"avg_context_frames": 16.0, "avg_micro_time_s": 5.151228193003044, "epoch": 14, "global_step": 860, "grad_norm_max": 0.0305479783564806, "hidden_mse": 0.011425260681426153, "loss": 0.0110642658197321, "lr_blocks": 6.545084971874738e-06, "lr_fusion": 6.545084971874738e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.009921739730634727} +{"avg_context_frames": 11.600000000000001, "avg_micro_time_s": 4.211736782052322, "epoch": 14, "global_step": 870, "grad_norm_max": 0.026409652084112167, "hidden_mse": 0.01238853510003537, "loss": 0.012299565935973079, "lr_blocks": 6.466250186922325e-06, "lr_fusion": 6.466250186922325e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.011060712434118615} +{"avg_context_frames": 13.600000000000001, "avg_micro_time_s": 4.639771432151611, "epoch": 14, "global_step": 880, "grad_norm_max": 0.019695278257131577, "hidden_mse": 0.01219881126889959, "loss": 0.011934288605698385, "lr_blocks": 6.387014543809224e-06, "lr_fusion": 6.387014543809223e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.010714407538762316} +{"avg_context_frames": 16.0, "avg_micro_time_s": 5.142290710139787, "epoch": 14, "global_step": 890, "grad_norm_max": 0.016247669234871864, "hidden_mse": 0.011143641523085536, "loss": 0.01048286504810676, "lr_blocks": 6.3073997047691e-06, "lr_fusion": 6.307399704769099e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.009368500870186836} +{"avg_context_frames": 15.200000000000001, "avg_micro_time_s": 4.987847860732291, "epoch": 15, "global_step": 900, "grad_norm_max": 0.022062597796320915, "hidden_mse": 0.010571928170975298, "loss": 0.009913909807801248, "lr_blocks": 6.227427435703997e-06, "lr_fusion": 6.227427435703997e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.008856717040180229} +{"avg_context_frames": 14.8, "avg_micro_time_s": 4.879689114679059, "epoch": 15, "global_step": 910, "grad_norm_max": 0.02047191932797432, "hidden_mse": 0.011311443225713447, "loss": 0.010882898489944637, "lr_blocks": 6.147119600233758e-06, "lr_fusion": 6.147119600233758e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.009751754155149684} +{"avg_context_frames": 14.8, "avg_micro_time_s": 4.884743447283109, "epoch": 15, "global_step": 920, "grad_norm_max": 0.02199728973209858, "hidden_mse": 0.011389071762096137, "loss": 0.010999922681367026, "lr_blocks": 6.066498153718735e-06, "lr_fusion": 6.066498153718735e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.009861015487695114} +{"avg_context_frames": 12.4, "avg_micro_time_s": 4.34978794335475, "epoch": 15, "global_step": 930, "grad_norm_max": 0.03067207895219326, "hidden_mse": 0.013153657829388977, "loss": 0.012736196257174016, "lr_blocks": 5.985585137257401e-06, "lr_fusion": 5.985585137257401e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.011420830507995562} +{"avg_context_frames": 11.600000000000001, "avg_micro_time_s": 4.1916746335220525, "epoch": 15, "global_step": 940, "grad_norm_max": 0.020803604274988174, "hidden_mse": 0.012272004928672687, "loss": 0.012140020501101391, "lr_blocks": 5.904402671660551e-06, "lr_fusion": 5.90440267166055e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.010912819992518053} +{"avg_context_frames": 14.0, "avg_micro_time_s": 4.730226397464867, "epoch": 15, "global_step": 950, "grad_norm_max": 0.01921531930565834, "hidden_mse": 0.011913212860235944, "loss": 0.011088083271170036, "lr_blocks": 5.82297295140367e-06, "lr_fusion": 5.8229729514036705e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.00989676196477376} +{"avg_context_frames": 17.2, "avg_micro_time_s": 5.408907661697595, "epoch": 15, "global_step": 960, "grad_norm_max": 0.022268736734986305, "hidden_mse": 0.011531175393611193, "loss": 0.010782257761457005, "lr_blocks": 5.74131823855921e-06, "lr_fusion": 5.74131823855921e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.00962914024712518} +{"avg_context_frames": 16.0, "avg_micro_time_s": 5.17663503959775, "epoch": 16, "global_step": 970, "grad_norm_max": 0.022820301353931427, "hidden_mse": 0.011508953530574219, "loss": 0.011060651397565381, "lr_blocks": 5.659460856710346e-06, "lr_fusion": 5.6594608567103456e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.0099097560683731} +{"avg_context_frames": 15.600000000000001, "avg_micro_time_s": 5.105509522710054, "epoch": 16, "global_step": 980, "grad_norm_max": 0.013454040512442589, "hidden_mse": 0.011161358148092404, "loss": 0.01038940447033383, "lr_blocks": 5.577423184847932e-06, "lr_fusion": 5.577423184847932e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.009273268666584044} +{"avg_context_frames": 13.600000000000001, "avg_micro_time_s": 4.661925639589026, "epoch": 16, "global_step": 990, "grad_norm_max": 0.020632155239582062, "hidden_mse": 0.01134271832415834, "loss": 0.01091669797897339, "lr_blocks": 5.495227651252315e-06, "lr_fusion": 5.495227651252315e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.009782426143647173} +{"avg_context_frames": 16.0, "avg_micro_time_s": 5.164481800935755, "epoch": 16, "global_step": 1000, "grad_norm_max": 0.024563465267419815, "hidden_mse": 0.010671592492144555, "loss": 0.010145315830595792, "lr_blocks": 5.412896727361663e-06, "lr_fusion": 5.4128967273616625e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.00907815657556057} +{"avg_context_frames": 12.8, "avg_micro_time_s": 4.506241119086917, "epoch": 16, "global_step": 1010, "grad_norm_max": 0.022344304248690605, "hidden_mse": 0.01228530397056602, "loss": 0.011641308473190294, "lr_blocks": 5.3304529216284974e-06, "lr_fusion": 5.330452921628497e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.01041277811746113} +{"avg_context_frames": 12.0, "avg_micro_time_s": 4.3087444583768955, "epoch": 16, "global_step": 1020, "grad_norm_max": 0.01708345301449299, "hidden_mse": 0.01275833914987743, "loss": 0.012676269648363814, "lr_blocks": 5.247918773366112e-06, "lr_fusion": 5.247918773366112e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.011400435719406233} +{"avg_context_frames": 14.4, "avg_micro_time_s": 4.827499430411263, "epoch": 17, "global_step": 1030, "grad_norm_max": 0.019867179915308952, "hidden_mse": 0.011946292250650005, "loss": 0.011273921706015246, "lr_blocks": 5.165316846586541e-06, "lr_fusion": 5.165316846586541e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.010079292510636152} +{"avg_context_frames": 12.8, "avg_micro_time_s": 4.478448233670497, "epoch": 17, "global_step": 1040, "grad_norm_max": 0.016002237796783447, "hidden_mse": 0.012245260900817812, "loss": 0.01189771867939271, "lr_blocks": 5.082669723831793e-06, "lr_fusion": 5.0826697238317935e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.010673192591639236} +{"avg_context_frames": 16.8, "avg_micro_time_s": 5.323054229211994, "epoch": 17, "global_step": 1050, "grad_norm_max": 0.016390278935432434, "hidden_mse": 0.01077720423345454, "loss": 0.010232555813854561, "lr_blocks": 5e-06, "lr_fusion": 5e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.009154835378285498} +{"avg_context_frames": 13.200000000000001, "avg_micro_time_s": 4.562606139799755, "epoch": 17, "global_step": 1060, "grad_norm_max": 0.026438388973474503, "hidden_mse": 0.012213680887361989, "loss": 0.011351821088464931, "lr_blocks": 4.917330276168208e-06, "lr_fusion": 4.917330276168208e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.010130453016608955} +{"avg_context_frames": 14.4, "avg_micro_time_s": 4.812389872428321, "epoch": 17, "global_step": 1070, "grad_norm_max": 0.019659731537103653, "hidden_mse": 0.010969395382562653, "loss": 0.010577247897163035, "lr_blocks": 4.8346831534134595e-06, "lr_fusion": 4.834683153413459e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.009480308392085135} +{"avg_context_frames": 16.400000000000002, "avg_micro_time_s": 5.238875152201217, "epoch": 17, "global_step": 1080, "grad_norm_max": 0.01787360943853855, "hidden_mse": 0.010859828657703475, "loss": 0.010085330903530122, "lr_blocks": 4.752081226633888e-06, "lr_fusion": 4.7520812266338885e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.008999348030192778} +{"avg_context_frames": 11.200000000000001, "avg_micro_time_s": 4.11885388498049, "epoch": 18, "global_step": 1090, "grad_norm_max": 0.01641595922410488, "hidden_mse": 0.012650443211896345, "loss": 0.012473293306538836, "lr_blocks": 4.669547078371503e-06, "lr_fusion": 4.669547078371504e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.011208248959155754} +{"avg_context_frames": 13.200000000000001, "avg_micro_time_s": 4.5457417811703635, "epoch": 18, "global_step": 1100, "grad_norm_max": 0.025272108614444733, "hidden_mse": 0.01180228542070836, "loss": 0.011049933396861888, "lr_blocks": 4.587103272638339e-06, "lr_fusion": 4.5871032726383386e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.00986970489320811} +{"avg_context_frames": 17.6, "avg_micro_time_s": 5.51286342581443, "epoch": 18, "global_step": 1110, "grad_norm_max": 0.014824707061052322, "hidden_mse": 0.010852158506168053, "loss": 0.010271983779966833, "lr_blocks": 4.504772348747687e-06, "lr_fusion": 4.504772348747687e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.009186767926439644} +{"avg_context_frames": 12.0, "avg_micro_time_s": 4.278515099761717, "epoch": 18, "global_step": 1120, "grad_norm_max": 0.01857677847146988, "hidden_mse": 0.012096728855976836, "loss": 0.011542611452750862, "lr_blocks": 4.42257681515207e-06, "lr_fusion": 4.4225768151520694e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.010332938583451324} +{"avg_context_frames": 13.200000000000001, "avg_micro_time_s": 4.584787677015993, "epoch": 18, "global_step": 1130, "grad_norm_max": 0.034388478845357895, "hidden_mse": 0.011803314241115005, "loss": 0.011482165241613985, "lr_blocks": 4.340539143289655e-06, "lr_fusion": 4.3405391432896555e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.010301833850098774} +{"avg_context_frames": 15.200000000000001, "avg_micro_time_s": 5.016715088812635, "epoch": 18, "global_step": 1140, "grad_norm_max": 0.018398752436041832, "hidden_mse": 0.01164366492885165, "loss": 0.011105831584427506, "lr_blocks": 4.25868176144079e-06, "lr_fusion": 4.2586817614407895e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.00994146508164704} +{"avg_context_frames": 17.2, "avg_micro_time_s": 5.437036600100692, "epoch": 18, "global_step": 1150, "grad_norm_max": 0.01763196289539337, "hidden_mse": 0.010850942041724921, "loss": 0.010408626560820268, "lr_blocks": 4.17702704859633e-06, "lr_fusion": 4.17702704859633e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.009323532332200558} +{"avg_context_frames": 15.200000000000001, "avg_micro_time_s": 4.991289191308898, "epoch": 19, "global_step": 1160, "grad_norm_max": 0.021377425640821457, "hidden_mse": 0.011213436967227609, "loss": 0.010921710572438315, "lr_blocks": 4.0955973283394525e-06, "lr_fusion": 4.095597328339452e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.009800366894342006} +{"avg_context_frames": 13.600000000000001, "avg_micro_time_s": 4.648795517331746, "epoch": 19, "global_step": 1170, "grad_norm_max": 0.018862634897232056, "hidden_mse": 0.011030561162624509, "loss": 0.010480100923450664, "lr_blocks": 4.0144148627426e-06, "lr_fusion": 4.0144148627425993e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.009377044794382528} +{"avg_context_frames": 11.600000000000001, "avg_micro_time_s": 4.211887062057213, "epoch": 19, "global_step": 1180, "grad_norm_max": 0.022378750145435333, "hidden_mse": 0.01176152649568394, "loss": 0.011280642729252577, "lr_blocks": 3.9335018462812664e-06, "lr_fusion": 3.933501846281267e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.010104490071535112} +{"avg_context_frames": 13.200000000000001, "avg_micro_time_s": 4.55724394696299, "epoch": 19, "global_step": 1190, "grad_norm_max": 0.015526598319411278, "hidden_mse": 0.01172978574759327, "loss": 0.01136860313708894, "lr_blocks": 3.852880399766243e-06, "lr_fusion": 3.852880399766243e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.010195624592597596} +{"avg_context_frames": 16.8, "avg_micro_time_s": 5.300758411127027, "epoch": 19, "global_step": 1200, "grad_norm_max": 0.01526639424264431, "hidden_mse": 0.010083798522828147, "loss": 0.00967812390299514, "lr_blocks": 3.7725725642960047e-06, "lr_fusion": 3.772572564296005e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.008669744129292667} +{"avg_context_frames": 14.0, "avg_micro_time_s": 4.762128529167967, "epoch": 19, "global_step": 1210, "grad_norm_max": 0.019311314448714256, "hidden_mse": 0.013179845898412168, "loss": 0.012192463618703187, "lr_blocks": 3.6926002952309015e-06, "lr_fusion": 3.6926002952309016e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.010874479071935639} +{"avg_context_frames": 16.400000000000002, "avg_micro_time_s": 5.284176959557226, "epoch": 20, "global_step": 1220, "grad_norm_max": 0.019998129457235336, "hidden_mse": 0.011850596603471786, "loss": 0.010842873307410628, "lr_blocks": 3.6129854561907786e-06, "lr_fusion": 3.612985456190778e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.00965781364357099} +{"avg_context_frames": 12.8, "avg_micro_time_s": 4.483776406795369, "epoch": 20, "global_step": 1230, "grad_norm_max": 0.017271578311920166, "hidden_mse": 0.011415444285375997, "loss": 0.011352180110407062, "lr_blocks": 3.533749813077677e-06, "lr_fusion": 3.533749813077677e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.010210635705152528} +{"avg_context_frames": 13.600000000000001, "avg_micro_time_s": 4.656213254862815, "epoch": 20, "global_step": 1240, "grad_norm_max": 0.014064237475395203, "hidden_mse": 0.01148239875328727, "loss": 0.010843799856957048, "lr_blocks": 3.4549150281252635e-06, "lr_fusion": 3.4549150281252636e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.009695559914689511} +{"avg_context_frames": 13.200000000000001, "avg_micro_time_s": 4.560206924116938, "epoch": 20, "global_step": 1250, "grad_norm_max": 0.017670955508947372, "hidden_mse": 0.011065705091459677, "loss": 0.01071744965738617, "lr_blocks": 3.3765026539765832e-06, "lr_fusion": 3.3765026539765834e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.009610879124375062} +{"avg_context_frames": 16.8, "avg_micro_time_s": 5.328506174344511, "epoch": 20, "global_step": 1260, "grad_norm_max": 0.01102686021476984, "hidden_mse": 0.010547005163971335, "loss": 0.009686331939883532, "lr_blocks": 3.298534127791785e-06, "lr_fusion": 3.298534127791785e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.008631631423486398} +{"avg_context_frames": 13.200000000000001, "avg_micro_time_s": 4.561444571141329, "epoch": 20, "global_step": 1270, "grad_norm_max": 0.012799439951777458, "hidden_mse": 0.01207845499739051, "loss": 0.011482512799557299, "lr_blocks": 3.2210307653874175e-06, "lr_fusion": 3.221030765387417e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.010274667281191797} +{"avg_context_frames": 16.0, "avg_micro_time_s": 5.188423505361425, "epoch": 20, "global_step": 1280, "grad_norm_max": 0.015083969570696354, "hidden_mse": 0.01155220395885408, "loss": 0.01088867765502073, "lr_blocks": 3.1440137554088957e-06, "lr_fusion": 3.144013755408895e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.009733457188121976} +{"avg_context_frames": 14.0, "avg_micro_time_s": 4.790371892452822, "epoch": 21, "global_step": 1290, "grad_norm_max": 0.012053855694830418, "hidden_mse": 0.01114994349773042, "loss": 0.010579595039598644, "lr_blocks": 3.06750415353774e-06, "lr_fusion": 3.0675041535377405e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.00946460070554167} +{"avg_context_frames": 16.400000000000002, "avg_micro_time_s": 5.309114452893846, "epoch": 21, "global_step": 1300, "grad_norm_max": 0.01286385953426361, "hidden_mse": 0.010883161297533661, "loss": 0.010473219823325053, "lr_blocks": 2.991522876735154e-06, "lr_fusion": 2.991522876735154e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.009384903698810378} +{"avg_context_frames": 14.8, "avg_micro_time_s": 4.965425504064479, "epoch": 21, "global_step": 1310, "grad_norm_max": 0.013713131658732891, "hidden_mse": 0.011455239210044967, "loss": 0.011120659590233117, "lr_blocks": 2.9160906975235493e-06, "lr_fusion": 2.916090697523549e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.009975135698914528} +{"avg_context_frames": 11.600000000000001, "avg_micro_time_s": 4.2894228708799345, "epoch": 21, "global_step": 1320, "grad_norm_max": 0.012657473795115948, "hidden_mse": 0.012887466332176701, "loss": 0.011980939749628307, "lr_blocks": 2.8412282383075362e-06, "lr_fusion": 2.8412282383075363e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.010692193114664406} +{"avg_context_frames": 13.200000000000001, "avg_micro_time_s": 4.574161896950682, "epoch": 21, "global_step": 1330, "grad_norm_max": 0.012245282530784607, "hidden_mse": 0.011927370924968273, "loss": 0.011377652312512511, "lr_blocks": 2.766955965735968e-06, "lr_fusion": 2.766955965735968e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.010184915183344856} +{"avg_context_frames": 15.600000000000001, "avg_micro_time_s": 5.076783360091213, "epoch": 21, "global_step": 1340, "grad_norm_max": 0.016588730737566948, "hidden_mse": 0.010310821456369013, "loss": 0.009638946934137494, "lr_blocks": 2.693294185106562e-06, "lr_fusion": 2.693294185106562e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.008607864749501459} +{"avg_context_frames": 14.4, "avg_micro_time_s": 4.878384872936294, "epoch": 22, "global_step": 1350, "grad_norm_max": 0.013409581035375595, "hidden_mse": 0.011179147643269972, "loss": 0.010377954313298687, "lr_blocks": 2.6202630348146323e-06, "lr_fusion": 2.6202630348146324e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.009260039543733} +{"avg_context_frames": 14.4, "avg_micro_time_s": 4.834770848561312, "epoch": 22, "global_step": 1360, "grad_norm_max": 0.01109087560325861, "hidden_mse": 0.01136967032798566, "loss": 0.010608441021759063, "lr_blocks": 2.5478824808474613e-06, "lr_fusion": 2.547882480847461e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.009471473982557654} +{"avg_context_frames": 12.4, "avg_micro_time_s": 4.429204055744049, "epoch": 22, "global_step": 1370, "grad_norm_max": 0.021064532920718193, "hidden_mse": 0.012530555069679395, "loss": 0.012223612191155553, "lr_blocks": 2.476172311325783e-06, "lr_fusion": 2.476172311325783e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.010970556689426304} +{"avg_context_frames": 16.0, "avg_micro_time_s": 5.20361854062503, "epoch": 22, "global_step": 1380, "grad_norm_max": 0.0111702810972929, "hidden_mse": 0.010823962202994154, "loss": 0.01013918417156674, "lr_blocks": 2.4051521310939258e-06, "lr_fusion": 2.405152131093926e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.009056787952431478} +{"avg_context_frames": 14.0, "avg_micro_time_s": 4.7770776948862474, "epoch": 22, "global_step": 1390, "grad_norm_max": 0.0121107567101717, "hidden_mse": 0.011095769720850513, "loss": 0.01053232909180224, "lr_blocks": 2.3348413563600324e-06, "lr_fusion": 2.3348413563600325e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.009422752124373802} +{"avg_context_frames": 14.8, "avg_micro_time_s": 4.950418756072759, "epoch": 22, "global_step": 1400, "grad_norm_max": 0.01475017610937357, "hidden_mse": 0.011290456139249727, "loss": 0.010794430889654906, "lr_blocks": 2.265259209387867e-06, "lr_fusion": 2.2652592093878666e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.00966538522625342} +{"avg_context_frames": 16.400000000000002, "avg_micro_time_s": 5.306543833778415, "epoch": 23, "global_step": 1410, "grad_norm_max": 0.009046899154782295, "hidden_mse": 0.010367092461092398, "loss": 0.010030270897550509, "lr_blocks": 2.1964247132416373e-06, "lr_fusion": 2.196424713241637e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.008993561641545966} +{"avg_context_frames": 11.600000000000001, "avg_micro_time_s": 4.248495539659053, "epoch": 23, "global_step": 1420, "grad_norm_max": 0.010985208675265312, "hidden_mse": 0.011904564360156656, "loss": 0.011507103824988008, "lr_blocks": 2.1283566865852824e-06, "lr_fusion": 2.128356686585282e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.01031664742040448} +{"avg_context_frames": 15.200000000000001, "avg_micro_time_s": 5.026380934593909, "epoch": 23, "global_step": 1430, "grad_norm_max": 0.010216421447694302, "hidden_mse": 0.010574286128394306, "loss": 0.009837693121517077, "lr_blocks": 2.061073738537635e-06, "lr_fusion": 2.061073738537635e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.008780264511005954} +{"avg_context_frames": 13.600000000000001, "avg_micro_time_s": 4.664036268649215, "epoch": 23, "global_step": 1440, "grad_norm_max": 0.015373562462627888, "hidden_mse": 0.010974857053952292, "loss": 0.010402807081118227, "lr_blocks": 1.9945942635848745e-06, "lr_fusion": 1.9945942635848748e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.009305321337888018} +{"avg_context_frames": 14.8, "avg_micro_time_s": 4.932774202720612, "epoch": 23, "global_step": 1450, "grad_norm_max": 0.010674063116312027, "hidden_mse": 0.011168300587451087, "loss": 0.010629237722605468, "lr_blocks": 1.928936436551661e-06, "lr_fusion": 1.928936436551661e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.009512407711008565} +{"avg_context_frames": 14.4, "avg_micro_time_s": 4.878843769291416, "epoch": 23, "global_step": 1460, "grad_norm_max": 0.013738292269408703, "hidden_mse": 0.012406620447291062, "loss": 0.011606505373492837, "lr_blocks": 1.864118207632315e-06, "lr_fusion": 1.8641182076323148e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.010365843336330727} +{"avg_context_frames": 14.4, "avg_micro_time_s": 4.891958764340962, "epoch": 23, "global_step": 1470, "grad_norm_max": 0.01728452742099762, "hidden_mse": 0.011697437893599272, "loss": 0.01084344630362466, "lr_blocks": 1.8001572974834169e-06, "lr_fusion": 1.800157297483417e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.0096737024752656} +{"avg_context_frames": 16.0, "avg_micro_time_s": 5.221281339987764, "epoch": 24, "global_step": 1480, "grad_norm_max": 0.011064918711781502, "hidden_mse": 0.010385561746079476, "loss": 0.009919141844147817, "lr_blocks": 1.7370711923791567e-06, "lr_fusion": 1.7370711923791567e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.008880585647420959} +{"avg_context_frames": 10.4, "avg_micro_time_s": 3.9976374572259377, "epoch": 24, "global_step": 1490, "grad_norm_max": 0.009658539667725563, "hidden_mse": 0.012219062284566463, "loss": 0.011569672427140177, "lr_blocks": 1.6748771394307584e-06, "lr_fusion": 1.6748771394307585e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.010347766242921353} +{"avg_context_frames": 14.4, "avg_micro_time_s": 4.841567431348086, "epoch": 24, "global_step": 1500, "grad_norm_max": 0.008413000963628292, "hidden_mse": 0.01065224792691879, "loss": 0.009852373186731711, "lr_blocks": 1.6135921418712959e-06, "lr_fusion": 1.6135921418712956e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.008787148431292736} +{"avg_context_frames": 14.8, "avg_micro_time_s": 4.961790508120612, "epoch": 24, "global_step": 1510, "grad_norm_max": 0.009142924100160599, "hidden_mse": 0.011196566792204977, "loss": 0.010748471223632806, "lr_blocks": 1.5532329544071712e-06, "lr_fusion": 1.553232954407171e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.009628814493771643} +{"avg_context_frames": 14.4, "avg_micro_time_s": 4.861143801423896, "epoch": 24, "global_step": 1520, "grad_norm_max": 0.009023429825901985, "hidden_mse": 0.010903132130624727, "loss": 0.010474244510987774, "lr_blocks": 1.4938160786375571e-06, "lr_fusion": 1.4938160786375572e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.009383931249612943} +{"avg_context_frames": 17.2, "avg_micro_time_s": 5.445800206036075, "epoch": 24, "global_step": 1530, "grad_norm_max": 0.013588802888989449, "hidden_mse": 0.010612551239319146, "loss": 0.009695019305218011, "lr_blocks": 1.4353577585430152e-06, "lr_fusion": 1.435357758543015e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.008633764219121076} +{"avg_context_frames": 16.400000000000002, "avg_micro_time_s": 5.331034036508936, "epoch": 25, "global_step": 1540, "grad_norm_max": 0.008687515743076801, "hidden_mse": 0.011636157328030096, "loss": 0.010721514513716102, "lr_blocks": 1.3778739760445552e-06, "lr_fusion": 1.3778739760445552e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.00955789877625648} +{"avg_context_frames": 12.0, "avg_micro_time_s": 4.372138765837008, "epoch": 25, "global_step": 1550, "grad_norm_max": 0.013105207122862339, "hidden_mse": 0.01253440240980126, "loss": 0.011710797075647861, "lr_blocks": 1.321380446634342e-06, "lr_fusion": 1.3213804466343421e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.010457356815459208} +{"avg_context_frames": 15.600000000000001, "avg_micro_time_s": 5.143927716847975, "epoch": 25, "global_step": 1560, "grad_norm_max": 0.008294758386909962, "hidden_mse": 0.009838198433863, "loss": 0.009285178387654015, "lr_blocks": 1.2658926150792321e-06, "lr_fusion": 1.2658926150792322e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.00830135855358094} +{"avg_context_frames": 12.8, "avg_micro_time_s": 4.503292883196264, "epoch": 25, "global_step": 1570, "grad_norm_max": 0.011292783543467522, "hidden_mse": 0.011039233871269972, "loss": 0.010640830238116905, "lr_blocks": 1.2114256511983274e-06, "lr_fusion": 1.2114256511983274e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.009536906861467288} +{"avg_context_frames": 14.8, "avg_micro_time_s": 4.956366715789773, "epoch": 25, "global_step": 1580, "grad_norm_max": 0.007793950382620096, "hidden_mse": 0.010693331772927195, "loss": 0.010046476300340147, "lr_blocks": 1.157994445715706e-06, "lr_fusion": 1.157994445715706e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.008977143152151257} +{"avg_context_frames": 12.0, "avg_micro_time_s": 4.309033639264817, "epoch": 25, "global_step": 1590, "grad_norm_max": 0.007273074705153704, "hidden_mse": 0.012181815743679181, "loss": 0.011497003358090297, "lr_blocks": 1.1056136061894386e-06, "lr_fusion": 1.1056136061894384e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.010278821783140303} +{"avg_context_frames": 17.2, "avg_micro_time_s": 5.496322584386508, "epoch": 25, "global_step": 1600, "grad_norm_max": 0.009046373888850212, "hidden_mse": 0.011640732182422654, "loss": 0.010705592529848219, "lr_blocks": 1.0542974530180327e-06, "lr_fusion": 1.0542974530180327e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.009541519344202243} +{"avg_context_frames": 16.0, "avg_micro_time_s": 5.262786305665213, "epoch": 26, "global_step": 1610, "grad_norm_max": 0.0073905144818127155, "hidden_mse": 0.010503500420600177, "loss": 0.009702961420407519, "lr_blocks": 1.0040600155253766e-06, "lr_fusion": 1.0040600155253765e-05, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.008652611391153187} +{"avg_context_frames": 11.200000000000001, "avg_micro_time_s": 4.192173269177147, "epoch": 26, "global_step": 1620, "grad_norm_max": 0.012138673104345798, "hidden_mse": 0.012977444223361091, "loss": 0.012084819219307975, "lr_blocks": 9.549150281252633e-07, "lr_fusion": 9.549150281252633e-06, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.010787074809195475} +{"avg_context_frames": 15.200000000000001, "avg_micro_time_s": 5.036922406271334, "epoch": 26, "global_step": 1630, "grad_norm_max": 0.010161138139665127, "hidden_mse": 0.011468991311267019, "loss": 0.010536264715483412, "lr_blocks": 9.068759265665384e-07, "lr_fusion": 9.068759265665384e-06, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.009389365604147316} +{"avg_context_frames": 16.8, "avg_micro_time_s": 5.377813266972954, "epoch": 26, "global_step": 1640, "grad_norm_max": 0.008084629662334919, "hidden_mse": 0.010284584056353198, "loss": 0.009920223971130328, "lr_blocks": 8.599558442598998e-07, "lr_fusion": 8.599558442598998e-06, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.008891765552107245} +{"avg_context_frames": 16.8, "avg_micro_time_s": 5.347511923548883, "epoch": 26, "global_step": 1650, "grad_norm_max": 0.006152788642793894, "hidden_mse": 0.009940642048604786, "loss": 0.009401344880461694, "lr_blocks": 8.141676086873574e-07, "lr_fusion": 8.141676086873572e-06, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.008407280687242747} +{"avg_context_frames": 8.8, "avg_micro_time_s": 3.593481087048713, "epoch": 26, "global_step": 1660, "grad_norm_max": 0.00933996681123972, "hidden_mse": 0.012596608337480575, "loss": 0.011661105655366556, "lr_blocks": 7.695237378953224e-07, "lr_fusion": 7.695237378953223e-06, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.010401444789022208} +{"avg_context_frames": 18.0, "avg_micro_time_s": 5.573880312051916, "epoch": 27, "global_step": 1670, "grad_norm_max": 0.00642797676846385, "hidden_mse": 0.009880980965681374, "loss": 0.009457560011651368, "lr_blocks": 7.260364370723044e-07, "lr_fusion": 7.260364370723044e-06, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.008469461940694601} +{"avg_context_frames": 14.0, "avg_micro_time_s": 4.761761988872604, "epoch": 27, "global_step": 1680, "grad_norm_max": 0.007301956415176392, "hidden_mse": 0.011654333135811613, "loss": 0.0107377879525302, "lr_blocks": 6.837175952121305e-07, "lr_fusion": 6.837175952121306e-06, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.009572354651754723} +{"avg_context_frames": 13.600000000000001, "avg_micro_time_s": 4.703425024136959, "epoch": 27, "global_step": 1690, "grad_norm_max": 0.009609649889171124, "hidden_mse": 0.011718985246261582, "loss": 0.011018212127964944, "lr_blocks": 6.425787818636131e-07, "lr_fusion": 6.425787818636131e-06, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.009846313542220743} +{"avg_context_frames": 10.8, "avg_micro_time_s": 4.081076828495134, "epoch": 27, "global_step": 1700, "grad_norm_max": 0.008875456638634205, "hidden_mse": 0.012598920735763387, "loss": 0.01166050879401155, "lr_blocks": 6.026312439675553e-07, "lr_fusion": 6.026312439675552e-06, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.010400616721017287} +{"avg_context_frames": 16.8, "avg_micro_time_s": 5.387506154696166, "epoch": 27, "global_step": 1710, "grad_norm_max": 0.00561999948695302, "hidden_mse": 0.009851854038424791, "loss": 0.00909091651556082, "lr_blocks": 5.63885902781941e-07, "lr_fusion": 5.6388590278194096e-06, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.00810573115304578} +{"avg_context_frames": 15.600000000000001, "avg_micro_time_s": 5.13343480929907, "epoch": 27, "global_step": 1720, "grad_norm_max": 0.005707027856260538, "hidden_mse": 0.010750477807596326, "loss": 0.0100424542411929, "lr_blocks": 5.263533508961827e-07, "lr_fusion": 5.263533508961827e-06, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.00896740642201621} +{"avg_context_frames": 12.8, "avg_micro_time_s": 4.506750443899364, "epoch": 28, "global_step": 1730, "grad_norm_max": 0.007734287995845079, "hidden_mse": 0.011775779718300329, "loss": 0.011105859687086196, "lr_blocks": 4.900438493352056e-07, "lr_fusion": 4.900438493352055e-06, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.009928281692555175} +{"avg_context_frames": 15.200000000000001, "avg_micro_time_s": 5.078928724987782, "epoch": 28, "global_step": 1740, "grad_norm_max": 0.005856911651790142, "hidden_mse": 0.010255668702302502, "loss": 0.00959837067348417, "lr_blocks": 4.549673247541875e-07, "lr_fusion": 4.549673247541875e-06, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.008572803792776541} +{"avg_context_frames": 15.200000000000001, "avg_micro_time_s": 5.039590949176636, "epoch": 28, "global_step": 1750, "grad_norm_max": 0.004726439248770475, "hidden_mse": 0.01022255591233261, "loss": 0.009764218580676243, "lr_blocks": 4.211333667247125e-07, "lr_fusion": 4.2113336672471245e-06, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.008741962967906148} +{"avg_context_frames": 16.400000000000002, "avg_micro_time_s": 5.352418518347259, "epoch": 28, "global_step": 1760, "grad_norm_max": 0.008246401324868202, "hidden_mse": 0.011989802384050563, "loss": 0.010823496733792126, "lr_blocks": 3.885512251130763e-07, "lr_fusion": 3.885512251130763e-06, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.009624516547773965} +{"avg_context_frames": 13.600000000000001, "avg_micro_time_s": 4.65755109241436, "epoch": 28, "global_step": 1770, "grad_norm_max": 0.007200377061963081, "hidden_mse": 0.011120803788071499, "loss": 0.010340929633821362, "lr_blocks": 3.572298075514652e-07, "lr_fusion": 3.5722980755146517e-06, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.009228849236387759} +{"avg_context_frames": 14.0, "avg_micro_time_s": 4.761123923411652, "epoch": 28, "global_step": 1780, "grad_norm_max": 0.004230953752994537, "hidden_mse": 0.01117156931431964, "loss": 0.010479365871287883, "lr_blocks": 3.271776770026963e-07, "lr_fusion": 3.271776770026963e-06, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.009362208913080395} +{"avg_context_frames": 13.600000000000001, "avg_micro_time_s": 4.712614896507875, "epoch": 28, "global_step": 1790, "grad_norm_max": 0.006735358387231827, "hidden_mse": 0.011185237631434575, "loss": 0.01065414056647569, "lr_blocks": 2.984030494191942e-07, "lr_fusion": 2.9840304941919415e-06, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.009535616784705781} +{"avg_context_frames": 13.600000000000001, "avg_micro_time_s": 4.649150823097443, "epoch": 29, "global_step": 1800, "grad_norm_max": 0.004072031006217003, "hidden_mse": 0.011939600616460667, "loss": 0.011137658683583141, "lr_blocks": 2.7091379149682683e-07, "lr_fusion": 2.7091379149682685e-06, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.009943698620190844} +{"avg_context_frames": 16.400000000000002, "avg_micro_time_s": 5.254102718135982, "epoch": 29, "global_step": 1810, "grad_norm_max": 0.003994398284703493, "hidden_mse": 0.010172534408047796, "loss": 0.009583780786488206, "lr_blocks": 2.447174185242324e-07, "lr_fusion": 2.4471741852423237e-06, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.008566527353832499} +{"avg_context_frames": 14.4, "avg_micro_time_s": 4.819457044982119, "epoch": 29, "global_step": 1820, "grad_norm_max": 0.004754683934152126, "hidden_mse": 0.01153004780644551, "loss": 0.010618696815799923, "lr_blocks": 2.198210923282118e-07, "lr_fusion": 2.1982109232821178e-06, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.009465691997320391} +{"avg_context_frames": 16.400000000000002, "avg_micro_time_s": 5.249644676691969, "epoch": 29, "global_step": 1830, "grad_norm_max": 0.00519185746088624, "hidden_mse": 0.010572508641052991, "loss": 0.009832778020063417, "lr_blocks": 1.962316193157593e-07, "lr_fusion": 1.962316193157593e-06, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.008775527140824125} +{"avg_context_frames": 14.8, "avg_micro_time_s": 4.898425450809009, "epoch": 29, "global_step": 1840, "grad_norm_max": 0.0037243792321532965, "hidden_mse": 0.010621813527541235, "loss": 0.010002896655350924, "lr_blocks": 1.7395544861325718e-07, "lr_fusion": 1.7395544861325718e-06, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.008940715284552425} +{"avg_context_frames": 10.4, "avg_micro_time_s": 3.950617023606901, "epoch": 29, "global_step": 1850, "grad_norm_max": 0.0035942576359957457, "hidden_mse": 0.012405866576591507, "loss": 0.011399441107641906, "lr_blocks": 1.5299867030334815e-07, "lr_fusion": 1.5299867030334814e-06, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.010158854472683743} +{"avg_context_frames": 13.600000000000001, "avg_micro_time_s": 4.62971022570855, "epoch": 30, "global_step": 1860, "grad_norm_max": 0.004638120532035828, "hidden_mse": 0.01178389951819554, "loss": 0.011056046473095195, "lr_blocks": 1.333670137599713e-07, "lr_fusion": 1.333670137599713e-06, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.009877656534081326} +{"avg_context_frames": 14.4, "avg_micro_time_s": 4.8619691123865785, "epoch": 30, "global_step": 1870, "grad_norm_max": 0.0066008660942316055, "hidden_mse": 0.011297193734208122, "loss": 0.010252417510491796, "lr_blocks": 1.1506584608200366e-07, "lr_fusion": 1.1506584608200367e-06, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.009122698125429452} +{"avg_context_frames": 14.4, "avg_micro_time_s": 4.790847748231317, "epoch": 30, "global_step": 1880, "grad_norm_max": 0.004440458957105875, "hidden_mse": 0.010870393394725397, "loss": 0.010131842328701169, "lr_blocks": 9.810017062595322e-08, "lr_fusion": 9.810017062595322e-07, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.009044802992139013} +{"avg_context_frames": 16.0, "avg_micro_time_s": 5.172321240675228, "epoch": 30, "global_step": 1890, "grad_norm_max": 0.005037728697061539, "hidden_mse": 0.011203817953355611, "loss": 0.010442562453681603, "lr_blocks": 8.247462563808816e-08, "lr_fusion": 8.247462563808817e-07, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.009322180645540357} +{"avg_context_frames": 14.0, "avg_micro_time_s": 4.722507163675618, "epoch": 30, "global_step": 1900, "grad_norm_max": 0.003680848516523838, "hidden_mse": 0.010782711213687436, "loss": 0.010399468149989844, "lr_blocks": 6.819348298638839e-08, "lr_fusion": 6.819348298638839e-07, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.009321197075769306} +{"avg_context_frames": 15.600000000000001, "avg_micro_time_s": 5.083531078226224, "epoch": 30, "global_step": 1910, "grad_norm_max": 0.004569054581224918, "hidden_mse": 0.010476762289181353, "loss": 0.009930167609127238, "lr_blocks": 5.526064699265754e-08, "lr_fusion": 5.526064699265753e-07, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.008882491441909224} +{"avg_context_frames": 11.600000000000001, "avg_micro_time_s": 4.2070195361593505, "epoch": 30, "global_step": 1920, "grad_norm_max": 0.005403668619692326, "hidden_mse": 0.012119897897355259, "loss": 0.011162532260641456, "lr_blocks": 4.367965336512403e-08, "lr_fusion": 4.367965336512403e-07, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.00995054248487577} +{"avg_context_frames": 15.200000000000001, "avg_micro_time_s": 4.990781178131875, "epoch": 31, "global_step": 1930, "grad_norm_max": 0.00453431298956275, "hidden_mse": 0.010618909657932819, "loss": 0.009979333373485133, "lr_blocks": 3.345366823180929e-08, "lr_fusion": 3.3453668231809286e-07, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.008917442383244633} +{"avg_context_frames": 15.600000000000001, "avg_micro_time_s": 5.11586886132427, "epoch": 31, "global_step": 1940, "grad_norm_max": 0.005244948901236057, "hidden_mse": 0.01191076728864573, "loss": 0.010789375845342876, "lr_blocks": 2.4585487274942922e-08, "lr_fusion": 2.458548727494292e-07, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.009598299133358523} +{"avg_context_frames": 12.8, "avg_micro_time_s": 4.473000544760725, "epoch": 31, "global_step": 1950, "grad_norm_max": 0.005132991820573807, "hidden_mse": 0.010976211878005416, "loss": 0.010630597785348073, "lr_blocks": 1.7077534966650767e-08, "lr_fusion": 1.7077534966650766e-07, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.00953297658415977} +{"avg_context_frames": 14.8, "avg_micro_time_s": 4.927452837210149, "epoch": 31, "global_step": 1960, "grad_norm_max": 0.005845134612172842, "hidden_mse": 0.01088259765529074, "loss": 0.010048519587144256, "lr_blocks": 1.0931863906127327e-08, "lr_fusion": 1.0931863906127327e-07, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.008960259851301089} +{"avg_context_frames": 14.8, "avg_micro_time_s": 4.893246184429154, "epoch": 31, "global_step": 1970, "grad_norm_max": 0.0039673615247011185, "hidden_mse": 0.011230864329263568, "loss": 0.010503764549503103, "lr_blocks": 6.150154258476315e-09, "lr_fusion": 6.150154258476315e-08, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.00938067812821828} +{"avg_context_frames": 14.8, "avg_micro_time_s": 4.937694618946989, "epoch": 31, "global_step": 1980, "grad_norm_max": 0.004802702460438013, "hidden_mse": 0.011374481348320843, "loss": 0.010405051708221437, "lr_blocks": 2.7337132953697555e-09, "lr_fusion": 2.7337132953697554e-08, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.00926760359434411} +{"avg_context_frames": 11.200000000000001, "avg_micro_time_s": 4.119106574963371, "epoch": 32, "global_step": 1990, "grad_norm_max": 0.004685183521360159, "hidden_mse": 0.011852678400464356, "loss": 0.011158295348286629, "lr_blocks": 6.834750376549793e-10, "lr_fusion": 6.834750376549792e-09, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.009973027501837352} +{"avg_context_frames": 13.200000000000001, "avg_micro_time_s": 4.538173634780105, "epoch": 32, "global_step": 2000, "grad_norm_max": 0.0032657969277352095, "hidden_mse": 0.011564509710296988, "loss": 0.011026101780589671, "lr_blocks": 0.0, "lr_fusion": 0.0, "peak_memory_gib": 66.09157276153564, "velocity_mse": 0.009869650864857249} +swanlab: 📁 View project at +https://swanlab.cn/@ZhuoChen/HY-WorldPlay-Predictor-v4 +swanlab: 🚀 View run pre...cks1-52-fp32master-8gpu-torch26-noflash at +https://swanlab.cn/@ZhuoChen/HY-WorldPlay-Predictor-v4/runs/7zvpuasj +swanlab: Upload complete: 77230 records diff --git a/logs/predictor_v4_stage1_previous_condition_ablation_seed0/environment.log b/logs/predictor_v4_stage1_previous_condition_ablation_seed0/environment.log new file mode 100644 index 0000000000000000000000000000000000000000..64fe9c0f63d7f1f5f5ac4246b22f379ba04e67ac --- /dev/null +++ b/logs/predictor_v4_stage1_previous_condition_ablation_seed0/environment.log @@ -0,0 +1 @@ +torch=2.6.0+cu124 cuda=12.4 flash_attn=False diff --git a/logs/predictor_v4_stage1_previous_condition_ablation_seed0/none.log b/logs/predictor_v4_stage1_previous_condition_ablation_seed0/none.log new file mode 100644 index 0000000000000000000000000000000000000000..4d711a05e8f8c599726f3e1bd052f795ea70bc8e --- /dev/null +++ b/logs/predictor_v4_stage1_previous_condition_ablation_seed0/none.log @@ -0,0 +1,304 @@ +[start] mode=none 2026-08-23T07:30:10Z +{ + "manifest": "/mnt/local_nvme/cz/projects/HY-WorldPlay-DEV-Predictor/datasets/hyworldplay_predictor_prefeature_v4/train_eval_manifest.jsonl", + "teacher_checkpoint": "/mnt/local_nvme/cz/checkpoints/hy_worldplay/huggingface/hub/models--tencent--HY-WorldPlay/snapshots/f4c29235647707b571479a69b569e4166f9f5bf8/ar_distilled_action_model/diffusion_pytorch_model.safetensors", + "output_dir": "/mnt/local_nvme/cz/projects/HY-WorldPlay-DEV-Predictor/checkpoints/predictor_v4_stage1_previous_condition_ablation_seed0/none", + "source_blocks": [ + 1, + 52 + ], + "predictor_variant": "v4", + "previous_condition_mode": "none", + "predictor_fusion_type": "concat", + "gate_hidden_dim": 256, + "gate_initial_probability": 0.8, + "predictor_arch": "baseline", + "atc_variant": "full", + "atc_mlp_hidden_dim": 4096, + "atc_gate_hidden_dim": 512, + "atc_gate_initial_probability": 0.3, + "atc_transport_heads": 8, + "atc_transport_window": 3, + "atc_transport_query_chunk_size": 128, + "atc_transport_initial_scale": 0.1, + "max_steps": 2000, + "batch_size": 16, + "gradient_accumulation_steps": 1, + "block_warmup_steps": 100, + "block_warmup_mode": "delayed", + "scheduler_warmup_steps": 100, + "fusion_lr": 0.0001, + "blocks_lr": 1e-05, + "weight_decay": 0.01, + "hidden_weight": 0.1, + "velocity_weight": 1.0, + "grad_clip": 1.0, + "log_every": 10, + "save_every": 100, + "keep_snapshots": 20, + "seed": 0, + "num_workers": 0, + "max_records": null, + "bucket_by_context": true, + "gradient_checkpointing": true, + "fp32_trainable_params": true, + "train_delta_only": false, + "train_fusion_residual_only": false, + "save_optimizer_state": true, + "init_training_state": null, + "resume": null, + "swanlab_project": "HY-WorldPlay-Predictor-v4", + "swanlab_workspace": null, + "swanlab_experiment": "predictor-v4-stage1-previous-none-seed0-blocks1-52-fp32master-8gpu-torch26-noflash", + "swanlab_mode": "cloud", + "predictor_version": "v4_prefeature", + "schema_version": "predictor_context_prefeature_bf16_v1", + "supervision_pairs": [ + [ + 0, + 1 + ], + [ + 1, + 2 + ], + [ + 2, + 3 + ] + ], + "previous_condition_source": "offline_full_teacher_hidden", + "records": 2800, + "dataset_pairs": 8400, + "steps_per_epoch_per_rank": 64, + "world_size": 8, + "per_rank_batch_size": 16, + "global_batch_size": 128, + "trainable_parameters": 188826112, + "frozen_projector_parameters": 16785664, + "dtype": "bfloat16", + "activation_dtype": "bfloat16", + "trainable_parameter_dtype": "float32", + "optimizer_moment_dtype": "float32", + "inference_checkpoint_dtype": "bfloat16", + "gpu": "NVIDIA H100 80GB HBM3" +} +swanlab: Currently logged in as: ZhuoChen to https://api.swanlab.cn. Use +`swanlab login --relogin` to force relogin +swanlab: Tracking run with swanlab version 0.9.0 +swanlab: 💾 Run data saved at +/mnt/local_nvme/cz/projects/HY-WorldPlay-DEV-Predictor/checkpoints/predictor_v4_ +stage1_previous_condition_ablation_seed0/none/swanlab/run-20260823_073035-bofinl +sd +swanlab: 📁 View project at +https://swanlab.cn/@ZhuoChen/HY-WorldPlay-Predictor-v4 +swanlab: 🚀 View run pre...cks1-52-fp32master-8gpu-torch26-noflash at +https://swanlab.cn/@ZhuoChen/HY-WorldPlay-Predictor-v4/runs/bofinlsd +{"swanlab_run_id": "bofinlsd"} +{"avg_context_frames": 4.0, "avg_micro_time_s": 4.762151292103226, "epoch": 1, "global_step": 1, "grad_norm_max": 0.03848670795559883, "hidden_mse": 0.019336470868438482, "loss": 0.018770750728435814, "lr_blocks": 0.0, "lr_fusion": 2.0000000000000003e-06, "peak_memory_gib": 37.596481800079346, "velocity_mse": 0.016837103641591966} +{"avg_context_frames": 14.666666666666666, "avg_micro_time_s": 4.981213239437137, "epoch": 1, "global_step": 10, "grad_norm_max": 0.04661709442734718, "hidden_mse": 0.015848037437535822, "loss": 0.012681794663270313, "lr_blocks": 0.0, "lr_fusion": 1.1000000000000001e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.011096990997126946} +{"avg_context_frames": 13.600000000000001, "avg_micro_time_s": 4.605663323387853, "epoch": 1, "global_step": 20, "grad_norm_max": 0.03799311816692352, "hidden_mse": 0.015780533803626897, "loss": 0.012886741809779778, "lr_blocks": 0.0, "lr_fusion": 2.1e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.011308688475401141} +{"avg_context_frames": 14.4, "avg_micro_time_s": 4.7802090390483505, "epoch": 1, "global_step": 30, "grad_norm_max": 0.03805028647184372, "hidden_mse": 0.016334849293343723, "loss": 0.01359585518948734, "lr_blocks": 0.0, "lr_fusion": 3.1e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.011962370277615264} +{"avg_context_frames": 14.4, "avg_micro_time_s": 4.773400909586053, "epoch": 1, "global_step": 40, "grad_norm_max": 0.05718250200152397, "hidden_mse": 0.017969358531991022, "loss": 0.015266653610160575, "lr_blocks": 0.0, "lr_fusion": 4.1e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.0134697177214548} +{"avg_context_frames": 14.0, "avg_micro_time_s": 4.6738942055992085, "epoch": 1, "global_step": 50, "grad_norm_max": 0.053489163517951965, "hidden_mse": 0.015367755567422137, "loss": 0.012481599277816714, "lr_blocks": 0.0, "lr_fusion": 5.1000000000000006e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.0109448236937169} +{"avg_context_frames": 16.0, "avg_micro_time_s": 5.08863685776887, "epoch": 1, "global_step": 60, "grad_norm_max": 0.09598032385110855, "hidden_mse": 0.0141681652283296, "loss": 0.01248905256506987, "lr_blocks": 0.0, "lr_fusion": 6.1e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.011072235979372637} +{"avg_context_frames": 14.0, "avg_micro_time_s": 4.678022270341171, "epoch": 2, "global_step": 70, "grad_norm_max": 0.04857872799038887, "hidden_mse": 0.014942161994986236, "loss": 0.013064558588666841, "lr_blocks": 0.0, "lr_fusion": 7.1e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.011570342408958822} +{"avg_context_frames": 14.8, "avg_micro_time_s": 4.849018460648949, "epoch": 2, "global_step": 80, "grad_norm_max": 0.0527622327208519, "hidden_mse": 0.01458810783806257, "loss": 0.012851519999094308, "lr_blocks": 0.0, "lr_fusion": 8.1e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.011392709182109684} +{"avg_context_frames": 15.600000000000001, "avg_micro_time_s": 5.005762742880325, "epoch": 2, "global_step": 90, "grad_norm_max": 0.062335677444934845, "hidden_mse": 0.014189064298989251, "loss": 0.01242592892376706, "lr_blocks": 0.0, "lr_fusion": 9.1e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.01100702252588235} +{"avg_context_frames": 16.0, "avg_micro_time_s": 5.077984425648174, "epoch": 2, "global_step": 100, "grad_norm_max": 0.05781257152557373, "hidden_mse": 0.013560525252250955, "loss": 0.011383339227177203, "lr_blocks": 1e-05, "lr_fusion": 0.0001, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.010027286675176585} +{"avg_context_frames": 13.600000000000001, "avg_micro_time_s": 4.5587942360682066, "epoch": 2, "global_step": 110, "grad_norm_max": 0.04622247815132141, "hidden_mse": 0.013832488603657113, "loss": 0.01227779100299813, "lr_blocks": 9.999316524962347e-06, "lr_fusion": 9.999316524962345e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.010894542164169253} +{"avg_context_frames": 12.4, "avg_micro_time_s": 4.3171984173706734, "epoch": 2, "global_step": 120, "grad_norm_max": 0.056430574506521225, "hidden_mse": 0.016518641996663065, "loss": 0.015070496289990843, "lr_blocks": 9.99726628670463e-06, "lr_fusion": 9.997266286704631e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.013418632053071634} +{"avg_context_frames": 14.0, "avg_micro_time_s": 4.650928204455704, "epoch": 3, "global_step": 130, "grad_norm_max": 0.056439921259880066, "hidden_mse": 0.01558381836512126, "loss": 0.014245172444498168, "lr_blocks": 9.993849845741525e-06, "lr_fusion": 9.993849845741524e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.012686790656880477} +{"avg_context_frames": 17.2, "avg_micro_time_s": 5.366527427906113, "epoch": 3, "global_step": 140, "grad_norm_max": 0.04949598014354706, "hidden_mse": 0.013716132723493502, "loss": 0.012433233385672794, "lr_blocks": 9.989068136093873e-06, "lr_fusion": 9.989068136093873e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.011061620101099835} +{"avg_context_frames": 12.0, "avg_micro_time_s": 4.2043128562843775, "epoch": 3, "global_step": 150, "grad_norm_max": 0.04599063843488693, "hidden_mse": 0.015013728273333983, "loss": 0.013747061375761406, "lr_blocks": 9.98292246503335e-06, "lr_fusion": 9.98292246503335e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.012245688500115648} +{"avg_context_frames": 13.600000000000001, "avg_micro_time_s": 4.572553740392323, "epoch": 3, "global_step": 160, "grad_norm_max": 0.040193863213062286, "hidden_mse": 0.014332780695986003, "loss": 0.013257606438128278, "lr_blocks": 9.975414512725058e-06, "lr_fusion": 9.975414512725057e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.011824328312650324} +{"avg_context_frames": 13.600000000000001, "avg_micro_time_s": 4.591992597263015, "epoch": 3, "global_step": 170, "grad_norm_max": 0.0306017454713583, "hidden_mse": 0.014302136655896903, "loss": 0.013464634178671987, "lr_blocks": 9.966546331768192e-06, "lr_fusion": 9.966546331768191e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.012034420511918144} +{"avg_context_frames": 15.200000000000001, "avg_micro_time_s": 4.934291811381991, "epoch": 3, "global_step": 180, "grad_norm_max": 0.03101331740617752, "hidden_mse": 0.013132135989144445, "loss": 0.012051337264711038, "lr_blocks": 9.956320346634877e-06, "lr_fusion": 9.956320346634876e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.010738123644841835} +{"avg_context_frames": 15.600000000000001, "avg_micro_time_s": 5.00324143433536, "epoch": 3, "global_step": 190, "grad_norm_max": 0.03519980609416962, "hidden_mse": 0.012750334659358487, "loss": 0.011265645967796445, "lr_blocks": 9.944739353007344e-06, "lr_fusion": 9.944739353007344e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.009990612440742553} +{"avg_context_frames": 12.8, "avg_micro_time_s": 4.381113779840234, "epoch": 4, "global_step": 200, "grad_norm_max": 0.030088158324360847, "hidden_mse": 0.013456535671139136, "loss": 0.012549652095185593, "lr_blocks": 9.931806517013612e-06, "lr_fusion": 9.931806517013612e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.011203998560085894} +{"avg_context_frames": 14.0, "avg_micro_time_s": 4.67490445387084, "epoch": 4, "global_step": 210, "grad_norm_max": 0.04261542856693268, "hidden_mse": 0.012976797402370722, "loss": 0.01214339662110433, "lr_blocks": 9.917525374361913e-06, "lr_fusion": 9.917525374361912e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.01084571679821238} +{"avg_context_frames": 14.4, "avg_micro_time_s": 4.7459421817591645, "epoch": 4, "global_step": 220, "grad_norm_max": 0.04161163046956062, "hidden_mse": 0.012690479517914355, "loss": 0.012131938087986783, "lr_blocks": 9.901899829374048e-06, "lr_fusion": 9.901899829374047e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.01086289011873305} +{"avg_context_frames": 13.600000000000001, "avg_micro_time_s": 4.58937808090559, "epoch": 4, "global_step": 230, "grad_norm_max": 0.03311352804303169, "hidden_mse": 0.014200211450224744, "loss": 0.012777648749761283, "lr_blocks": 9.884934153917998e-06, "lr_fusion": 9.884934153917997e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.011357627657707781} +{"avg_context_frames": 13.200000000000001, "avg_micro_time_s": 4.532511411502492, "epoch": 4, "global_step": 240, "grad_norm_max": 0.037929292768239975, "hidden_mse": 0.014411382481921465, "loss": 0.013795509340707213, "lr_blocks": 9.86663298624003e-06, "lr_fusion": 9.86663298624003e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.01235437106806785} +{"avg_context_frames": 16.0, "avg_micro_time_s": 5.087296562727715, "epoch": 4, "global_step": 250, "grad_norm_max": 0.023749908432364464, "hidden_mse": 0.01199757379363291, "loss": 0.011047354724723846, "lr_blocks": 9.847001329696653e-06, "lr_fusion": 9.847001329696653e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.009847597335465252} +{"avg_context_frames": 16.8, "avg_micro_time_s": 5.266301933936484, "epoch": 5, "global_step": 260, "grad_norm_max": 0.027942156419157982, "hidden_mse": 0.012866420729551464, "loss": 0.012588273215806113, "lr_blocks": 9.826044551386743e-06, "lr_fusion": 9.826044551386744e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.011301631235983224} +{"avg_context_frames": 10.4, "avg_micro_time_s": 3.8817506656370826, "epoch": 5, "global_step": 270, "grad_norm_max": 0.04069827124476433, "hidden_mse": 0.014713287935592234, "loss": 0.014208584395237268, "lr_blocks": 9.803768380684242e-06, "lr_fusion": 9.803768380684242e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.012737255566753447} +{"avg_context_frames": 14.0, "avg_micro_time_s": 4.658292368704861, "epoch": 5, "global_step": 280, "grad_norm_max": 0.034294240176677704, "hidden_mse": 0.013367986475350336, "loss": 0.012552671547746287, "lr_blocks": 9.780178907671788e-06, "lr_fusion": 9.780178907671789e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.011215872876346112} +{"avg_context_frames": 15.200000000000001, "avg_micro_time_s": 4.913184907866526, "epoch": 5, "global_step": 290, "grad_norm_max": 0.07727838307619095, "hidden_mse": 0.01224559135735035, "loss": 0.01196555041242391, "lr_blocks": 9.755282581475769e-06, "lr_fusion": 9.755282581475769e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.010740991288912483} +{"avg_context_frames": 14.8, "avg_micro_time_s": 4.834281374403509, "epoch": 5, "global_step": 300, "grad_norm_max": 0.03901295363903046, "hidden_mse": 0.011805333098163829, "loss": 0.011570752266561613, "lr_blocks": 9.729086208503174e-06, "lr_fusion": 9.729086208503174e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.010390219005057589} +{"avg_context_frames": 16.8, "avg_micro_time_s": 5.270738072787935, "epoch": 5, "global_step": 310, "grad_norm_max": 0.032428890466690063, "hidden_mse": 0.01129992362111807, "loss": 0.010603717243066059, "lr_blocks": 9.701596950580807e-06, "lr_fusion": 9.701596950580806e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.009473724919371308} +{"avg_context_frames": 14.4, "avg_micro_time_s": 4.739742369580199, "epoch": 5, "global_step": 320, "grad_norm_max": 0.03079734742641449, "hidden_mse": 0.012718154658796266, "loss": 0.01167728099389933, "lr_blocks": 9.672822322997305e-06, "lr_fusion": 9.672822322997305e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.010405465489020571} +{"avg_context_frames": 15.600000000000001, "avg_micro_time_s": 4.99633021933405, "epoch": 6, "global_step": 330, "grad_norm_max": 0.02266876772046089, "hidden_mse": 0.012140661856392398, "loss": 0.011163931613555179, "lr_blocks": 9.642770192448537e-06, "lr_fusion": 9.642770192448536e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.00994986537261866} +{"avg_context_frames": 16.8, "avg_micro_time_s": 5.253954757991597, "epoch": 6, "global_step": 340, "grad_norm_max": 0.028467698022723198, "hidden_mse": 0.011551170586608352, "loss": 0.010992423456627876, "lr_blocks": 9.611448774886925e-06, "lr_fusion": 9.611448774886924e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.009837306442204864} +{"avg_context_frames": 16.0, "avg_micro_time_s": 5.079899284025306, "epoch": 6, "global_step": 350, "grad_norm_max": 0.022100435569882393, "hidden_mse": 0.011371495696948841, "loss": 0.010794030729448423, "lr_blocks": 9.578866633275289e-06, "lr_fusion": 9.578866633275288e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.009656881145201624} +{"avg_context_frames": 12.8, "avg_micro_time_s": 4.398697693928262, "epoch": 6, "global_step": 360, "grad_norm_max": 0.028619898483157158, "hidden_mse": 0.013821229001041503, "loss": 0.013357635354623199, "lr_blocks": 9.545032675245814e-06, "lr_fusion": 9.545032675245813e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.011975512426579371} +{"avg_context_frames": 14.0, "avg_micro_time_s": 4.687615850758448, "epoch": 6, "global_step": 370, "grad_norm_max": 0.03337932378053665, "hidden_mse": 0.013186370226321742, "loss": 0.01299535048310645, "lr_blocks": 9.509956150664796e-06, "lr_fusion": 9.509956150664796e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.01167671352159232} +{"avg_context_frames": 10.8, "avg_micro_time_s": 3.9727940955897796, "epoch": 6, "global_step": 380, "grad_norm_max": 0.026334108784794807, "hidden_mse": 0.013070544303627685, "loss": 0.012638726265868172, "lr_blocks": 9.473646649103819e-06, "lr_fusion": 9.473646649103818e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.011331671840162018} +{"avg_context_frames": 14.4, "avg_micro_time_s": 4.744302426747163, "epoch": 7, "global_step": 390, "grad_norm_max": 0.024069467559456825, "hidden_mse": 0.011971065355464817, "loss": 0.01143742444110103, "lr_blocks": 9.43611409721806e-06, "lr_fusion": 9.43611409721806e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.010240317904390395} +{"avg_context_frames": 15.200000000000001, "avg_micro_time_s": 4.9360351602110315, "epoch": 7, "global_step": 400, "grad_norm_max": 0.026781490072607994, "hidden_mse": 0.011864381947088987, "loss": 0.011240192881086842, "lr_blocks": 9.397368756032445e-06, "lr_fusion": 9.397368756032445e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.010053754679393024} +{"avg_context_frames": 16.8, "avg_micro_time_s": 5.265632639810793, "epoch": 7, "global_step": 410, "grad_norm_max": 0.02910347282886505, "hidden_mse": 0.011687422875547783, "loss": 0.011406643321970479, "lr_blocks": 9.357421218136387e-06, "lr_fusion": 9.357421218136386e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.010237901023356245} +{"avg_context_frames": 15.200000000000001, "avg_micro_time_s": 4.92420130607934, "epoch": 7, "global_step": 420, "grad_norm_max": 0.02800738625228405, "hidden_mse": 0.011864811775740237, "loss": 0.011400661279913038, "lr_blocks": 9.31628240478787e-06, "lr_fusion": 9.316282404787871e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.010214180074399337} +{"avg_context_frames": 10.0, "avg_micro_time_s": 3.7944296595873315, "epoch": 7, "global_step": 430, "grad_norm_max": 0.03242500126361847, "hidden_mse": 0.013470440090168269, "loss": 0.012844176247017459, "lr_blocks": 9.273963562927695e-06, "lr_fusion": 9.273963562927695e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.01149713220074773} +{"avg_context_frames": 14.4, "avg_micro_time_s": 4.772981342750427, "epoch": 7, "global_step": 440, "grad_norm_max": 0.023965537548065186, "hidden_mse": 0.01230547489831224, "loss": 0.012043614720460028, "lr_blocks": 9.230476262104678e-06, "lr_fusion": 9.230476262104677e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.010813067201524973} +{"avg_context_frames": 14.8, "avg_micro_time_s": 4.847352599287115, "epoch": 8, "global_step": 450, "grad_norm_max": 0.021985908970236778, "hidden_mse": 0.011981550813652576, "loss": 0.011436267773387954, "lr_blocks": 9.185832391312644e-06, "lr_fusion": 9.185832391312644e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.010238112704246306} +{"avg_context_frames": 12.4, "avg_micro_time_s": 4.342389860184631, "epoch": 8, "global_step": 460, "grad_norm_max": 0.02265697903931141, "hidden_mse": 0.013463902147486807, "loss": 0.01312111959559843, "lr_blocks": 9.140044155740102e-06, "lr_fusion": 9.140044155740101e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.011774729361059146} +{"avg_context_frames": 12.0, "avg_micro_time_s": 4.245438282878604, "epoch": 8, "global_step": 470, "grad_norm_max": 0.02227902226150036, "hidden_mse": 0.013429917034227402, "loss": 0.013380561507074163, "lr_blocks": 9.093124073433464e-06, "lr_fusion": 9.093124073433463e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.012037569790845738} +{"avg_context_frames": 13.600000000000001, "avg_micro_time_s": 4.559097037784523, "epoch": 8, "global_step": 480, "grad_norm_max": 0.026094423606991768, "hidden_mse": 0.011505403096089141, "loss": 0.010543525658431464, "lr_blocks": 9.045084971874738e-06, "lr_fusion": 9.045084971874738e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.00939298538141884} +{"avg_context_frames": 16.0, "avg_micro_time_s": 5.087354506402335, "epoch": 8, "global_step": 490, "grad_norm_max": 0.03045101836323738, "hidden_mse": 0.011917435383656994, "loss": 0.011521349981194362, "lr_blocks": 8.995939984474624e-06, "lr_fusion": 8.995939984474624e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.01032960648299195} +{"avg_context_frames": 17.6, "avg_micro_time_s": 5.446768684605195, "epoch": 8, "global_step": 500, "grad_norm_max": 0.02475646138191223, "hidden_mse": 0.010780319571495056, "loss": 0.010038511926541106, "lr_blocks": 8.94570254698197e-06, "lr_fusion": 8.945702546981969e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.008960480001405813} +{"avg_context_frames": 14.8, "avg_micro_time_s": 4.82343084863096, "epoch": 8, "global_step": 510, "grad_norm_max": 0.025624647736549377, "hidden_mse": 0.011794449377339334, "loss": 0.011315560806542635, "lr_blocks": 8.894386393810563e-06, "lr_fusion": 8.894386393810563e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.010136115885688924} +{"avg_context_frames": 14.0, "avg_micro_time_s": 4.666595962841529, "epoch": 9, "global_step": 520, "grad_norm_max": 0.0418974906206131, "hidden_mse": 0.011345733067719266, "loss": 0.010787713079480454, "lr_blocks": 8.842005554284296e-06, "lr_fusion": 8.842005554284296e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.009653139841975645} +{"avg_context_frames": 15.600000000000001, "avg_micro_time_s": 5.021413427664084, "epoch": 9, "global_step": 530, "grad_norm_max": 0.023907843977212906, "hidden_mse": 0.012732068361947313, "loss": 0.011850910563953222, "lr_blocks": 8.788574348801676e-06, "lr_fusion": 8.788574348801675e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.010577703718445264} +{"avg_context_frames": 17.2, "avg_micro_time_s": 5.380881479952951, "epoch": 9, "global_step": 540, "grad_norm_max": 0.023843156173825264, "hidden_mse": 0.01102459091343917, "loss": 0.01041046335012652, "lr_blocks": 8.734107384920771e-06, "lr_fusion": 8.73410738492077e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.009308004222111777} +{"avg_context_frames": 14.4, "avg_micro_time_s": 4.759774723411829, "epoch": 9, "global_step": 550, "grad_norm_max": 0.022636793553829193, "hidden_mse": 0.012126436637481676, "loss": 0.011906641122186557, "lr_blocks": 8.67861955336566e-06, "lr_fusion": 8.678619553365659e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.01069399748230353} +{"avg_context_frames": 14.0, "avg_micro_time_s": 4.677346480736742, "epoch": 9, "global_step": 560, "grad_norm_max": 0.01930987276136875, "hidden_mse": 0.01131048125680536, "loss": 0.010936896729981528, "lr_blocks": 8.622126023955446e-06, "lr_fusion": 8.622126023955446e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.009805848641553894} +{"avg_context_frames": 9.600000000000001, "avg_micro_time_s": 3.708226036078122, "epoch": 9, "global_step": 570, "grad_norm_max": 0.01599816046655178, "hidden_mse": 0.012620575691107661, "loss": 0.012525929091498257, "lr_blocks": 8.564642241456986e-06, "lr_fusion": 8.564642241456986e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.01126387152762618} +{"avg_context_frames": 13.600000000000001, "avg_micro_time_s": 4.575639447307913, "epoch": 10, "global_step": 580, "grad_norm_max": 0.02985103614628315, "hidden_mse": 0.012438407313311473, "loss": 0.011925903224619106, "lr_blocks": 8.506183921362443e-06, "lr_fusion": 8.506183921362443e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.01068206245545298} +{"avg_context_frames": 16.8, "avg_micro_time_s": 5.270001177744416, "epoch": 10, "global_step": 590, "grad_norm_max": 0.02321651019155979, "hidden_mse": 0.010367738391505555, "loss": 0.01006148760789074, "lr_blocks": 8.446767045592829e-06, "lr_fusion": 8.44676704559283e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.009024713741382584} +{"avg_context_frames": 14.0, "avg_micro_time_s": 4.69658444152883, "epoch": 10, "global_step": 600, "grad_norm_max": 0.023342672735452652, "hidden_mse": 0.012530325655825438, "loss": 0.012597436143551022, "lr_blocks": 8.386407858128707e-06, "lr_fusion": 8.386407858128706e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.011344403569819406} +{"avg_context_frames": 14.8, "avg_micro_time_s": 4.8497321871400345, "epoch": 10, "global_step": 610, "grad_norm_max": 0.028735198080539703, "hidden_mse": 0.011818257078994067, "loss": 0.011157826776616276, "lr_blocks": 8.325122860569241e-06, "lr_fusion": 8.32512286056924e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.009976001037284733} +{"avg_context_frames": 11.600000000000001, "avg_micro_time_s": 4.142936893693696, "epoch": 10, "global_step": 620, "grad_norm_max": 0.02332930453121662, "hidden_mse": 0.012512056261766703, "loss": 0.011937863257480787, "lr_blocks": 8.262928807620843e-06, "lr_fusion": 8.262928807620843e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.01068665764760226} +{"avg_context_frames": 15.600000000000001, "avg_micro_time_s": 5.010561677979422, "epoch": 10, "global_step": 630, "grad_norm_max": 0.021765615791082382, "hidden_mse": 0.011172114312648774, "loss": 0.0105506592663005, "lr_blocks": 8.199842702516584e-06, "lr_fusion": 8.199842702516583e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.009433447840274313} +{"avg_context_frames": 15.600000000000001, "avg_micro_time_s": 5.009309411895811, "epoch": 10, "global_step": 640, "grad_norm_max": 0.019307268783450127, "hidden_mse": 0.012035261397249997, "loss": 0.011505382036557422, "lr_blocks": 8.135881792367686e-06, "lr_fusion": 8.135881792367686e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.010301855864236133} +{"avg_context_frames": 13.200000000000001, "avg_micro_time_s": 4.486667810486688, "epoch": 11, "global_step": 650, "grad_norm_max": 0.02247830480337143, "hidden_mse": 0.01202990037854761, "loss": 0.011464118404546753, "lr_blocks": 8.071063563448341e-06, "lr_fusion": 8.07106356344834e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.010261128345155157} +{"avg_context_frames": 17.6, "avg_micro_time_s": 5.451789160753833, "epoch": 11, "global_step": 660, "grad_norm_max": 0.020574290305376053, "hidden_mse": 0.011748683999758215, "loss": 0.011060564510989935, "lr_blocks": 8.005405736415127e-06, "lr_fusion": 8.005405736415126e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.009885696164565161} +{"avg_context_frames": 15.600000000000001, "avg_micro_time_s": 5.028745436429745, "epoch": 11, "global_step": 670, "grad_norm_max": 0.03385557606816292, "hidden_mse": 0.011973587877582759, "loss": 0.011442645109491424, "lr_blocks": 7.938926261462366e-06, "lr_fusion": 7.938926261462366e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.010245286370627583} +{"avg_context_frames": 14.0, "avg_micro_time_s": 4.635213445122645, "epoch": 11, "global_step": 680, "grad_norm_max": 0.034260962158441544, "hidden_mse": 0.011264913744525984, "loss": 0.010931266855914147, "lr_blocks": 7.871643313414718e-06, "lr_fusion": 7.871643313414718e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.009804775458178483} +{"avg_context_frames": 10.8, "avg_micro_time_s": 3.9661532087702653, "epoch": 11, "global_step": 690, "grad_norm_max": 0.016513360664248466, "hidden_mse": 0.012141201627673582, "loss": 0.011850291746668518, "lr_blocks": 7.803575286758365e-06, "lr_fusion": 7.803575286758364e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.010636171617079526} +{"avg_context_frames": 14.8, "avg_micro_time_s": 4.853634571342264, "epoch": 11, "global_step": 700, "grad_norm_max": 0.019081976264715195, "hidden_mse": 0.011432281270390377, "loss": 0.011112847685581073, "lr_blocks": 7.734740790612137e-06, "lr_fusion": 7.734740790612136e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.00996961952187121} +{"avg_context_frames": 12.4, "avg_micro_time_s": 4.295945443461824, "epoch": 12, "global_step": 710, "grad_norm_max": 0.016820289194583893, "hidden_mse": 0.012068879511207343, "loss": 0.011634094815235585, "lr_blocks": 7.66515864363997e-06, "lr_fusion": 7.66515864363997e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.010427206897293219} +{"avg_context_frames": 16.400000000000002, "avg_micro_time_s": 5.1936479735901235, "epoch": 12, "global_step": 720, "grad_norm_max": 0.028356177732348442, "hidden_mse": 0.011086813308065758, "loss": 0.010511533479439096, "lr_blocks": 7.594847868906076e-06, "lr_fusion": 7.594847868906076e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.009402852153289133} +{"avg_context_frames": 13.600000000000001, "avg_micro_time_s": 4.590956429393555, "epoch": 12, "global_step": 730, "grad_norm_max": 0.016953621059656143, "hidden_mse": 0.011208402027841659, "loss": 0.011220439762109891, "lr_blocks": 7.52382768867422e-06, "lr_fusion": 7.52382768867422e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.010099599545355886} +{"avg_context_frames": 16.8, "avg_micro_time_s": 5.289492794175749, "epoch": 12, "global_step": 740, "grad_norm_max": 0.019774697721004486, "hidden_mse": 0.011336562712676823, "loss": 0.010762492561480031, "lr_blocks": 7.452117519152542e-06, "lr_fusion": 7.452117519152542e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.009628836321644485} +{"avg_context_frames": 16.0, "avg_micro_time_s": 5.127239219890908, "epoch": 12, "global_step": 750, "grad_norm_max": 0.01952684484422207, "hidden_mse": 0.012110512907383963, "loss": 0.011263733165105805, "lr_blocks": 7.379736965185369e-06, "lr_fusion": 7.379736965185368e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.01005268187436741} +{"avg_context_frames": 13.600000000000001, "avg_micro_time_s": 4.563850575075776, "epoch": 12, "global_step": 760, "grad_norm_max": 0.01640239544212818, "hidden_mse": 0.01110696673858911, "loss": 0.010564620979130268, "lr_blocks": 7.30670581489344e-06, "lr_fusion": 7.30670581489344e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.009453924276749605} +{"avg_context_frames": 12.4, "avg_micro_time_s": 4.31717497419595, "epoch": 13, "global_step": 770, "grad_norm_max": 0.01790228858590126, "hidden_mse": 0.011304982082219795, "loss": 0.0111156627710443, "lr_blocks": 7.233044034264034e-06, "lr_fusion": 7.233044034264034e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.009985164558747784} +{"avg_context_frames": 15.600000000000001, "avg_micro_time_s": 4.996845347913041, "epoch": 13, "global_step": 780, "grad_norm_max": 0.022648947313427925, "hidden_mse": 0.012829933851026, "loss": 0.011906621127855033, "lr_blocks": 7.158771761692464e-06, "lr_fusion": 7.158771761692464e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.010623627714812756} +{"avg_context_frames": 15.200000000000001, "avg_micro_time_s": 4.960352646200045, "epoch": 13, "global_step": 790, "grad_norm_max": 0.02009998820722103, "hidden_mse": 0.011243926925817506, "loss": 0.010759754033642821, "lr_blocks": 7.083909302476453e-06, "lr_fusion": 7.083909302476453e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.009635361333494075} +{"avg_context_frames": 11.200000000000001, "avg_micro_time_s": 4.087250665559259, "epoch": 13, "global_step": 800, "grad_norm_max": 0.01856914721429348, "hidden_mse": 0.011759783216984943, "loss": 0.01159047317923978, "lr_blocks": 7.008477123264849e-06, "lr_fusion": 7.008477123264848e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.010414494830183686} +{"avg_context_frames": 14.4, "avg_micro_time_s": 4.780655866609595, "epoch": 13, "global_step": 810, "grad_norm_max": 0.011573858559131622, "hidden_mse": 0.010595428838860244, "loss": 0.010448452265700327, "lr_blocks": 6.932495846462262e-06, "lr_fusion": 6.932495846462261e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.009388909372501076} +{"avg_context_frames": 14.4, "avg_micro_time_s": 4.79745815643546, "epoch": 13, "global_step": 820, "grad_norm_max": 0.019255926832556725, "hidden_mse": 0.010919769579777494, "loss": 0.01056575881666504, "lr_blocks": 6.855986244591104e-06, "lr_fusion": 6.855986244591104e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.009473781805718319} +{"avg_context_frames": 15.200000000000001, "avg_micro_time_s": 4.9791276299147285, "epoch": 13, "global_step": 830, "grad_norm_max": 0.023429743945598602, "hidden_mse": 0.011723412259016186, "loss": 0.011230392346624285, "lr_blocks": 6.778969234612583e-06, "lr_fusion": 6.778969234612584e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.010058051193482243} +{"avg_context_frames": 15.600000000000001, "avg_micro_time_s": 5.051606317168626, "epoch": 14, "global_step": 840, "grad_norm_max": 0.01930163986980915, "hidden_mse": 0.01251281897420995, "loss": 0.011743449076311663, "lr_blocks": 6.701465872208216e-06, "lr_fusion": 6.701465872208216e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.010492167191114277} +{"avg_context_frames": 14.0, "avg_micro_time_s": 4.679657639107609, "epoch": 14, "global_step": 850, "grad_norm_max": 0.022194625809788704, "hidden_mse": 0.011261694296263159, "loss": 0.010776551192975603, "lr_blocks": 6.6234973460234184e-06, "lr_fusion": 6.623497346023418e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.00965038180584088} +{"avg_context_frames": 16.0, "avg_micro_time_s": 5.120263557357248, "epoch": 14, "global_step": 860, "grad_norm_max": 0.02509160153567791, "hidden_mse": 0.010993058123858646, "loss": 0.010667429296881893, "lr_blocks": 6.545084971874738e-06, "lr_fusion": 6.545084971874738e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.00956812349613756} +{"avg_context_frames": 11.600000000000001, "avg_micro_time_s": 4.161493936098123, "epoch": 14, "global_step": 870, "grad_norm_max": 0.021270187571644783, "hidden_mse": 0.011933282716199756, "loss": 0.011852303263731302, "lr_blocks": 6.466250186922325e-06, "lr_fusion": 6.466250186922325e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.010658974983380178} +{"avg_context_frames": 13.600000000000001, "avg_micro_time_s": 4.610631607458345, "epoch": 14, "global_step": 880, "grad_norm_max": 0.021937578916549683, "hidden_mse": 0.01181057332432829, "loss": 0.01163646246714052, "lr_blocks": 6.387014543809224e-06, "lr_fusion": 6.387014543809223e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.010455405167886057} +{"avg_context_frames": 16.0, "avg_micro_time_s": 5.129588767088717, "epoch": 14, "global_step": 890, "grad_norm_max": 0.01230375561863184, "hidden_mse": 0.010734989197226243, "loss": 0.010184086137451233, "lr_blocks": 6.3073997047691e-06, "lr_fusion": 6.307399704769099e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.009110587218310684} +{"avg_context_frames": 15.200000000000001, "avg_micro_time_s": 4.982359376977548, "epoch": 15, "global_step": 900, "grad_norm_max": 0.020702483132481575, "hidden_mse": 0.010174744419055061, "loss": 0.009603622334543616, "lr_blocks": 6.227427435703997e-06, "lr_fusion": 6.227427435703997e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.008586147893220186} +{"avg_context_frames": 14.8, "avg_micro_time_s": 4.8980372467282, "epoch": 15, "global_step": 910, "grad_norm_max": 0.02189629338681698, "hidden_mse": 0.010901349381310866, "loss": 0.010486193912220188, "lr_blocks": 6.147119600233758e-06, "lr_fusion": 6.147119600233758e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.009396058972924948} +{"avg_context_frames": 14.8, "avg_micro_time_s": 4.854985787571059, "epoch": 15, "global_step": 920, "grad_norm_max": 0.015580134466290474, "hidden_mse": 0.010983562044566499, "loss": 0.010625599836930634, "lr_blocks": 6.066498153718735e-06, "lr_fusion": 6.066498153718735e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.009527243601041846} +{"avg_context_frames": 12.4, "avg_micro_time_s": 4.332862402533647, "epoch": 15, "global_step": 930, "grad_norm_max": 0.01973402313888073, "hidden_mse": 0.01268377808155492, "loss": 0.012295961583731696, "lr_blocks": 5.985585137257401e-06, "lr_fusion": 5.985585137257401e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.011027583794202656} +{"avg_context_frames": 11.600000000000001, "avg_micro_time_s": 4.169335038342979, "epoch": 15, "global_step": 940, "grad_norm_max": 0.01587819494307041, "hidden_mse": 0.011831260821782054, "loss": 0.011753488145768643, "lr_blocks": 5.904402671660551e-06, "lr_fusion": 5.90440267166055e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.010570362099679188} +{"avg_context_frames": 14.0, "avg_micro_time_s": 4.67880463568581, "epoch": 15, "global_step": 950, "grad_norm_max": 0.016212571412324905, "hidden_mse": 0.011496761871967465, "loss": 0.010780829522991553, "lr_blocks": 5.82297295140367e-06, "lr_fusion": 5.8229729514036705e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.00963115331833251} +{"avg_context_frames": 17.2, "avg_micro_time_s": 5.373770519137906, "epoch": 15, "global_step": 960, "grad_norm_max": 0.017012957483530045, "hidden_mse": 0.011156657518586145, "loss": 0.010438891049125232, "lr_blocks": 5.74131823855921e-06, "lr_fusion": 5.74131823855921e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.00932322524022311} +{"avg_context_frames": 16.0, "avg_micro_time_s": 5.120276839516009, "epoch": 16, "global_step": 970, "grad_norm_max": 0.017096003517508507, "hidden_mse": 0.011139071505749599, "loss": 0.0107249743596185, "lr_blocks": 5.659460856710346e-06, "lr_fusion": 5.6594608567103456e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.00961106718168594} +{"avg_context_frames": 15.600000000000001, "avg_micro_time_s": 5.034382682020078, "epoch": 16, "global_step": 980, "grad_norm_max": 0.013145791366696358, "hidden_mse": 0.010801950376480818, "loss": 0.01008177847834304, "lr_blocks": 5.577423184847932e-06, "lr_fusion": 5.577423184847932e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.009001583445933648} +{"avg_context_frames": 13.600000000000001, "avg_micro_time_s": 4.5841599470659276, "epoch": 16, "global_step": 990, "grad_norm_max": 0.015255802311003208, "hidden_mse": 0.010951224283780904, "loss": 0.010562052042223513, "lr_blocks": 5.495227651252315e-06, "lr_fusion": 5.495227651252315e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.009466929678455928} +{"avg_context_frames": 16.0, "avg_micro_time_s": 5.1199407061983955, "epoch": 16, "global_step": 1000, "grad_norm_max": 0.017258524894714355, "hidden_mse": 0.010333122673910112, "loss": 0.009837530920049176, "lr_blocks": 5.412896727361663e-06, "lr_fusion": 5.4128967273616625e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.008804218709701673} +{"avg_context_frames": 12.8, "avg_micro_time_s": 4.414588269837259, "epoch": 16, "global_step": 1010, "grad_norm_max": 0.020221570506691933, "hidden_mse": 0.011863487324444578, "loss": 0.011262252024607733, "lr_blocks": 5.3304529216284974e-06, "lr_fusion": 5.330452921628497e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.010075903238612228} +{"avg_context_frames": 12.0, "avg_micro_time_s": 4.241363725326664, "epoch": 16, "global_step": 1020, "grad_norm_max": 0.02329360321164131, "hidden_mse": 0.012356764008291067, "loss": 0.01230824298108928, "lr_blocks": 5.247918773366112e-06, "lr_fusion": 5.247918773366112e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.011072566581424326} +{"avg_context_frames": 14.4, "avg_micro_time_s": 4.7706073836627185, "epoch": 17, "global_step": 1030, "grad_norm_max": 0.014604498632252216, "hidden_mse": 0.01155251243035309, "loss": 0.010980011516949162, "lr_blocks": 5.165316846586541e-06, "lr_fusion": 5.165316846586541e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.009824760264018552} +{"avg_context_frames": 12.8, "avg_micro_time_s": 4.439029383803427, "epoch": 17, "global_step": 1040, "grad_norm_max": 0.014422290027141571, "hidden_mse": 0.011864299909211696, "loss": 0.011584639694774523, "lr_blocks": 5.082669723831793e-06, "lr_fusion": 5.0826697238317935e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.010398209671257064} +{"avg_context_frames": 16.8, "avg_micro_time_s": 5.287972552498104, "epoch": 17, "global_step": 1050, "grad_norm_max": 0.012556794099509716, "hidden_mse": 0.010417429771041499, "loss": 0.009942337241955103, "lr_blocks": 5e-06, "lr_fusion": 5e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.00890059424273204} +{"avg_context_frames": 13.200000000000001, "avg_micro_time_s": 4.531524181371788, "epoch": 17, "global_step": 1060, "grad_norm_max": 0.01162427756935358, "hidden_mse": 0.011846417229389773, "loss": 0.010972627240698786, "lr_blocks": 4.917330276168208e-06, "lr_fusion": 4.917330276168208e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.009787985519506038} +{"avg_context_frames": 14.4, "avg_micro_time_s": 4.781121213025472, "epoch": 17, "global_step": 1070, "grad_norm_max": 0.016112878918647766, "hidden_mse": 0.010615628608502448, "loss": 0.010271769168321045, "lr_blocks": 4.8346831534134595e-06, "lr_fusion": 4.834683153413459e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.009210206341231241} +{"avg_context_frames": 16.400000000000002, "avg_micro_time_s": 5.215458749645041, "epoch": 17, "global_step": 1080, "grad_norm_max": 0.012125704437494278, "hidden_mse": 0.010486891097389163, "loss": 0.009762267925543711, "lr_blocks": 4.752081226633888e-06, "lr_fusion": 4.7520812266338885e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.008713578869355842} +{"avg_context_frames": 11.200000000000001, "avg_micro_time_s": 4.11017642889783, "epoch": 18, "global_step": 1090, "grad_norm_max": 0.01271024439483881, "hidden_mse": 0.012256077287020162, "loss": 0.012049886648310349, "lr_blocks": 4.669547078371503e-06, "lr_fusion": 4.669547078371504e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.010824278951622546} +{"avg_context_frames": 13.200000000000001, "avg_micro_time_s": 4.560629206844896, "epoch": 18, "global_step": 1100, "grad_norm_max": 0.017399823293089867, "hidden_mse": 0.011418378917733209, "loss": 0.010645095928339288, "lr_blocks": 4.587103272638339e-06, "lr_fusion": 4.5871032726383386e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.009503258054610343} +{"avg_context_frames": 17.6, "avg_micro_time_s": 5.535053869942203, "epoch": 18, "global_step": 1110, "grad_norm_max": 0.010781140998005867, "hidden_mse": 0.010519632406067105, "loss": 0.009955240786075592, "lr_blocks": 4.504772348747687e-06, "lr_fusion": 4.504772348747687e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.00890327756642364} +{"avg_context_frames": 12.0, "avg_micro_time_s": 4.286398434638977, "epoch": 18, "global_step": 1120, "grad_norm_max": 0.013983622193336487, "hidden_mse": 0.01173855439410545, "loss": 0.011235184589168057, "lr_blocks": 4.42257681515207e-06, "lr_fusion": 4.4225768151520694e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.010061329221935013} +{"avg_context_frames": 13.200000000000001, "avg_micro_time_s": 4.54721834321972, "epoch": 18, "global_step": 1130, "grad_norm_max": 0.02523583546280861, "hidden_mse": 0.011431433603866026, "loss": 0.011173607682576404, "lr_blocks": 4.340539143289655e-06, "lr_fusion": 4.3405391432896555e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.010030464339070023} +{"avg_context_frames": 15.200000000000001, "avg_micro_time_s": 5.076442328169652, "epoch": 18, "global_step": 1140, "grad_norm_max": 0.013606994412839413, "hidden_mse": 0.011298774502938614, "loss": 0.010791620519012213, "lr_blocks": 4.25868176144079e-06, "lr_fusion": 4.2586817614407895e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.009661743068136275} +{"avg_context_frames": 17.2, "avg_micro_time_s": 5.4701070228125905, "epoch": 18, "global_step": 1150, "grad_norm_max": 0.015463371761143208, "hidden_mse": 0.010527873021783308, "loss": 0.010150764521677048, "lr_blocks": 4.17702704859633e-06, "lr_fusion": 4.17702704859633e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.00909797728236299} +{"avg_context_frames": 15.200000000000001, "avg_micro_time_s": 4.967304617713672, "epoch": 19, "global_step": 1160, "grad_norm_max": 0.018161801621317863, "hidden_mse": 0.01088655036292039, "loss": 0.010653895721770824, "lr_blocks": 4.0955973283394525e-06, "lr_fusion": 4.095597328339452e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.009565240668598563} +{"avg_context_frames": 13.600000000000001, "avg_micro_time_s": 4.584564515487, "epoch": 19, "global_step": 1170, "grad_norm_max": 0.012305768206715584, "hidden_mse": 0.0107141392538324, "loss": 0.01025349728879519, "lr_blocks": 4.0144148627426e-06, "lr_fusion": 4.0144148627425993e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.009182083341875114} +{"avg_context_frames": 11.600000000000001, "avg_micro_time_s": 4.151958814366663, "epoch": 19, "global_step": 1180, "grad_norm_max": 0.02170264720916748, "hidden_mse": 0.011395214148797095, "loss": 0.010994421708164738, "lr_blocks": 3.9335018462812664e-06, "lr_fusion": 3.933501846281267e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.009854900260688738} +{"avg_context_frames": 13.200000000000001, "avg_micro_time_s": 4.563660281444027, "epoch": 19, "global_step": 1190, "grad_norm_max": 0.012783233076334, "hidden_mse": 0.011410185007844121, "loss": 0.011043253616662696, "lr_blocks": 3.852880399766243e-06, "lr_fusion": 3.852880399766243e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.00990223514090758} +{"avg_context_frames": 16.8, "avg_micro_time_s": 5.282186317091692, "epoch": 19, "global_step": 1200, "grad_norm_max": 0.014000105671584606, "hidden_mse": 0.009784756315639243, "loss": 0.009384909871732816, "lr_blocks": 3.7725725642960047e-06, "lr_fusion": 3.772572564296005e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.008406434213975444} +{"avg_context_frames": 14.0, "avg_micro_time_s": 4.62493295618915, "epoch": 19, "global_step": 1210, "grad_norm_max": 0.017361069098114967, "hidden_mse": 0.012824688112596051, "loss": 0.011821632797364146, "lr_blocks": 3.6926002952309015e-06, "lr_fusion": 3.6926002952309016e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.010539164001238534} +{"avg_context_frames": 16.400000000000002, "avg_micro_time_s": 5.155958832954639, "epoch": 20, "global_step": 1220, "grad_norm_max": 0.016311736777424812, "hidden_mse": 0.011495744867715985, "loss": 0.010484484833432362, "lr_blocks": 3.6129854561907786e-06, "lr_fusion": 3.612985456190778e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.009334910369943827} +{"avg_context_frames": 12.8, "avg_micro_time_s": 4.363768623696524, "epoch": 20, "global_step": 1230, "grad_norm_max": 0.013402091339230537, "hidden_mse": 0.011093755182810128, "loss": 0.011019175554974937, "lr_blocks": 3.533749813077677e-06, "lr_fusion": 3.533749813077677e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.009909800018067473} +{"avg_context_frames": 13.600000000000001, "avg_micro_time_s": 4.529210444715864, "epoch": 20, "global_step": 1240, "grad_norm_max": 0.013183310627937317, "hidden_mse": 0.011160887347068639, "loss": 0.010557603818597272, "lr_blocks": 3.4549150281252635e-06, "lr_fusion": 3.4549150281252636e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.009441515142680146} +{"avg_context_frames": 13.200000000000001, "avg_micro_time_s": 4.4451388211673475, "epoch": 20, "global_step": 1250, "grad_norm_max": 0.018259108066558838, "hidden_mse": 0.010740913456538694, "loss": 0.010403757268795744, "lr_blocks": 3.3765026539765832e-06, "lr_fusion": 3.3765026539765834e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.009329665964469314} +{"avg_context_frames": 16.8, "avg_micro_time_s": 5.222734866509564, "epoch": 20, "global_step": 1260, "grad_norm_max": 0.009890066459774971, "hidden_mse": 0.010259215673431755, "loss": 0.009419662499567494, "lr_blocks": 3.298534127791785e-06, "lr_fusion": 3.298534127791785e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.008393740962492303} +{"avg_context_frames": 13.200000000000001, "avg_micro_time_s": 4.45880435997824, "epoch": 20, "global_step": 1270, "grad_norm_max": 0.01061070617288351, "hidden_mse": 0.01173077785060741, "loss": 0.011168135888874532, "lr_blocks": 3.2210307653874175e-06, "lr_fusion": 3.221030765387417e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.009995058050844819} +{"avg_context_frames": 16.0, "avg_micro_time_s": 5.060942972618796, "epoch": 20, "global_step": 1280, "grad_norm_max": 0.01174140628427267, "hidden_mse": 0.011257574008777738, "loss": 0.010587803629459814, "lr_blocks": 3.1440137554088957e-06, "lr_fusion": 3.144013755408895e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.00946204626816325} +{"avg_context_frames": 14.0, "avg_micro_time_s": 4.617468223371543, "epoch": 21, "global_step": 1290, "grad_norm_max": 0.009279873222112656, "hidden_mse": 0.010843503143405542, "loss": 0.010285751457558945, "lr_blocks": 3.06750415353774e-06, "lr_fusion": 3.0675041535377405e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.009201401108293794} +{"avg_context_frames": 16.400000000000002, "avg_micro_time_s": 5.146299298723171, "epoch": 21, "global_step": 1300, "grad_norm_max": 0.010406007058918476, "hidden_mse": 0.010595145722618327, "loss": 0.010198809206485748, "lr_blocks": 2.991522876735154e-06, "lr_fusion": 2.991522876735154e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.00913929465750698} +{"avg_context_frames": 14.8, "avg_micro_time_s": 4.806147188332398, "epoch": 21, "global_step": 1310, "grad_norm_max": 0.016675801947712898, "hidden_mse": 0.0111664138850756, "loss": 0.010829223383916543, "lr_blocks": 2.9160906975235493e-06, "lr_fusion": 2.916090697523549e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.009712582029169426} +{"avg_context_frames": 11.600000000000001, "avg_micro_time_s": 4.106639125540096, "epoch": 21, "global_step": 1320, "grad_norm_max": 0.015449466183781624, "hidden_mse": 0.012570883164880798, "loss": 0.011704751214710997, "lr_blocks": 2.8412282383075362e-06, "lr_fusion": 2.8412282383075363e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.010447662870865316} +{"avg_context_frames": 13.200000000000001, "avg_micro_time_s": 4.4643723360684815, "epoch": 21, "global_step": 1330, "grad_norm_max": 0.013478947803378105, "hidden_mse": 0.011603340087458492, "loss": 0.011096158871077934, "lr_blocks": 2.766955965735968e-06, "lr_fusion": 2.766955965735968e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.009935824858257548} +{"avg_context_frames": 15.600000000000001, "avg_micro_time_s": 4.987799538315449, "epoch": 21, "global_step": 1340, "grad_norm_max": 0.015309369191527367, "hidden_mse": 0.010036737454356626, "loss": 0.009409548994153739, "lr_blocks": 2.693294185106562e-06, "lr_fusion": 2.693294185106562e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.008405875254538842} +{"avg_context_frames": 14.4, "avg_micro_time_s": 4.722995287810045, "epoch": 22, "global_step": 1350, "grad_norm_max": 0.011092848144471645, "hidden_mse": 0.010885873914230616, "loss": 0.010092562483623625, "lr_blocks": 2.6202630348146323e-06, "lr_fusion": 2.6202630348146324e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.009003975088126026} +{"avg_context_frames": 14.4, "avg_micro_time_s": 4.6997389111653325, "epoch": 22, "global_step": 1360, "grad_norm_max": 0.01276668906211853, "hidden_mse": 0.011074058577651159, "loss": 0.010334288160083816, "lr_blocks": 2.5478824808474613e-06, "lr_fusion": 2.547882480847461e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.009226882312214002} +{"avg_context_frames": 12.4, "avg_micro_time_s": 4.299101527153107, "epoch": 22, "global_step": 1370, "grad_norm_max": 0.016638632863759995, "hidden_mse": 0.012231126398546621, "loss": 0.01194828191655688, "lr_blocks": 2.476172311325783e-06, "lr_fusion": 2.476172311325783e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.010725169291254133} +{"avg_context_frames": 16.0, "avg_micro_time_s": 5.046185013758077, "epoch": 22, "global_step": 1380, "grad_norm_max": 0.013440818525850773, "hidden_mse": 0.01054119669715874, "loss": 0.009862290491582826, "lr_blocks": 2.4051521310939258e-06, "lr_fusion": 2.405152131093926e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.00880817076540552} +{"avg_context_frames": 14.0, "avg_micro_time_s": 4.605366279740702, "epoch": 22, "global_step": 1390, "grad_norm_max": 0.011775731109082699, "hidden_mse": 0.010806745773879812, "loss": 0.010260508145438508, "lr_blocks": 2.3348413563600324e-06, "lr_fusion": 2.3348413563600325e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.00917983366525732} +{"avg_context_frames": 14.8, "avg_micro_time_s": 4.799859227264824, "epoch": 22, "global_step": 1400, "grad_norm_max": 0.013671238906681538, "hidden_mse": 0.011007250455440954, "loss": 0.01046952257747762, "lr_blocks": 2.265259209387867e-06, "lr_fusion": 2.2652592093878666e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.009368797481874936} +{"avg_context_frames": 16.400000000000002, "avg_micro_time_s": 5.13914484778652, "epoch": 23, "global_step": 1410, "grad_norm_max": 0.009764766320586205, "hidden_mse": 0.010092604951933027, "loss": 0.009709037846187131, "lr_blocks": 2.1964247132416373e-06, "lr_fusion": 2.196424713241637e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.00869977737311274} +{"avg_context_frames": 11.600000000000001, "avg_micro_time_s": 4.116821014993184, "epoch": 23, "global_step": 1420, "grad_norm_max": 0.010225928388535976, "hidden_mse": 0.011598363640950994, "loss": 0.011211766191991046, "lr_blocks": 2.1283566865852824e-06, "lr_fusion": 2.128356686585282e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.010051929799374193} +{"avg_context_frames": 15.200000000000001, "avg_micro_time_s": 4.890675671151257, "epoch": 23, "global_step": 1430, "grad_norm_max": 0.009803097695112228, "hidden_mse": 0.010295259795384482, "loss": 0.009591977734817193, "lr_blocks": 2.061073738537635e-06, "lr_fusion": 2.061073738537635e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.008562451711622999} +{"avg_context_frames": 13.600000000000001, "avg_micro_time_s": 4.535590662746109, "epoch": 23, "global_step": 1440, "grad_norm_max": 0.013143915683031082, "hidden_mse": 0.01068424681434408, "loss": 0.010162599239265546, "lr_blocks": 1.9945942635848745e-06, "lr_fusion": 1.9945942635848748e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.009094174538040534} +{"avg_context_frames": 14.8, "avg_micro_time_s": 4.799071120620647, "epoch": 23, "global_step": 1450, "grad_norm_max": 0.00843881256878376, "hidden_mse": 0.010900036030216144, "loss": 0.010352569341193885, "lr_blocks": 1.928936436551661e-06, "lr_fusion": 1.928936436551661e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.009262565764947795} +{"avg_context_frames": 14.4, "avg_micro_time_s": 4.724029757530661, "epoch": 23, "global_step": 1460, "grad_norm_max": 0.014737945981323719, "hidden_mse": 0.0121111216256395, "loss": 0.011325870431028307, "lr_blocks": 1.864118207632315e-06, "lr_fusion": 1.8641182076323148e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.010114758275449276} +{"avg_context_frames": 14.4, "avg_micro_time_s": 4.714366234390764, "epoch": 23, "global_step": 1470, "grad_norm_max": 0.013767022639513016, "hidden_mse": 0.011411064601270483, "loss": 0.010587109171319754, "lr_blocks": 1.8001572974834169e-06, "lr_fusion": 1.800157297483417e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.0094460026914021} +{"avg_context_frames": 16.0, "avg_micro_time_s": 5.035403382673394, "epoch": 24, "global_step": 1480, "grad_norm_max": 0.007507711183279753, "hidden_mse": 0.010140616929857061, "loss": 0.009673510334687309, "lr_blocks": 1.7370711923791567e-06, "lr_fusion": 1.7370711923791567e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.008659448669641278} +{"avg_context_frames": 10.4, "avg_micro_time_s": 3.835159823195136, "epoch": 24, "global_step": 1490, "grad_norm_max": 0.007216795347630978, "hidden_mse": 0.011915904295165092, "loss": 0.01125910658738576, "lr_blocks": 1.6748771394307584e-06, "lr_fusion": 1.6748771394307585e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.010067516157869251} +{"avg_context_frames": 14.4, "avg_micro_time_s": 4.681469369717525, "epoch": 24, "global_step": 1500, "grad_norm_max": 0.008854463696479797, "hidden_mse": 0.010383950447430835, "loss": 0.00962356278905645, "lr_blocks": 1.6135921418712959e-06, "lr_fusion": 1.6135921418712956e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.00858516778680496} +{"avg_context_frames": 14.8, "avg_micro_time_s": 4.79766893490305, "epoch": 24, "global_step": 1510, "grad_norm_max": 0.009236050769686699, "hidden_mse": 0.010941975563764573, "loss": 0.0105033388157608, "lr_blocks": 1.5532329544071712e-06, "lr_fusion": 1.553232954407171e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.00940914125822019} +{"avg_context_frames": 14.4, "avg_micro_time_s": 4.705061392278004, "epoch": 24, "global_step": 1520, "grad_norm_max": 0.011460250243544579, "hidden_mse": 0.010644898447208107, "loss": 0.01024511760333553, "lr_blocks": 1.4938160786375571e-06, "lr_fusion": 1.4938160786375572e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.009180627774912865} +{"avg_context_frames": 17.2, "avg_micro_time_s": 5.304795087805542, "epoch": 24, "global_step": 1530, "grad_norm_max": 0.013605614192783833, "hidden_mse": 0.010371161543298514, "loss": 0.009473931818502025, "lr_blocks": 1.4353577585430152e-06, "lr_fusion": 1.435357758543015e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.00843681563565042} +{"avg_context_frames": 16.400000000000002, "avg_micro_time_s": 5.148552063370881, "epoch": 25, "global_step": 1540, "grad_norm_max": 0.007240370847284794, "hidden_mse": 0.011363890016218648, "loss": 0.01046161290141754, "lr_blocks": 1.3778739760445552e-06, "lr_fusion": 1.3778739760445552e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.009325223966152407} +{"avg_context_frames": 12.0, "avg_micro_time_s": 4.218317999898864, "epoch": 25, "global_step": 1550, "grad_norm_max": 0.011495298705995083, "hidden_mse": 0.012243018404114993, "loss": 0.011384196765720845, "lr_blocks": 1.321380446634342e-06, "lr_fusion": 1.3213804466343421e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.010159894882235676} +{"avg_context_frames": 15.600000000000001, "avg_micro_time_s": 4.956380099902162, "epoch": 25, "global_step": 1560, "grad_norm_max": 0.006335103884339333, "hidden_mse": 0.009599077462917195, "loss": 0.009054656280204654, "lr_blocks": 1.2658926150792321e-06, "lr_fusion": 1.2658926150792322e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.008094748534495011} +{"avg_context_frames": 12.8, "avg_micro_time_s": 4.36600677229726, "epoch": 25, "global_step": 1570, "grad_norm_max": 0.007134098559617996, "hidden_mse": 0.010783782729413362, "loss": 0.010396900365594776, "lr_blocks": 1.2114256511983274e-06, "lr_fusion": 1.2114256511983274e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.009318522072862835} +{"avg_context_frames": 14.8, "avg_micro_time_s": 4.802309349030839, "epoch": 25, "global_step": 1580, "grad_norm_max": 0.006028773728758097, "hidden_mse": 0.01044910394703038, "loss": 0.009790748899104074, "lr_blocks": 1.157994445715706e-06, "lr_fusion": 1.157994445715706e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.00874583853292279} +{"avg_context_frames": 12.0, "avg_micro_time_s": 4.182760251735454, "epoch": 25, "global_step": 1590, "grad_norm_max": 0.005331455264240503, "hidden_mse": 0.011886735871667043, "loss": 0.011212147842161358, "lr_blocks": 1.1056136061894386e-06, "lr_fusion": 1.1056136061894384e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.01002347426256165} +{"avg_context_frames": 17.2, "avg_micro_time_s": 5.326669098653657, "epoch": 25, "global_step": 1600, "grad_norm_max": 0.008312598802149296, "hidden_mse": 0.011391867243219167, "loss": 0.010427710262592883, "lr_blocks": 1.0542974530180327e-06, "lr_fusion": 1.0542974530180327e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.009288523538270966} +{"avg_context_frames": 16.0, "avg_micro_time_s": 5.0552997812847025, "epoch": 26, "global_step": 1610, "grad_norm_max": 0.006056332495063543, "hidden_mse": 0.01026600631303154, "loss": 0.009472076126257889, "lr_blocks": 1.0040600155253766e-06, "lr_fusion": 1.0040600155253765e-05, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.008445475430926308} +{"avg_context_frames": 11.200000000000001, "avg_micro_time_s": 4.02838661234855, "epoch": 26, "global_step": 1620, "grad_norm_max": 0.009777577593922615, "hidden_mse": 0.012680578959407286, "loss": 0.011785311339190231, "lr_blocks": 9.549150281252633e-07, "lr_fusion": 9.549150281252633e-06, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.010517253482248634} +{"avg_context_frames": 15.200000000000001, "avg_micro_time_s": 4.874200343768462, "epoch": 26, "global_step": 1630, "grad_norm_max": 0.010693206451833248, "hidden_mse": 0.011203744309023024, "loss": 0.010288827487966046, "lr_blocks": 9.068759265665384e-07, "lr_fusion": 9.068759265665384e-06, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.009168453025631606} +{"avg_context_frames": 16.8, "avg_micro_time_s": 5.21346646660968, "epoch": 26, "global_step": 1640, "grad_norm_max": 0.0067598153837025166, "hidden_mse": 0.010066380561329425, "loss": 0.009677897690562532, "lr_blocks": 8.599558442598998e-07, "lr_fusion": 8.599558442598998e-06, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.008671259629772976} +{"avg_context_frames": 16.8, "avg_micro_time_s": 5.216752200851625, "epoch": 26, "global_step": 1650, "grad_norm_max": 0.004959532991051674, "hidden_mse": 0.009719162248075008, "loss": 0.009188506694044918, "lr_blocks": 8.141676086873574e-07, "lr_fusion": 8.141676086873572e-06, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.008216590463416652} +{"avg_context_frames": 8.8, "avg_micro_time_s": 3.4956692137304346, "epoch": 26, "global_step": 1660, "grad_norm_max": 0.007046353071928024, "hidden_mse": 0.012303299980703742, "loss": 0.011386787856463344, "lr_blocks": 7.695237378953224e-07, "lr_fusion": 7.695237378953223e-06, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.010156457903212868} +{"avg_context_frames": 18.0, "avg_micro_time_s": 5.475170267379145, "epoch": 27, "global_step": 1670, "grad_norm_max": 0.005601448938250542, "hidden_mse": 0.00966342295287177, "loss": 0.0092592045199126, "lr_blocks": 7.260364370723044e-07, "lr_fusion": 7.260364370723044e-06, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.008292862269445323} +{"avg_context_frames": 14.0, "avg_micro_time_s": 4.638306342525175, "epoch": 27, "global_step": 1680, "grad_norm_max": 0.006744225509464741, "hidden_mse": 0.011391530511900783, "loss": 0.01047203232010361, "lr_blocks": 6.837175952121305e-07, "lr_fusion": 6.837175952121306e-06, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.009332879236899317} +{"avg_context_frames": 13.600000000000001, "avg_micro_time_s": 4.554895031701017, "epoch": 27, "global_step": 1690, "grad_norm_max": 0.009354707784950733, "hidden_mse": 0.011478202714351939, "loss": 0.01076030233525671, "lr_blocks": 6.425787818636131e-07, "lr_fusion": 6.425787818636131e-06, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.009612482012016699} +{"avg_context_frames": 10.8, "avg_micro_time_s": 3.9689012693590486, "epoch": 27, "global_step": 1700, "grad_norm_max": 0.01021617092192173, "hidden_mse": 0.012309554487001152, "loss": 0.011392151605105028, "lr_blocks": 6.026312439675553e-07, "lr_fusion": 6.026312439675552e-06, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.010161196207627655} +{"avg_context_frames": 16.8, "avg_micro_time_s": 5.210915003839182, "epoch": 27, "global_step": 1710, "grad_norm_max": 0.006314720027148724, "hidden_mse": 0.009632948698708788, "loss": 0.008895453257719056, "lr_blocks": 5.63885902781941e-07, "lr_fusion": 5.6388590278194096e-06, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.007932158350013197} +{"avg_context_frames": 15.600000000000001, "avg_micro_time_s": 4.9713362104739645, "epoch": 27, "global_step": 1720, "grad_norm_max": 0.006035194266587496, "hidden_mse": 0.010518019236042165, "loss": 0.009827163099544123, "lr_blocks": 5.263533508961827e-07, "lr_fusion": 5.263533508961827e-06, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.008775361176230945} +{"avg_context_frames": 12.8, "avg_micro_time_s": 4.366136288931012, "epoch": 28, "global_step": 1730, "grad_norm_max": 0.009497205726802349, "hidden_mse": 0.011518516438081862, "loss": 0.010834899492328987, "lr_blocks": 4.900438493352056e-07, "lr_fusion": 4.900438493352055e-06, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.009683047863654794} +{"avg_context_frames": 15.200000000000001, "avg_micro_time_s": 4.901215536496602, "epoch": 28, "global_step": 1740, "grad_norm_max": 0.005356828216463327, "hidden_mse": 0.010027159715536983, "loss": 0.009365687874378636, "lr_blocks": 4.549673247541875e-07, "lr_fusion": 4.549673247541875e-06, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.008362971886526793} +{"avg_context_frames": 15.200000000000001, "avg_micro_time_s": 4.884086235142604, "epoch": 28, "global_step": 1750, "grad_norm_max": 0.005230274982750416, "hidden_mse": 0.010005503549473362, "loss": 0.00955009061144665, "lr_blocks": 4.211333667247125e-07, "lr_fusion": 4.2113336672471245e-06, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.008549540268722922} +{"avg_context_frames": 16.400000000000002, "avg_micro_time_s": 5.224043913454807, "epoch": 28, "global_step": 1760, "grad_norm_max": 0.009236546233296394, "hidden_mse": 0.011735450301785022, "loss": 0.010584994731470943, "lr_blocks": 3.885512251130763e-07, "lr_fusion": 3.885512251130763e-06, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.009411449654726312} +{"avg_context_frames": 13.600000000000001, "avg_micro_time_s": 4.530024908542692, "epoch": 28, "global_step": 1770, "grad_norm_max": 0.008108831010758877, "hidden_mse": 0.01087133648688905, "loss": 0.010107534783310258, "lr_blocks": 3.572298075514652e-07, "lr_fusion": 3.5722980755146517e-06, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.009020401127054356} +{"avg_context_frames": 14.0, "avg_micro_time_s": 4.617836628292571, "epoch": 28, "global_step": 1780, "grad_norm_max": 0.004343056585639715, "hidden_mse": 0.010929477814352141, "loss": 0.010256368457339704, "lr_blocks": 3.271776770026963e-07, "lr_fusion": 3.271776770026963e-06, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.009163420682307332} +{"avg_context_frames": 13.600000000000001, "avg_micro_time_s": 4.531201763309946, "epoch": 28, "global_step": 1790, "grad_norm_max": 0.006929009687155485, "hidden_mse": 0.010940699506318197, "loss": 0.010395804056315683, "lr_blocks": 2.984030494191942e-07, "lr_fusion": 2.9840304941919415e-06, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.009301734124892392} +{"avg_context_frames": 13.600000000000001, "avg_micro_time_s": 4.540558878425509, "epoch": 29, "global_step": 1800, "grad_norm_max": 0.004829120356589556, "hidden_mse": 0.011692805972415954, "loss": 0.010900972344097683, "lr_blocks": 2.7091379149682683e-07, "lr_fusion": 2.7091379149682685e-06, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.00973169170320034} +{"avg_context_frames": 16.400000000000002, "avg_micro_time_s": 5.134762143029366, "epoch": 29, "global_step": 1810, "grad_norm_max": 0.004480123519897461, "hidden_mse": 0.009953124972525984, "loss": 0.00938678826787509, "lr_blocks": 2.447174185242324e-07, "lr_fusion": 2.4471741852423237e-06, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.008391475753160195} +{"avg_context_frames": 14.4, "avg_micro_time_s": 4.720543994214677, "epoch": 29, "global_step": 1820, "grad_norm_max": 0.00631394237279892, "hidden_mse": 0.011283233191352339, "loss": 0.010356933396542445, "lr_blocks": 2.198210923282118e-07, "lr_fusion": 2.1982109232821178e-06, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.009228610066929833} +{"avg_context_frames": 16.400000000000002, "avg_micro_time_s": 5.155054196060519, "epoch": 29, "global_step": 1830, "grad_norm_max": 0.00500832311809063, "hidden_mse": 0.010350234835641459, "loss": 0.009602258197264747, "lr_blocks": 1.962316193157593e-07, "lr_fusion": 1.962316193157593e-06, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.00856723468459677} +{"avg_context_frames": 14.8, "avg_micro_time_s": 4.7913876567719855, "epoch": 29, "global_step": 1840, "grad_norm_max": 0.003995540086179972, "hidden_mse": 0.0103910195233766, "loss": 0.009777978260535747, "lr_blocks": 1.7395544861325718e-07, "lr_fusion": 1.7395544861325718e-06, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.008738876375718975} +{"avg_context_frames": 10.4, "avg_micro_time_s": 3.8436150688430644, "epoch": 29, "global_step": 1850, "grad_norm_max": 0.004223161842674017, "hidden_mse": 0.0121317976154387, "loss": 0.011148306314134972, "lr_blocks": 1.5299867030334815e-07, "lr_fusion": 1.5299867030334814e-06, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.0099351265525911} +{"avg_context_frames": 13.600000000000001, "avg_micro_time_s": 4.560986260355276, "epoch": 30, "global_step": 1860, "grad_norm_max": 0.005154220853000879, "hidden_mse": 0.011537798447534442, "loss": 0.01080962223932147, "lr_blocks": 1.333670137599713e-07, "lr_fusion": 1.333670137599713e-06, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.009655842377105728} +{"avg_context_frames": 14.4, "avg_micro_time_s": 4.755338394652063, "epoch": 30, "global_step": 1870, "grad_norm_max": 0.0062322006560862064, "hidden_mse": 0.011064839229220524, "loss": 0.010007060886709952, "lr_blocks": 1.1506584608200366e-07, "lr_fusion": 1.1506584608200367e-06, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.008900576969608665} +{"avg_context_frames": 14.4, "avg_micro_time_s": 4.738156871790125, "epoch": 30, "global_step": 1880, "grad_norm_max": 0.004222824238240719, "hidden_mse": 0.010634913231479005, "loss": 0.00989298215135932, "lr_blocks": 9.810017062595322e-08, "lr_fusion": 9.810017062595322e-07, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.008829490849166177} +{"avg_context_frames": 16.0, "avg_micro_time_s": 5.092115807252412, "epoch": 30, "global_step": 1890, "grad_norm_max": 0.00543114822357893, "hidden_mse": 0.01096829564194195, "loss": 0.010194446030072869, "lr_blocks": 8.247462563808816e-08, "lr_fusion": 8.247462563808817e-07, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.00909761647344567} +{"avg_context_frames": 14.0, "avg_micro_time_s": 4.643113673427433, "epoch": 30, "global_step": 1900, "grad_norm_max": 0.004269456956535578, "hidden_mse": 0.010559354658471421, "loss": 0.010193631250876934, "lr_blocks": 6.819348298638839e-08, "lr_fusion": 6.819348298638839e-07, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.009137695806566625} +{"avg_context_frames": 15.600000000000001, "avg_micro_time_s": 4.964571412671649, "epoch": 30, "global_step": 1910, "grad_norm_max": 0.0045198313891887665, "hidden_mse": 0.010255125840194524, "loss": 0.009715685714036228, "lr_blocks": 5.526064699265754e-08, "lr_fusion": 5.526064699265753e-07, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.008690173109062017} +{"avg_context_frames": 11.600000000000001, "avg_micro_time_s": 4.112253642309224, "epoch": 30, "global_step": 1920, "grad_norm_max": 0.0056124115362763405, "hidden_mse": 0.011844803846906872, "loss": 0.01091694308561273, "lr_blocks": 4.367965336512403e-08, "lr_fusion": 4.367965336512403e-07, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.009732462704414503} +{"avg_context_frames": 15.200000000000001, "avg_micro_time_s": 4.925167639796564, "epoch": 31, "global_step": 1930, "grad_norm_max": 0.004744055215269327, "hidden_mse": 0.010398254019673914, "loss": 0.009776062949094923, "lr_blocks": 3.345366823180929e-08, "lr_fusion": 3.3453668231809286e-07, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.008736237505218014} +{"avg_context_frames": 15.600000000000001, "avg_micro_time_s": 5.018330485402839, "epoch": 31, "global_step": 1940, "grad_norm_max": 0.005914777517318726, "hidden_mse": 0.011659513332415373, "loss": 0.010540930891875178, "lr_blocks": 2.4585487274942922e-08, "lr_fusion": 2.458548727494292e-07, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.009374979493441061} +{"avg_context_frames": 12.8, "avg_micro_time_s": 4.36818097357027, "epoch": 31, "global_step": 1950, "grad_norm_max": 0.005412289407104254, "hidden_mse": 0.010736750171054155, "loss": 0.010404293605824934, "lr_blocks": 1.7077534966650767e-08, "lr_fusion": 1.7077534966650766e-07, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.009330618553212844} +{"avg_context_frames": 14.8, "avg_micro_time_s": 4.79289449652133, "epoch": 31, "global_step": 1960, "grad_norm_max": 0.005767333786934614, "hidden_mse": 0.010644643427804114, "loss": 0.00981854532728903, "lr_blocks": 1.0931863906127327e-08, "lr_fusion": 1.0931863906127327e-07, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.00875408091233112} +{"avg_context_frames": 14.8, "avg_micro_time_s": 4.790249129706353, "epoch": 31, "global_step": 1970, "grad_norm_max": 0.004202915821224451, "hidden_mse": 0.011000800353940576, "loss": 0.010241292067803442, "lr_blocks": 6.150154258476315e-09, "lr_fusion": 6.150154258476315e-08, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.009141211968380959} +{"avg_context_frames": 14.8, "avg_micro_time_s": 4.816118484268372, "epoch": 31, "global_step": 1980, "grad_norm_max": 0.005114747676998377, "hidden_mse": 0.011142879974795506, "loss": 0.010160063934745268, "lr_blocks": 2.7337132953697555e-09, "lr_fusion": 2.7337132953697554e-08, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.009045775950653479} +{"avg_context_frames": 11.200000000000001, "avg_micro_time_s": 4.012313273070322, "epoch": 32, "global_step": 1990, "grad_norm_max": 0.005168226547539234, "hidden_mse": 0.011596153286518529, "loss": 0.01093838744563982, "lr_blocks": 6.834750376549793e-10, "lr_fusion": 6.834750376549792e-09, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.0097787721111672} +{"avg_context_frames": 13.200000000000001, "avg_micro_time_s": 4.46597387889633, "epoch": 32, "global_step": 2000, "grad_norm_max": 0.003432836849242449, "hidden_mse": 0.01132609474589117, "loss": 0.010788679530378431, "lr_blocks": 0.0, "lr_fusion": 0.0, "peak_memory_gib": 62.76183795928955, "velocity_mse": 0.009656070088385605} +swanlab: 📁 View project at +https://swanlab.cn/@ZhuoChen/HY-WorldPlay-Predictor-v4 +swanlab: 🚀 View run pre...cks1-52-fp32master-8gpu-torch26-noflash at +https://swanlab.cn/@ZhuoChen/HY-WorldPlay-Predictor-v4/runs/bofinlsd +swanlab: Upload complete: 72910 records +[complete] mode=none 2026-08-23T11:08:25Z diff --git a/logs/predictor_v4_stage1_previous_condition_ablation_seed0/previous_step.log b/logs/predictor_v4_stage1_previous_condition_ablation_seed0/previous_step.log new file mode 100644 index 0000000000000000000000000000000000000000..15e20c44e963075069db0d759596099b6df4b2c4 --- /dev/null +++ b/logs/predictor_v4_stage1_previous_condition_ablation_seed0/previous_step.log @@ -0,0 +1,304 @@ +[start] mode=previous_step 2026-08-23T11:08:25Z +{ + "manifest": "/mnt/local_nvme/cz/projects/HY-WorldPlay-DEV-Predictor/datasets/hyworldplay_predictor_prefeature_v4/train_eval_manifest.jsonl", + "teacher_checkpoint": "/mnt/local_nvme/cz/checkpoints/hy_worldplay/huggingface/hub/models--tencent--HY-WorldPlay/snapshots/f4c29235647707b571479a69b569e4166f9f5bf8/ar_distilled_action_model/diffusion_pytorch_model.safetensors", + "output_dir": "/mnt/local_nvme/cz/projects/HY-WorldPlay-DEV-Predictor/checkpoints/predictor_v4_stage1_previous_condition_ablation_seed0/previous_step", + "source_blocks": [ + 1, + 52 + ], + "predictor_variant": "v4", + "previous_condition_mode": "previous_step", + "predictor_fusion_type": "concat", + "gate_hidden_dim": 256, + "gate_initial_probability": 0.8, + "predictor_arch": "baseline", + "atc_variant": "full", + "atc_mlp_hidden_dim": 4096, + "atc_gate_hidden_dim": 512, + "atc_gate_initial_probability": 0.3, + "atc_transport_heads": 8, + "atc_transport_window": 3, + "atc_transport_query_chunk_size": 128, + "atc_transport_initial_scale": 0.1, + "max_steps": 2000, + "batch_size": 16, + "gradient_accumulation_steps": 1, + "block_warmup_steps": 100, + "block_warmup_mode": "delayed", + "scheduler_warmup_steps": 100, + "fusion_lr": 0.0001, + "blocks_lr": 1e-05, + "weight_decay": 0.01, + "hidden_weight": 0.1, + "velocity_weight": 1.0, + "grad_clip": 1.0, + "log_every": 10, + "save_every": 100, + "keep_snapshots": 20, + "seed": 0, + "num_workers": 0, + "max_records": null, + "bucket_by_context": true, + "gradient_checkpointing": true, + "fp32_trainable_params": true, + "train_delta_only": false, + "train_fusion_residual_only": false, + "save_optimizer_state": true, + "init_training_state": null, + "resume": null, + "swanlab_project": "HY-WorldPlay-Predictor-v4", + "swanlab_workspace": null, + "swanlab_experiment": "predictor-v4-stage1-previous-previous_step-seed0-blocks1-52-fp32master-8gpu-torch26-noflash", + "swanlab_mode": "cloud", + "predictor_version": "v4_prefeature", + "schema_version": "predictor_context_prefeature_bf16_v1", + "supervision_pairs": [ + [ + 0, + 1 + ], + [ + 1, + 2 + ], + [ + 2, + 3 + ] + ], + "previous_condition_source": "offline_full_teacher_hidden", + "records": 2800, + "dataset_pairs": 8400, + "steps_per_epoch_per_rank": 64, + "world_size": 8, + "per_rank_batch_size": 16, + "global_batch_size": 128, + "trainable_parameters": 197218816, + "frozen_projector_parameters": 16785664, + "dtype": "bfloat16", + "activation_dtype": "bfloat16", + "trainable_parameter_dtype": "float32", + "optimizer_moment_dtype": "float32", + "inference_checkpoint_dtype": "bfloat16", + "gpu": "NVIDIA H100 80GB HBM3" +} +swanlab: Currently logged in as: ZhuoChen to https://api.swanlab.cn. Use +`swanlab login --relogin` to force relogin +swanlab: Tracking run with swanlab version 0.9.0 +swanlab: 💾 Run data saved at +/mnt/local_nvme/cz/projects/HY-WorldPlay-DEV-Predictor/checkpoints/predictor_v4_ +stage1_previous_condition_ablation_seed0/previous_step/swanlab/run-20260823_1108 +51-5cggdz0m +swanlab: 📁 View project at +https://swanlab.cn/@ZhuoChen/HY-WorldPlay-Predictor-v4 +swanlab: 🚀 View run pre...cks1-52-fp32master-8gpu-torch26-noflash at +https://swanlab.cn/@ZhuoChen/HY-WorldPlay-Predictor-v4/runs/5cggdz0m +{"swanlab_run_id": "5cggdz0m"} +{"avg_context_frames": 4.0, "avg_micro_time_s": 5.390003433145466, "epoch": 1, "global_step": 1, "grad_norm_max": 0.04213421419262886, "hidden_mse": 0.019336470868438482, "loss": 0.018770750728435814, "lr_blocks": 0.0, "lr_fusion": 2.0000000000000003e-06, "peak_memory_gib": 39.19881868362427, "velocity_mse": 0.016837103641591966} +{"avg_context_frames": 14.666666666666666, "avg_micro_time_s": 4.947693011501946, "epoch": 1, "global_step": 10, "grad_norm_max": 0.04090447351336479, "hidden_mse": 0.015850999483114313, "loss": 0.01267727701148639, "lr_blocks": 0.0, "lr_fusion": 1.1000000000000001e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.011092177087751526} +{"avg_context_frames": 13.600000000000001, "avg_micro_time_s": 4.613336005959718, "epoch": 1, "global_step": 20, "grad_norm_max": 0.08625926822423935, "hidden_mse": 0.015807461051736027, "loss": 0.012926212040474639, "lr_blocks": 0.0, "lr_fusion": 2.1e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.011345465909107589} +{"avg_context_frames": 14.4, "avg_micro_time_s": 4.792831727927842, "epoch": 1, "global_step": 30, "grad_norm_max": 0.053188733756542206, "hidden_mse": 0.01640594428172335, "loss": 0.01366034388775006, "lr_blocks": 0.0, "lr_fusion": 3.1e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.012019749393220992} +{"avg_context_frames": 14.4, "avg_micro_time_s": 4.7873840484855466, "epoch": 1, "global_step": 40, "grad_norm_max": 0.06775817275047302, "hidden_mse": 0.017974023893475532, "loss": 0.01532769956975244, "lr_blocks": 0.0, "lr_fusion": 4.1e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.013530297199031338} +{"avg_context_frames": 14.0, "avg_micro_time_s": 4.7087282816472, "epoch": 1, "global_step": 50, "grad_norm_max": 0.07651187479496002, "hidden_mse": 0.015489266213262454, "loss": 0.01270688243675977, "lr_blocks": 0.0, "lr_fusion": 5.1000000000000006e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.011157955881208182} +{"avg_context_frames": 16.0, "avg_micro_time_s": 5.109921197060612, "epoch": 1, "global_step": 60, "grad_norm_max": 0.12943489849567413, "hidden_mse": 0.014434293005615474, "loss": 0.012779192253947259, "lr_blocks": 0.0, "lr_fusion": 6.1e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.01133576286956668} +{"avg_context_frames": 14.0, "avg_micro_time_s": 4.675526716551395, "epoch": 2, "global_step": 70, "grad_norm_max": 0.15710951387882233, "hidden_mse": 0.015274828427936883, "loss": 0.013460440433118493, "lr_blocks": 0.0, "lr_fusion": 7.1e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.011932957632234321} +{"avg_context_frames": 14.8, "avg_micro_time_s": 4.849272131502222, "epoch": 2, "global_step": 80, "grad_norm_max": 0.2156461924314499, "hidden_mse": 0.014942835085093975, "loss": 0.013359106972347946, "lr_blocks": 0.0, "lr_fusion": 8.1e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.011864823434734717} +{"avg_context_frames": 15.600000000000001, "avg_micro_time_s": 5.006391294996138, "epoch": 2, "global_step": 90, "grad_norm_max": 0.14938706159591675, "hidden_mse": 0.014649465586990118, "loss": 0.012930308206705377, "lr_blocks": 0.0, "lr_fusion": 9.1e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.011465361632872373} +{"avg_context_frames": 16.0, "avg_micro_time_s": 5.083120605624572, "epoch": 2, "global_step": 100, "grad_norm_max": 0.06058469042181969, "hidden_mse": 0.014045216620434077, "loss": 0.011741607694420965, "lr_blocks": 1e-05, "lr_fusion": 0.0001, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.01033708602481056} +{"avg_context_frames": 13.600000000000001, "avg_micro_time_s": 4.57161300743901, "epoch": 2, "global_step": 110, "grad_norm_max": 0.06875701248645782, "hidden_mse": 0.01438696948462166, "loss": 0.012956620211480186, "lr_blocks": 9.999316524962347e-06, "lr_fusion": 9.999316524962345e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.01151792330783792} +{"avg_context_frames": 12.4, "avg_micro_time_s": 4.3366182038997065, "epoch": 2, "global_step": 120, "grad_norm_max": 0.03919781744480133, "hidden_mse": 0.017330362647771835, "loss": 0.015402053430443631, "lr_blocks": 9.99726628670463e-06, "lr_fusion": 9.997266286704631e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.01366901712026447} +{"avg_context_frames": 14.0, "avg_micro_time_s": 4.67345496796479, "epoch": 3, "global_step": 130, "grad_norm_max": 0.08746759593486786, "hidden_mse": 0.016191604419145734, "loss": 0.01447714706300758, "lr_blocks": 9.993849845741525e-06, "lr_fusion": 9.993849845741524e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.012857986622839235} +{"avg_context_frames": 17.2, "avg_micro_time_s": 5.369903985538986, "epoch": 3, "global_step": 140, "grad_norm_max": 0.027580276131629944, "hidden_mse": 0.014352006383705885, "loss": 0.012679693673271687, "lr_blocks": 9.989068136093873e-06, "lr_fusion": 9.989068136093873e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.011244493047706784} +{"avg_context_frames": 12.0, "avg_micro_time_s": 4.224316095960967, "epoch": 3, "global_step": 150, "grad_norm_max": 0.04737461730837822, "hidden_mse": 0.0157106681203004, "loss": 0.014020476379664616, "lr_blocks": 9.98292246503335e-06, "lr_fusion": 9.98292246503335e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.012449409486725927} +{"avg_context_frames": 13.600000000000001, "avg_micro_time_s": 4.585182291832462, "epoch": 3, "global_step": 160, "grad_norm_max": 0.05160961672663689, "hidden_mse": 0.015099020081106574, "loss": 0.013598884991370143, "lr_blocks": 9.975414512725058e-06, "lr_fusion": 9.975414512725057e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.012088982941349968} +{"avg_context_frames": 13.600000000000001, "avg_micro_time_s": 4.584030968713342, "epoch": 3, "global_step": 170, "grad_norm_max": 0.040993254631757736, "hidden_mse": 0.015086026093922556, "loss": 0.013857662526424975, "lr_blocks": 9.966546331768192e-06, "lr_fusion": 9.966546331768191e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.012349059846019373} +{"avg_context_frames": 15.200000000000001, "avg_micro_time_s": 4.929474897681212, "epoch": 3, "global_step": 180, "grad_norm_max": 0.04446394741535187, "hidden_mse": 0.013900127890519798, "loss": 0.012457838148111479, "lr_blocks": 9.956320346634877e-06, "lr_fusion": 9.956320346634876e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.011067825433565305} +{"avg_context_frames": 15.600000000000001, "avg_micro_time_s": 5.00047430912382, "epoch": 3, "global_step": 190, "grad_norm_max": 0.04856764152646065, "hidden_mse": 0.013513659156160429, "loss": 0.011597990058362484, "lr_blocks": 9.944739353007344e-06, "lr_fusion": 9.944739353007344e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.01024662415147759} +{"avg_context_frames": 12.8, "avg_micro_time_s": 4.404858514116495, "epoch": 4, "global_step": 200, "grad_norm_max": 0.04151467606425285, "hidden_mse": 0.014193975599482656, "loss": 0.012880070065148175, "lr_blocks": 9.931806517013612e-06, "lr_fusion": 9.931806517013612e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.011460672531393358} +{"avg_context_frames": 14.0, "avg_micro_time_s": 4.665488255413948, "epoch": 4, "global_step": 210, "grad_norm_max": 0.042730752378702164, "hidden_mse": 0.013703356962651014, "loss": 0.012539418519008905, "lr_blocks": 9.917525374361913e-06, "lr_fusion": 9.917525374361912e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.011169082770356909} +{"avg_context_frames": 14.4, "avg_micro_time_s": 4.751063931632962, "epoch": 4, "global_step": 220, "grad_norm_max": 0.06264649331569672, "hidden_mse": 0.01350449207238853, "loss": 0.012608876824378968, "lr_blocks": 9.901899829374048e-06, "lr_fusion": 9.901899829374047e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.011258427664870396} +{"avg_context_frames": 13.600000000000001, "avg_micro_time_s": 4.60076371262694, "epoch": 4, "global_step": 230, "grad_norm_max": 0.047696635127067566, "hidden_mse": 0.015162470564246178, "loss": 0.013359621650306509, "lr_blocks": 9.884934153917998e-06, "lr_fusion": 9.884934153917997e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.01184337460435927} +{"avg_context_frames": 13.200000000000001, "avg_micro_time_s": 4.526023490203079, "epoch": 4, "global_step": 240, "grad_norm_max": 0.04687022045254707, "hidden_mse": 0.015321134566329421, "loss": 0.014358370885020122, "lr_blocks": 9.86663298624003e-06, "lr_fusion": 9.86663298624003e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.012826257414417341} +{"avg_context_frames": 16.0, "avg_micro_time_s": 5.095935432029364, "epoch": 4, "global_step": 250, "grad_norm_max": 0.05396527424454689, "hidden_mse": 0.012721611332381144, "loss": 0.011482204304775223, "lr_blocks": 9.847001329696653e-06, "lr_fusion": 9.847001329696653e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.01021004317153711} +{"avg_context_frames": 16.8, "avg_micro_time_s": 5.2886908607950325, "epoch": 5, "global_step": 260, "grad_norm_max": 0.030838878825306892, "hidden_mse": 0.013655089685926214, "loss": 0.013163776032160969, "lr_blocks": 9.826044551386743e-06, "lr_fusion": 9.826044551386744e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.011798267054837198} +{"avg_context_frames": 10.4, "avg_micro_time_s": 3.908949061020394, "epoch": 5, "global_step": 270, "grad_norm_max": 0.05804785341024399, "hidden_mse": 0.015608888387214394, "loss": 0.01483464890625328, "lr_blocks": 9.803768380684242e-06, "lr_fusion": 9.803768380684242e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.013273760152515024} +{"avg_context_frames": 14.0, "avg_micro_time_s": 4.669440998102073, "epoch": 5, "global_step": 280, "grad_norm_max": 0.04710798338055611, "hidden_mse": 0.01408187875058502, "loss": 0.012788709381129594, "lr_blocks": 9.780178907671788e-06, "lr_fusion": 9.780178907671789e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.011380521551473066} +{"avg_context_frames": 15.200000000000001, "avg_micro_time_s": 4.926285554229981, "epoch": 5, "global_step": 290, "grad_norm_max": 0.04940410330891609, "hidden_mse": 0.01285569365718402, "loss": 0.01215036086505279, "lr_blocks": 9.755282581475769e-06, "lr_fusion": 9.755282581475769e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.010864791538915598} +{"avg_context_frames": 14.8, "avg_micro_time_s": 4.850944496688317, "epoch": 5, "global_step": 300, "grad_norm_max": 0.0355144701898098, "hidden_mse": 0.012357702222652735, "loss": 0.011785703935311177, "lr_blocks": 9.729086208503174e-06, "lr_fusion": 9.729086208503174e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.010549933667061851} +{"avg_context_frames": 16.8, "avg_micro_time_s": 5.26801302296517, "epoch": 5, "global_step": 310, "grad_norm_max": 0.023449573665857315, "hidden_mse": 0.011787765641929583, "loss": 0.010818741767434404, "lr_blocks": 9.701596950580807e-06, "lr_fusion": 9.701596950580806e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.00963996523350943} +{"avg_context_frames": 14.4, "avg_micro_time_s": 4.727107053964573, "epoch": 5, "global_step": 320, "grad_norm_max": 0.057214442640542984, "hidden_mse": 0.013266852096421644, "loss": 0.012109356699511409, "lr_blocks": 9.672822322997305e-06, "lr_fusion": 9.672822322997305e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.010782671463675798} +{"avg_context_frames": 15.600000000000001, "avg_micro_time_s": 5.008041609400244, "epoch": 6, "global_step": 330, "grad_norm_max": 0.03418257087469101, "hidden_mse": 0.012710116291418673, "loss": 0.011506931739859283, "lr_blocks": 9.642770192448537e-06, "lr_fusion": 9.642770192448536e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.010235920111881569} +{"avg_context_frames": 16.8, "avg_micro_time_s": 5.281815978346277, "epoch": 6, "global_step": 340, "grad_norm_max": 0.02486269548535347, "hidden_mse": 0.012145551387220621, "loss": 0.011343726865015925, "lr_blocks": 9.611448774886925e-06, "lr_fusion": 9.611448774886924e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.010129171732114629} +{"avg_context_frames": 16.0, "avg_micro_time_s": 5.102710058123922, "epoch": 6, "global_step": 350, "grad_norm_max": 0.032991405576467514, "hidden_mse": 0.011948788695735858, "loss": 0.01111817192286253, "lr_blocks": 9.578866633275289e-06, "lr_fusion": 9.578866633275288e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.009923293109750375} +{"avg_context_frames": 12.8, "avg_micro_time_s": 4.426730060341652, "epoch": 6, "global_step": 360, "grad_norm_max": 0.06703214347362518, "hidden_mse": 0.014436629868578167, "loss": 0.013832362671382726, "lr_blocks": 9.545032675245814e-06, "lr_fusion": 9.545032675245813e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.012388699749135413} +{"avg_context_frames": 14.0, "avg_micro_time_s": 4.665024829235335, "epoch": 6, "global_step": 370, "grad_norm_max": 0.04031626135110855, "hidden_mse": 0.013715448719449342, "loss": 0.013370293017942458, "lr_blocks": 9.509956150664796e-06, "lr_fusion": 9.509956150664796e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.011998748118639924} +{"avg_context_frames": 10.8, "avg_micro_time_s": 3.967238826741232, "epoch": 6, "global_step": 380, "grad_norm_max": 0.031028706580400467, "hidden_mse": 0.013640649657463656, "loss": 0.013025015045423062, "lr_blocks": 9.473646649103819e-06, "lr_fusion": 9.473646649103818e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.011660950054647401} +{"avg_context_frames": 14.4, "avg_micro_time_s": 4.765819489619753, "epoch": 7, "global_step": 390, "grad_norm_max": 0.04073986038565636, "hidden_mse": 0.012500616494799034, "loss": 0.01180085794767365, "lr_blocks": 9.43611409721806e-06, "lr_fusion": 9.43611409721806e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.010550796275492758} +{"avg_context_frames": 15.200000000000001, "avg_micro_time_s": 4.908283748821123, "epoch": 7, "global_step": 400, "grad_norm_max": 0.029762687161564827, "hidden_mse": 0.012378795922268183, "loss": 0.011574207741068676, "lr_blocks": 9.397368756032445e-06, "lr_fusion": 9.397368756032445e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.010336328152334318} +{"avg_context_frames": 16.8, "avg_micro_time_s": 5.27210545877897, "epoch": 7, "global_step": 410, "grad_norm_max": 0.024780524894595146, "hidden_mse": 0.01213957951986231, "loss": 0.011697162035852672, "lr_blocks": 9.357421218136387e-06, "lr_fusion": 9.357421218136386e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.010483204090269283} +{"avg_context_frames": 15.200000000000001, "avg_micro_time_s": 4.912542160760495, "epoch": 7, "global_step": 420, "grad_norm_max": 0.023539738729596138, "hidden_mse": 0.012292333890218289, "loss": 0.011726830241968855, "lr_blocks": 9.31628240478787e-06, "lr_fusion": 9.316282404787871e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.010497596912318842} +{"avg_context_frames": 10.0, "avg_micro_time_s": 3.7861226167311544, "epoch": 7, "global_step": 430, "grad_norm_max": 0.04219616577029228, "hidden_mse": 0.013949477346614003, "loss": 0.013240219914587216, "lr_blocks": 9.273963562927695e-06, "lr_fusion": 9.273963562927695e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.011845272145001218} +{"avg_context_frames": 14.4, "avg_micro_time_s": 4.77231790015212, "epoch": 7, "global_step": 440, "grad_norm_max": 0.035215962678194046, "hidden_mse": 0.012785457988502459, "loss": 0.012359452788950877, "lr_blocks": 9.230476262104678e-06, "lr_fusion": 9.230476262104677e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.011080906935967506} +{"avg_context_frames": 14.8, "avg_micro_time_s": 4.8527173098293135, "epoch": 8, "global_step": 450, "grad_norm_max": 0.031801987439394, "hidden_mse": 0.012391461082734168, "loss": 0.011733510129852222, "lr_blocks": 9.185832391312644e-06, "lr_fusion": 9.185832391312644e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.010494364015175962} +{"avg_context_frames": 12.4, "avg_micro_time_s": 4.337323385721539, "epoch": 8, "global_step": 460, "grad_norm_max": 0.03215407580137253, "hidden_mse": 0.013910783943720162, "loss": 0.01358282583532855, "lr_blocks": 9.140044155740102e-06, "lr_fusion": 9.140044155740101e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.012191747489850968} +{"avg_context_frames": 12.0, "avg_micro_time_s": 4.239174249922508, "epoch": 8, "global_step": 470, "grad_norm_max": 0.02904793620109558, "hidden_mse": 0.013880216283723712, "loss": 0.013772513944422827, "lr_blocks": 9.093124073433464e-06, "lr_fusion": 9.093124073433463e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.012384492304408924} +{"avg_context_frames": 13.600000000000001, "avg_micro_time_s": 4.552743449634, "epoch": 8, "global_step": 480, "grad_norm_max": 0.034129779785871506, "hidden_mse": 0.011890934873372318, "loss": 0.01080229061190039, "lr_blocks": 9.045084971874738e-06, "lr_fusion": 9.045084971874738e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.00961319708148949} +{"avg_context_frames": 16.0, "avg_micro_time_s": 5.087259579749662, "epoch": 8, "global_step": 490, "grad_norm_max": 0.03165849670767784, "hidden_mse": 0.012337448331527412, "loss": 0.011864739895099775, "lr_blocks": 8.995939984474624e-06, "lr_fusion": 8.995939984474624e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.010630995099199937} +{"avg_context_frames": 17.6, "avg_micro_time_s": 5.428945595084224, "epoch": 8, "global_step": 500, "grad_norm_max": 0.03432052955031395, "hidden_mse": 0.011195689387386666, "loss": 0.0102499351487495, "lr_blocks": 8.94570254698197e-06, "lr_fusion": 8.945702546981969e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.009130366204772145} +{"avg_context_frames": 14.8, "avg_micro_time_s": 4.833534198156849, "epoch": 8, "global_step": 510, "grad_norm_max": 0.03290513902902603, "hidden_mse": 0.01217188026639633, "loss": 0.011646660038968549, "lr_blocks": 8.894386393810563e-06, "lr_fusion": 8.894386393810563e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.010429472033865751} +{"avg_context_frames": 14.0, "avg_micro_time_s": 4.637755968683632, "epoch": 9, "global_step": 520, "grad_norm_max": 0.04613647609949112, "hidden_mse": 0.011698566656559706, "loss": 0.011001769837457688, "lr_blocks": 8.842005554284296e-06, "lr_fusion": 8.842005554284296e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.009831913141533734} +{"avg_context_frames": 15.600000000000001, "avg_micro_time_s": 5.024616871058242, "epoch": 9, "global_step": 530, "grad_norm_max": 0.04497421905398369, "hidden_mse": 0.013158760668011383, "loss": 0.012170639366377146, "lr_blocks": 8.788574348801676e-06, "lr_fusion": 8.788574348801675e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.010854763246607037} +{"avg_context_frames": 17.2, "avg_micro_time_s": 5.317051677909331, "epoch": 9, "global_step": 540, "grad_norm_max": 0.03815830498933792, "hidden_mse": 0.011408550007035957, "loss": 0.010689407592872159, "lr_blocks": 8.734107384920771e-06, "lr_fusion": 8.73410738492077e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.009548552596243098} +{"avg_context_frames": 14.4, "avg_micro_time_s": 4.733377268258482, "epoch": 9, "global_step": 550, "grad_norm_max": 0.02301616780459881, "hidden_mse": 0.01252606079215184, "loss": 0.012292950798291714, "lr_blocks": 8.67861955336566e-06, "lr_fusion": 8.678619553365659e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.01104034468298778} +{"avg_context_frames": 14.0, "avg_micro_time_s": 4.6243225870988685, "epoch": 9, "global_step": 560, "grad_norm_max": 0.021490655839443207, "hidden_mse": 0.011698554427130149, "loss": 0.011247829266358167, "lr_blocks": 8.622126023955446e-06, "lr_fusion": 8.622126023955446e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.010077973821898923} +{"avg_context_frames": 9.600000000000001, "avg_micro_time_s": 3.705859788171074, "epoch": 9, "global_step": 570, "grad_norm_max": 0.02617492526769638, "hidden_mse": 0.013040747831109912, "loss": 0.01295601183664985, "lr_blocks": 8.564642241456986e-06, "lr_fusion": 8.564642241456986e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.011651937061105856} +{"avg_context_frames": 13.600000000000001, "avg_micro_time_s": 4.582612262750627, "epoch": 10, "global_step": 580, "grad_norm_max": 0.02891497313976288, "hidden_mse": 0.012813387328060345, "loss": 0.012273600336629898, "lr_blocks": 8.506183921362443e-06, "lr_fusion": 8.506183921362443e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.01099226160440594} +{"avg_context_frames": 16.8, "avg_micro_time_s": 5.26017247944983, "epoch": 10, "global_step": 590, "grad_norm_max": 0.017202794551849365, "hidden_mse": 0.010690513148438186, "loss": 0.010219289845554159, "lr_blocks": 8.446767045592829e-06, "lr_fusion": 8.44676704559283e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.009150238515576349} +{"avg_context_frames": 14.0, "avg_micro_time_s": 4.6805535545252495, "epoch": 10, "global_step": 600, "grad_norm_max": 0.025328058749437332, "hidden_mse": 0.01287782230647281, "loss": 0.01281475629657507, "lr_blocks": 8.386407858128707e-06, "lr_fusion": 8.386407858128706e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.01152697402285412} +{"avg_context_frames": 14.8, "avg_micro_time_s": 4.827596270953655, "epoch": 10, "global_step": 610, "grad_norm_max": 0.03863966837525368, "hidden_mse": 0.012177846976555884, "loss": 0.011386544402921573, "lr_blocks": 8.325122860569241e-06, "lr_fusion": 8.32512286056924e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.010168759737280198} +{"avg_context_frames": 11.600000000000001, "avg_micro_time_s": 4.139076747800573, "epoch": 10, "global_step": 620, "grad_norm_max": 0.02639959380030632, "hidden_mse": 0.012848934688372539, "loss": 0.012194167141569779, "lr_blocks": 8.262928807620843e-06, "lr_fusion": 8.262928807620843e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.010909273673314603} +{"avg_context_frames": 15.600000000000001, "avg_micro_time_s": 5.024057594381157, "epoch": 10, "global_step": 630, "grad_norm_max": 0.020170163363218307, "hidden_mse": 0.011495722172548996, "loss": 0.010790128057124093, "lr_blocks": 8.199842702516584e-06, "lr_fusion": 8.199842702516583e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.009640555875375868} +{"avg_context_frames": 15.600000000000001, "avg_micro_time_s": 5.0195043452695245, "epoch": 10, "global_step": 640, "grad_norm_max": 0.029749007895588875, "hidden_mse": 0.012404954043449835, "loss": 0.011846381274517627, "lr_blocks": 8.135881792367686e-06, "lr_fusion": 8.135881792367686e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.010605885938275605} +{"avg_context_frames": 13.200000000000001, "avg_micro_time_s": 4.493526067626954, "epoch": 11, "global_step": 650, "grad_norm_max": 0.021860528737306595, "hidden_mse": 0.012394975079223514, "loss": 0.011755301407538355, "lr_blocks": 8.071063563448341e-06, "lr_fusion": 8.07106356344834e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.010515803954331205} +{"avg_context_frames": 17.6, "avg_micro_time_s": 5.4495809333529905, "epoch": 11, "global_step": 660, "grad_norm_max": 0.016494564712047577, "hidden_mse": 0.012093369249487297, "loss": 0.011364454805152491, "lr_blocks": 8.005405736415127e-06, "lr_fusion": 8.005405736415126e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.01015511795412749} +{"avg_context_frames": 15.600000000000001, "avg_micro_time_s": 5.038696090436133, "epoch": 11, "global_step": 670, "grad_norm_max": 0.03731521591544151, "hidden_mse": 0.012297756201587618, "loss": 0.01171095430618152, "lr_blocks": 7.938926261462366e-06, "lr_fusion": 7.938926261462366e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.01048117867903784} +{"avg_context_frames": 14.0, "avg_micro_time_s": 4.643289411079604, "epoch": 11, "global_step": 680, "grad_norm_max": 0.03519211336970329, "hidden_mse": 0.011629881529370324, "loss": 0.011244232172612102, "lr_blocks": 7.871643313414718e-06, "lr_fusion": 7.871643313414718e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.010081244053435513} +{"avg_context_frames": 10.8, "avg_micro_time_s": 3.969643171239295, "epoch": 11, "global_step": 690, "grad_norm_max": 0.02735678292810917, "hidden_mse": 0.012486814300063999, "loss": 0.012232138693798335, "lr_blocks": 7.803575286758365e-06, "lr_fusion": 7.803575286758364e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.01098345724749379} +{"avg_context_frames": 14.8, "avg_micro_time_s": 4.833224603638519, "epoch": 11, "global_step": 700, "grad_norm_max": 0.026330161839723587, "hidden_mse": 0.011770539358258248, "loss": 0.011451999266864733, "lr_blocks": 7.734740790612137e-06, "lr_fusion": 7.734740790612136e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.010274945315904916} +{"avg_context_frames": 12.4, "avg_micro_time_s": 4.299038258772635, "epoch": 12, "global_step": 710, "grad_norm_max": 0.018607286736369133, "hidden_mse": 0.01245503923855722, "loss": 0.011929811246227474, "lr_blocks": 7.66515864363997e-06, "lr_fusion": 7.66515864363997e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.01068430726591032} +{"avg_context_frames": 16.400000000000002, "avg_micro_time_s": 5.1718356425539245, "epoch": 12, "global_step": 720, "grad_norm_max": 0.03193839266896248, "hidden_mse": 0.011414412455633283, "loss": 0.010722295579034835, "lr_blocks": 7.594847868906076e-06, "lr_fusion": 7.594847868906076e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.009580854314845056} +{"avg_context_frames": 13.600000000000001, "avg_micro_time_s": 4.583939254962025, "epoch": 12, "global_step": 730, "grad_norm_max": 0.019162647426128387, "hidden_mse": 0.011507620714837685, "loss": 0.01141413410077803, "lr_blocks": 7.52382768867422e-06, "lr_fusion": 7.52382768867422e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.010263372020563112} +{"avg_context_frames": 16.8, "avg_micro_time_s": 5.280904379184358, "epoch": 12, "global_step": 740, "grad_norm_max": 0.020085055381059647, "hidden_mse": 0.011626391165191308, "loss": 0.010950101126218215, "lr_blocks": 7.452117519152542e-06, "lr_fusion": 7.452117519152542e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.009787461993983016} +{"avg_context_frames": 16.0, "avg_micro_time_s": 5.104417345493857, "epoch": 12, "global_step": 750, "grad_norm_max": 0.030951593071222305, "hidden_mse": 0.012416702212067322, "loss": 0.011506420187652112, "lr_blocks": 7.379736965185369e-06, "lr_fusion": 7.379736965185368e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.010264749958878383} +{"avg_context_frames": 13.600000000000001, "avg_micro_time_s": 4.568630446221505, "epoch": 12, "global_step": 760, "grad_norm_max": 0.02326587215065956, "hidden_mse": 0.011415151465917006, "loss": 0.010812057845760137, "lr_blocks": 7.30670581489344e-06, "lr_fusion": 7.30670581489344e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.009670542762614788} +{"avg_context_frames": 12.4, "avg_micro_time_s": 4.337388903547253, "epoch": 13, "global_step": 770, "grad_norm_max": 0.021099479869008064, "hidden_mse": 0.011649939353810625, "loss": 0.011352432222338393, "lr_blocks": 7.233044034264034e-06, "lr_fusion": 7.233044034264034e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.010187438334105537} +{"avg_context_frames": 15.600000000000001, "avg_micro_time_s": 5.040023005229887, "epoch": 13, "global_step": 780, "grad_norm_max": 0.025230588391423225, "hidden_mse": 0.01315962041844614, "loss": 0.012169644638197497, "lr_blocks": 7.158771761692464e-06, "lr_fusion": 7.158771761692464e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.010853682598099114} +{"avg_context_frames": 15.200000000000001, "avg_micro_time_s": 4.945494659984252, "epoch": 13, "global_step": 790, "grad_norm_max": 0.023723840713500977, "hidden_mse": 0.011543380294460804, "loss": 0.011000581196276471, "lr_blocks": 7.083909302476453e-06, "lr_fusion": 7.083909302476453e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.009846243204083295} +{"avg_context_frames": 11.200000000000001, "avg_micro_time_s": 4.066284731389897, "epoch": 13, "global_step": 800, "grad_norm_max": 0.0216158889234066, "hidden_mse": 0.012074235372710974, "loss": 0.011836985149420798, "lr_blocks": 7.008477123264849e-06, "lr_fusion": 7.008477123264848e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.010629561630776153} +{"avg_context_frames": 14.4, "avg_micro_time_s": 4.7482081787777135, "epoch": 13, "global_step": 810, "grad_norm_max": 0.013302837498486042, "hidden_mse": 0.010893627465702594, "loss": 0.010638147918507458, "lr_blocks": 6.932495846462262e-06, "lr_fusion": 6.932495846462261e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.009548785159131512} +{"avg_context_frames": 14.4, "avg_micro_time_s": 4.735536746906291, "epoch": 13, "global_step": 820, "grad_norm_max": 0.01727539300918579, "hidden_mse": 0.011193052888847888, "loss": 0.01073916400782764, "lr_blocks": 6.855986244591104e-06, "lr_fusion": 6.855986244591104e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.009619858697988093} +{"avg_context_frames": 15.200000000000001, "avg_micro_time_s": 4.8915618092956725, "epoch": 13, "global_step": 830, "grad_norm_max": 0.02317405492067337, "hidden_mse": 0.011999397148611025, "loss": 0.01142525791656226, "lr_blocks": 6.778969234612583e-06, "lr_fusion": 6.778969234612584e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.0102253182529239} +{"avg_context_frames": 15.600000000000001, "avg_micro_time_s": 4.987799059993995, "epoch": 14, "global_step": 840, "grad_norm_max": 0.018023265525698662, "hidden_mse": 0.012787919671973215, "loss": 0.01203137548873201, "lr_blocks": 6.701465872208216e-06, "lr_fusion": 6.701465872208216e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.010752583493012936} +{"avg_context_frames": 14.0, "avg_micro_time_s": 4.640337211434963, "epoch": 14, "global_step": 850, "grad_norm_max": 0.02690989151597023, "hidden_mse": 0.011521955434000121, "loss": 0.010968233057064937, "lr_blocks": 6.6234973460234184e-06, "lr_fusion": 6.623497346023418e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.00981603755208198} +{"avg_context_frames": 16.0, "avg_micro_time_s": 5.073578205090598, "epoch": 14, "global_step": 860, "grad_norm_max": 0.022481247782707214, "hidden_mse": 0.011246695619774982, "loss": 0.010851394085329959, "lr_blocks": 6.545084971874738e-06, "lr_fusion": 6.545084971874738e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.009726724459324033} +{"avg_context_frames": 11.600000000000001, "avg_micro_time_s": 4.134275454042654, "epoch": 14, "global_step": 870, "grad_norm_max": 0.0329272598028183, "hidden_mse": 0.012194143160013483, "loss": 0.012055486364988611, "lr_blocks": 6.466250186922325e-06, "lr_fusion": 6.466250186922325e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.010836072091478854} +{"avg_context_frames": 13.600000000000001, "avg_micro_time_s": 4.588835718957125, "epoch": 14, "global_step": 880, "grad_norm_max": 0.02120060846209526, "hidden_mse": 0.012053120130440222, "loss": 0.011847794888308272, "lr_blocks": 6.387014543809224e-06, "lr_fusion": 6.387014543809223e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.010642482934053989} +{"avg_context_frames": 16.0, "avg_micro_time_s": 5.060962021512387, "epoch": 14, "global_step": 890, "grad_norm_max": 0.021079517900943756, "hidden_mse": 0.011012707208283247, "loss": 0.010395187401445584, "lr_blocks": 6.3073997047691e-06, "lr_fusion": 6.307399704769099e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.009293916664319113} +{"avg_context_frames": 15.200000000000001, "avg_micro_time_s": 4.899482249228459, "epoch": 15, "global_step": 900, "grad_norm_max": 0.02221139520406723, "hidden_mse": 0.010432708833832294, "loss": 0.009815370175056161, "lr_blocks": 6.227427435703997e-06, "lr_fusion": 6.227427435703997e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.008772099303314462} +{"avg_context_frames": 14.8, "avg_micro_time_s": 4.815964195190464, "epoch": 15, "global_step": 910, "grad_norm_max": 0.019223596900701523, "hidden_mse": 0.011160227772779764, "loss": 0.010657096741488204, "lr_blocks": 6.147119600233758e-06, "lr_fusion": 6.147119600233758e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.009541073997388594} +{"avg_context_frames": 14.8, "avg_micro_time_s": 4.833462211224833, "epoch": 15, "global_step": 920, "grad_norm_max": 0.011919276788830757, "hidden_mse": 0.011219456040998922, "loss": 0.010750112525420264, "lr_blocks": 6.066498153718735e-06, "lr_fusion": 6.066498153718735e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.009628166953916661} +{"avg_context_frames": 12.4, "avg_micro_time_s": 4.323704674710462, "epoch": 15, "global_step": 930, "grad_norm_max": 0.022854363545775414, "hidden_mse": 0.01295729247503914, "loss": 0.012480229430366307, "lr_blocks": 5.985585137257401e-06, "lr_fusion": 5.985585137257401e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.011184500143281186} +{"avg_context_frames": 11.600000000000001, "avg_micro_time_s": 4.153113881387981, "epoch": 15, "global_step": 940, "grad_norm_max": 0.020989449694752693, "hidden_mse": 0.012097615422680974, "loss": 0.011943326191976668, "lr_blocks": 5.904402671660551e-06, "lr_fusion": 5.90440267166055e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.010733564657857643} +{"avg_context_frames": 14.0, "avg_micro_time_s": 4.649320444492332, "epoch": 15, "global_step": 950, "grad_norm_max": 0.014516905881464481, "hidden_mse": 0.01173671457800083, "loss": 0.010940490674693139, "lr_blocks": 5.82297295140367e-06, "lr_fusion": 5.8229729514036705e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.009766819223295899} +{"avg_context_frames": 17.2, "avg_micro_time_s": 5.343339977180586, "epoch": 15, "global_step": 960, "grad_norm_max": 0.018263204023241997, "hidden_mse": 0.011373553518205883, "loss": 0.010623812276753597, "lr_blocks": 5.74131823855921e-06, "lr_fusion": 5.74131823855921e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.009486456861486659} +{"avg_context_frames": 16.0, "avg_micro_time_s": 5.099267087162298, "epoch": 16, "global_step": 970, "grad_norm_max": 0.019088026136159897, "hidden_mse": 0.011353406246053056, "loss": 0.010902073944453151, "lr_blocks": 5.659460856710346e-06, "lr_fusion": 5.6594608567103456e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.009766733349533752} +{"avg_context_frames": 15.600000000000001, "avg_micro_time_s": 5.015965533703275, "epoch": 16, "global_step": 980, "grad_norm_max": 0.013443585485219955, "hidden_mse": 0.011005152174038814, "loss": 0.010242818156257272, "lr_blocks": 5.577423184847932e-06, "lr_fusion": 5.577423184847932e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.00914230292546563} +{"avg_context_frames": 13.600000000000001, "avg_micro_time_s": 4.574561873804487, "epoch": 16, "global_step": 990, "grad_norm_max": 0.016379451379179955, "hidden_mse": 0.011182169872336091, "loss": 0.010733105830149726, "lr_blocks": 5.495227651252315e-06, "lr_fusion": 5.495227651252315e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.00961488881148398} +{"avg_context_frames": 16.0, "avg_micro_time_s": 5.080968237727939, "epoch": 16, "global_step": 1000, "grad_norm_max": 0.014817033894360065, "hidden_mse": 0.010536393959773705, "loss": 0.010006206337129698, "lr_blocks": 5.412896727361663e-06, "lr_fusion": 5.4128967273616625e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.008952566963853315} +{"avg_context_frames": 12.8, "avg_micro_time_s": 4.420772327139275, "epoch": 16, "global_step": 1010, "grad_norm_max": 0.022306101396679878, "hidden_mse": 0.012128665659110993, "loss": 0.011479068046901375, "lr_blocks": 5.3304529216284974e-06, "lr_fusion": 5.330452921628497e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.010266201506601647} +{"avg_context_frames": 12.0, "avg_micro_time_s": 4.231940373257385, "epoch": 16, "global_step": 1020, "grad_norm_max": 0.019634846597909927, "hidden_mse": 0.012609264912316577, "loss": 0.012515229941345752, "lr_blocks": 5.247918773366112e-06, "lr_fusion": 5.247918773366112e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.011254303413443268} +{"avg_context_frames": 14.4, "avg_micro_time_s": 4.736853080859873, "epoch": 17, "global_step": 1030, "grad_norm_max": 0.01501576416194439, "hidden_mse": 0.011774868553038687, "loss": 0.0111176616337616, "lr_blocks": 5.165316846586541e-06, "lr_fusion": 5.165316846586541e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.009940174798248337} +{"avg_context_frames": 12.8, "avg_micro_time_s": 4.3904326512347325, "epoch": 17, "global_step": 1040, "grad_norm_max": 0.014378438703715801, "hidden_mse": 0.012088660977315159, "loss": 0.011727701360359789, "lr_blocks": 5.082669723831793e-06, "lr_fusion": 5.0826697238317935e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.010518835310358556} +{"avg_context_frames": 16.8, "avg_micro_time_s": 5.247016504725616, "epoch": 17, "global_step": 1050, "grad_norm_max": 0.017493458464741707, "hidden_mse": 0.010637813928769902, "loss": 0.010126318220864051, "lr_blocks": 5e-06, "lr_fusion": 5e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.009062536846613511} +{"avg_context_frames": 13.200000000000001, "avg_micro_time_s": 4.510812513409474, "epoch": 17, "global_step": 1060, "grad_norm_max": 0.020490048453211784, "hidden_mse": 0.012075224169529976, "loss": 0.011202040506759658, "lr_blocks": 4.917330276168208e-06, "lr_fusion": 4.917330276168208e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.009994518093299121} +{"avg_context_frames": 14.4, "avg_micro_time_s": 4.7775565871183066, "epoch": 17, "global_step": 1070, "grad_norm_max": 0.018479550257325172, "hidden_mse": 0.010845704510575161, "loss": 0.010490530083188787, "lr_blocks": 4.8346831534134595e-06, "lr_fusion": 4.834683153413459e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.009405959653668106} +{"avg_context_frames": 16.400000000000002, "avg_micro_time_s": 5.1785991128403115, "epoch": 17, "global_step": 1080, "grad_norm_max": 0.01840595155954361, "hidden_mse": 0.010716999846044928, "loss": 0.009974862285889686, "lr_blocks": 4.752081226633888e-06, "lr_fusion": 4.7520812266338885e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.008903162309434266} +{"avg_context_frames": 11.200000000000001, "avg_micro_time_s": 4.081637502633384, "epoch": 18, "global_step": 1090, "grad_norm_max": 0.014469847083091736, "hidden_mse": 0.012514767673565075, "loss": 0.012284942425321788, "lr_blocks": 4.669547078371503e-06, "lr_fusion": 4.669547078371504e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.011033465678337961} +{"avg_context_frames": 13.200000000000001, "avg_micro_time_s": 4.504346966695448, "epoch": 18, "global_step": 1100, "grad_norm_max": 0.021681908518075943, "hidden_mse": 0.011659391515422612, "loss": 0.010850486005074345, "lr_blocks": 4.587103272638339e-06, "lr_fusion": 4.5871032726383386e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.00968454690591898} +{"avg_context_frames": 17.6, "avg_micro_time_s": 5.466380042651145, "epoch": 18, "global_step": 1110, "grad_norm_max": 0.01313855405896902, "hidden_mse": 0.010718481853837149, "loss": 0.010107062436873093, "lr_blocks": 4.504772348747687e-06, "lr_fusion": 4.504772348747687e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.00903521423751954} +{"avg_context_frames": 12.0, "avg_micro_time_s": 4.252443635032978, "epoch": 18, "global_step": 1120, "grad_norm_max": 0.016378507018089294, "hidden_mse": 0.011972847417928279, "loss": 0.011409764789277688, "lr_blocks": 4.42257681515207e-06, "lr_fusion": 4.4225768151520694e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.010212480070185848} +{"avg_context_frames": 13.200000000000001, "avg_micro_time_s": 4.510066485889547, "epoch": 18, "global_step": 1130, "grad_norm_max": 0.022022085264325142, "hidden_mse": 0.011651095526758581, "loss": 0.011329390259925277, "lr_blocks": 4.340539143289655e-06, "lr_fusion": 4.3405391432896555e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.010164280675235206} +{"avg_context_frames": 15.200000000000001, "avg_micro_time_s": 4.95837510052952, "epoch": 18, "global_step": 1140, "grad_norm_max": 0.012510515749454498, "hidden_mse": 0.011499336030101405, "loss": 0.010953272943152115, "lr_blocks": 4.25868176144079e-06, "lr_fusion": 4.2586817614407895e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.009803339379141109} +{"avg_context_frames": 17.2, "avg_micro_time_s": 5.390902116279176, "epoch": 18, "global_step": 1150, "grad_norm_max": 0.013637686148285866, "hidden_mse": 0.010717973666032776, "loss": 0.010293309297412635, "lr_blocks": 4.17702704859633e-06, "lr_fusion": 4.17702704859633e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.009221511951182038} +{"avg_context_frames": 15.200000000000001, "avg_micro_time_s": 4.938525357053732, "epoch": 19, "global_step": 1160, "grad_norm_max": 0.01935427077114582, "hidden_mse": 0.011080202634911985, "loss": 0.010779957432532683, "lr_blocks": 4.0955973283394525e-06, "lr_fusion": 4.095597328339452e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.009671937179518864} +{"avg_context_frames": 13.600000000000001, "avg_micro_time_s": 4.565018052158121, "epoch": 19, "global_step": 1170, "grad_norm_max": 0.01671302504837513, "hidden_mse": 0.01090292414301075, "loss": 0.01035338577348739, "lr_blocks": 4.0144148627426e-06, "lr_fusion": 4.0144148627425993e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.009263093312620185} +{"avg_context_frames": 11.600000000000001, "avg_micro_time_s": 4.144031305231329, "epoch": 19, "global_step": 1180, "grad_norm_max": 0.022863034158945084, "hidden_mse": 0.011614868015749381, "loss": 0.011124649096745999, "lr_blocks": 3.9335018462812664e-06, "lr_fusion": 3.933501846281267e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.00996316227829084} +{"avg_context_frames": 13.200000000000001, "avg_micro_time_s": 4.5215470430353895, "epoch": 19, "global_step": 1190, "grad_norm_max": 0.019639724865555763, "hidden_mse": 0.011618092842400075, "loss": 0.011214426264632494, "lr_blocks": 3.852880399766243e-06, "lr_fusion": 3.852880399766243e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.010052616955363192} +{"avg_context_frames": 16.8, "avg_micro_time_s": 5.278701306048607, "epoch": 19, "global_step": 1200, "grad_norm_max": 0.013881155289709568, "hidden_mse": 0.009965649386867882, "loss": 0.009543122234754266, "lr_blocks": 3.7725725642960047e-06, "lr_fusion": 3.772572564296005e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.008546557306544855} +{"avg_context_frames": 14.0, "avg_micro_time_s": 4.690072318990133, "epoch": 19, "global_step": 1210, "grad_norm_max": 0.01823275536298752, "hidden_mse": 0.013038989173946903, "loss": 0.012048358720494435, "lr_blocks": 3.6926002952309015e-06, "lr_fusion": 3.6926002952309016e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.010744459758279847} +{"avg_context_frames": 16.400000000000002, "avg_micro_time_s": 5.208265690266853, "epoch": 20, "global_step": 1220, "grad_norm_max": 0.01877760700881481, "hidden_mse": 0.011719083180651069, "loss": 0.010691058769589291, "lr_blocks": 3.6129854561907786e-06, "lr_fusion": 3.612985456190778e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.009519150495179929} +{"avg_context_frames": 12.8, "avg_micro_time_s": 4.422553172298649, "epoch": 20, "global_step": 1230, "grad_norm_max": 0.017899416387081146, "hidden_mse": 0.011300698283594103, "loss": 0.011195075674913825, "lr_blocks": 3.533749813077677e-06, "lr_fusion": 3.533749813077677e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.010065005859360099} +{"avg_context_frames": 13.600000000000001, "avg_micro_time_s": 4.580692743565305, "epoch": 20, "global_step": 1240, "grad_norm_max": 0.013124460354447365, "hidden_mse": 0.011365904164267705, "loss": 0.010706490487791598, "lr_blocks": 3.4549150281252635e-06, "lr_fusion": 3.4549150281252636e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.009569900057977067} +{"avg_context_frames": 13.200000000000001, "avg_micro_time_s": 4.507879159919685, "epoch": 20, "global_step": 1250, "grad_norm_max": 0.01686941832304001, "hidden_mse": 0.010944274376379327, "loss": 0.010535373043967412, "lr_blocks": 3.3765026539765832e-06, "lr_fusion": 3.3765026539765834e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.00944094563019462} +{"avg_context_frames": 16.8, "avg_micro_time_s": 5.269363912683912, "epoch": 20, "global_step": 1260, "grad_norm_max": 0.011800975538790226, "hidden_mse": 0.010452765604713933, "loss": 0.00955662438645959, "lr_blocks": 3.298534127791785e-06, "lr_fusion": 3.298534127791785e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.008511347760213539} +{"avg_context_frames": 13.200000000000001, "avg_micro_time_s": 4.486637803389749, "epoch": 20, "global_step": 1270, "grad_norm_max": 0.00913545023649931, "hidden_mse": 0.0119449045159854, "loss": 0.011296011722879485, "lr_blocks": 3.2210307653874175e-06, "lr_fusion": 3.221030765387417e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.010101521265460179} +{"avg_context_frames": 16.0, "avg_micro_time_s": 5.1098935861125945, "epoch": 20, "global_step": 1280, "grad_norm_max": 0.012479772791266441, "hidden_mse": 0.01143988748663105, "loss": 0.010748607415007428, "lr_blocks": 3.1440137554088957e-06, "lr_fusion": 3.144013755408895e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.00960461861686781} +{"avg_context_frames": 14.0, "avg_micro_time_s": 4.66468191168824, "epoch": 21, "global_step": 1290, "grad_norm_max": 0.010951487347483635, "hidden_mse": 0.011044421943370254, "loss": 0.01045076870941557, "lr_blocks": 3.06750415353774e-06, "lr_fusion": 3.0675041535377405e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.009346326530794613} +{"avg_context_frames": 16.400000000000002, "avg_micro_time_s": 5.196936400650884, "epoch": 21, "global_step": 1300, "grad_norm_max": 0.012817778624594212, "hidden_mse": 0.010785160388331862, "loss": 0.010344182967673988, "lr_blocks": 2.991522876735154e-06, "lr_fusion": 2.991522876735154e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.009265666973078623} +{"avg_context_frames": 14.8, "avg_micro_time_s": 4.844198746807525, "epoch": 21, "global_step": 1310, "grad_norm_max": 0.01746842823922634, "hidden_mse": 0.011365688865771518, "loss": 0.010998867708258331, "lr_blocks": 2.9160906975235493e-06, "lr_fusion": 2.916090697523549e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.009862298879306765} +{"avg_context_frames": 11.600000000000001, "avg_micro_time_s": 4.145829289100948, "epoch": 21, "global_step": 1320, "grad_norm_max": 0.014280302450060844, "hidden_mse": 0.012770673574414105, "loss": 0.011853273032465951, "lr_blocks": 2.8412282383075362e-06, "lr_fusion": 2.8412282383075363e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.010576205688994379} +{"avg_context_frames": 13.200000000000001, "avg_micro_time_s": 4.498126996241626, "epoch": 21, "global_step": 1330, "grad_norm_max": 0.012146072462201118, "hidden_mse": 0.011811680562095717, "loss": 0.011255794862518088, "lr_blocks": 2.766955965735968e-06, "lr_fusion": 2.766955965735968e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.01007462683483027} +{"avg_context_frames": 15.600000000000001, "avg_micro_time_s": 5.000488996443164, "epoch": 21, "global_step": 1340, "grad_norm_max": 0.016375984996557236, "hidden_mse": 0.010209381650201977, "loss": 0.009517519114888275, "lr_blocks": 2.693294185106562e-06, "lr_fusion": 2.693294185106562e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.008496580872451887} +{"avg_context_frames": 14.4, "avg_micro_time_s": 4.752854582459259, "epoch": 22, "global_step": 1350, "grad_norm_max": 0.015220776200294495, "hidden_mse": 0.011078161501791329, "loss": 0.010252770740771668, "lr_blocks": 2.6202630348146323e-06, "lr_fusion": 2.6202630348146324e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.009144954636576585} +{"avg_context_frames": 14.4, "avg_micro_time_s": 4.733257904538187, "epoch": 22, "global_step": 1360, "grad_norm_max": 0.011605209670960903, "hidden_mse": 0.011268787307199091, "loss": 0.010491596191423015, "lr_blocks": 2.5478824808474613e-06, "lr_fusion": 2.547882480847461e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.00936471747700125} +{"avg_context_frames": 12.4, "avg_micro_time_s": 4.309442731441232, "epoch": 22, "global_step": 1370, "grad_norm_max": 0.018279148265719414, "hidden_mse": 0.012431519536767155, "loss": 0.01210868107737042, "lr_blocks": 2.476172311325783e-06, "lr_fusion": 2.476172311325783e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.01086552903871052} +{"avg_context_frames": 16.0, "avg_micro_time_s": 5.085443596898404, "epoch": 22, "global_step": 1380, "grad_norm_max": 0.013926972635090351, "hidden_mse": 0.010723891278030351, "loss": 0.01000883829256054, "lr_blocks": 2.4051521310939258e-06, "lr_fusion": 2.405152131093926e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.008936449119937606} +{"avg_context_frames": 14.0, "avg_micro_time_s": 4.653221165735158, "epoch": 22, "global_step": 1390, "grad_norm_max": 0.010899711400270462, "hidden_mse": 0.010991557216038929, "loss": 0.01038530458463356, "lr_blocks": 2.3348413563600324e-06, "lr_fusion": 2.3348413563600325e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.009286148854880594} +{"avg_context_frames": 14.8, "avg_micro_time_s": 4.840354088667664, "epoch": 22, "global_step": 1400, "grad_norm_max": 0.01484893448650837, "hidden_mse": 0.01119777720887214, "loss": 0.010615348658757285, "lr_blocks": 2.265259209387867e-06, "lr_fusion": 2.2652592093878666e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.009495570906437933} +{"avg_context_frames": 16.400000000000002, "avg_micro_time_s": 5.197918331775873, "epoch": 23, "global_step": 1410, "grad_norm_max": 0.0081826476380229, "hidden_mse": 0.010273119009798394, "loss": 0.009885615069651977, "lr_blocks": 2.1964247132416373e-06, "lr_fusion": 2.196424713241637e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.00885830314946361} +{"avg_context_frames": 11.600000000000001, "avg_micro_time_s": 4.165223089905339, "epoch": 23, "global_step": 1420, "grad_norm_max": 0.011765100061893463, "hidden_mse": 0.011794531159102917, "loss": 0.011367314646486194, "lr_blocks": 2.1283566865852824e-06, "lr_fusion": 2.128356686585282e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.01018786151544191} +{"avg_context_frames": 15.200000000000001, "avg_micro_time_s": 4.9317244929407025, "epoch": 23, "global_step": 1430, "grad_norm_max": 0.013556716963648796, "hidden_mse": 0.010485906671965495, "loss": 0.00974137083394453, "lr_blocks": 2.061073738537635e-06, "lr_fusion": 2.061073738537635e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.008692780195269733} +{"avg_context_frames": 13.600000000000001, "avg_micro_time_s": 4.572829652384098, "epoch": 23, "global_step": 1440, "grad_norm_max": 0.014423741959035397, "hidden_mse": 0.010887607344193384, "loss": 0.01029668509727344, "lr_blocks": 1.9945942635848745e-06, "lr_fusion": 1.9945942635848748e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.00920792439719662} +{"avg_context_frames": 14.8, "avg_micro_time_s": 4.804273775946058, "epoch": 23, "global_step": 1450, "grad_norm_max": 0.009066827595233917, "hidden_mse": 0.01108798424829729, "loss": 0.010514156869612636, "lr_blocks": 1.928936436551661e-06, "lr_fusion": 1.928936436551661e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.009405358476215042} +{"avg_context_frames": 14.4, "avg_micro_time_s": 4.782503490094678, "epoch": 23, "global_step": 1460, "grad_norm_max": 0.012391151860356331, "hidden_mse": 0.012302883324446158, "loss": 0.011482411832548679, "lr_blocks": 1.864118207632315e-06, "lr_fusion": 1.8641182076323148e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.010252123483223841} +{"avg_context_frames": 14.4, "avg_micro_time_s": 4.763748030239367, "epoch": 23, "global_step": 1470, "grad_norm_max": 0.01666567288339138, "hidden_mse": 0.011601725989021362, "loss": 0.010708705123397522, "lr_blocks": 1.8001572974834169e-06, "lr_fusion": 1.800157297483417e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.009548532468033956} +{"avg_context_frames": 16.0, "avg_micro_time_s": 5.105029701559397, "epoch": 24, "global_step": 1480, "grad_norm_max": 0.007869450375437737, "hidden_mse": 0.010309066070476547, "loss": 0.009799729031510652, "lr_blocks": 1.7370711923791567e-06, "lr_fusion": 1.7370711923791567e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.008768822392448783} +{"avg_context_frames": 10.4, "avg_micro_time_s": 3.8783476755881567, "epoch": 24, "global_step": 1490, "grad_norm_max": 0.008603546768426895, "hidden_mse": 0.012117054517148063, "loss": 0.011406626860843972, "lr_blocks": 1.6748771394307584e-06, "lr_fusion": 1.6748771394307585e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.010194921452784911} +{"avg_context_frames": 14.4, "avg_micro_time_s": 4.747819963515212, "epoch": 24, "global_step": 1500, "grad_norm_max": 0.009113138541579247, "hidden_mse": 0.01056220151949674, "loss": 0.009745723713422194, "lr_blocks": 1.6135921418712959e-06, "lr_fusion": 1.6135921418712956e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.008689503525965847} +{"avg_context_frames": 14.8, "avg_micro_time_s": 4.8422974339220675, "epoch": 24, "global_step": 1510, "grad_norm_max": 0.009046677500009537, "hidden_mse": 0.011112269636942075, "loss": 0.01060579260229133, "lr_blocks": 1.5532329544071712e-06, "lr_fusion": 1.553232954407171e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.009494565633940511} +{"avg_context_frames": 14.4, "avg_micro_time_s": 4.757075529414578, "epoch": 24, "global_step": 1520, "grad_norm_max": 0.010557533241808414, "hidden_mse": 0.010822936653858051, "loss": 0.010363288567168638, "lr_blocks": 1.4938160786375571e-06, "lr_fusion": 1.4938160786375572e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.009280994889559224} +{"avg_context_frames": 17.2, "avg_micro_time_s": 5.357483462701203, "epoch": 24, "global_step": 1530, "grad_norm_max": 0.011946751736104488, "hidden_mse": 0.010540719516575337, "loss": 0.009585928963497282, "lr_blocks": 1.4353577585430152e-06, "lr_fusion": 1.435357758543015e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.00853185698215384} +{"avg_context_frames": 16.400000000000002, "avg_micro_time_s": 5.185359577655618, "epoch": 25, "global_step": 1540, "grad_norm_max": 0.007965869270265102, "hidden_mse": 0.011549287679372355, "loss": 0.010612667800160125, "lr_blocks": 1.3778739760445552e-06, "lr_fusion": 1.3778739760445552e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.009457739000208677} +{"avg_context_frames": 12.0, "avg_micro_time_s": 4.2216648488363715, "epoch": 25, "global_step": 1550, "grad_norm_max": 0.011087382212281227, "hidden_mse": 0.012428120122058317, "loss": 0.011559291259618477, "lr_blocks": 1.321380446634342e-06, "lr_fusion": 1.3213804466343421e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.010316479293396697} +{"avg_context_frames": 15.600000000000001, "avg_micro_time_s": 5.008699878887274, "epoch": 25, "global_step": 1560, "grad_norm_max": 0.007003052160143852, "hidden_mse": 0.009763904713327066, "loss": 0.00917876220191829, "lr_blocks": 1.2658926150792321e-06, "lr_fusion": 1.2658926150792322e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.008202371705556289} +{"avg_context_frames": 12.8, "avg_micro_time_s": 4.416036189952865, "epoch": 25, "global_step": 1570, "grad_norm_max": 0.009147959761321545, "hidden_mse": 0.010971030162181705, "loss": 0.010505453264340759, "lr_blocks": 1.2114256511983274e-06, "lr_fusion": 1.2114256511983274e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.009408350259764121} +{"avg_context_frames": 14.8, "avg_micro_time_s": 4.854656426908332, "epoch": 25, "global_step": 1580, "grad_norm_max": 0.006209497340023518, "hidden_mse": 0.010618329688441008, "loss": 0.009915703616570681, "lr_blocks": 1.157994445715706e-06, "lr_fusion": 1.157994445715706e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.008853870621533134} +{"avg_context_frames": 12.0, "avg_micro_time_s": 4.221170920622535, "epoch": 25, "global_step": 1590, "grad_norm_max": 0.006901085376739502, "hidden_mse": 0.01208687485777773, "loss": 0.011354214960010723, "lr_blocks": 1.1056136061894386e-06, "lr_fusion": 1.1056136061894384e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.010145527502754704} +{"avg_context_frames": 17.2, "avg_micro_time_s": 5.375108856055886, "epoch": 25, "global_step": 1600, "grad_norm_max": 0.00799157377332449, "hidden_mse": 0.011562452139332892, "loss": 0.010567418026039378, "lr_blocks": 1.0542974530180327e-06, "lr_fusion": 1.0542974530180327e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.009411172760883347} +{"avg_context_frames": 16.0, "avg_micro_time_s": 5.114862356413505, "epoch": 26, "global_step": 1610, "grad_norm_max": 0.006925714667886496, "hidden_mse": 0.010430932935560123, "loss": 0.00958031289628707, "lr_blocks": 1.0040600155253766e-06, "lr_fusion": 1.0040600155253765e-05, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.008537219595746137} +{"avg_context_frames": 11.200000000000001, "avg_micro_time_s": 4.076742993789958, "epoch": 26, "global_step": 1620, "grad_norm_max": 0.010778370313346386, "hidden_mse": 0.012875790236284957, "loss": 0.011928505630930886, "lr_blocks": 9.549150281252633e-07, "lr_fusion": 9.549150281252633e-06, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.01064092650776729} +{"avg_context_frames": 15.200000000000001, "avg_micro_time_s": 4.91690728977701, "epoch": 26, "global_step": 1630, "grad_norm_max": 0.010149696841835976, "hidden_mse": 0.01137813255772926, "loss": 0.010421179106924684, "lr_blocks": 9.068759265665384e-07, "lr_fusion": 9.068759265665384e-06, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.009283365862211212} +{"avg_context_frames": 16.8, "avg_micro_time_s": 5.251534129875655, "epoch": 26, "global_step": 1640, "grad_norm_max": 0.007494920864701271, "hidden_mse": 0.010229651187546552, "loss": 0.009787540562683716, "lr_blocks": 8.599558442598998e-07, "lr_fusion": 8.599558442598998e-06, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.008764575445093214} +{"avg_context_frames": 16.8, "avg_micro_time_s": 5.289970108147827, "epoch": 26, "global_step": 1650, "grad_norm_max": 0.005772795993834734, "hidden_mse": 0.0098832230665721, "loss": 0.00929704400477931, "lr_blocks": 8.141676086873574e-07, "lr_fusion": 8.141676086873572e-06, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.008308721653884276} +{"avg_context_frames": 8.8, "avg_micro_time_s": 3.549413197900867, "epoch": 26, "global_step": 1660, "grad_norm_max": 0.008459553122520447, "hidden_mse": 0.012503058323636651, "loss": 0.011526797455735505, "lr_blocks": 7.695237378953224e-07, "lr_fusion": 7.695237378953223e-06, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.010276491596596316} +{"avg_context_frames": 18.0, "avg_micro_time_s": 5.540167348842079, "epoch": 27, "global_step": 1670, "grad_norm_max": 0.006040719337761402, "hidden_mse": 0.00982662042370066, "loss": 0.009351571032311767, "lr_blocks": 7.260364370723044e-07, "lr_fusion": 7.260364370723044e-06, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.008368908986449241} +{"avg_context_frames": 14.0, "avg_micro_time_s": 4.678109683087678, "epoch": 27, "global_step": 1680, "grad_norm_max": 0.007613856811076403, "hidden_mse": 0.011569939384935425, "loss": 0.01060179262130987, "lr_blocks": 6.837175952121305e-07, "lr_fusion": 6.837175952121306e-06, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.00944479865138419} +{"avg_context_frames": 13.600000000000001, "avg_micro_time_s": 4.5720506857542205, "epoch": 27, "global_step": 1690, "grad_norm_max": 0.009097845293581486, "hidden_mse": 0.011648505093762652, "loss": 0.010882016102550553, "lr_blocks": 6.425787818636131e-07, "lr_fusion": 6.425787818636131e-06, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.009717165620531888} +{"avg_context_frames": 10.8, "avg_micro_time_s": 3.9544836690925877, "epoch": 27, "global_step": 1700, "grad_norm_max": 0.008241798728704453, "hidden_mse": 0.012498434266308324, "loss": 0.01152691089664586, "lr_blocks": 6.026312439675553e-07, "lr_fusion": 6.026312439675552e-06, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.010277067520655693} +{"avg_context_frames": 16.8, "avg_micro_time_s": 5.252977297447797, "epoch": 27, "global_step": 1710, "grad_norm_max": 0.005965256132185459, "hidden_mse": 0.009790840366622434, "loss": 0.008979868702590465, "lr_blocks": 5.63885902781941e-07, "lr_fusion": 5.6388590278194096e-06, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.008000784640898928} +{"avg_context_frames": 15.600000000000001, "avg_micro_time_s": 5.01817140855419, "epoch": 27, "global_step": 1720, "grad_norm_max": 0.006612634751945734, "hidden_mse": 0.010684299760032446, "loss": 0.009940559454844334, "lr_blocks": 5.263533508961827e-07, "lr_fusion": 5.263533508961827e-06, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.008872129442170263} +{"avg_context_frames": 12.8, "avg_micro_time_s": 4.401973593690491, "epoch": 28, "global_step": 1730, "grad_norm_max": 0.008472484536468983, "hidden_mse": 0.01170709988800809, "loss": 0.010971345892176033, "lr_blocks": 4.900438493352056e-07, "lr_fusion": 4.900438493352055e-06, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.00980063591268845} +{"avg_context_frames": 15.200000000000001, "avg_micro_time_s": 4.898669266035722, "epoch": 28, "global_step": 1740, "grad_norm_max": 0.005134311970323324, "hidden_mse": 0.010188091878080742, "loss": 0.009483447211096063, "lr_blocks": 4.549673247541875e-07, "lr_fusion": 4.549673247541875e-06, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.008464638039004059} +{"avg_context_frames": 15.200000000000001, "avg_micro_time_s": 4.953151851585426, "epoch": 28, "global_step": 1750, "grad_norm_max": 0.005264210980385542, "hidden_mse": 0.010168692405568436, "loss": 0.009657599416095765, "lr_blocks": 4.211333667247125e-07, "lr_fusion": 4.2113336672471245e-06, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.008640730229672045} +{"avg_context_frames": 16.400000000000002, "avg_micro_time_s": 5.226515496625507, "epoch": 28, "global_step": 1760, "grad_norm_max": 0.00838795118033886, "hidden_mse": 0.011909916769945995, "loss": 0.01069888652127702, "lr_blocks": 3.885512251130763e-07, "lr_fusion": 3.885512251130763e-06, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.009507894856506029} +{"avg_context_frames": 13.600000000000001, "avg_micro_time_s": 4.632364567874174, "epoch": 28, "global_step": 1770, "grad_norm_max": 0.00759158143773675, "hidden_mse": 0.011039437243016438, "loss": 0.010217314178589731, "lr_blocks": 3.572298075514652e-07, "lr_fusion": 3.5722980755146517e-06, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.009113370382692665} +{"avg_context_frames": 14.0, "avg_micro_time_s": 4.687877335623489, "epoch": 28, "global_step": 1780, "grad_norm_max": 0.004666333552449942, "hidden_mse": 0.011103744938736781, "loss": 0.010361632268177346, "lr_blocks": 3.271776770026963e-07, "lr_fusion": 3.271776770026963e-06, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.009251257806317881} +{"avg_context_frames": 13.600000000000001, "avg_micro_time_s": 4.599278081623197, "epoch": 28, "global_step": 1790, "grad_norm_max": 0.006053748074918985, "hidden_mse": 0.011113211989868433, "loss": 0.01052120756939985, "lr_blocks": 2.984030494191942e-07, "lr_fusion": 2.9840304941919415e-06, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.009409886403591373} +{"avg_context_frames": 13.600000000000001, "avg_micro_time_s": 4.588346884711064, "epoch": 29, "global_step": 1800, "grad_norm_max": 0.004144392907619476, "hidden_mse": 0.0118655061989557, "loss": 0.011013782626832835, "lr_blocks": 2.7091379149682683e-07, "lr_fusion": 2.7091379149682685e-06, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.009827231944655068} +{"avg_context_frames": 16.400000000000002, "avg_micro_time_s": 5.210386208684941, "epoch": 29, "global_step": 1810, "grad_norm_max": 0.004174100700765848, "hidden_mse": 0.01011652717133984, "loss": 0.009480330155929551, "lr_blocks": 2.447174185242324e-07, "lr_fusion": 2.4471741852423237e-06, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.008468677412020042} +{"avg_context_frames": 14.4, "avg_micro_time_s": 4.730244041526748, "epoch": 29, "global_step": 1820, "grad_norm_max": 0.00522144278511405, "hidden_mse": 0.01145612609689124, "loss": 0.010481840348802507, "lr_blocks": 2.198210923282118e-07, "lr_fusion": 2.1982109232821178e-06, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.009336227763560601} +{"avg_context_frames": 16.400000000000002, "avg_micro_time_s": 5.165751515641751, "epoch": 29, "global_step": 1830, "grad_norm_max": 0.0051048253662884235, "hidden_mse": 0.010516874660970644, "loss": 0.009727180041954853, "lr_blocks": 1.962316193157593e-07, "lr_fusion": 1.962316193157593e-06, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.008675492575275712} +{"avg_context_frames": 14.8, "avg_micro_time_s": 4.7976421843559365, "epoch": 29, "global_step": 1840, "grad_norm_max": 0.003483401145786047, "hidden_mse": 0.010557679162593559, "loss": 0.00988672422245145, "lr_blocks": 1.7395544861325718e-07, "lr_fusion": 1.7395544861325718e-06, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.008830956279416569} +{"avg_context_frames": 10.4, "avg_micro_time_s": 3.883136258776358, "epoch": 29, "global_step": 1850, "grad_norm_max": 0.0037173114251345396, "hidden_mse": 0.01231392920599319, "loss": 0.011277897394029424, "lr_blocks": 1.5299867030334815e-07, "lr_fusion": 1.5299867030334814e-06, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.010046504443744199} +{"avg_context_frames": 13.600000000000001, "avg_micro_time_s": 4.582784790352162, "epoch": 30, "global_step": 1860, "grad_norm_max": 0.004942863713949919, "hidden_mse": 0.011713505216175691, "loss": 0.010921512806089596, "lr_blocks": 1.333670137599713e-07, "lr_fusion": 1.333670137599713e-06, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.009750162286218257} +{"avg_context_frames": 14.4, "avg_micro_time_s": 4.775307224848075, "epoch": 30, "global_step": 1870, "grad_norm_max": 0.0059623210690915585, "hidden_mse": 0.011227280355524272, "loss": 0.010118972533382476, "lr_blocks": 1.1506584608200366e-07, "lr_fusion": 1.1506584608200367e-06, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.008996244543232024} +{"avg_context_frames": 14.4, "avg_micro_time_s": 4.764360447485524, "epoch": 30, "global_step": 1880, "grad_norm_max": 0.004404996521770954, "hidden_mse": 0.010805027768947185, "loss": 0.010003475996199995, "lr_blocks": 9.810017062595322e-08, "lr_fusion": 9.810017062595322e-07, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.008922973214066588} +{"avg_context_frames": 16.0, "avg_micro_time_s": 5.1204566784697825, "epoch": 30, "global_step": 1890, "grad_norm_max": 0.005264414008706808, "hidden_mse": 0.011139299487695098, "loss": 0.010331200476503, "lr_blocks": 8.247462563808816e-08, "lr_fusion": 8.247462563808817e-07, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.009217270533554256} +{"avg_context_frames": 14.0, "avg_micro_time_s": 4.662073467359005, "epoch": 30, "global_step": 1900, "grad_norm_max": 0.004001558758318424, "hidden_mse": 0.010723561415215955, "loss": 0.01027929904521443, "lr_blocks": 6.819348298638839e-08, "lr_fusion": 6.819348298638839e-07, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.009206942893797532} +{"avg_context_frames": 15.600000000000001, "avg_micro_time_s": 5.013501656531298, "epoch": 30, "global_step": 1910, "grad_norm_max": 0.004428073298186064, "hidden_mse": 0.01041651072446257, "loss": 0.00982943093404174, "lr_blocks": 5.526064699265754e-08, "lr_fusion": 5.526064699265753e-07, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.008787779847625644} +{"avg_context_frames": 11.600000000000001, "avg_micro_time_s": 4.14697117824835, "epoch": 30, "global_step": 1920, "grad_norm_max": 0.00527539337053895, "hidden_mse": 0.012029773188987748, "loss": 0.011034077405929566, "lr_blocks": 4.367965336512403e-08, "lr_fusion": 4.367965336512403e-07, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.009831100079463796} +{"avg_context_frames": 15.200000000000001, "avg_micro_time_s": 4.901597832782136, "epoch": 31, "global_step": 1930, "grad_norm_max": 0.0042945914901793, "hidden_mse": 0.010561493248678744, "loss": 0.009865093388361857, "lr_blocks": 3.345366823180929e-08, "lr_fusion": 3.3453668231809286e-07, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.00880894406582229} +{"avg_context_frames": 15.600000000000001, "avg_micro_time_s": 4.995177410470205, "epoch": 31, "global_step": 1940, "grad_norm_max": 0.005610039457678795, "hidden_mse": 0.011831889807945119, "loss": 0.010663101513637231, "lr_blocks": 2.4585487274942922e-08, "lr_fusion": 2.458548727494292e-07, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.009479912539245561} +{"avg_context_frames": 12.8, "avg_micro_time_s": 4.408782520698151, "epoch": 31, "global_step": 1950, "grad_norm_max": 0.0050836289301514626, "hidden_mse": 0.010904357407707722, "loss": 0.010504080465761945, "lr_blocks": 1.7077534966650767e-08, "lr_fusion": 1.7077534966650766e-07, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.009413644747110084} +{"avg_context_frames": 14.8, "avg_micro_time_s": 4.828281990058895, "epoch": 31, "global_step": 1960, "grad_norm_max": 0.0056594074703752995, "hidden_mse": 0.010813442018115894, "loss": 0.009940294205443935, "lr_blocks": 1.0931863906127327e-08, "lr_fusion": 1.0931863906127327e-07, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.008858949999557808} +{"avg_context_frames": 14.8, "avg_micro_time_s": 4.812497291009641, "epoch": 31, "global_step": 1970, "grad_norm_max": 0.0038862251676619053, "hidden_mse": 0.011173089244402946, "loss": 0.010378605034202338, "lr_blocks": 6.150154258476315e-09, "lr_fusion": 6.150154258476315e-08, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.009261296124896035} +{"avg_context_frames": 14.8, "avg_micro_time_s": 4.86070041981875, "epoch": 31, "global_step": 1980, "grad_norm_max": 0.005016204435378313, "hidden_mse": 0.011308084259508178, "loss": 0.010282004851615057, "lr_blocks": 2.7337132953697555e-09, "lr_fusion": 2.7337132953697554e-08, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.009151196412858553} +{"avg_context_frames": 11.200000000000001, "avg_micro_time_s": 4.063614819588838, "epoch": 32, "global_step": 1990, "grad_norm_max": 0.0047661722637712955, "hidden_mse": 0.011776553199160844, "loss": 0.01103718699887395, "lr_blocks": 6.834750376549793e-10, "lr_fusion": 6.834750376549792e-09, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.009859531658003108} +{"avg_context_frames": 13.200000000000001, "avg_micro_time_s": 4.505251754174242, "epoch": 32, "global_step": 2000, "grad_norm_max": 0.003149358555674553, "hidden_mse": 0.011502530233701692, "loss": 0.010889919585315511, "lr_blocks": 0.0, "lr_fusion": 0.0, "peak_memory_gib": 64.4267053604126, "velocity_mse": 0.009739666487439536} +swanlab: 📁 View project at +https://swanlab.cn/@ZhuoChen/HY-WorldPlay-Predictor-v4 +swanlab: 🚀 View run pre...cks1-52-fp32master-8gpu-torch26-noflash at +https://swanlab.cn/@ZhuoChen/HY-WorldPlay-Predictor-v4/runs/5cggdz0m +swanlab: Upload complete: 74477 records +[complete] mode=previous_step 2026-08-23T14:51:32Z diff --git a/logs/stage1_previous_condition_validation_torch26-retrain_20260824/all.completed b/logs/stage1_previous_condition_validation_torch26-retrain_20260824/all.completed new file mode 100644 index 0000000000000000000000000000000000000000..2f6e3a2b10c1e9f8589e0fe28ace8ddf9005a5a5 --- /dev/null +++ b/logs/stage1_previous_condition_validation_torch26-retrain_20260824/all.completed @@ -0,0 +1 @@ +2026-08-24T04:15:37Z diff --git a/logs/stage1_previous_condition_validation_torch26-retrain_20260824/sequential.log b/logs/stage1_previous_condition_validation_torch26-retrain_20260824/sequential.log new file mode 100644 index 0000000000000000000000000000000000000000..6654c6b3942cdd5dafa8da1b6bae0fc841b7abf5 --- /dev/null +++ b/logs/stage1_previous_condition_validation_torch26-retrain_20260824/sequential.log @@ -0,0 +1,7 @@ +torch=2.6.0+cu124 cuda=12.4 flash_attn=False +Validation evaluation complete: /mnt/local_nvme/cz/projects/HY-WorldPlay-DEV-Predictor/datasets/hyworldplay_predictor_vbench_val25_test50/evaluations/eval_validation25_previous-condition-none_torch26-retrain_20260824 +Durable copy: s3://s3-us-west2-default/zoubin/cz/projects/HY-WorldPlay-DEV-Predictor/datasets/hyworldplay_predictor_vbench_val25_test50/evaluations/eval_validation25_previous-condition-none_torch26-retrain_20260824 +Validation evaluation complete: /mnt/local_nvme/cz/projects/HY-WorldPlay-DEV-Predictor/datasets/hyworldplay_predictor_vbench_val25_test50/evaluations/eval_validation25_previous-condition-previous-step_torch26-retrain_20260824 +Durable copy: s3://s3-us-west2-default/zoubin/cz/projects/HY-WorldPlay-DEV-Predictor/datasets/hyworldplay_predictor_vbench_val25_test50/evaluations/eval_validation25_previous-condition-previous-step_torch26-retrain_20260824 +Validation evaluation complete: /mnt/local_nvme/cz/projects/HY-WorldPlay-DEV-Predictor/datasets/hyworldplay_predictor_vbench_val25_test50/evaluations/eval_validation25_previous-condition-both_torch26-retrain_20260824 +Durable copy: s3://s3-us-west2-default/zoubin/cz/projects/HY-WorldPlay-DEV-Predictor/datasets/hyworldplay_predictor_vbench_val25_test50/evaluations/eval_validation25_previous-condition-both_torch26-retrain_20260824 diff --git a/logs/stage1_previous_condition_validation_torch26-retrain_20260824/status.tsv b/logs/stage1_previous_condition_validation_torch26-retrain_20260824/status.tsv new file mode 100644 index 0000000000000000000000000000000000000000..564c13a43935c85109aa0483ba333c9a8ab3cae7 --- /dev/null +++ b/logs/stage1_previous_condition_validation_torch26-retrain_20260824/status.tsv @@ -0,0 +1,8 @@ +preflight 2026-08-24T03:03:37Z torch=2.6.0+cu124 cuda=12.4 flash_attn=False +start 2026-08-24T03:03:37Z eval_validation25_previous-condition-none_torch26-retrain_20260824 +complete 2026-08-24T03:27:19Z eval_validation25_previous-condition-none_torch26-retrain_20260824 +start 2026-08-24T03:27:19Z eval_validation25_previous-condition-previous-step_torch26-retrain_20260824 +complete 2026-08-24T03:49:40Z eval_validation25_previous-condition-previous-step_torch26-retrain_20260824 +start 2026-08-24T03:49:40Z eval_validation25_previous-condition-both_torch26-retrain_20260824 +complete 2026-08-24T04:15:37Z eval_validation25_previous-condition-both_torch26-retrain_20260824 +complete 2026-08-24T04:15:37Z all diff --git a/model.md b/model.md new file mode 100644 index 0000000000000000000000000000000000000000..49ac0e8c8bf8169f61f9539f3e9804055a84b703 --- /dev/null +++ b/model.md @@ -0,0 +1,158 @@ +# Model + +## 1. 共同目标 + +HY-WorldPlay 使用四步去噪。Predictor 的目标是在运行时替代一个或多个完整 54-layer DiT step： + +```text +Full step: 54 个 Teacher blocks + final layer +Predictor step: 2 个 Predictor blocks + frozen Teacher final layer +``` + +所有版本都使用目标 step 的 65 通道 I2V 输入、timestep/action frame condition、相机参数和 RoPE； +最终均预测 final-layer 前的 hidden，并复用冻结的 Teacher final layer 得到 velocity。residual head +零初始化，使初始状态满足 `pred_hidden = anchor_hidden`，等价于 Direct Reuse。 + +## 2. 版本总览 + +| 版本 | image fusion | 文本/Context | previous chunk | Predictor blocks | 参数量 | +| --- | --- | --- | --- | --- | ---: | +| v1 | current + same-chunk anchor | dense current/cached text，双流预测 | 无 | 完整 double-stream | 339,893,248 | +| v2 | current + same-chunk anchor | Text K/V + exact history Vision K/V | 无 | AR vision-only | 176,241,152 | +| v3 | current + anchor + previous chunk | Text K/V + exact history Vision K/V | 有 | AR vision-only | 197,218,816 | +| v4 | current + anchor + previous chunk | Text K/V + BF16 pre-feature 重建 Vision K/V | 有 | AR vision-only | 197,218,816 | +| predictor-disca | current + same-chunk anchor | 仅 Text K/V | 无 | AR vision-only | 176,241,152 | + +参数量为可训练参数。v4 另含 16,785,664 个冻结 Context projector 参数。 + +## 3. Predictor v1 + +实现：`models/predictor.py`。 + +- `img_in`、final layer 从 Teacher 加载并冻结； +- current image tokens 与 same-chunk Full-DiT anchor hidden 经过 image fusion； +- current text 与 text-only prefill 后的 cached text 经过独立 text fusion； +- 两个完整 image/text double-stream blocks 默认从 Teacher `[1,52]` 初始化； +- 不读取 Text K/V 或 history Vision K/V； +- 输出 `anchor_hidden + residual_out(predictor_hidden)`。 + +v1 是早期可行性验证代码路径。旧机器上的实验结果不再纳入当前项目报告；当前可核验的结果统一见 +`experiment.md`。 + +## 4. Predictor v2 + +实现：`models/predictor_v2.py`。 + +v2 将 v1 的 dense text double-stream 改成 WorldPlay 原生 AR vision inference： + +- current image tokens 与 same-chunk anchor hidden 做双输入 fusion； +- Predictor 内不执行 text stream； +- 每个 Predictor block 读取该 Teacher 层的 Text K/V 和 history Vision K/V； +- Vision K/V 包含 regular RoPE 与 ProPE 两个分支； +- 支持 `[0,53]` 或 `[1,52]` 两种 Teacher 初始化； +- `img_in`、final layer和 block text 参数冻结。 + +v2 数据直接保存每层 history Vision K/V，I/O 和磁盘开销较大。代码仍保留，但当前后续实验统一使用 +v4 pre-feature schema。 + +## 5. Predictor v3 + +实现：`models/predictor_v3.py`。 + +v3 在 v2 上增加 previous-chunk same-timestep hidden： + +```text +current image tokens +same-chunk anchor final hidden +previous-chunk target-step final hidden + -> TripleFeatureFusion -> two AR vision blocks +``` + +TripleFeatureFusion 分别 LayerNorm 三路输入，拼接后经过 `6144→4096→2048` MLP。因为需要前一 +chunk，chunk 0 不进入训练。其 Context 仍是 v2 的 direct Vision K/V。 + +## 6. Predictor v4 + +实现：`models/predictor_v4.py`。 + +v4 保留 v3 的三路 image fusion，但将磁盘中的 direct Vision K/V 替换为 BF16 Context +pre-feature： + +```text +img_modulated + -> frozen Teacher K/V projections + K norm + -> Teacher-compatible RoPE/ProPE + -> Context Vision K/V +``` + +训练时从 pre-feature 在线重建 K/V；推理 pipeline 已经持有精确 Teacher Context K/V，因此直接传入， +不重复运行 projector。动态 Context mask 会同时作用于 regular/ProPE 分支。 + +v4 的实验性 `predictor_fusion_type` 另支持： + +- `concat`：默认且与旧 checkpoint 严格兼容的三路 LN + concat MLP； +- `projection_add`：三路 identity-init 独立投影、无 affine source norm、相加后 FusionNorm； +- `token_gate_prev`：在 `projection_add` 上，用 `current/previous/Teacher time+action condition` + 预测 `[B,N,1]` 的 previous-chunk gate。第一版只支持已验证最好的 `same_step` previous condition， + 不 gate same-chunk anchor。 + +正式分辨率下三路 fusion tensor 均为 `[B,6240,2048]`，其中 `6240=4×30×52`。 + +当前已完成两套正式权重： + +- blocks `[0,53]`：`checkpoints/predictor_v4_prefeature_blocks0-53_bf16_8gpu_flr1e-4_blr1e-5_schedsteps2000_swarmup100_bwarmup100_swanlab`； +- blocks `[1,52]`：`checkpoints/predictor_v4_prefeature_blocks1-52_bf16_8gpu_flr1e-4_blr1e-5_schedsteps2000_swarmup100_bwarmup100_swanlab`。 + +v4 能训练所有相邻转换 `0→1`、`1→2`、`2→3`。推理时可任意指定一个或多个替代 step；连续替代 +时，前一个 Predictor 输出成为下一个 Predictor 的 same-chunk anchor。 + +## 7. Predictor-DisCa + +实现：`models/predictor_disca.py`。 + +这是当前项目中与 v1、v4 并列的 `predictor-disca` 版本，不是新 HY-WorldPlay 主模型： + +- current image tokens 与 same-chunk anchor hidden 做双输入 fusion； +- 不注入 previous-chunk hidden； +- 不注入 Context Vision K/V； +- 不加载或运行 frozen Context projector； +- 保留静态 Text K/V，因为 WorldPlay AR vision block 仍需要文本条件； +- 使用两个 Teacher-initialized AR vision blocks； +- 输出仍为 `anchor_hidden + zero-init residual`。 + +当前正式权重为 blocks `[0,53]`： + +```text +checkpoints/predictor_disca_blocks0-53_bf16_8gpu_flr1e-4_blr1e-5_schedsteps2000_swarmup100_bwarmup100_swanlab +``` + +它复现了 DisCa 的“两块轻量 Predictor 替代 Full DiT”思想，但没有复现原始 DisCa 的 dense text +cache/fusion、固定 full-step cache 语义或对抗训练。因此准确定位是 DisCa-inspired WorldPlay ablation， +不是严格的 WorldPlay DisCa 复现。 + +## 8. 当前推荐对比 + +当前主要实验比较： + +```text +Full-DiT: F-F-F-F +Direct Reuse: F-R-R-F # Full steps 0、3；steps 1、2 reuse +Predictor v4: F-P-P-F +Predictor-DisCa: F-P-P-F +``` + +v4 与 predictor-disca 使用相同 Teacher、数据、监督 pair、训练步数和评测集。两者的核心差异只在 +previous-chunk hidden 与 Context Vision K/V，可用于分析 WorldPlay AR Context 的贡献。 + +## ATC Transport 对齐 Causal-Forcing（2026-09-06） + +`models/predictor_atc.py` 的 State Transport 现在默认使用与 Causal-Forcing-a `predictor_training/atc_fusion.py` +相同的实现（`GlobalStateTransport`）：当前 chunk 的 token 对上一 chunk 做**全局** cross-attention， +query/key 分别套用 Teacher 的 3-D RoPE（当前 chunk 帧位置 / 上一 chunk 帧位置），无可学习偏置， +残差加在上一 chunk 特征上；`atc_previous_scope=last_frame` 时 key 只取上一 chunk 最后一帧。 +Transport 头数默认等于 Teacher 的 `heads_num`（head_dim 128 = `sum(rope_dim_list)`），其余维度 +（hidden 2048、MLP 4096、gate 512）不变；`anchor_only / raw_prev / no_gate / no_distance` 消融开关保留。 +原 3×3 局部窗口实现保留为 `--atc_transport_mode local`，2026-09-06 之前训练的 ATC checkpoint 都属于 +该模式，续训或评估时需显式指定。诊断量新增 `transport_attention_entropy_normalized` 与 +`transport_argmax_displacement`。实测 B=2、6240 token：global/chunk 25 ms、global/last_frame 15 ms、 +local 40 ms（fwd+bwd，SDPA 后端，本环境无 flash_attn）。 diff --git a/models/__init__.py b/models/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..1051084609d45c6048f626c888e3a368a22eee89 --- /dev/null +++ b/models/__init__.py @@ -0,0 +1,21 @@ +"""Project-local Predictor models.""" + +from .predictor import HYWorldPlayPredictor, PredictorConfig +from .predictor_v2 import HYWorldPlayPredictorV2, PredictorV2Config +from .predictor_v3 import HYWorldPlayPredictorV3, TripleFeatureFusion +from .predictor_v4 import HYWorldPlayPredictorV4 +from .predictor_atc import ATCFeatureFusion, LocalCrossChunkTransport +from .predictor_disca import HYWorldPlayPredictorDisCa + +__all__ = [ + "HYWorldPlayPredictor", + "PredictorConfig", + "HYWorldPlayPredictorV2", + "PredictorV2Config", + "HYWorldPlayPredictorV3", + "TripleFeatureFusion", + "HYWorldPlayPredictorV4", + "ATCFeatureFusion", + "LocalCrossChunkTransport", + "HYWorldPlayPredictorDisCa", +] diff --git a/models/confidence_head.py b/models/confidence_head.py new file mode 100644 index 0000000000000000000000000000000000000000..c26c40df4744c66622e1912a52e8997b8c024f41 --- /dev/null +++ b/models/confidence_head.py @@ -0,0 +1,43 @@ +"""Lightweight error heads for frozen HY-WorldPlay Predictor outputs.""" + +from __future__ import annotations + +import torch +from torch import nn + + +class PredictorConfidenceHead(nn.Module): + """Regress log Predictor risk from pooled hidden, step, and residual size.""" + + def __init__( + self, + hidden_dim: int = 2048, + timestep_dim: int = 32, + mlp_dim: int = 256, + mlp_dims: tuple[int, ...] | None = None, + ): + super().__init__() + if mlp_dims is None: + mlp_dims = (mlp_dim,) + if not mlp_dims or any(dim <= 0 for dim in mlp_dims): + raise ValueError(f"mlp_dims must contain positive dimensions, got {mlp_dims}") + self.pooled_scale = nn.Parameter(torch.ones(hidden_dim)) + self.pooled_bias = nn.Parameter(torch.zeros(hidden_dim)) + self.step_embedding = nn.Embedding(3, timestep_dim) + layers: list[nn.Module] = [] + input_dim = hidden_dim + timestep_dim + 1 + for output_dim in mlp_dims: + layers.extend((nn.Linear(input_dim, output_dim), nn.SiLU())) + input_dim = output_dim + layers.append(nn.Linear(input_dim, 1)) + self.mlp = nn.Sequential(*layers) + + def forward(self, pooled_hidden, step_id, log_residual): + pooled_hidden = pooled_hidden * self.pooled_scale + self.pooled_bias + step_feature = self.step_embedding(step_id.long() - 1) + return self.mlp( + torch.cat((pooled_hidden, step_feature, log_residual[:, None]), dim=-1) + ).squeeze(-1) + + +__all__ = ["PredictorConfidenceHead"] diff --git a/models/context_prefeature_projector.py b/models/context_prefeature_projector.py new file mode 100644 index 0000000000000000000000000000000000000000..bc53962b98d31306165bd8c1e2837482af3f127e --- /dev/null +++ b/models/context_prefeature_projector.py @@ -0,0 +1,130 @@ +"""Frozen Teacher projectors that rebuild regular-RoPE and ProPE Context K/V.""" + +from __future__ import annotations + +import copy + +import torch +from einops import rearrange +from torch import nn + +from hyvideo.models.transformers.modules.posemb_layers import ( + apply_rotary_emb, + get_nd_rotary_pos_embed, +) +from hyvideo.prope.camera_rope import _prepare_apply_fns_all_dim + + +class FrozenTeacherContextProjector(nn.Module): + """Rebuild the exact cache layout ``[regular batches, ProPE batches]``.""" + + def __init__(self, teacher_transformer: nn.Module, block_ids: tuple[int, ...]) -> None: + super().__init__() + self.block_ids = tuple(int(value) for value in block_ids) + self.heads = int(teacher_transformer.heads_num) + self.head_dim = int(teacher_transformer.hidden_size // self.heads) + self.rope_dim_list = tuple(teacher_transformer.rope_dim_list) + self.rope_theta = float(teacher_transformer.rope_theta) + self.patch_size = tuple(teacher_transformer.patch_size) + self.latent_height = int(getattr(teacher_transformer, "latent_height", 30)) + self.latent_width = int(getattr(teacher_transformer, "latent_width", 52)) + self.k_proj = nn.ModuleDict() + self.v_proj = nn.ModuleDict() + self.k_norm = nn.ModuleDict() + for block_id in self.block_ids: + block = teacher_transformer.double_blocks[block_id] + key = str(block_id) + self.k_proj[key] = copy.deepcopy(block.img_attn_k) + self.v_proj[key] = copy.deepcopy(block.img_attn_v) + self.k_norm[key] = copy.deepcopy(block.img_attn_k_norm) + self.eval().requires_grad_(False) + + def _rope(self, frames: int, height: int, width: int, device: torch.device): + cos, sin = get_nd_rotary_pos_embed( + list(self.rope_dim_list), + (frames // self.patch_size[0], height, width), + theta=self.rope_theta, + use_real=True, + theta_rescale_factor=1, + ) + return cos.to(device), sin.to(device) + + def forward_block( + self, + block_id: int, + img_modulated: torch.Tensor, + context_viewmats: torch.Tensor, + context_Ks: torch.Tensor, + context_valid_mask: torch.Tensor, + *, + latent_height: int | None = None, + latent_width: int | None = None, + ) -> dict[str, torch.Tensor]: + key = str(int(block_id)) + if key not in self.k_proj: + raise ValueError(f"Projector has no block {block_id}") + latent_height = self.latent_height if latent_height is None else int(latent_height) + latent_width = self.latent_width if latent_width is None else int(latent_width) + batch, tokens, _ = img_modulated.shape + tokens_per_frame = latent_height * latent_width + if tokens % tokens_per_frame: + raise ValueError("Context features are not frame-aligned") + frames = tokens // tokens_per_frame + if tuple(context_valid_mask.shape) != (batch, tokens): + raise ValueError("Context valid mask shape mismatch") + if tuple(context_viewmats.shape) != (batch, frames, 4, 4): + raise ValueError("Context viewmats shape mismatch") + if tuple(context_Ks.shape) != (batch, frames, 3, 3): + raise ValueError("Context Ks shape mismatch") + if tokens == 0: + empty = img_modulated.new_empty((2 * batch, self.heads, 0, self.head_dim)) + return {"k_vision": empty, "v_vision": empty.clone()} + + k = rearrange( + self.k_proj[key](img_modulated), + "b s (h d) -> b s h d", + h=self.heads, + ) + v = rearrange( + self.v_proj[key](img_modulated), + "b s (h d) -> b s h d", + h=self.heads, + ) + k = self.k_norm[key](k).to(v) + cos, sin = self._rope(frames, latent_height, latent_width, img_modulated.device) + _, regular_k = apply_rotary_emb(k, k, (cos, sin), head_first=False) + regular_k = regular_k.permute(0, 2, 1, 3) + regular_v = v.permute(0, 2, 1, 3) + + token_viewmats = context_viewmats.repeat_interleave(tokens_per_frame, dim=1) + token_Ks = context_Ks.repeat_interleave(tokens_per_frame, dim=1) + _, apply_kv, _ = _prepare_apply_fns_all_dim( + head_dim=self.head_dim, + viewmats=token_viewmats, + Ks=token_Ks, + patches_x=None, + patches_y=None, + image_width=None, + image_height=None, + ) + prope_k = apply_kv(k.permute(0, 2, 1, 3)) + prope_v = apply_kv(v.permute(0, 2, 1, 3)) + k_cache = torch.cat([regular_k, prope_k], dim=0) + v_cache = torch.cat([regular_v, prope_v], dim=0) + branch_mask = torch.cat([context_valid_mask, context_valid_mask], dim=0) + branch_mask = branch_mask[:, None, :, None] + k_cache = k_cache.masked_fill(~branch_mask, 0) + v_cache = v_cache.masked_fill(~branch_mask, 0) + return {"k_vision": k_cache, "v_vision": v_cache} + + def forward(self, context_by_block: dict[int, dict[str, torch.Tensor]]): + return { + int(block_id): self.forward_block( + int(block_id), + values["img_modulated"], + values["context_viewmats"], + values["context_Ks"], + values["context_valid_mask"], + ) + for block_id, values in context_by_block.items() + } diff --git a/models/predictor.py b/models/predictor.py new file mode 100644 index 0000000000000000000000000000000000000000..2dd29d9163e228802306492cad6abb94059835b1 --- /dev/null +++ b/models/predictor.py @@ -0,0 +1,363 @@ +"""DisCa-style dense image/text feature Predictor for HY-WorldPlay AR.""" + +from __future__ import annotations + +from dataclasses import asdict, dataclass +from math import prod +from pathlib import Path +from typing import Any + +import torch +from einops import repeat +from safetensors import safe_open +from torch import nn +from torch.utils.checkpoint import checkpoint + +from hyvideo.models.transformers.modules.activation_layers import get_activation_layer +from hyvideo.models.transformers.modules.embed_layers import PatchEmbed +from hyvideo.models.transformers.modules.mlp_layers import FinalLayer +from hyvideo.models.transformers.modules.posemb_layers import get_nd_rotary_pos_embed +from hyvideo.models.transformers.worldplay_1_5_transformer import MMDoubleStreamBlock + + +@dataclass(frozen=True) +class PredictorConfig: + hidden_size: int = 2048 + heads_num: int = 16 + mlp_width_ratio: float = 4.0 + mlp_act_type: str = "gelu_tanh" + qkv_bias: bool = True + qk_norm: bool = True + qk_norm_type: str = "rms" + attn_mode: str = "flash" + patch_size: tuple[int, int, int] = (1, 1, 1) + in_channels: int = 32 + out_channels: int = 32 + concat_condition: bool = True + rope_dim_list: tuple[int, int, int] = (16, 56, 56) + rope_theta: float = 256.0 + source_block_ids: tuple[int, int] = (1, 52) + latent_height: int = 30 + latent_width: int = 52 + latent_frames: int = 4 + + +class FeatureFusion(nn.Module): + def __init__(self, hidden_size: int) -> None: + super().__init__() + self.current_norm = nn.LayerNorm(hidden_size, eps=1e-6) + self.cached_norm = nn.LayerNorm(hidden_size, eps=1e-6) + self.mlp = nn.Sequential( + nn.Linear(2 * hidden_size, hidden_size), + nn.SiLU(), + nn.Linear(hidden_size, hidden_size), + ) + + def forward(self, current: torch.Tensor, cached: torch.Tensor) -> torch.Tensor: + if current.shape != cached.shape: + raise ValueError(f"Fusion shape mismatch: {current.shape} != {cached.shape}") + return self.mlp( + torch.cat([self.current_norm(current), self.cached_norm(cached)], dim=-1) + ) + + +class HYWorldPlayPredictor(nn.Module): + """Two full double-stream blocks over fused current/cached image and txt features.""" + + def __init__(self, config: PredictorConfig | None = None) -> None: + super().__init__() + self.predictor_config = config or PredictorConfig() + cfg = self.predictor_config + self.img_in = PatchEmbed( + list(cfg.patch_size), + cfg.in_channels, + cfg.hidden_size, + is_reshape_temporal_channels=False, + concat_condition=cfg.concat_condition, + ) + self.img_fusion = FeatureFusion(cfg.hidden_size) + self.txt_fusion = FeatureFusion(cfg.hidden_size) + self.double_blocks = nn.ModuleList( + [ + MMDoubleStreamBlock( + cfg.hidden_size, + cfg.heads_num, + mlp_width_ratio=cfg.mlp_width_ratio, + mlp_act_type=cfg.mlp_act_type, + attn_mode=cfg.attn_mode, + qk_norm=cfg.qk_norm, + qk_norm_type=cfg.qk_norm_type, + qkv_bias=cfg.qkv_bias, + ) + for _ in cfg.source_block_ids + ] + ) + # HY-WorldPlay action checkpoints add the ProPE output projection after + # constructing the base Hunyuan blocks (see transformer.add_action_parameters). + for block in self.double_blocks: + block.img_attn_prope_proj = nn.Linear( + cfg.hidden_size, cfg.hidden_size, bias=cfg.qkv_bias + ) + self.residual_out = nn.Linear(cfg.hidden_size, cfg.hidden_size) + self.final_layer = FinalLayer( + cfg.hidden_size, + list(cfg.patch_size), + cfg.out_channels, + get_activation_layer("silu"), + ) + nn.init.zeros_(self.residual_out.weight) + nn.init.zeros_(self.residual_out.bias) + self.gradient_checkpointing = False + self.attn_param: dict[str, Any] = { + "thw": [cfg.latent_frames, cfg.latent_height, cfg.latent_width], + "win_type": "fixed", + "win_ratio": 0, + } + self.img_in.requires_grad_(False) + self.final_layer.requires_grad_(False) + + @property + def config_dict(self) -> dict[str, Any]: + return asdict(self.predictor_config) + + def enable_gradient_checkpointing(self, enabled: bool = True) -> None: + self.gradient_checkpointing = enabled + + def train(self, mode: bool = True): + super().train(mode) + self.img_in.eval() + self.final_layer.eval() + return self + + @staticmethod + def _load_prefixed_module( + module: nn.Module, + handle, + prefix: str, + ) -> None: + target = module.state_dict() + loaded = {} + for name in target: + key = prefix + name + if key not in handle.keys(): + raise KeyError(f"Missing Teacher checkpoint key: {key}") + loaded[name] = handle.get_tensor(key) + result = module.load_state_dict(loaded, strict=True) + if result.missing_keys or result.unexpected_keys: + raise RuntimeError(f"Unexpected load result for {prefix}: {result}") + + def load_teacher_initialization(self, checkpoint_path: str | Path) -> None: + path = str(Path(checkpoint_path).resolve()) + with safe_open(path, framework="pt", device="cpu") as handle: + self._load_prefixed_module(self.img_in, handle, "img_in.") + self._load_prefixed_module(self.final_layer, handle, "final_layer.") + for predictor_block, teacher_id in zip( + self.double_blocks, self.predictor_config.source_block_ids + ): + self._load_prefixed_module( + predictor_block, handle, f"double_blocks.{teacher_id}." + ) + self.img_in.requires_grad_(False).eval() + self.final_layer.requires_grad_(False).eval() + nn.init.zeros_(self.residual_out.weight) + nn.init.zeros_(self.residual_out.bias) + + def _vision_rope( + self, + rope_temporal_size: int, + start_rope_start_idx: int, + *, + device: torch.device, + dtype: torch.dtype, + ) -> tuple[torch.Tensor, torch.Tensor]: + cfg = self.predictor_config + cos, sin = get_nd_rotary_pos_embed( + list(cfg.rope_dim_list), + (rope_temporal_size, cfg.latent_height, cfg.latent_width), + theta=cfg.rope_theta, + use_real=True, + theta_rescale_factor=1, + ) + tokens_per_frame = cfg.latent_height * cfg.latent_width + start = start_rope_start_idx * tokens_per_frame + end = (start_rope_start_idx + cfg.latent_frames) * tokens_per_frame + cos = cos[start:end].to(device=device, dtype=dtype) + sin = sin[start:end].to(device=device, dtype=dtype) + expected = cfg.latent_frames * tokens_per_frame + if cos.shape[0] != expected: + raise ValueError(f"RoPE tokens {cos.shape[0]} != {expected}") + return cos, sin + + def _run_block( + self, + block: MMDoubleStreamBlock, + source_block_id: int, + img: torch.Tensor, + txt: torch.Tensor, + vec: torch.Tensor, + vec_txt: torch.Tensor, + freqs_cis: tuple[torch.Tensor, torch.Tensor], + viewmats: torch.Tensor, + Ks: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor]: + def block_forward(img_arg: torch.Tensor, txt_arg: torch.Tensor): + return block( + bi_inference=True, + ar_txt_inference=False, + ar_vision_inference=False, + img=img_arg, + txt=txt_arg, + vec_txt=vec_txt, + vec=vec, + freqs_cis=freqs_cis, + text_mask=None, + attn_param=self.attn_param, + is_flash=False, + block_idx=source_block_id, + viewmats=viewmats, + Ks=Ks, + ) + + if self.gradient_checkpointing and self.training: + return checkpoint(block_forward, img, txt, use_reentrant=False) + return block_forward(img, txt) + + def unpatchify(self, x: torch.Tensor) -> torch.Tensor: + cfg = self.predictor_config + batch = x.shape[0] + expected = cfg.latent_frames * cfg.latent_height * cfg.latent_width + if x.shape[1] != expected: + raise ValueError(f"Output tokens {x.shape[1]} != {expected}") + x = x.reshape( + batch, + cfg.latent_frames, + cfg.latent_height, + cfg.latent_width, + cfg.out_channels, + *cfg.patch_size, + ) + x = torch.einsum("nthwcopq->nctohpwq", x) + return x.reshape( + batch, + cfg.out_channels, + cfg.latent_frames * cfg.patch_size[0], + cfg.latent_height * cfg.patch_size[1], + cfg.latent_width * cfg.patch_size[2], + ) + + def hidden_to_velocity( + self, + hidden: torch.Tensor, + frame_condition: torch.Tensor, + ) -> torch.Tensor: + cfg = self.predictor_config + spatial_tokens = cfg.latent_height * cfg.latent_width + condition_tokens = frame_condition.repeat_interleave(spatial_tokens, dim=1) + vec = condition_tokens.reshape(-1, cfg.hidden_size) + return self.unpatchify(self.final_layer(hidden, vec)) + + def forward( + self, + *, + target_model_input: torch.Tensor, + anchor_hidden: torch.Tensor, + current_txt: torch.Tensor, + cached_txt: torch.Tensor, + target_frame_condition: torch.Tensor, + vec_txt: torch.Tensor, + target_viewmats: torch.Tensor, + target_Ks: torch.Tensor, + rope_temporal_size: torch.Tensor | int, + start_rope_start_idx: torch.Tensor | int, + ) -> dict[str, torch.Tensor]: + cfg = self.predictor_config + batch = target_model_input.shape[0] + if batch != 1: + raise ValueError("Predictor v1 currently requires micro-batch 1") + if target_model_input.shape[1:] != ( + 65, + cfg.latent_frames, + cfg.latent_height, + cfg.latent_width, + ): + raise ValueError(f"Unexpected target_model_input: {target_model_input.shape}") + + with torch.no_grad(): + current_img = self.img_in(target_model_input) + if current_img.shape != anchor_hidden.shape: + raise ValueError(f"Current/anchor mismatch: {current_img.shape} != {anchor_hidden.shape}") + + if current_txt.shape[0] != batch: + current_txt = current_txt.expand(batch, -1, -1) + if cached_txt.shape[0] != batch: + cached_txt = cached_txt.expand(batch, -1, -1) + if vec_txt.shape[0] != batch: + vec_txt = vec_txt.expand(batch, -1) + + img = self.img_fusion(current_img, anchor_hidden) + txt = self.txt_fusion(current_txt, cached_txt) + spatial_tokens = cfg.latent_height * cfg.latent_width + if target_frame_condition.shape != (batch, cfg.latent_frames, cfg.hidden_size): + raise ValueError(f"Unexpected frame condition: {target_frame_condition.shape}") + condition_tokens = target_frame_condition.repeat_interleave(spatial_tokens, dim=1) + vec = condition_tokens.reshape(-1, cfg.hidden_size) + viewmats = repeat( + target_viewmats, + "B T M N -> B (T H W) M N", + H=cfg.latent_height, + W=cfg.latent_width, + ) + Ks = repeat( + target_Ks, + "B T M N -> B (T H W) M N", + H=cfg.latent_height, + W=cfg.latent_width, + ) + rope_size = int(rope_temporal_size.reshape(-1)[0].item()) if torch.is_tensor(rope_temporal_size) else int(rope_temporal_size) + rope_start = int(start_rope_start_idx.reshape(-1)[0].item()) if torch.is_tensor(start_rope_start_idx) else int(start_rope_start_idx) + freqs_cis = self._vision_rope( + rope_size, + rope_start, + device=img.device, + dtype=img.dtype, + ) + self.attn_param["thw"] = [cfg.latent_frames, cfg.latent_height, cfg.latent_width] + for block, source_id in zip(self.double_blocks, cfg.source_block_ids): + img, txt = self._run_block( + block, + source_id, + img, + txt, + vec, + vec_txt, + freqs_cis, + viewmats, + Ks, + ) + + delta_hidden = self.residual_out(img) + pred_hidden = anchor_hidden + delta_hidden + # Frozen parameters still allow the velocity loss to backpropagate to pred_hidden. + pred_tokens = self.final_layer(pred_hidden, vec) + pred_velocity = self.unpatchify(pred_tokens) + return { + "pred_hidden": pred_hidden, + "pred_velocity": pred_velocity, + "delta_hidden": delta_hidden, + "pred_txt": txt, + } + + def trainable_parameter_count(self) -> int: + return sum(parameter.numel() for parameter in self.parameters() if parameter.requires_grad) + + def trainable_parameter_breakdown(self) -> dict[str, int]: + groups = { + "img_fusion": self.img_fusion, + "txt_fusion": self.txt_fusion, + "double_blocks": self.double_blocks, + "residual_out": self.residual_out, + } + return { + name: sum(parameter.numel() for parameter in module.parameters() if parameter.requires_grad) + for name, module in groups.items() + } diff --git a/models/predictor_atc.py b/models/predictor_atc.py new file mode 100644 index 0000000000000000000000000000000000000000..f8b2ebdeee69db717531acb9e9310976b6f74126 --- /dev/null +++ b/models/predictor_atc.py @@ -0,0 +1,586 @@ +"""Anchor--Transport--Correct feature modules for Predictor-v4. + +Two Transport implementations are available: + +* ``global`` (default) -- the Causal-Forcing ``StateTransport``: RoPE'd global + attention from the current chunk's tokens to the previous chunk (or only its + last frame, ``previous_scope="last_frame"``), residual on the previous + features. Query and key positions come from the Teacher's 3-D RoPE so the + alignment is position-aware without learned biases. +* ``local`` -- the original 3x3 local spatiotemporal cross-attention with learned + temporal/spatial biases (kept for checkpoints trained before the switch). +""" + +from __future__ import annotations + +import math + +import torch +from torch import nn +from torch.utils.checkpoint import checkpoint + +from hyvideo.models.transformers.modules.embed_layers import TimestepEmbedder +from hyvideo.models.transformers.modules.posemb_layers import apply_rotary_emb + +try: # flash-attn supports different query/key lengths; the repo's attention() does not. + from flash_attn import flash_attn_func as _flash_attn_func +except Exception: # pragma: no cover - depends on the environment + _flash_attn_func = None + + +class MultiInputMLP(nn.Module): + """A concat-equivalent MLP without materializing a very wide token tensor.""" + + def __init__(self, input_count: int, hidden_size: int, mlp_hidden_size: int) -> None: + super().__init__() + self.inputs = nn.ModuleList( + nn.Linear(hidden_size, mlp_hidden_size) for _ in range(input_count) + ) + self.activation = nn.SiLU() + self.out = nn.Linear(mlp_hidden_size, hidden_size) + + def forward(self, *values: torch.Tensor) -> torch.Tensor: + if len(values) != len(self.inputs): + raise ValueError(f"Expected {len(self.inputs)} MLP inputs, got {len(values)}") + shape = values[0].shape + if any(value.shape != shape for value in values): + raise ValueError(f"Multi-input MLP shape mismatch: {[value.shape for value in values]}") + hidden = self.inputs[0](values[0]) + for projection, value in zip(self.inputs[1:], values[1:]): + hidden = hidden + projection(value) + return self.out(self.activation(hidden)) + + +class ATCTokenGate(nn.Module): + """Predict a scalar reliability gate for each token.""" + + def __init__( + self, + input_count: int, + hidden_size: int, + gate_hidden_size: int, + initial_probability: float, + ) -> None: + super().__init__() + if not 0.0 < initial_probability < 1.0: + raise ValueError("ATC gate initial probability must be in (0, 1)") + self.inputs = nn.ModuleList( + nn.Linear(hidden_size, gate_hidden_size) for _ in range(input_count) + ) + self.activation = nn.SiLU() + self.out = nn.Linear(gate_hidden_size, 1) + nn.init.normal_(self.out.weight, mean=0.0, std=1e-3) + nn.init.constant_( + self.out.bias, + math.log(initial_probability / (1.0 - initial_probability)), + ) + + def forward(self, *values: torch.Tensor) -> torch.Tensor: + if len(values) != len(self.inputs): + raise ValueError(f"Expected {len(self.inputs)} gate inputs, got {len(values)}") + hidden = self.inputs[0](values[0]) + for projection, value in zip(self.inputs[1:], values[1:]): + hidden = hidden + projection(value) + return torch.sigmoid(self.out(self.activation(hidden))) + + +class LocalCrossChunkTransport(nn.Module): + """Full-width local spatiotemporal cross-attention over the previous chunk.""" + + def __init__( + self, + hidden_size: int, + heads: int, + frames: int, + height: int, + width: int, + window: int, + query_chunk_size: int, + initial_scale: float, + ) -> None: + super().__init__() + if hidden_size % heads: + raise ValueError("ATC transport hidden size must be divisible by heads") + if window <= 0 or window % 2 != 1: + raise ValueError("ATC transport window must be a positive odd number") + if query_chunk_size <= 0: + raise ValueError("ATC transport query chunk size must be positive") + self.hidden_size = hidden_size + self.heads = heads + self.head_dim = hidden_size // heads + self.frames = frames + self.height = height + self.width = width + self.window = window + self.query_chunk_size = query_chunk_size + self.q_proj = nn.Linear(hidden_size, hidden_size) + self.k_proj = nn.Linear(hidden_size, hidden_size) + self.v_proj = nn.Linear(hidden_size, hidden_size) + self.out_proj = nn.Linear(hidden_size, hidden_size) + self.residual_scale = nn.Parameter(torch.tensor(float(initial_scale))) + self.temporal_bias = nn.Parameter(torch.zeros(heads, frames, frames)) + self.spatial_bias = nn.Parameter(torch.zeros(heads, window * window)) + + radius = window // 2 + indices: list[list[int]] = [] + valid: list[list[bool]] = [] + for y in range(height): + for x in range(width): + token_indices: list[int] = [] + token_valid: list[bool] = [] + for dy in range(-radius, radius + 1): + for dx in range(-radius, radius + 1): + yy, xx = y + dy, x + dx + is_valid = 0 <= yy < height and 0 <= xx < width + token_indices.append(yy * width + xx if is_valid else 0) + token_valid.append(is_valid) + indices.append(token_indices) + valid.append(token_valid) + self.register_buffer( + "neighbor_indices", torch.tensor(indices, dtype=torch.long), persistent=False + ) + self.register_buffer( + "neighbor_valid", torch.tensor(valid, dtype=torch.bool), persistent=False + ) + self.latest_attention_entropy = torch.tensor(0.0) + + def _attention(self, current: torch.Tensor, previous: torch.Tensor): + batch, tokens, channels = current.shape + spatial_tokens = self.height * self.width + expected = self.frames * spatial_tokens + if tokens != expected or channels != self.hidden_size: + raise ValueError( + f"ATC transport expected [B,{expected},{self.hidden_size}], got {current.shape}" + ) + q = self.q_proj(current).reshape( + batch, self.frames, spatial_tokens, self.heads, self.head_dim + ) + k = self.k_proj(previous).reshape( + batch, self.frames, spatial_tokens, self.heads, self.head_dim + ) + v = self.v_proj(previous).reshape( + batch, self.frames, spatial_tokens, self.heads, self.head_dim + ) + scale = self.head_dim**-0.5 + outputs: list[torch.Tensor] = [] + entropy_sum = current.new_zeros((), dtype=torch.float32) + window_tokens = self.window * self.window + bias = ( + self.temporal_bias[:, :, :, None] + + self.spatial_bias[:, None, None, :] + ).reshape(self.heads, self.frames, self.frames * window_tokens) + + for start in range(0, spatial_tokens, self.query_chunk_size): + end = min(spatial_tokens, start + self.query_chunk_size) + neighbor_indices = self.neighbor_indices[start:end] + neighbor_valid = self.neighbor_valid[start:end] + query = q[:, :, start:end].permute(0, 2, 3, 1, 4) + key = k[:, :, neighbor_indices].permute(0, 2, 4, 1, 3, 5) + value = v[:, :, neighbor_indices].permute(0, 2, 4, 1, 3, 5) + chunk_tokens = end - start + key = key.reshape( + batch, + chunk_tokens, + self.heads, + self.frames * window_tokens, + self.head_dim, + ) + value = value.reshape_as(key) + scores = torch.einsum("bnhqd,bnhkd->bnhqk", query, key) * scale + scores = scores + bias.to(dtype=scores.dtype)[None, None] + valid = ( + neighbor_valid[:, None] + .expand(chunk_tokens, self.frames, window_tokens) + .reshape(chunk_tokens, self.frames * window_tokens) + ) + scores = scores.masked_fill(~valid[None, :, None, None, :], -torch.inf) + attention = torch.softmax(scores.float(), dim=-1).to(dtype=value.dtype) + entropy = -( + attention.float() + * attention.float().clamp_min(1e-8).log() + ).sum(dim=-1).mean() + entropy_sum = entropy_sum + entropy * (chunk_tokens / spatial_tokens) + transported = torch.einsum("bnhqk,bnhkd->bnhqd", attention, value) + outputs.append(transported.permute(0, 3, 1, 2, 4)) + + output = torch.cat(outputs, dim=2).reshape(batch, tokens, channels) + return self.out_proj(output), entropy_sum + + def forward(self, current: torch.Tensor, previous: torch.Tensor) -> torch.Tensor: + if self.training and torch.is_grad_enabled(): + transported, entropy = checkpoint( + self._attention, current, previous, use_reentrant=False + ) + else: + transported, entropy = self._attention(current, previous) + self.latest_attention_entropy = entropy.detach() + scale = self.residual_scale.to(dtype=transported.dtype) + return previous + scale * transported + + +def _cross_attention( + q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, attn_mode: str +) -> torch.Tensor: + """[B,L,H,D] x [B,S,H,D] -> [B,L,H,D]; flash-attn when possible, else SDPA.""" + use_flash = ( + attn_mode == "flash" + and _flash_attn_func is not None + and q.is_cuda + and q.dtype in (torch.float16, torch.bfloat16) + ) + if use_flash: + return _flash_attn_func(q, k, v) + out = torch.nn.functional.scaled_dot_product_attention( + q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2) + ) + return out.transpose(1, 2) + + +class GlobalStateTransport(nn.Module): + """RoPE'd global cross-chunk attention (Causal-Forcing ``StateTransport``). + + ``forward(current, previous, query_freqs, key_freqs)`` aligns the previous + chunk's same-step hidden to the current tokens. ``query_freqs`` are the + (cos, sin) RoPE tables of the current chunk, ``key_freqs`` those of the + previous chunk (``previous_scope="chunk"``) or of its last frame only + (``previous_scope="last_frame"``), both as [S, D] or batched [B, S, D]. + """ + + VALID_SCOPES = ("chunk", "last_frame") + + def __init__( + self, + hidden_size: int, + heads: int, + frames: int, + height: int, + width: int, + previous_scope: str, + initial_scale: float, + attn_mode: str = "flash", + diagnostic_queries: int = 128, + collect_diagnostics: bool = True, + ) -> None: + super().__init__() + if hidden_size % heads: + raise ValueError("ATC transport hidden size must be divisible by heads") + if previous_scope not in self.VALID_SCOPES: + raise ValueError(f"Unknown ATC previous_scope {previous_scope!r}") + self.hidden_size = hidden_size + self.heads = heads + self.head_dim = hidden_size // heads + self.frames = frames + self.height = height + self.width = width + self.previous_scope = previous_scope + self.attn_mode = attn_mode + self.diagnostic_queries = int(diagnostic_queries) + self.collect_diagnostics = bool(collect_diagnostics) + self.q_proj = nn.Linear(hidden_size, hidden_size) + self.k_proj = nn.Linear(hidden_size, hidden_size) + self.v_proj = nn.Linear(hidden_size, hidden_size) + self.out_proj = nn.Linear(hidden_size, hidden_size) + self.residual_scale = nn.Parameter(torch.tensor(float(initial_scale))) + self.latest_attention_entropy = torch.tensor(0.0) + self.latest_diagnostics: dict[str, torch.Tensor] = {} + + def key_frames(self) -> int: + return self.frames if self.previous_scope == "chunk" else 1 + + def _scope_inputs(self, previous: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + """(keys/values source, residual base) for the configured scope.""" + batch, tokens, channels = previous.shape + spatial = self.height * self.width + if tokens != self.frames * spatial or channels != self.hidden_size: + raise ValueError( + f"ATC transport expected [B,{self.frames * spatial},{self.hidden_size}], " + f"got {previous.shape}" + ) + if self.previous_scope == "chunk": + return previous, previous + last = previous.reshape(batch, self.frames, spatial, channels)[:, -1] + base = last[:, None].expand(batch, self.frames, spatial, channels).reshape( + batch, tokens, channels + ) + return last, base + + @staticmethod + def _rope(x: torch.Tensor, freqs) -> torch.Tensor: + # apply_rotary_emb rotates a (query, key) pair of equal length; feed the + # same tensor twice to rotate a single stream. + return apply_rotary_emb(x, x, freqs)[0] + + def _attention(self, current, source, query_freqs, key_freqs): + batch, length, _ = current.shape + q = self.q_proj(current).reshape(batch, length, self.heads, self.head_dim) + k = self.k_proj(source).reshape(batch, source.shape[1], self.heads, self.head_dim) + v = self.v_proj(source).reshape(batch, source.shape[1], self.heads, self.head_dim) + q = self._rope(q, query_freqs) + k = self._rope(k, key_freqs) + out = _cross_attention(q, k, v, self.attn_mode) + return self.out_proj(out.reshape(batch, length, self.hidden_size)) + + @torch.no_grad() + def _probe(self, current, source, query_freqs, key_freqs) -> dict[str, torch.Tensor]: + """Entropy and argmax spatial displacement on a few sampled queries.""" + spatial = self.height * self.width + count = min(self.diagnostic_queries, current.shape[1]) + index = torch.linspace(0, current.shape[1] - 1, count, device=current.device).round().long() + first = lambda f: (f[0][:1], f[1][:1]) if f[0].ndim == 3 else f # batched -> sample 0 + q = self._rope( + self.q_proj(current[:1]).reshape(1, current.shape[1], self.heads, self.head_dim), + first(query_freqs), + )[:, index] + k = self._rope( + self.k_proj(source[:1]).reshape(1, source.shape[1], self.heads, self.head_dim), + first(key_freqs), + ) + scores = torch.einsum("bqhd,bkhd->bhqk", q.float(), k.float()) / math.sqrt(self.head_dim) + prob = scores.softmax(dim=-1) + entropy = -(prob * prob.clamp_min(1e-12).log()).sum(-1).mean() + selected = prob.mean(dim=1).argmax(dim=-1).squeeze(0) + qy, qx = torch.div(index.remainder(spatial), self.width, rounding_mode="floor"), index.remainder(self.width) + ky, kx = torch.div(selected.remainder(spatial), self.width, rounding_mode="floor"), selected.remainder(self.width) + displacement = torch.sqrt((qy - ky).float().square() + (qx - kx).float().square()).mean() + return { + "transport_attention_entropy": entropy.detach(), + "transport_attention_entropy_normalized": (entropy / math.log(source.shape[1])).detach(), + "transport_argmax_displacement": displacement.detach(), + } + + def forward( + self, + current: torch.Tensor, + previous: torch.Tensor, + query_freqs, + key_freqs, + ) -> torch.Tensor: + if query_freqs is None or key_freqs is None: + raise ValueError("GlobalStateTransport requires query_freqs and key_freqs") + source, base = self._scope_inputs(previous) + if self.training and torch.is_grad_enabled(): + transported = checkpoint( + self._attention, current, source, query_freqs, key_freqs, use_reentrant=False + ) + else: + transported = self._attention(current, source, query_freqs, key_freqs) + aligned = base + self.residual_scale.to(dtype=transported.dtype) * transported + self.latest_diagnostics = {} + if self.collect_diagnostics: + self.latest_diagnostics = self._probe(current, source, query_freqs, key_freqs) + self.latest_attention_entropy = self.latest_diagnostics["transport_attention_entropy"] + return aligned + + +class ATCFeatureFusion(nn.Module): + """Explicit Anchor Evolution, State Transport, and Causal Correction.""" + + VALID_VARIANTS = {"full", "anchor_only", "raw_prev", "no_gate", "no_distance"} + VALID_TRANSPORT_MODES = {"global", "local"} + is_atc = True + + def __init__( + self, + *, + hidden_size: int, + mlp_hidden_size: int, + gate_hidden_size: int, + gate_initial_probability: float, + transport_heads: int, + transport_window: int, + transport_query_chunk_size: int, + transport_initial_scale: float, + frames: int, + height: int, + width: int, + variant: str, + transport_mode: str = "global", + previous_scope: str = "chunk", + attn_mode: str = "flash", + collect_diagnostics: bool = True, + ) -> None: + super().__init__() + if variant not in self.VALID_VARIANTS: + raise ValueError( + f"Unknown ATC variant {variant!r}; expected one of {sorted(self.VALID_VARIANTS)}" + ) + if transport_mode not in self.VALID_TRANSPORT_MODES: + raise ValueError(f"Unknown ATC transport_mode {transport_mode!r}") + self.variant = variant + self.transport_mode = transport_mode + self.previous_scope = previous_scope + self.collect_diagnostics = bool(collect_diagnostics) + self.current_norm = nn.LayerNorm(hidden_size, eps=1e-6) + self.anchor_norm = nn.LayerNorm(hidden_size, eps=1e-6) + self.previous_norm = nn.LayerNorm(hidden_size, eps=1e-6) + self.condition_norm = nn.LayerNorm(hidden_size, eps=1e-6) + self.proj_e = nn.Linear(hidden_size, hidden_size) + self.proj_anchor = nn.Linear(hidden_size, hidden_size) + self.proj_prev = nn.Linear(hidden_size, hidden_size) + for projection in (self.proj_e, self.proj_anchor, self.proj_prev): + nn.init.eye_(projection.weight) + nn.init.zeros_(projection.bias) + self.proj_norm_e = nn.LayerNorm(hidden_size, eps=1e-6, elementwise_affine=False) + self.proj_norm_anchor = nn.LayerNorm( + hidden_size, eps=1e-6, elementwise_affine=False + ) + self.proj_norm_prev = nn.LayerNorm( + hidden_size, eps=1e-6, elementwise_affine=False + ) + self.distance_embed = TimestepEmbedder( + mlp_hidden_size, nn.SiLU, out_size=hidden_size + ) + self.distance_norm = nn.LayerNorm( + hidden_size, eps=1e-6, elementwise_affine=False + ) + self.anchor_evolution = MultiInputMLP(3, hidden_size, mlp_hidden_size) + if transport_mode == "local": + self.transport = LocalCrossChunkTransport( + hidden_size, + transport_heads, + frames, + height, + width, + transport_window, + transport_query_chunk_size, + transport_initial_scale, + ) + else: + self.transport = GlobalStateTransport( + hidden_size, + transport_heads, + frames, + height, + width, + previous_scope, + transport_initial_scale, + attn_mode=attn_mode, + collect_diagnostics=collect_diagnostics, + ) + self.causal_correction = MultiInputMLP(5, hidden_size, mlp_hidden_size) + nn.init.zeros_(self.causal_correction.out.weight) + nn.init.zeros_(self.causal_correction.out.bias) + self.gate_head = ATCTokenGate( + 5, + hidden_size, + gate_hidden_size, + gate_initial_probability, + ) + self.latest_gate_stats: dict[str, torch.Tensor] = {} + self.latest_atc_stats: dict[str, torch.Tensor] = {} + + @staticmethod + def _rms(value: torch.Tensor) -> torch.Tensor: + return value.detach().float().square().mean().sqrt() + + def prepare_features( + self, + current: torch.Tensor, + anchor: torch.Tensor, + previous: torch.Tensor | None, + condition_tokens: torch.Tensor, + anchor_distance: torch.Tensor | None, + query_freqs=None, + key_freqs=None, + ) -> dict[str, torch.Tensor]: + if previous is None: + raise ValueError("ATC requires previous-chunk same-timestep hidden") + if current.shape != anchor.shape or current.shape != previous.shape: + raise ValueError( + "ATC source shapes must match: " + f"current={current.shape}, anchor={anchor.shape}, previous={previous.shape}" + ) + batch, tokens, _ = current.shape + if condition_tokens.shape != current.shape: + raise ValueError("ATC condition tokens must match image-token shape") + if anchor_distance is None: + raise ValueError("ATC requires explicit anchor_distance") + distance_value = anchor_distance.reshape(-1) + if distance_value.numel() == 1 and batch != 1: + distance_value = distance_value.expand(batch) + if distance_value.numel() != batch: + raise ValueError( + f"ATC anchor_distance has {distance_value.numel()} values for batch {batch}" + ) + if self.variant == "no_distance": + distance_value = distance_value * 0.0 + + e = self.proj_norm_e(self.proj_e(self.current_norm(current))) + a = self.proj_norm_anchor(self.proj_anchor(self.anchor_norm(anchor))) + p = self.proj_norm_prev(self.proj_prev(self.previous_norm(previous))) + condition = self.condition_norm(condition_tokens) + distance = self.distance_norm(self.distance_embed(distance_value)).unsqueeze(1) + distance = distance.expand(batch, tokens, -1) + backbone_input = self.anchor_evolution(e, a, distance) + + if self.transport_mode == "local": + transported_candidate = self.transport(e, p) + else: + transported_candidate = self.transport(e, p, query_freqs, key_freqs) + if self.variant in {"raw_prev", "anchor_only"}: + transported = p + 0.0 * (transported_candidate - p) + else: + transported = transported_candidate + correction = self.causal_correction(e, a, transported, condition, distance) + predicted_gate = self.gate_head(e, a, transported, condition, distance) + if self.variant == "no_gate": + gate = 1.0 + 0.0 * predicted_gate + else: + gate = predicted_gate + if self.variant == "anchor_only": + correction = correction * 0.0 + + flat_gate = gate.detach().float().reshape(-1) + quantiles = torch.quantile( + flat_gate, + torch.tensor((0.1, 0.5, 0.9), device=gate.device, dtype=torch.float32), + ) + self.latest_gate_stats = { + "gate_p_mean": flat_gate.mean(), + "gate_p_std": flat_gate.std(unbiased=False), + "gate_p_p10": quantiles[0], + "gate_p_p50": quantiles[1], + "gate_p_p90": quantiles[2], + "gate_p_near_zero": (flat_gate < 0.05).float().mean(), + "gate_p_near_one": (flat_gate > 0.95).float().mean(), + } + return { + "backbone_input": backbone_input, + "correction": correction, + "gate": gate, + "transported_prev": transported, + "projected_prev": p, + "anchor_distance": distance_value, + } + + def finalize( + self, + anchor_hidden: torch.Tensor, + delta_anchor: torch.Tensor, + prepared: dict[str, torch.Tensor], + ) -> dict[str, torch.Tensor | dict[str, torch.Tensor]]: + correction = prepared["correction"] + gate = prepared["gate"] + pred_hidden = anchor_hidden + delta_anchor + gate * correction + transport_residual = prepared["transported_prev"] - prepared["projected_prev"] + self.latest_atc_stats = { + "delta_h_d_norm": self._rms(delta_anchor), + "delta_h_c_norm": self._rms(correction), + "gated_delta_h_c_norm": self._rms(gate * correction), + "transport_residual_norm": self._rms(transport_residual), + "anchor_distance_mean": prepared["anchor_distance"].detach().float().mean(), + "transport_attention_entropy": self.transport.latest_attention_entropy.float(), + } + extra = getattr(self.transport, "latest_diagnostics", None) + if extra: + self.latest_atc_stats.update( + {key: value.float() for key, value in extra.items() + if key != "transport_attention_entropy"} + ) + return { + "pred_hidden": pred_hidden, + "delta_correction": correction, + "gate": gate, + "transported_prev": prepared["transported_prev"], + "atc_stats": self.latest_atc_stats, + } diff --git a/models/predictor_disca.py b/models/predictor_disca.py new file mode 100644 index 0000000000000000000000000000000000000000..a909e0fbabe94b157e7d40f110179b6ddc58ec5c --- /dev/null +++ b/models/predictor_disca.py @@ -0,0 +1,80 @@ +"""Predictor-DisCa: current/anchor fusion without AR Context Vision K/V.""" + +from __future__ import annotations + +from typing import Mapping + +import torch + +from .predictor_v2 import HYWorldPlayPredictorV2, PredictorV2Config + + +class HYWorldPlayPredictorDisCa(HYWorldPlayPredictorV2): + """Two-block Predictor using only current/anchor image features and text K/V. + + This is the project-local ``predictor-disca`` variant. Unlike Predictor-v4, + it neither fuses previous-chunk hidden states nor injects Context Vision K/V. + The cached text K/V remains available because HY-WorldPlay AR vision blocks + require text conditioning. + """ + + uses_context_kv = True + uses_context_vision_kv = False + uses_context_prefeature = False + requires_previous_chunk_hidden = False + + def _text_only_cache( + self, + values: Mapping[int, Mapping[str, torch.Tensor]], + ) -> dict[int, dict[str, torch.Tensor | None]]: + result: dict[int, dict[str, torch.Tensor | None]] = {} + for block_id in self.predictor_config.source_block_ids: + if block_id not in values: + raise ValueError(f"Missing Predictor-DisCa text block {block_id}") + block = values[block_id] + missing = {"k_txt", "v_txt"}.difference(block) + if missing: + raise ValueError( + f"Predictor-DisCa block {block_id} lacks {sorted(missing)}" + ) + result[block_id] = { + "k_txt": block["k_txt"], + "v_txt": block["v_txt"], + "text_valid_mask": block.get("text_valid_mask"), + # Deliberately discard online Context Vision K/V and construct + # zero-length vision caches in HYWorldPlayPredictorV2._make_cache. + "k_vision": None, + "v_vision": None, + } + return result + + def forward( + self, + *, + target_model_input: torch.Tensor, + anchor_hidden: torch.Tensor, + target_frame_condition: torch.Tensor, + target_viewmats: torch.Tensor, + target_Ks: torch.Tensor, + rope_temporal_size: torch.Tensor | int, + start_rope_start_idx: torch.Tensor | int, + text_kv: Mapping[int, Mapping[str, torch.Tensor]] | None = None, + context_kv: Mapping[int, Mapping[str, torch.Tensor]] | None = None, + previous_chunk_hidden: torch.Tensor | None = None, + ) -> dict[str, torch.Tensor]: + if previous_chunk_hidden is not None: + raise ValueError("Predictor-DisCa does not accept previous_chunk_hidden") + source = text_kv if text_kv is not None else context_kv + if source is None: + raise ValueError("Predictor-DisCa requires text_kv or online context_kv") + return super().forward( + target_model_input=target_model_input, + anchor_hidden=anchor_hidden, + target_frame_condition=target_frame_condition, + target_viewmats=target_viewmats, + target_Ks=target_Ks, + rope_temporal_size=rope_temporal_size, + start_rope_start_idx=start_rope_start_idx, + context_kv=self._text_only_cache(source), + previous_chunk_hidden=None, + ) diff --git a/models/predictor_v2.py b/models/predictor_v2.py new file mode 100644 index 0000000000000000000000000000000000000000..fbb039b139c09a2df94d1416e8d8b4fa524c9001 --- /dev/null +++ b/models/predictor_v2.py @@ -0,0 +1,599 @@ +"""AR-context Predictor-v2 for HY-WorldPlay skipped denoising steps.""" + +from __future__ import annotations + +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any, Mapping + +import torch +from einops import repeat +from safetensors import safe_open +from torch import nn +from torch.utils.checkpoint import checkpoint + +from hyvideo.models.transformers.modules.activation_layers import get_activation_layer +from hyvideo.models.transformers.modules.embed_layers import PatchEmbed +from hyvideo.models.transformers.modules.mlp_layers import FinalLayer +from hyvideo.models.transformers.modules.posemb_layers import get_nd_rotary_pos_embed +from hyvideo.models.transformers.worldplay_1_5_transformer import MMDoubleStreamBlock + +from .predictor import FeatureFusion + + +@dataclass(frozen=True) +class PredictorV2Config: + hidden_size: int = 2048 + heads_num: int = 16 + mlp_width_ratio: float = 4.0 + mlp_act_type: str = "gelu_tanh" + qkv_bias: bool = True + qk_norm: bool = True + qk_norm_type: str = "rms" + attn_mode: str = "flash" + patch_size: tuple[int, int, int] = (1, 1, 1) + in_channels: int = 32 + out_channels: int = 32 + concat_condition: bool = True + rope_dim_list: tuple[int, int, int] = (16, 56, 56) + rope_theta: float = 256.0 + source_block_ids: tuple[int, int] = (1, 52) + teacher_block_count: int = 54 + latent_height: int = 30 + latent_width: int = 52 + latent_frames: int = 4 + previous_condition_mode: str = "same_step" + predictor_fusion_type: str = "concat" + gate_hidden_dim: int = 256 + gate_initial_probability: float = 0.8 + predictor_arch: str = "baseline" + atc_variant: str = "full" + atc_mlp_hidden_dim: int = 4096 + atc_gate_hidden_dim: int = 512 + atc_gate_initial_probability: float = 0.3 + # Transport heads default to the Teacher's head count so head_dim equals + # sum(rope_dim_list) and the Teacher RoPE can be reused (global mode). + atc_transport_heads: int | None = None + atc_transport_window: int = 3 + atc_transport_query_chunk_size: int = 128 + atc_transport_initial_scale: float = 0.1 + # "global": Causal-Forcing RoPE'd global cross-chunk attention (default); + # "local": the original 3x3 local window transport (legacy checkpoints). + atc_transport_mode: str = "global" + # Keys of the global transport: the whole previous chunk or its last frame. + atc_previous_scope: str = "chunk" + + +class HYWorldPlayPredictorV2(nn.Module): + """Two Teacher-initialized AR vision blocks reading exact selected-layer memory.""" + + uses_context_kv = True + + def __init__(self, config: PredictorV2Config | None = None) -> None: + super().__init__() + self.predictor_config = config or PredictorV2Config() + cfg = self.predictor_config + if len(cfg.source_block_ids) != 2 or len(set(cfg.source_block_ids)) != 2: + raise ValueError("Predictor v2 requires exactly two distinct source blocks") + if any(block_id < 0 or block_id >= cfg.teacher_block_count for block_id in cfg.source_block_ids): + raise ValueError(f"Invalid source blocks: {cfg.source_block_ids}") + + self.img_in = PatchEmbed( + list(cfg.patch_size), + cfg.in_channels, + cfg.hidden_size, + is_reshape_temporal_channels=False, + concat_condition=cfg.concat_condition, + ) + self.img_fusion = FeatureFusion(cfg.hidden_size) + self.double_blocks = nn.ModuleList( + [ + MMDoubleStreamBlock( + cfg.hidden_size, + cfg.heads_num, + mlp_width_ratio=cfg.mlp_width_ratio, + mlp_act_type=cfg.mlp_act_type, + attn_mode=cfg.attn_mode, + qk_norm=cfg.qk_norm, + qk_norm_type=cfg.qk_norm_type, + qkv_bias=cfg.qkv_bias, + ) + for _ in cfg.source_block_ids + ] + ) + for block in self.double_blocks: + block.img_attn_prope_proj = nn.Linear( + cfg.hidden_size, cfg.hidden_size, bias=cfg.qkv_bias + ) + # AR vision inference consumes cached text K/V and never executes the + # text stream. Keep those Teacher weights frozen and out of checkpoints. + for name, parameter in block.named_parameters(): + if name.startswith("txt_"): + parameter.requires_grad_(False) + + self.residual_out = nn.Linear(cfg.hidden_size, cfg.hidden_size) + self.final_layer = FinalLayer( + cfg.hidden_size, + list(cfg.patch_size), + cfg.out_channels, + get_activation_layer("silu"), + ) + nn.init.zeros_(self.residual_out.weight) + nn.init.zeros_(self.residual_out.bias) + self.gradient_checkpointing = False + self.attn_param: dict[str, Any] = { + "thw": [cfg.latent_frames, cfg.latent_height, cfg.latent_width], + "win_type": "fixed", + "win_ratio": 0, + "enable_sparse": False, + "enable_decode_sparse": False, + "enable_prefill_sparse": False, + "append_vision_cache": False, + } + self.img_in.requires_grad_(False) + self.final_layer.requires_grad_(False) + + @property + def config_dict(self) -> dict[str, Any]: + return asdict(self.predictor_config) + + def enable_gradient_checkpointing(self, enabled: bool = True) -> None: + self.gradient_checkpointing = enabled + + def train(self, mode: bool = True): + super().train(mode) + self.img_in.eval() + self.final_layer.eval() + return self + + @staticmethod + def _load_prefixed_module(module: nn.Module, handle, prefix: str) -> None: + target = module.state_dict() + loaded = {} + keys = set(handle.keys()) + for name in target: + key = prefix + name + if key not in keys: + raise KeyError(f"Missing Teacher checkpoint key: {key}") + loaded[name] = handle.get_tensor(key) + result = module.load_state_dict(loaded, strict=True) + if result.missing_keys or result.unexpected_keys: + raise RuntimeError(f"Unexpected Teacher load result for {prefix}: {result}") + + def load_teacher_initialization(self, checkpoint_path: str | Path) -> None: + path = str(Path(checkpoint_path).resolve()) + with safe_open(path, framework="pt", device="cpu") as handle: + self._load_prefixed_module(self.img_in, handle, "img_in.") + self._load_prefixed_module(self.final_layer, handle, "final_layer.") + for predictor_block, teacher_id in zip( + self.double_blocks, self.predictor_config.source_block_ids + ): + self._load_prefixed_module( + predictor_block, handle, f"double_blocks.{teacher_id}." + ) + self.img_in.requires_grad_(False).eval() + self.final_layer.requires_grad_(False).eval() + for block in self.double_blocks: + for name, parameter in block.named_parameters(): + if name.startswith("txt_"): + parameter.requires_grad_(False) + nn.init.zeros_(self.residual_out.weight) + nn.init.zeros_(self.residual_out.bias) + + def _vision_rope_single( + self, + rope_temporal_size: int, + start_rope_start_idx: int, + *, + device: torch.device, + dtype: torch.dtype, + ) -> tuple[torch.Tensor, torch.Tensor]: + cfg = self.predictor_config + cos, sin = get_nd_rotary_pos_embed( + list(cfg.rope_dim_list), + (rope_temporal_size, cfg.latent_height, cfg.latent_width), + theta=cfg.rope_theta, + use_real=True, + theta_rescale_factor=1, + ) + tokens_per_frame = cfg.latent_height * cfg.latent_width + start = start_rope_start_idx * tokens_per_frame + end = (start_rope_start_idx + cfg.latent_frames) * tokens_per_frame + cos = cos[start:end].to(device=device, dtype=dtype) + sin = sin[start:end].to(device=device, dtype=dtype) + expected = cfg.latent_frames * tokens_per_frame + if cos.shape[0] != expected: + raise ValueError(f"RoPE tokens {cos.shape[0]} != {expected}") + return cos, sin + + def _vision_rope_span( + self, + rope_temporal_size: int, + start_frame: int, + frames: int, + *, + device: torch.device, + dtype: torch.dtype, + ) -> tuple[torch.Tensor, torch.Tensor]: + """RoPE tables for ``frames`` latent frames starting at ``start_frame``.""" + cfg = self.predictor_config + if start_frame < 0 or start_frame + frames > rope_temporal_size: + raise ValueError( + f"RoPE span [{start_frame}, {start_frame + frames}) outside " + f"temporal size {rope_temporal_size}" + ) + cos, sin = get_nd_rotary_pos_embed( + list(cfg.rope_dim_list), + (rope_temporal_size, cfg.latent_height, cfg.latent_width), + theta=cfg.rope_theta, + use_real=True, + theta_rescale_factor=1, + ) + tokens_per_frame = cfg.latent_height * cfg.latent_width + start = start_frame * tokens_per_frame + end = (start_frame + frames) * tokens_per_frame + return cos[start:end].to(device=device, dtype=dtype), sin[start:end].to(device=device, dtype=dtype) + + def _atc_key_rope_batch( + self, + rope_temporal_size: torch.Tensor | int, + start_rope_start_idx: torch.Tensor | int, + *, + batch: int, + device: torch.device, + dtype: torch.dtype, + ) -> tuple[torch.Tensor, torch.Tensor]: + """Batched RoPE tables for the ATC transport keys (previous chunk or its last frame).""" + cfg = self.predictor_config + frames = cfg.latent_frames if cfg.atc_previous_scope == "chunk" else 1 + def values(value, name): + result = ([int(v) for v in value.detach().reshape(-1).cpu().tolist()] + if torch.is_tensor(value) else [int(value)]) + if len(result) == 1: + result *= batch + if len(result) != batch: + raise ValueError(f"{name} contains {len(result)} values for batch {batch}") + return result + sizes = values(rope_temporal_size, "rope_temporal_size") + starts = values(start_rope_start_idx, "start_rope_start_idx") + pairs = [] + for size, start in zip(sizes, starts): + key_start = start - frames + if key_start < 0: + raise ValueError( + "ATC transport needs a real previous chunk; " + f"start_rope_start_idx={start} < key frames {frames}" + ) + pairs.append(self._vision_rope_span(size, key_start, frames, device=device, dtype=dtype)) + return (torch.stack([p[0] for p in pairs]), torch.stack([p[1] for p in pairs])) + + def _vision_rope_batch( + self, + rope_temporal_size: torch.Tensor | int, + start_rope_start_idx: torch.Tensor | int, + *, + batch: int, + device: torch.device, + dtype: torch.dtype, + ) -> tuple[torch.Tensor, torch.Tensor]: + def values(value: torch.Tensor | int, name: str) -> list[int]: + if torch.is_tensor(value): + result = [int(item) for item in value.detach().reshape(-1).cpu().tolist()] + else: + result = [int(value)] + if len(result) == 1: + result *= batch + if len(result) != batch: + raise ValueError(f"{name} contains {len(result)} values for batch {batch}") + return result + + sizes = values(rope_temporal_size, "rope_temporal_size") + starts = values(start_rope_start_idx, "start_rope_start_idx") + pairs = [ + self._vision_rope_single(size, start, device=device, dtype=dtype) + for size, start in zip(sizes, starts) + ] + return ( + torch.stack([pair[0] for pair in pairs], dim=0), + torch.stack([pair[1] for pair in pairs], dim=0), + ) + + def _make_cache( + self, + context_kv: Mapping[int, Mapping[str, torch.Tensor]], + ) -> list[dict[str, torch.Tensor] | None]: + cfg = self.predictor_config + result: list[dict[str, torch.Tensor] | None] = [None] * cfg.teacher_block_count + for block_id in cfg.source_block_ids: + if block_id not in context_kv: + raise ValueError(f"Missing Predictor context block {block_id}") + cache = context_kv[block_id] + missing = {"k_txt", "v_txt", "k_vision", "v_vision"}.difference(cache) + if missing: + raise ValueError(f"Block {block_id} missing context tensors: {sorted(missing)}") + if cache["k_txt"].shape != cache["v_txt"].shape: + raise ValueError(f"Block {block_id} text K/V mismatch") + k_vision = cache["k_vision"] + v_vision = cache["v_vision"] + if (k_vision is None) != (v_vision is None): + raise ValueError(f"Block {block_id} has incomplete vision KV") + if k_vision is None: + k_txt = cache["k_txt"] + empty_shape = ( + 2 * int(k_txt.shape[0]), + int(k_txt.shape[1]), + 0, + int(k_txt.shape[3]), + ) + k_vision = k_txt.new_empty(empty_shape) + v_vision = cache["v_txt"].new_empty(empty_shape) + if k_vision.shape != v_vision.shape: + raise ValueError(f"Block {block_id} vision K/V mismatch") + batch = int(cache["k_txt"].shape[0]) + text_valid_mask = cache.get("text_valid_mask") + if text_valid_mask is None: + text_valid_mask = torch.ones( + batch, + cache["k_txt"].shape[2], + dtype=torch.bool, + device=cache["k_txt"].device, + ) + context_valid_mask = cache.get("context_valid_mask") + if context_valid_mask is None: + context_valid_mask = torch.ones( + batch, + k_vision.shape[2], + dtype=torch.bool, + device=k_vision.device, + ) + if tuple(text_valid_mask.shape) != (batch, cache["k_txt"].shape[2]): + raise ValueError(f"Block {block_id} text mask/KV mismatch") + if tuple(context_valid_mask.shape) != (batch, k_vision.shape[2]): + raise ValueError(f"Block {block_id} context mask/KV mismatch") + if k_vision.shape[0] != 2 * batch: + raise ValueError( + f"Block {block_id} vision KV batch {k_vision.shape[0]} != {2 * batch}" + ) + result[block_id] = { + "k_txt": cache["k_txt"], + "v_txt": cache["v_txt"], + "k_vision": k_vision, + "v_vision": v_vision, + "text_valid_mask": text_valid_mask.bool(), + "context_valid_mask": context_valid_mask.bool(), + } + return result + + def _run_block( + self, + block: MMDoubleStreamBlock, + source_block_id: int, + img: torch.Tensor, + vec: torch.Tensor, + freqs_cis: tuple[torch.Tensor, torch.Tensor], + viewmats: torch.Tensor, + Ks: torch.Tensor, + kv_cache: list[dict[str, torch.Tensor] | None], + ) -> torch.Tensor: + def block_forward(img_arg: torch.Tensor) -> torch.Tensor: + output, _ = block( + bi_inference=False, + ar_txt_inference=False, + ar_vision_inference=True, + img=img_arg, + vec=vec, + freqs_cis=freqs_cis, + attn_param=self.attn_param, + block_idx=source_block_id, + viewmats=viewmats, + Ks=Ks, + kv_cache=kv_cache, + cache_vision=False, + ) + return output + + if self.gradient_checkpointing and self.training: + return checkpoint(block_forward, img, use_reentrant=False) + return block_forward(img) + + def unpatchify(self, x: torch.Tensor) -> torch.Tensor: + cfg = self.predictor_config + batch = x.shape[0] + expected = cfg.latent_frames * cfg.latent_height * cfg.latent_width + if x.shape[1] != expected: + raise ValueError(f"Output tokens {x.shape[1]} != {expected}") + x = x.reshape( + batch, + cfg.latent_frames, + cfg.latent_height, + cfg.latent_width, + cfg.out_channels, + *cfg.patch_size, + ) + x = torch.einsum("nthwcopq->nctohpwq", x) + return x.reshape( + batch, + cfg.out_channels, + cfg.latent_frames * cfg.patch_size[0], + cfg.latent_height * cfg.patch_size[1], + cfg.latent_width * cfg.patch_size[2], + ) + + def hidden_to_velocity( + self, + hidden: torch.Tensor, + frame_condition: torch.Tensor, + ) -> torch.Tensor: + cfg = self.predictor_config + spatial_tokens = cfg.latent_height * cfg.latent_width + condition_tokens = frame_condition.repeat_interleave(spatial_tokens, dim=1) + vec = condition_tokens.reshape(-1, cfg.hidden_size) + return self.unpatchify(self.final_layer(hidden, vec)) + + def fuse_features( + self, + current_img: torch.Tensor, + anchor_hidden: torch.Tensor, + previous_chunk_hidden: torch.Tensor | None = None, + previous_step_chunk_hidden: torch.Tensor | None = None, + condition_tokens: torch.Tensor | None = None, + ) -> torch.Tensor: + if previous_chunk_hidden is not None or previous_step_chunk_hidden is not None: + raise ValueError("Predictor v2 does not consume previous-chunk hidden states") + return self.img_fusion(current_img, anchor_hidden) + + def forward( + self, + *, + target_model_input: torch.Tensor, + anchor_hidden: torch.Tensor, + target_frame_condition: torch.Tensor, + target_viewmats: torch.Tensor, + target_Ks: torch.Tensor, + rope_temporal_size: torch.Tensor | int, + start_rope_start_idx: torch.Tensor | int, + context_kv: Mapping[int, Mapping[str, torch.Tensor]], + previous_chunk_hidden: torch.Tensor | None = None, + previous_step_chunk_hidden: torch.Tensor | None = None, + anchor_distance: torch.Tensor | None = None, + return_debug: bool = False, + ) -> dict[str, torch.Tensor]: + cfg = self.predictor_config + batch = target_model_input.shape[0] + expected_input = ( + 65, + cfg.latent_frames, + cfg.latent_height, + cfg.latent_width, + ) + if tuple(target_model_input.shape[1:]) != expected_input: + raise ValueError(f"Unexpected target_model_input: {target_model_input.shape}") + + # ``img_in`` is frozen, but its input must remain differentiable for + # recursive rollout training: later-step losses backpropagate through + # the scheduler sample into earlier Predictor calls. With ordinary + # offline inputs ``requires_grad`` is false, so this creates no extra + # graph there. + current_img = self.img_in(target_model_input) + if current_img.shape != anchor_hidden.shape: + raise ValueError(f"Current/anchor mismatch: {current_img.shape} != {anchor_hidden.shape}") + if tuple(target_frame_condition.shape) != ( + batch, + cfg.latent_frames, + cfg.hidden_size, + ): + raise ValueError(f"Unexpected frame condition: {target_frame_condition.shape}") + + spatial_tokens = cfg.latent_height * cfg.latent_width + condition_tokens = target_frame_condition.repeat_interleave( + spatial_tokens, dim=1 + ) + freqs_cis = self._vision_rope_batch( + rope_temporal_size, + start_rope_start_idx, + batch=batch, + device=current_img.device, + dtype=current_img.dtype, + ) + atc_prepared = None + if getattr(self.img_fusion, "is_atc", False): + key_freqs = None + if getattr(self.img_fusion, "transport_mode", "local") == "global": + key_freqs = self._atc_key_rope_batch( + rope_temporal_size, + start_rope_start_idx, + batch=batch, + device=current_img.device, + dtype=current_img.dtype, + ) + atc_prepared = self.img_fusion.prepare_features( + current_img, + anchor_hidden, + previous_chunk_hidden, + condition_tokens, + anchor_distance, + query_freqs=freqs_cis, + key_freqs=key_freqs, + ) + img = atc_prepared["backbone_input"] + else: + img = self.fuse_features( + current_img, + anchor_hidden, + previous_chunk_hidden=previous_chunk_hidden, + previous_step_chunk_hidden=previous_step_chunk_hidden, + condition_tokens=condition_tokens, + ) + vec = condition_tokens.reshape(-1, cfg.hidden_size) + viewmats = repeat( + target_viewmats, + "B T M N -> B (T H W) M N", + H=cfg.latent_height, + W=cfg.latent_width, + ) + Ks = repeat( + target_Ks, + "B T M N -> B (T H W) M N", + H=cfg.latent_height, + W=cfg.latent_width, + ) + kv_cache = self._make_cache(context_kv) + for source_id in cfg.source_block_ids: + if kv_cache[source_id]["k_txt"].shape[0] != batch: + raise ValueError(f"Block {source_id} context batch does not match input batch") + self.attn_param["thw"] = [cfg.latent_frames, cfg.latent_height, cfg.latent_width] + for block, source_id in zip(self.double_blocks, cfg.source_block_ids): + img = self._run_block( + block, + source_id, + img, + vec, + freqs_cis, + viewmats, + Ks, + kv_cache, + ) + + delta_hidden = self.residual_out(img) + atc_result = None + if atc_prepared is None: + pred_hidden = anchor_hidden + delta_hidden + else: + atc_result = self.img_fusion.finalize( + anchor_hidden, delta_hidden, atc_prepared + ) + pred_hidden = atc_result["pred_hidden"] + pred_tokens = self.final_layer(pred_hidden, vec) + pred_velocity = self.unpatchify(pred_tokens) + result = { + "pred_hidden": pred_hidden, + "pred_velocity": pred_velocity, + "delta_hidden": delta_hidden, + } + gate_stats = getattr(self.img_fusion, "latest_gate_stats", None) + if gate_stats: + result["gate_stats"] = gate_stats + if atc_result is not None: + result["atc_stats"] = atc_result["atc_stats"] + if return_debug: + result.update( + delta_anchor=delta_hidden, + delta_correction=atc_result["delta_correction"], + gate=atc_result["gate"], + transported_prev=atc_result["transported_prev"], + ) + return result + + def trainable_parameter_count(self) -> int: + return sum(parameter.numel() for parameter in self.parameters() if parameter.requires_grad) + + def trainable_parameter_breakdown(self) -> dict[str, int]: + groups = { + "img_fusion": self.img_fusion, + "ar_vision_blocks": self.double_blocks, + "residual_out": self.residual_out, + } + return { + name: sum(parameter.numel() for parameter in module.parameters() if parameter.requires_grad) + for name, module in groups.items() + } diff --git a/models/predictor_v3.py b/models/predictor_v3.py new file mode 100644 index 0000000000000000000000000000000000000000..e4660015fc2289ec03227e4b155f774a3d6ce828 --- /dev/null +++ b/models/predictor_v3.py @@ -0,0 +1,335 @@ +"""Predictor-v3 with configurable previous-chunk feature fusion.""" + +from __future__ import annotations + +import math + +import torch +from torch import nn + +from .predictor_v2 import HYWorldPlayPredictorV2, PredictorV2Config +from .predictor_atc import ATCFeatureFusion + + +class SourceConditionedTokenGate(nn.Module): + """Predict one scalar gate per token without materializing a 3C concat.""" + + def __init__( + self, + hidden_size: int, + gate_hidden_dim: int, + initial_probability: float, + ) -> None: + super().__init__() + if gate_hidden_dim <= 0: + raise ValueError("gate_hidden_dim must be positive") + if not 0.0 < initial_probability < 1.0: + raise ValueError("gate_initial_probability must be in (0, 1)") + self.current_in = nn.Linear(hidden_size, gate_hidden_dim) + self.source_in = nn.Linear(hidden_size, gate_hidden_dim) + self.condition_in = nn.Linear(hidden_size, gate_hidden_dim) + self.activation = nn.SiLU() + self.out = nn.Linear(gate_hidden_dim, 1) + nn.init.normal_(self.out.weight, mean=0.0, std=1e-3) + nn.init.constant_( + self.out.bias, + math.log(initial_probability / (1.0 - initial_probability)), + ) + + def forward( + self, + current: torch.Tensor, + source: torch.Tensor, + condition: torch.Tensor, + ) -> torch.Tensor: + if current.shape != source.shape or current.shape != condition.shape: + raise ValueError( + "Token gate inputs must have identical [B, N, C] shapes; " + f"current={current.shape}, source={source.shape}, " + f"condition={condition.shape}" + ) + hidden = ( + self.current_in(current) + + self.source_in(source) + + self.condition_in(condition) + ) + return torch.sigmoid(self.out(self.activation(hidden))) + + +class TripleFeatureFusion(nn.Module): + """Fuse current/anchor tokens with configurable previous-chunk states.""" + + VALID_MODES = { + "none", "same_step", "previous_step", "both", "both_delta_gated" + } + VALID_FUSION_TYPES = { + "concat", + "projection_add", + "token_gate_prev", + "concat_residual_gate_prev", + } + + def __init__( + self, + hidden_size: int, + previous_condition_mode: str = "same_step", + predictor_fusion_type: str = "concat", + gate_hidden_dim: int = 256, + gate_initial_probability: float = 0.8, + ) -> None: + super().__init__() + mode = str(previous_condition_mode).lower() + if mode not in self.VALID_MODES: + raise ValueError( + f"Unknown previous_condition_mode {mode!r}; " + f"expected one of {sorted(self.VALID_MODES)}" + ) + fusion_type = str(predictor_fusion_type).lower() + if fusion_type not in self.VALID_FUSION_TYPES: + raise ValueError( + f"Unknown predictor_fusion_type {fusion_type!r}; " + f"expected one of {sorted(self.VALID_FUSION_TYPES)}" + ) + if fusion_type != "concat" and mode != "same_step": + raise ValueError( + f"predictor_fusion_type={fusion_type!r} currently requires " + "previous_condition_mode='same_step'" + ) + self.previous_condition_mode = mode + self.predictor_fusion_type = fusion_type + self.latest_gate_stats: dict[str, torch.Tensor] = {} + self.current_norm = nn.LayerNorm(hidden_size, eps=1e-6) + self.anchor_norm = nn.LayerNorm(hidden_size, eps=1e-6) + if mode != "none": + self.previous_norm = nn.LayerNorm(hidden_size, eps=1e-6) + if mode in {"both", "both_delta_gated"}: + self.previous_step_norm = nn.LayerNorm(hidden_size, eps=1e-6) + input_count = { + "none": 2, + "same_step": 3, + "previous_step": 3, + "both": 4, + # Keep the proven same-step fusion as the base. The extra + # previous-step condition enters only through a zero-initialized + # gated residual below, so initialization can be exactly + # equivalent to a trained same-step checkpoint. + "both_delta_gated": 3, + }[mode] + if fusion_type in {"concat", "concat_residual_gate_prev"}: + self.mlp = nn.Sequential( + nn.Linear(input_count * hidden_size, 2 * hidden_size), + nn.SiLU(), + nn.Linear(2 * hidden_size, hidden_size), + ) + if fusion_type == "concat_residual_gate_prev": + self.gate_p = SourceConditionedTokenGate( + hidden_size, + gate_hidden_dim, + gate_initial_probability, + ) + self.residual_mlp = nn.Sequential( + nn.Linear(hidden_size, hidden_size), + nn.SiLU(), + nn.Linear(hidden_size, hidden_size), + ) + nn.init.zeros_(self.residual_mlp[-1].weight) + nn.init.zeros_(self.residual_mlp[-1].bias) + else: + self.proj_e = nn.Linear(hidden_size, hidden_size) + self.proj_a = nn.Linear(hidden_size, hidden_size) + self.proj_p = nn.Linear(hidden_size, hidden_size) + for projection in (self.proj_e, self.proj_a, self.proj_p): + nn.init.eye_(projection.weight) + nn.init.zeros_(projection.bias) + self.proj_norm_e = nn.LayerNorm( + hidden_size, eps=1e-6, elementwise_affine=False + ) + self.proj_norm_a = nn.LayerNorm( + hidden_size, eps=1e-6, elementwise_affine=False + ) + self.proj_norm_p = nn.LayerNorm( + hidden_size, eps=1e-6, elementwise_affine=False + ) + self.fusion_norm = nn.LayerNorm(hidden_size, eps=1e-6) + if fusion_type == "token_gate_prev": + self.gate_p = SourceConditionedTokenGate( + hidden_size, + gate_hidden_dim, + gate_initial_probability, + ) + if fusion_type == "concat" and mode == "both_delta_gated": + self.delta_mlp = nn.Sequential( + nn.Linear(hidden_size, hidden_size), + nn.SiLU(), + nn.Linear(hidden_size, hidden_size), + ) + self.delta_gate = nn.Parameter(torch.zeros(hidden_size)) + nn.init.zeros_(self.delta_mlp[-1].weight) + nn.init.zeros_(self.delta_mlp[-1].bias) + + def forward( + self, + current: torch.Tensor, + anchor: torch.Tensor, + previous_same_step: torch.Tensor | None = None, + previous_previous_step: torch.Tensor | None = None, + condition_tokens: torch.Tensor | None = None, + ) -> torch.Tensor: + if current.shape != anchor.shape: + raise ValueError( + "Fusion current/anchor shape mismatch: " + f"current={current.shape}, anchor={anchor.shape}" + ) + features = [self.current_norm(current), self.anchor_norm(anchor)] + mode = self.previous_condition_mode + if mode in {"same_step", "both", "both_delta_gated"}: + if previous_same_step is None or previous_same_step.shape != current.shape: + raise ValueError( + "same-step previous hidden must match current shape; got " + f"{None if previous_same_step is None else previous_same_step.shape} " + f"versus {current.shape}" + ) + features.append(self.previous_norm(previous_same_step)) + if mode in {"previous_step", "both", "both_delta_gated"}: + if ( + previous_previous_step is None + or previous_previous_step.shape != current.shape + ): + raise ValueError( + "previous-step previous hidden must match current shape; got " + f"{None if previous_previous_step is None else previous_previous_step.shape} " + f"versus {current.shape}" + ) + normalizer = ( + self.previous_step_norm + if mode in {"both", "both_delta_gated"} + else self.previous_norm + ) + normalized_previous_step = normalizer(previous_previous_step) + if mode in {"previous_step", "both"}: + features.append(normalized_previous_step) + if self.predictor_fusion_type in {"projection_add", "token_gate_prev"}: + self.latest_gate_stats = {} + z_e = self.proj_norm_e(self.proj_e(features[0])) + z_a = self.proj_norm_a(self.proj_a(features[1])) + z_p = self.proj_norm_p(self.proj_p(features[2])) + if self.predictor_fusion_type == "projection_add": + return self.fusion_norm(z_e + z_a + z_p) + if condition_tokens is None: + raise ValueError("token_gate_prev requires condition_tokens") + gate_p = self.gate_p(z_e, z_p, condition_tokens) + self._update_gate_stats(gate_p) + return self.fusion_norm(z_e + z_a + gate_p * z_p) + + self.latest_gate_stats = {} + base = self.mlp(torch.cat(features, dim=-1)) + if self.predictor_fusion_type == "concat_residual_gate_prev": + if condition_tokens is None: + raise ValueError( + "concat_residual_gate_prev requires condition_tokens" + ) + previous_anchor_delta = features[2] - features[1] + gate_p = self.gate_p( + features[0], previous_anchor_delta, condition_tokens + ) + self._update_gate_stats(gate_p) + return base + gate_p * self.residual_mlp(previous_anchor_delta) + if mode != "both_delta_gated": + return base + delta = normalized_previous_step - self.previous_norm(previous_same_step) + gate = torch.sigmoid(self.delta_gate).view(1, 1, -1) + return base + gate * self.delta_mlp(delta) + + def _update_gate_stats(self, gate_p: torch.Tensor) -> None: + flat_gate = gate_p.detach().float().reshape(-1) + quantiles = torch.quantile( + flat_gate, + torch.tensor( + (0.1, 0.5, 0.9), device=flat_gate.device, dtype=flat_gate.dtype + ), + ) + self.latest_gate_stats = { + "gate_p_mean": flat_gate.mean(), + "gate_p_std": flat_gate.std(unbiased=False), + "gate_p_p10": quantiles[0], + "gate_p_p50": quantiles[1], + "gate_p_p90": quantiles[2], + "gate_p_near_zero": (flat_gate < 0.05).float().mean(), + "gate_p_near_one": (flat_gate > 0.95).float().mean(), + } + + +class HYWorldPlayPredictorV3(HYWorldPlayPredictorV2): + """Fresh v3 Predictor with configurable previous-chunk conditions.""" + + requires_previous_chunk_hidden = True + + def __init__(self, config: PredictorV2Config | None = None) -> None: + super().__init__(config) + self.previous_condition_mode = self.predictor_config.previous_condition_mode + arch = self.predictor_config.predictor_arch.lower() + if arch not in {"baseline", "atc"}: + raise ValueError("predictor_arch must be 'baseline' or 'atc'") + self.predictor_arch = arch + if arch == "atc": + if self.previous_condition_mode != "same_step": + raise ValueError("ATC requires previous_condition_mode='same_step'") + self.requires_previous_chunk_hidden = True + cfg = self.predictor_config + transport_heads = cfg.atc_transport_heads or cfg.heads_num + if cfg.atc_transport_mode == "global": + head_dim = cfg.hidden_size // transport_heads + if cfg.hidden_size % transport_heads or head_dim != sum(cfg.rope_dim_list): + raise ValueError( + "Global ATC transport reuses the Teacher RoPE, so " + f"hidden_size/heads ({cfg.hidden_size}/{transport_heads}) must equal " + f"sum(rope_dim_list)={sum(cfg.rope_dim_list)}" + ) + self.img_fusion = ATCFeatureFusion( + hidden_size=self.predictor_config.hidden_size, + mlp_hidden_size=self.predictor_config.atc_mlp_hidden_dim, + gate_hidden_size=self.predictor_config.atc_gate_hidden_dim, + gate_initial_probability=( + self.predictor_config.atc_gate_initial_probability + ), + transport_heads=transport_heads, + transport_window=self.predictor_config.atc_transport_window, + transport_query_chunk_size=( + self.predictor_config.atc_transport_query_chunk_size + ), + transport_initial_scale=( + self.predictor_config.atc_transport_initial_scale + ), + frames=self.predictor_config.latent_frames, + height=self.predictor_config.latent_height, + width=self.predictor_config.latent_width, + variant=self.predictor_config.atc_variant, + transport_mode=self.predictor_config.atc_transport_mode, + previous_scope=self.predictor_config.atc_previous_scope, + attn_mode=self.predictor_config.attn_mode, + ) + else: + self.requires_previous_chunk_hidden = self.previous_condition_mode != "none" + self.img_fusion = TripleFeatureFusion( + self.predictor_config.hidden_size, + previous_condition_mode=self.previous_condition_mode, + predictor_fusion_type=self.predictor_config.predictor_fusion_type, + gate_hidden_dim=self.predictor_config.gate_hidden_dim, + gate_initial_probability=self.predictor_config.gate_initial_probability, + ) + + def fuse_features( + self, + current_img: torch.Tensor, + anchor_hidden: torch.Tensor, + previous_chunk_hidden: torch.Tensor | None = None, + previous_step_chunk_hidden: torch.Tensor | None = None, + condition_tokens: torch.Tensor | None = None, + ) -> torch.Tensor: + return self.img_fusion( + current_img, + anchor_hidden, + previous_same_step=previous_chunk_hidden, + previous_previous_step=previous_step_chunk_hidden, + condition_tokens=condition_tokens, + ) diff --git a/models/predictor_v4.py b/models/predictor_v4.py new file mode 100644 index 0000000000000000000000000000000000000000..c95ee481c384cf124acf0e44a49bbedb9db4de83 --- /dev/null +++ b/models/predictor_v4.py @@ -0,0 +1,126 @@ +"""Predictor-v4 with BF16 Context prefeature reconstruction.""" + +from __future__ import annotations + +from typing import Mapping + +import torch + +from .context_prefeature_projector import FrozenTeacherContextProjector +from .predictor_v2 import PredictorV2Config +from .predictor_v3 import HYWorldPlayPredictorV3 + + +class _TeacherProjectionAdapter: + """Expose Predictor source blocks with the Teacher transformer's indexing.""" + + def __init__(self, model: HYWorldPlayPredictorV3) -> None: + cfg = model.predictor_config + self.heads_num = cfg.heads_num + self.hidden_size = cfg.hidden_size + self.rope_dim_list = cfg.rope_dim_list + self.rope_theta = cfg.rope_theta + self.patch_size = cfg.patch_size + self.latent_height = cfg.latent_height + self.latent_width = cfg.latent_width + self.double_blocks = [None] * cfg.teacher_block_count + for source_id, block in zip(cfg.source_block_ids, model.double_blocks): + self.double_blocks[source_id] = block + + +class HYWorldPlayPredictorV4(HYWorldPlayPredictorV3): + """v3 previous-chunk fusion plus frozen Teacher Context K/V projectors.""" + + # The online rollout already owns the exact Teacher Context K/V cache, so it + # can pass that cache directly. Training continues to use the BF16 + # prefeature + frozen-projector path below. + uses_context_kv = True + uses_context_prefeature = True + + def __init__(self, config: PredictorV2Config | None = None) -> None: + super().__init__(config) + self.context_projector = FrozenTeacherContextProjector( + _TeacherProjectionAdapter(self), self.predictor_config.source_block_ids + ) + self.context_projector.eval().requires_grad_(False) + + def _sync_frozen_projectors(self) -> None: + """Copy the just-loaded Teacher source projections into frozen modules.""" + for source_id, block in zip( + self.predictor_config.source_block_ids, self.double_blocks + ): + key = str(source_id) + self.context_projector.k_proj[key].load_state_dict( + block.img_attn_k.state_dict(), strict=True + ) + self.context_projector.v_proj[key].load_state_dict( + block.img_attn_v.state_dict(), strict=True + ) + self.context_projector.k_norm[key].load_state_dict( + block.img_attn_k_norm.state_dict(), strict=True + ) + self.context_projector.eval().requires_grad_(False) + + def load_teacher_initialization(self, checkpoint_path) -> None: + super().load_teacher_initialization(checkpoint_path) + self._sync_frozen_projectors() + + def train(self, mode: bool = True): + super().train(mode) + self.context_projector.eval() + return self + + def forward( + self, + *, + target_model_input: torch.Tensor, + anchor_hidden: torch.Tensor, + target_frame_condition: torch.Tensor, + target_viewmats: torch.Tensor, + target_Ks: torch.Tensor, + rope_temporal_size: torch.Tensor | int, + start_rope_start_idx: torch.Tensor | int, + context_prefeature: Mapping[int, Mapping[str, torch.Tensor]] | None = None, + text_kv: Mapping[int, Mapping[str, torch.Tensor]] | None = None, + context_kv: Mapping[int, Mapping[str, torch.Tensor]] | None = None, + previous_chunk_hidden: torch.Tensor | None = None, + previous_step_chunk_hidden: torch.Tensor | None = None, + anchor_distance: torch.Tensor | None = None, + return_debug: bool = False, + ) -> dict[str, torch.Tensor]: + if context_kv is None: + if context_prefeature is None or text_kv is None: + raise ValueError( + "Predictor-v4 requires either direct context_kv or both " + "context_prefeature and text_kv" + ) + rebuilt_context_kv: dict[int, dict[str, torch.Tensor]] = {} + # These are exact, fixed Teacher transformations. Avoid constructing + # an autograd graph for several GiB of padded Context K/V. + with torch.no_grad(): + rebuilt = self.context_projector(dict(context_prefeature)) + for source_id in self.predictor_config.source_block_ids: + if source_id not in text_kv or source_id not in rebuilt: + raise ValueError(f"Missing Predictor-v4 Context block {source_id}") + rebuilt_context_kv[source_id] = { + **dict(text_kv[source_id]), + **rebuilt[source_id], + "context_valid_mask": context_prefeature[source_id][ + "context_valid_mask" + ], + } + context_kv = rebuilt_context_kv + return super().forward( + target_model_input=target_model_input, + anchor_hidden=anchor_hidden, + target_frame_condition=target_frame_condition, + target_viewmats=target_viewmats, + target_Ks=target_Ks, + rope_temporal_size=rope_temporal_size, + start_rope_start_idx=start_rope_start_idx, + context_kv=dict(context_kv), + previous_chunk_hidden=previous_chunk_hidden, + previous_step_chunk_hidden=previous_step_chunk_hidden, + anchor_distance=anchor_distance, + return_debug=return_debug, + ) diff --git a/outputs/c1_threshold_smoke/confidence_decisions/worker_0.jsonl b/outputs/c1_threshold_smoke/confidence_decisions/worker_0.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..1922073c93ab0d02965e70d27acb9b6d45e9d90e --- /dev/null +++ b/outputs/c1_threshold_smoke/confidence_decisions/worker_0.jsonl @@ -0,0 +1,21 @@ +{"action_id": 0, "action_name": "w_s", "c1": 0.006530642043799162, "case_id": 0, "chunk_id": 1, "denoise_step": 1, "fallback_to_full": false, "log_c1": -5.03125, "threshold_log_c1": -4.329442501, "timestep": 937.5} +{"action_id": 0, "action_name": "w_s", "c1": 0.00865169521421194, "case_id": 0, "chunk_id": 1, "denoise_step": 2, "fallback_to_full": false, "log_c1": -4.75, "threshold_log_c1": -4.329442501, "timestep": 833.3333129882812} +{"action_id": 0, "action_name": "w_s", "c1": 0.030672922730445862, "case_id": 0, "chunk_id": 1, "denoise_step": 3, "fallback_to_full": true, "log_c1": -3.484375, "threshold_log_c1": -4.329442501, "timestep": 625.0} +{"action_id": 0, "action_name": "w_s", "c1": 0.005946217104792595, "case_id": 0, "chunk_id": 2, "denoise_step": 1, "fallback_to_full": false, "log_c1": -5.125, "threshold_log_c1": -4.329442501, "timestep": 937.5} +{"action_id": 0, "action_name": "w_s", "c1": 0.008385511115193367, "case_id": 0, "chunk_id": 2, "denoise_step": 2, "fallback_to_full": false, "log_c1": -4.78125, "threshold_log_c1": -4.329442501, "timestep": 833.3333129882812} +{"action_id": 0, "action_name": "w_s", "c1": 0.03019738383591175, "case_id": 0, "chunk_id": 2, "denoise_step": 3, "fallback_to_full": true, "log_c1": -3.5, "threshold_log_c1": -4.329442501, "timestep": 625.0} +{"action_id": 0, "action_name": "w_s", "c1": 0.005414092447608709, "case_id": 0, "chunk_id": 3, "denoise_step": 1, "fallback_to_full": false, "log_c1": -5.21875, "threshold_log_c1": -4.329442501, "timestep": 937.5} +{"action_id": 0, "action_name": "w_s", "c1": 0.007635093759745359, "case_id": 0, "chunk_id": 3, "denoise_step": 2, "fallback_to_full": false, "log_c1": -4.875, "threshold_log_c1": -4.329442501, "timestep": 833.3333129882812} +{"action_id": 0, "action_name": "w_s", "c1": 0.028367817401885986, "case_id": 0, "chunk_id": 3, "denoise_step": 3, "fallback_to_full": true, "log_c1": -3.5625, "threshold_log_c1": -4.329442501, "timestep": 625.0} +{"action_id": 0, "action_name": "w_s", "c1": 0.005414092447608709, "case_id": 0, "chunk_id": 4, "denoise_step": 1, "fallback_to_full": false, "log_c1": -5.21875, "threshold_log_c1": -4.329442501, "timestep": 937.5} +{"action_id": 0, "action_name": "w_s", "c1": 0.007635093759745359, "case_id": 0, "chunk_id": 4, "denoise_step": 2, "fallback_to_full": false, "log_c1": -4.875, "threshold_log_c1": -4.329442501, "timestep": 833.3333129882812} +{"action_id": 0, "action_name": "w_s", "c1": 0.02792801521718502, "case_id": 0, "chunk_id": 4, "denoise_step": 3, "fallback_to_full": true, "log_c1": -3.578125, "threshold_log_c1": -4.329442501, "timestep": 625.0} +{"action_id": 0, "action_name": "w_s", "c1": 0.005946217104792595, "case_id": 0, "chunk_id": 5, "denoise_step": 1, "fallback_to_full": false, "log_c1": -5.125, "threshold_log_c1": -4.329442501, "timestep": 937.5} +{"action_id": 0, "action_name": "w_s", "c1": 0.008127516135573387, "case_id": 0, "chunk_id": 5, "denoise_step": 2, "fallback_to_full": false, "log_c1": -4.8125, "threshold_log_c1": -4.329442501, "timestep": 833.3333129882812} +{"action_id": 0, "action_name": "w_s", "c1": 0.029268307611346245, "case_id": 0, "chunk_id": 5, "denoise_step": 3, "fallback_to_full": true, "log_c1": -3.53125, "threshold_log_c1": -4.329442501, "timestep": 625.0} +{"action_id": 0, "action_name": "w_s", "c1": 0.005946217104792595, "case_id": 0, "chunk_id": 6, "denoise_step": 1, "fallback_to_full": false, "log_c1": -5.125, "threshold_log_c1": -4.329442501, "timestep": 937.5} +{"action_id": 0, "action_name": "w_s", "c1": 0.008385511115193367, "case_id": 0, "chunk_id": 6, "denoise_step": 2, "fallback_to_full": false, "log_c1": -4.78125, "threshold_log_c1": -4.329442501, "timestep": 833.3333129882812} +{"action_id": 0, "action_name": "w_s", "c1": 0.029729217290878296, "case_id": 0, "chunk_id": 6, "denoise_step": 3, "fallback_to_full": true, "log_c1": -3.515625, "threshold_log_c1": -4.329442501, "timestep": 625.0} +{"action_id": 0, "action_name": "w_s", "c1": 0.005585954524576664, "case_id": 0, "chunk_id": 7, "denoise_step": 1, "fallback_to_full": false, "log_c1": -5.1875, "threshold_log_c1": -4.329442501, "timestep": 937.5} +{"action_id": 0, "action_name": "w_s", "c1": 0.007877458818256855, "case_id": 0, "chunk_id": 7, "denoise_step": 2, "fallback_to_full": false, "log_c1": -4.84375, "threshold_log_c1": -4.329442501, "timestep": 833.3333129882812} +{"action_id": 0, "action_name": "w_s", "c1": 0.028814544901251793, "case_id": 0, "chunk_id": 7, "denoise_step": 3, "fallback_to_full": true, "log_c1": -3.546875, "threshold_log_c1": -4.329442501, "timestep": 625.0} diff --git a/outputs/c1_threshold_smoke/logs/worker_0.jsonl b/outputs/c1_threshold_smoke/logs/worker_0.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..54be9d65df686b89415007358268b634b8923ff7 --- /dev/null +++ b/outputs/c1_threshold_smoke/logs/worker_0.jsonl @@ -0,0 +1 @@ +{"action_id": 0, "action_name": "w_s", "case_id": 0, "confidence_candidates": 21, "confidence_checkpoint": "/mnt/local_nvme/cz/projects/HY-WorldPlay-DEV-Predictor/confidence_validation/checkpoints/confidence_c1.pt", "confidence_fallback_count": 7, "confidence_fallback_rate": 0.3333333333333333, "confidence_threshold_log_c1": -4.329442501, "elapsed_s": 93.39685152299353, "end_to_end_s": 93.39685152299353, "mode": "predictor", "peak_memory_gib": 71.97628927230835, "predictor_fusion_type": "concat", "predictor_variant": "v4", "previous_condition_mode": "same_step", "replace_steps": [1, 2, 3], "seed": 0, "source_blocks": [1, 52], "timing": {}, "video_encode_s": 2.2333505289861932, "video_path": "/mnt/local_nvme/cz/projects/HY-WorldPlay-DEV-Predictor/outputs/c1_threshold_smoke/predictor/case_0000_action_00.mp4", "video_validate_s": 0.16869094499270432, "worker_id": 0} diff --git a/outputs/c1_threshold_smoke/predictor/case_0000_action_00.mp4 b/outputs/c1_threshold_smoke/predictor/case_0000_action_00.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..58b39ade95dcda668a5026a7803e6e22b5fcf860 --- /dev/null +++ b/outputs/c1_threshold_smoke/predictor/case_0000_action_00.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:749c3e6b33ec73809d246f106e027e2694c64778a3b7e41dd7539969e383f567 +size 704000 diff --git a/outputs/hf_snapshot_repaired_9b49404/scheduler/scheduler_config.json b/outputs/hf_snapshot_repaired_9b49404/scheduler/scheduler_config.json new file mode 100644 index 0000000000000000000000000000000000000000..a0f8a3a40d306aede296e4f8dd19808e93cc0b3c --- /dev/null +++ b/outputs/hf_snapshot_repaired_9b49404/scheduler/scheduler_config.json @@ -0,0 +1,15 @@ +{ + "_class_name": "FlowMatchDiscreteScheduler", + "_diffusers_version": "0.35.0", + "flux_base_shift": 0.5, + "flux_base_token": 256.0, + "flux_max_shift": 1.15, + "flux_max_token": 4096.0, + "flux_shift_factor": 1.0, + "n_tokens": null, + "num_train_timesteps": 1000, + "reverse": true, + "shift": 7.0, + "solver": "euler", + "use_flux_shift": false +} diff --git a/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/.gitattributes b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/.gitattributes new file mode 100644 index 0000000000000000000000000000000000000000..a6344aac8c09253b3b630fb776ae94478aa0275b --- /dev/null +++ b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/.gitattributes @@ -0,0 +1,35 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text diff --git a/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/README.md b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/README.md new file mode 100644 index 0000000000000000000000000000000000000000..1a54f9550937c220d62bc35d0a38f4004c8f6159 --- /dev/null +++ b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/README.md @@ -0,0 +1,87 @@ +--- +language: +- en +library_name: glyph-byt5 +--- + +# Glyph-ByT5-v2: A Strong Aesthetic Baseline for Accurate Multilingual Visual Text Rendering + +We introduce **Glyph-ByT5-v2**, a customized text encoder for accurate **multilingual** visual text rendering and improved aesthetics. +As an extension of **Glyph-SDXL**, our multilingual version supports visual text rendering for up to 10 different languages: English, Chinese, Japanese, Korean, French, German, Spanish, Italian, Portuguese and Russian. +Combined with SDXL, our proposed **Glyph-SDXL-v2** achieves accurate multilingual design image visual text rendering. + + +> [**Glyph-ByT5-v2: A Strong Aesthetic Baseline for Accurate Multilingual Visual Text Rendering**](https://glyph-byt5-v2.github.io/) +> [Zeyu Liu](https://github.com/lzy-tony), [Weicong Liang](https://scholar.google.com/citations?user=QvHDIygAAAAJ&hl=zh-CN), [Yiming Zhao](https://scholar.google.com.hk/citations?user=_knPaYsAAAAJ&hl=zh-CN), [Bohan Chen](https://github.com/BHCHENGIT), [Ji Li](https://sites.google.com/a/usc.edu/jili/), [Yuhui Yuan](https://www.microsoft.com/en-us/research/people/yuyua/) +> Microsoft Research Asia; Tsinghua University; Peking University; University of Liverpool +> Preprint + +## Model Sources + + + +- **Repository:** [https://github.com/AIGText/Glyph-ByT5] +- **Paper:** [https://arxiv.org/abs/2406.10208] +- **Project Page:** [https://glyph-byt5-v2.github.io/] + + +## Model Description + +Please check our [paper](https://arxiv.org/abs/2406.10208) and [project page](https://glyph-byt5-v2.github.io/) for more details. Detail usage and inference code can be found [here](https://github.com/AIGText/Glyph-ByT5). + +## Visualization + + + + + + + + +
example 1example 2example 3example 4
+ +## Quick Usage + +``` +python inference_v2.py configs/glyph_sdxl_v2_albedo.py checkpoints examples/xiaoman.json --out_folder work_dirs/xiaoman --device cuda --sampler dpm +``` + +## More Configurations + +We list some more useful configurations for easy usage: + +| Argument/Config | Place | Default | Description | +| ----------------------------- | ---------- | ----------------------------------- | ------------------------------------------------------------ | +| cfg | argument | 5.0 | Classifier-free guidance | +| sampler | argument | dpm | Sampler, provide support for dpm (DPM++ 2M Karras) and euler (EulerDiscreteScheduler) | +| pretrained_model_name_or_path | config | stablediffusionapi/albedobase-xl-20 | Base model | +| seed | annotation | None | Seed for inference | + + +## Citation + +If you find our work useful in your research, please consider citing: + +``` +@misc{liu2024glyphbyt5v2, + title={Glyph-ByT5-v2: A Strong Aesthetic Baseline for Accurate Multilingual Visual Text Rendering}, + author={Zeyu Liu and Weicong Liang and Yiming Zhao and Bohan Chen and Ji Li and Yuhui Yuan}, + year={2024}, + eprint={2406.10208}, + archivePrefix={arXiv}, + primaryClass={cs.CV} +} +``` + +and + +``` +@misc{liu2024glyphbyt5, + title={Glyph-ByT5: A Customized Text Encoder for Accurate Visual Text Rendering}, + author={Zeyu Liu and Weicong Liang and Zhanhao Liang and Chong Luo and Ji Li and Gao Huang and Yuhui Yuan}, + year={2024}, + eprint={2403.09622}, + archivePrefix={arXiv}, + primaryClass={cs.CV} +} +``` \ No newline at end of file diff --git a/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/assets/Arial.ttf b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/assets/Arial.ttf new file mode 100644 index 0000000000000000000000000000000000000000..fa960f48d5545af1391c0fcb24f223e37f22863e --- /dev/null +++ b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/assets/Arial.ttf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35c0f3559d8db569e36c31095b8a60d441643d95f59139de40e23fada819b833 +size 275572 diff --git a/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/assets/chinese_char.txt b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/assets/chinese_char.txt new file mode 100644 index 0000000000000000000000000000000000000000..2506e2a28c78a58fd60905d170533bf1caf9103a --- /dev/null +++ b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/assets/chinese_char.txt @@ -0,0 +1,1000 @@ +的 +一 +是 +不 +了 +在 +人 +有 +我 +他 +这 +个 +们 +中 +来 +上 +大 +为 +和 +国 +地 +到 +以 +说 +时 +要 +就 +出 +会 +可 +也 +你 +对 +生 +能 +而 +子 +那 +得 +于 +着 +下 +自 +之 +年 +过 +发 +后 +作 +里 +用 +道 +行 +所 +然 +家 +种 +事 +成 +方 +多 +经 +么 +去 +法 +学 +如 +都 +同 +现 +当 +没 +动 +面 +起 +看 +定 +天 +分 +还 +进 +好 +小 +部 +其 +些 +主 +样 +理 +心 +她 +本 +前 +开 +但 +因 +只 +从 +想 +实 +日 +军 +者 +意 +无 +力 +它 +与 +长 +把 +机 +十 +民 +第 +公 +此 +已 +工 +使 +情 +明 +性 +知 +全 +三 +又 +关 +点 +正 +业 +外 +将 +两 +高 +间 +由 +问 +很 +最 +重 +并 +物 +手 +应 +战 +向 +头 +文 +体 +政 +美 +相 +见 +被 +利 +什 +二 +等 +产 +或 +新 +己 +制 +身 +果 +加 +西 +斯 +月 +话 +合 +回 +特 +代 +内 +信 +表 +化 +老 +给 +世 +位 +次 +度 +门 +任 +常 +先 +海 +通 +教 +儿 +原 +东 +声 +提 +立 +及 +比 +员 +解 +水 +名 +真 +论 +处 +走 +义 +各 +入 +几 +口 +认 +条 +平 +系 +气 +题 +活 +尔 +更 +别 +打 +女 +变 +四 +神 +总 +何 +电 +数 +安 +少 +报 +才 +结 +反 +受 +目 +太 +量 +再 +感 +建 +务 +做 +接 +必 +场 +件 +计 +管 +期 +市 +直 +德 +资 +命 +山 +金 +指 +克 +许 +统 +区 +保 +至 +队 +形 +社 +便 +空 +决 +治 +展 +马 +科 +司 +五 +基 +眼 +书 +非 +则 +听 +白 +却 +界 +达 +光 +放 +强 +即 +像 +难 +且 +权 +思 +王 +象 +完 +设 +式 +色 +路 +记 +南 +品 +住 +告 +类 +求 +据 +程 +北 +边 +死 +张 +该 +交 +规 +万 +取 +拉 +格 +望 +觉 +术 +领 +共 +确 +传 +师 +观 +清 +今 +切 +院 +让 +识 +候 +带 +导 +争 +运 +笑 +飞 +风 +步 +改 +收 +根 +干 +造 +言 +联 +持 +组 +每 +济 +车 +亲 +极 +林 +服 +快 +办 +议 +往 +元 +英 +士 +证 +近 +失 +转 +夫 +令 +准 +布 +始 +怎 +呢 +存 +未 +远 +叫 +台 +单 +影 +具 +罗 +字 +爱 +击 +流 +备 +兵 +连 +调 +深 +商 +算 +质 +团 +集 +百 +需 +价 +花 +党 +华 +城 +石 +级 +整 +府 +离 +况 +亚 +请 +技 +际 +约 +示 +复 +病 +息 +究 +线 +似 +官 +火 +断 +精 +满 +支 +视 +消 +越 +器 +容 +照 +须 +九 +增 +研 +写 +称 +企 +八 +功 +吗 +包 +片 +史 +委 +乎 +查 +轻 +易 +早 +曾 +除 +农 +找 +装 +广 +显 +吧 +阿 +李 +标 +谈 +吃 +图 +念 +六 +引 +历 +首 +医 +局 +突 +专 +费 +号 +尽 +另 +周 +较 +注 +语 +仅 +考 +落 +青 +随 +选 +列 +武 +红 +响 +虽 +推 +势 +参 +希 +古 +众 +构 +房 +半 +节 +土 +投 +某 +案 +黑 +维 +革 +划 +敌 +致 +陈 +律 +足 +态 +护 +七 +兴 +派 +孩 +验 +责 +营 +星 +够 +章 +音 +跟 +志 +底 +站 +严 +巴 +例 +防 +族 +供 +效 +续 +施 +留 +讲 +型 +料 +终 +答 +紧 +黄 +绝 +奇 +察 +母 +京 +段 +依 +批 +群 +项 +故 +按 +河 +米 +围 +江 +织 +害 +斗 +双 +境 +客 +纪 +采 +举 +杀 +攻 +父 +苏 +密 +低 +朝 +友 +诉 +止 +细 +愿 +千 +值 +仍 +男 +钱 +破 +网 +热 +助 +倒 +育 +属 +坐 +帝 +限 +船 +脸 +职 +速 +刻 +乐 +否 +刚 +威 +毛 +状 +率 +甚 +独 +球 +般 +普 +怕 +弹 +校 +苦 +创 +假 +久 +错 +承 +印 +晚 +兰 +试 +股 +拿 +脑 +预 +谁 +益 +阳 +若 +哪 +微 +尼 +继 +送 +急 +血 +惊 +伤 +素 +药 +适 +波 +夜 +省 +初 +喜 +卫 +源 +食 +险 +待 +述 +陆 +习 +置 +居 +劳 +财 +环 +排 +福 +纳 +欢 +雷 +警 +获 +模 +充 +负 +云 +停 +木 +游 +龙 +树 +疑 +层 +冷 +洲 +冲 +射 +略 +范 +竟 +句 +室 +异 +激 +汉 +村 +哈 +策 +演 +简 +卡 +罪 +判 +担 +州 +静 +退 +既 +衣 +您 +宗 +积 +余 +痛 +检 +差 +富 +灵 +协 +角 +占 +配 +征 +修 +皮 +挥 +胜 +降 +阶 +审 +沉 +坚 +善 +妈 +刘 +读 +啊 +超 +免 +压 +银 +买 +皇 +养 +伊 +怀 +执 +副 +乱 +抗 +犯 +追 +帮 +宣 +佛 +岁 +航 +优 +怪 +香 +著 +田 +铁 +控 +税 +左 +右 +份 +穿 +艺 +背 +阵 +草 +脚 +概 +恶 +块 +顿 +敢 +守 +酒 +岛 +托 +央 +户 +烈 +洋 +哥 +索 +胡 +款 +靠 +评 +版 +宝 +座 +释 +景 +顾 +弟 +登 +货 +互 +付 +伯 +慢 +欧 +换 +闻 +危 +忙 +核 +暗 +姐 +介 +坏 +讨 +丽 +良 +序 +升 +监 +临 +亮 +露 +永 +呼 +味 +野 +架 +域 +沙 +掉 +括 +舰 +鱼 +杂 +误 +湾 +吉 +减 +编 +楚 +肯 +测 +败 +屋 +跑 +梦 +散 +温 +困 +剑 +渐 +封 +救 +贵 +枪 +缺 +楼 +县 +尚 +毫 +移 +娘 +朋 +画 +班 +智 +亦 +耳 +恩 +短 +掌 +恐 +遗 +固 +席 +松 +秘 +谢 +鲁 +遇 +康 +虑 +幸 +均 +销 +钟 +诗 +藏 +赶 +剧 +票 +损 +忽 +巨 +炮 +旧 +端 +探 +湖 +录 +叶 +春 +乡 +附 +吸 +予 +礼 +港 +雨 +呀 +板 +庭 +妇 +归 +睛 +饭 +额 +含 +顺 +输 +摇 +招 +婚 +脱 +补 +谓 +督 +毒 +油 +疗 +旅 +泽 +材 +灭 +逐 +莫 +笔 +亡 +鲜 +词 +圣 +择 +寻 +厂 +睡 +博 +勒 +烟 +授 +诺 +伦 +岸 +奥 +唐 +卖 +俄 +炸 +载 +洛 +健 +堂 +旁 +宫 +喝 +借 +君 +禁 +阴 +园 +谋 +宋 +避 +抓 +荣 +姑 +孙 +逃 +牙 +束 +跳 +顶 \ No newline at end of file diff --git a/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/assets/color_idx.json b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/assets/color_idx.json new file mode 100644 index 0000000000000000000000000000000000000000..8181bd294a73465b691d125ad439c8d3d85ff9fc --- /dev/null +++ b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/assets/color_idx.json @@ -0,0 +1 @@ +{"white": 0, "black": 1, "darkslategray": 2, "dimgray": 3, "darkolivegreen": 4, "midnightblue": 5, "saddlebrown": 6, "sienna": 7, "whitesmoke": 8, "darkslateblue": 9, "indianred": 10, "linen": 11, "maroon": 12, "khaki": 13, "sandybrown": 14, "gray": 15, "gainsboro": 16, "teal": 17, "peru": 18, "gold": 19, "snow": 20, "firebrick": 21, "crimson": 22, "chocolate": 23, "tomato": 24, "brown": 25, "goldenrod": 26, "antiquewhite": 27, "rosybrown": 28, "steelblue": 29, "floralwhite": 30, "seashell": 31, "darkgreen": 32, "oldlace": 33, "darkkhaki": 34, "burlywood": 35, "red": 36, "darkgray": 37, "orange": 38, "royalblue": 39, "seagreen": 40, "lightgray": 41, "tan": 42, "coral": 43, "beige": 44, "palevioletred": 45, "wheat": 46, "lavender": 47, "darkcyan": 48, "slateblue": 49, "slategray": 50, "orangered": 51, "silver": 52, "olivedrab": 53, "forestgreen": 54, "darkgoldenrod": 55, "ivory": 56, "darkorange": 57, "yellow": 58, "hotpink": 59, "ghostwhite": 60, "lightcoral": 61, "indigo": 62, "bisque": 63, "darkred": 64, "darksalmon": 65, "lightslategray": 66, "dodgerblue": 67, "lightpink": 68, "mistyrose": 69, "mediumvioletred": 70, "cadetblue": 71, "deeppink": 72, "salmon": 73, "palegoldenrod": 74, "blanchedalmond": 75, "lightseagreen": 76, "cornflowerblue": 77, "yellowgreen": 78, "greenyellow": 79, "navajowhite": 80, "papayawhip": 81, "mediumslateblue": 82, "purple": 83, "blueviolet": 84, "pink": 85, "cornsilk": 86, "lightsalmon": 87, "mediumpurple": 88, "moccasin": 89, "turquoise": 90, "mediumseagreen": 91, "lavenderblush": 92, "mediumblue": 93, "darkseagreen": 94, "mediumturquoise": 95, "paleturquoise": 96, "skyblue": 97, "lemonchiffon": 98, "olive": 99, "peachpuff": 100, "lightyellow": 101, "lightsteelblue": 102, "mediumorchid": 103, "plum": 104, "darkturquoise": 105, "aliceblue": 106, "mediumaquamarine": 107, "orchid": 108, "powderblue": 109, "blue": 110, "darkorchid": 111, "violet": 112, "lightskyblue": 113, "lightcyan": 114, "lightgoldenrodyellow": 115, "navy": 116, "thistle": 117, "honeydew": 118, "mintcream": 119, "lightblue": 120, "darkblue": 121, "darkmagenta": 122, "deepskyblue": 123, "magenta": 124, "limegreen": 125, "darkviolet": 126, "cyan": 127, "palegreen": 128, "aquamarine": 129, "lawngreen": 130, "lightgreen": 131, "azure": 132, "chartreuse": 133, "green": 134, "mediumspringgreen": 135, "lime": 136, "springgreen": 137} \ No newline at end of file diff --git a/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/assets/font_idx_512.json b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/assets/font_idx_512.json new file mode 100644 index 0000000000000000000000000000000000000000..9b40ddd4b31d3235b5b5b9200121f43d0264282f --- /dev/null +++ b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/assets/font_idx_512.json @@ -0,0 +1 @@ +{"Montserrat-Regular": 0, "Poppins-Italic": 1, "GlacialIndifference-Regular": 2, "OpenSans-ExtraBoldItalic": 3, "Montserrat-Bold": 4, "Now-Regular": 5, "Garet-Regular": 6, "LeagueSpartan-Bold": 7, "DMSans-Regular": 8, "OpenSauceOne-Regular": 9, "OpenSans-ExtraBold": 10, "KGPrimaryPenmanship": 11, "Anton-Regular": 12, "Aileron-BlackItalic": 13, "Quicksand-Light": 14, "Roboto-BoldItalic": 15, "TheSeasons-It": 16, "Kollektif": 17, "Inter-BoldItalic": 18, "Poppins-Medium": 19, "Poppins-Light": 20, "RoxboroughCF-RegularItalic": 21, "PlayfairDisplay-SemiBold": 22, "Agrandir-Italic": 23, "Lato-Regular": 24, "MoreSugarRegular": 25, "CanvaSans-RegularItalic": 26, "PublicSans-Italic": 27, "CodePro-NormalLC": 28, "Belleza-Regular": 29, "JosefinSans-Bold": 30, "HKGrotesk-Bold": 31, "Telegraf-Medium": 32, "BrittanySignatureRegular": 33, "Raleway-ExtraBoldItalic": 34, "Mont-RegularItalic": 35, "Arimo-BoldItalic": 36, "Lora-Italic": 37, "ArchivoBlack-Regular": 38, "Poppins": 39, "Barlow-Black": 40, "CormorantGaramond-Bold": 41, "LibreBaskerville-Regular": 42, "CanvaSchoolFontRegular": 43, "BebasNeueBold": 44, "LazydogRegular": 45, "FredokaOne-Regular": 46, "Horizon-Bold": 47, "Nourd-Regular": 48, "Hatton-Regular": 49, "Nunito-ExtraBoldItalic": 50, "CerebriSans-Regular": 51, "Montserrat-Light": 52, "TenorSans": 53, "Norwester-Regular": 54, "ClearSans-Bold": 55, "Cardo-Regular": 56, "Alice-Regular": 57, "Oswald-Regular": 58, "Gaegu-Bold": 59, "Muli-Black": 60, "TAN-PEARL-Regular": 61, "CooperHewitt-Book": 62, "Agrandir-Grand": 63, "BlackMango-Thin": 64, "DMSerifDisplay-Regular": 65, "Antonio-Bold": 66, "Sniglet-Regular": 67, "BeVietnam-Regular": 68, "NunitoSans10pt-BlackItalic": 69, "AbhayaLibre-ExtraBold": 70, "Rubik-Regular": 71, "PPNeueMachina-Regular": 72, "TAN - MON CHERI-Regular": 73, "Jua-Regular": 74, "Playlist-Script": 75, "SourceSansPro-BoldItalic": 76, "MoonTime-Regular": 77, "Eczar-ExtraBold": 78, "Gatwick-Regular": 79, "MonumentExtended-Regular": 80, "BarlowSemiCondensed-Regular": 81, "BarlowCondensed-Regular": 82, "Alegreya-Regular": 83, "DreamAvenue": 84, "RobotoCondensed-Italic": 85, "BobbyJones-Regular": 86, "Garet-ExtraBold": 87, "YesevaOne-Regular": 88, "Dosis-ExtraBold": 89, "LeagueGothic-Regular": 90, "OpenSans-Italic": 91, "TANAEGEAN-Regular": 92, "Maharlika-Regular": 93, "MarykateRegular": 94, "Cinzel-Regular": 95, "Agrandir-Wide": 96, "Chewy-Regular": 97, "BodoniFLF-BoldItalic": 98, "Nunito-BlackItalic": 99, "LilitaOne": 100, "HandyCasualCondensed-Regular": 101, "Ovo": 102, "Livvic-Regular": 103, "Agrandir-Narrow": 104, "CrimsonPro-Italic": 105, "AnonymousPro-Bold": 106, "NF-OneLittleFont-Bold": 107, "RedHatDisplay-BoldItalic": 108, "CodecPro-Regular": 109, "HalimunRegular": 110, "LibreFranklin-Black": 111, "TeXGyreTermes-BoldItalic": 112, "Shrikhand-Regular": 113, "TTNormsPro-Italic": 114, "Gagalin-Regular": 115, "OpenSans-Bold": 116, "GreatVibes-Regular": 117, "Breathing": 118, "HeroLight-Regular": 119, "KGPrimaryDots": 120, "Quicksand-Bold": 121, "Brice-ExtraLightSemiExpanded": 122, "Lato-BoldItalic": 123, "Fraunces9pt-Italic": 124, "AbrilFatface-Regular": 125, "BerkshireSwash-Regular": 126, "Atma-Bold": 127, "HolidayRegular": 128, "BebasNeueCyrillic": 129, "IntroRust-Base": 130, "Gistesy": 131, "BDScript-Regular": 132, "ApricotsRegular": 133, "Prompt-Black": 134, "TAN MERINGUE": 135, "Sukar Regular": 136, "GentySans-Regular": 137, "NeueEinstellung-Normal": 138, "Garet-Bold": 139, "FiraSans-Black": 140, "BantayogLight": 141, "NotoSerifDisplay-Black": 142, "TTChocolates-Regular": 143, "Ubuntu-Regular": 144, "Assistant-Bold": 145, "ABeeZee-Regular": 146, "LexendDeca-Regular": 147, "KingredSerif": 148, "Radley-Regular": 149, "BrownSugar": 150, "MigraItalic-ExtraboldItalic": 151, "ChildosArabic-Regular": 152, "PeaceSans": 153, "LondrinaSolid-Black": 154, "SpaceMono-BoldItalic": 155, "RobotoMono-Light": 156, "CourierPrime-Regular": 157, "Alata-Regular": 158, "Amsterdam-One": 159, "IreneFlorentina-Regular": 160, "CatchyMager": 161, "Alta_regular": 162, "ArticulatCF-Regular": 163, "Raleway-Regular": 164, "BrasikaDisplay": 165, "TANAngleton-Italic": 166, "NotoSerifDisplay-ExtraCondensedItalic": 167, "Bryndan Write": 168, "TTCommonsPro-It": 169, "AlexBrush-Regular": 170, "Antic-Regular": 171, "TTHoves-Bold": 172, "DroidSerif": 173, "AblationRegular": 174, "Marcellus-Regular": 175, "Sanchez-Italic": 176, "JosefinSans": 177, "Afrah-Regular": 178, "PinyonScript": 179, "TTInterphases-BoldItalic": 180, "Yellowtail-Regular": 181, "Gliker-Regular": 182, "BobbyJonesSoft-Regular": 183, "IBMPlexSans": 184, "Amsterdam-Three": 185, "Amsterdam-FourSlant": 186, "TTFors-Regular": 187, "Quattrocento": 188, "Sifonn-Basic": 189, "AlegreyaSans-Black": 190, "Daydream": 191, "AristotelicaProTx-Rg": 192, "NotoSerif": 193, "EBGaramond-Italic": 194, "HammersmithOne-Regular": 195, "RobotoSlab-Regular": 196, "DO-Sans-Regular": 197, "KGPrimaryDotsLined": 198, "Blinker-Regular": 199, "TAN NIMBUS": 200, "Blueberry-Regular": 201, "Rosario-Regular": 202, "Forum": 203, "MistrullyRegular": 204, "SourceSerifPro-Regular": 205, "Bugaki-Regular": 206, "CMUSerif-Roman": 207, "GulfsDisplay-NormalItalic": 208, "PTSans-Bold": 209, "Sensei-Medium": 210, "SquadaOne-Regular": 211, "Arapey-Italic": 212, "Parisienne-Regular": 213, "Aleo-Italic": 214, "QuicheDisplay-Italic": 215, "RocaOne-It": 216, "Funtastic-Regular": 217, "PTSerif-BoldItalic": 218, "Muller-RegularItalic": 219, "ArgentCF-Regular": 220, "Brightwall-Italic": 221, "Knewave-Regular": 222, "TYSerif-D": 223, "Agrandir-Tight": 224, "AlfaSlabOne-Regular": 225, "TANTangkiwood-Display": 226, "Kief-Montaser-Regular": 227, "Gotham-Book": 228, "JuliusSansOne-Regular": 229, "CocoGothic-Italic": 230, "SairaCondensed-Regular": 231, "DellaRespira-Regular": 232, "Questrial-Regular": 233, "BukhariScript-Regular": 234, "HelveticaWorld-Bold": 235, "TANKINDRED-Display": 236, "CinzelDecorative-Regular": 237, "Vidaloka-Regular": 238, "AlegreyaSansSC-Black": 239, "FeelingPassionate-Regular": 240, "QuincyCF-Regular": 241, "FiraCode-Regular": 242, "Genty-Regular": 243, "Nickainley-Normal": 244, "RubikOne-Regular": 245, "Gidole-Regular": 246, "Borsok": 247, "Gordita-RegularItalic": 248, "Scripter-Regular": 249, "Buffalo-Regular": 250, "KleinText-Regular": 251, "Creepster-Regular": 252, "Arvo-Bold": 253, "GabrielSans-NormalItalic": 254, "Heebo-Black": 255, "LexendExa-Regular": 256, "BrixtonSansTC-Regular": 257, "GildaDisplay-Regular": 258, "ChunkFive-Roman": 259, "Amaranth-BoldItalic": 260, "BubbleboddyNeue-Regular": 261, "MavenPro-Bold": 262, "TTDrugs-Italic": 263, "CyGrotesk-KeyRegular": 264, "VarelaRound-Regular": 265, "Ruda-Black": 266, "SafiraMarch": 267, "BloggerSans": 268, "TANHEADLINE-Regular": 269, "SloopScriptPro-Regular": 270, "NeueMontreal-Regular": 271, "Schoolbell-Regular": 272, "SigherRegular": 273, "InriaSerif-Regular": 274, "JetBrainsMono-Regular": 275, "MADEEvolveSans": 276, "Dekko": 277, "Handyman-Regular": 278, "Aileron-BoldItalic": 279, "Bright-Italic": 280, "Solway-Regular": 281, "Higuen-Regular": 282, "WedgesItalic": 283, "TANASHFORD-BOLD": 284, "IBMPlexMono": 285, "RacingSansOne-Regular": 286, "RegularBrush": 287, "OpenSans-LightItalic": 288, "SpecialElite-Regular": 289, "FuturaLTPro-Medium": 290, "MaragsaDisplay": 291, "BigShouldersDisplay-Regular": 292, "BDSans-Regular": 293, "RasputinRegular": 294, "Yvesyvesdrawing-BoldItalic": 295, "Bitter-Regular": 296, "LuckiestGuy-Regular": 297, "CanvaSchoolFontDotted": 298, "TTFirsNeue-Italic": 299, "Sunday-Regular": 300, "HKGothic-MediumItalic": 301, "CaveatBrush-Regular": 302, "HeliosExt": 303, "ArchitectsDaughter-Regular": 304, "Angelina": 305, "Calistoga-Regular": 306, "ArchivoNarrow-Regular": 307, "ObjectSans-MediumSlanted": 308, "AyrLucidityCondensed-Regular": 309, "Nexa-RegularItalic": 310, "Lustria-Regular": 311, "Amsterdam-TwoSlant": 312, "Virtual-Regular": 313, "Brusher-Regular": 314, "NF-Lepetitcochon-Regular": 315, "TANTWINKLE": 316, "LeJour-Serif": 317, "Prata-Regular": 318, "PPWoodland-Regular": 319, "PlayfairDisplay-BoldItalic": 320, "AmaticSC-Regular": 321, "Cabin-Regular": 322, "Manjari-Bold": 323, "MrDafoe-Regular": 324, "TTRamillas-Italic": 325, "Luckybones-Bold": 326, "DarkerGrotesque-Light": 327, "BellabooRegular": 328, "CormorantSC-Bold": 329, "GochiHand-Regular": 330, "Atteron": 331, "RocaTwo-Lt": 332, "ZCOOLXiaoWei-Regular": 333, "TANSONGBIRD": 334, "HeadingNow-74Regular": 335, "Luthier-BoldItalic": 336, "Oregano-Regular": 337, "AyrTropikaIsland-Int": 338, "Mali-Regular": 339, "DidactGothic-Regular": 340, "Lovelace-Regular": 341, "BakerieSmooth-Regular": 342, "CarterOne": 343, "HussarBd": 344, "OldStandard-Italic": 345, "TAN-ASTORIA-Display": 346, "rugratssans-Regular": 347, "BMHANNA": 348, "BetterSaturday": 349, "AdigianaToybox": 350, "Sailors": 351, "PlayfairDisplaySC-Italic": 352, "Etna-Regular": 353, "Revive80Signature": 354, "CAGenerated": 355, "Poppins-Regular": 356, "Jonathan-Regular": 357, "Pacifico-Regular": 358, "Saira-Black": 359, "Loubag-Regular": 360, "Decalotype-Black": 361, "Mansalva-Regular": 362, "Allura-Regular": 363, "ProximaNova-Bold": 364, "TANMIGNON-DISPLAY": 365, "ArsenicaAntiqua-Regular": 366, "BreulGroteskA-RegularItalic": 367, "HKModular-Bold": 368, "TANNightingale-Regular": 369, "AristotelicaProCndTxt-Rg": 370, "Aprila-Regular": 371, "Tomorrow-Regular": 372, "AngellaWhite": 373, "KaushanScript-Regular": 374, "NotoSans": 375, "LeJour-Script": 376, "BrixtonTC-Regular": 377, "OleoScript-Regular": 378, "Cakerolli-Regular": 379, "Lobster-Regular": 380, "FrunchySerif-Regular": 381, "PorcelainRegular": 382, "AlojaExtended": 383, "SergioTrendy-Italic": 384, "LovelaceText-Bold": 385, "Anaktoria": 386, "JimmyScript-Light": 387, "IBMPlexSerif": 388, "Marta": 389, "Mango-Regular": 390, "Overpass-Italic": 391, "Hagrid-Regular": 392, "ElikaGorica": 393, "Amiko-Regular": 394, "EFCOBrookshire-Regular": 395, "Caladea-Regular": 396, "MoonlightBold": 397, "Staatliches-Regular": 398, "Helios-Bold": 399, "Satisfy-Regular": 400, "NexaScript-Regular": 401, "Trocchi-Regular": 402, "March": 403, "IbarraRealNova-Regular": 404, "Nectarine-Regular": 405, "Overpass-Light": 406, "TruetypewriterPolyglOTT": 407, "Bangers-Regular": 408, "Lazord-BoldExpandedItalic": 409, "Chloe-Regular": 410, "BaskervilleDisplayPT-Regular": 411, "Bright-Regular": 412, "Vollkorn-Regular": 413, "Harmattan": 414, "SortsMillGoudy-Regular": 415, "Biryani-Bold": 416, "SugoProDisplay-Italic": 417, "Lazord-BoldItalic": 418, "Alike-Regular": 419, "PermanentMarker-Regular": 420, "Sacramento-Regular": 421, "HKGroteskPro-Italic": 422, "Aleo-BoldItalic": 423, "Noot": 424, "TANGARLAND-Regular": 425, "Twister": 426, "Arsenal-Italic": 427, "Bogart-Italic": 428, "BethEllen-Regular": 429, "Caveat-Regular": 430, "BalsamiqSans-Bold": 431, "BreeSerif-Regular": 432, "CodecPro-ExtraBold": 433, "Pierson-Light": 434, "CyGrotesk-WideRegular": 435, "Lumios-Marker": 436, "Comfortaa-Bold": 437, "TraceFontRegular": 438, "RTL-AdamScript-Regular": 439, "EastmanGrotesque-Italic": 440, "Kalam-Bold": 441, "ChauPhilomeneOne-Regular": 442, "Coiny-Regular": 443, "Lovera": 444, "Gellatio": 445, "TitilliumWeb-Bold": 446, "OilvareBase-Italic": 447, "Catamaran-Black": 448, "Anteb-Italic": 449, "SueEllenFrancisco": 450, "SweetApricot": 451, "BrightSunshine": 452, "IM_FELL_Double_Pica_Italic": 453, "Granaina-limpia": 454, "TANPARFAIT": 455, "AcherusGrotesque-Regular": 456, "AwesomeLathusca-Italic": 457, "Signika-Bold": 458, "Andasia": 459, "DO-AllCaps-Slanted": 460, "Zenaida-Regular": 461, "Fahkwang-Regular": 462, "Play-Regular": 463, "BERNIERRegular-Regular": 464, "PlumaThin-Regular": 465, "SportsWorld": 466, "Garet-Black": 467, "CarolloPlayscript-BlackItalic": 468, "Cheque-Regular": 469, "SEGO": 470, "BobbyJones-Condensed": 471, "NexaSlab-RegularItalic": 472, "DancingScript-Regular": 473, "PaalalabasDisplayWideBETA": 474, "Magnolia-Script": 475, "OpunMai-400It": 476, "MadelynFill-Regular": 477, "ZingRust-Base": 478, "FingerPaint-Regular": 479, "BostonAngel-Light": 480, "Gliker-RegularExpanded": 481, "Ahsing": 482, "Engagement-Regular": 483, "EyesomeScript": 484, "LibraSerifModern-Regular": 485, "London-Regular": 486, "AtkinsonHyperlegible-Regular": 487, "StadioNow-TextItalic": 488, "Aniyah": 489, "ITCAvantGardePro-Bold": 490, "Comica-Regular": 491, "Coustard-Regular": 492, "Brice-BoldCondensed": 493, "TANNEWYORK-Bold": 494, "TANBUSTER-Bold": 495, "Alatsi-Regular": 496, "TYSerif-Book": 497, "Jingleberry": 498, "Rajdhani-Bold": 499, "LobsterTwo-BoldItalic": 500, "BestLight-Medium": 501, "Hitchcut-Regular": 502, "GermaniaOne-Regular": 503, "Emitha-Script": 504, "LemonTuesday": 505, "Cubao_Free_Regular": 506, "MonterchiSerif-Regular": 507, "AllertaStencil-Regular": 508, "RTL-Sondos-Regular": 509, "HomemadeApple-Regular": 510, "CosmicOcto-Medium": 511} \ No newline at end of file diff --git a/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/assets/multi_fonts/cn.json b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/assets/multi_fonts/cn.json new file mode 100644 index 0000000000000000000000000000000000000000..7a357682d54ff5065c7ca13ca82eb5c542bbf796 --- /dev/null +++ b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/assets/multi_fonts/cn.json @@ -0,0 +1 @@ +[["cn-HelloFont-FangHuaTi", 0], ["cn-HelloFont-ID-DianFangSong-Bold", 1], ["cn-HelloFont-ID-DianFangSong", 2], ["cn-HelloFont-ID-DianHei-CEJ", 3], ["cn-HelloFont-ID-DianHei-DEJ", 4], ["cn-HelloFont-ID-DianHei-EEJ", 5], ["cn-HelloFont-ID-DianHei-FEJ", 6], ["cn-HelloFont-ID-DianHei-GEJ", 7], ["cn-HelloFont-ID-DianKai-Bold", 8], ["cn-HelloFont-ID-DianKai", 9], ["cn-HelloFont-WenYiHei", 10], ["cn-Hellofont-ID-ChenYanXingKai", 11], ["cn-Hellofont-ID-DaZiBao", 12], ["cn-Hellofont-ID-DaoCaoRen", 13], ["cn-Hellofont-ID-JianSong", 14], ["cn-Hellofont-ID-JiangHuZhaoPaiHei", 15], ["cn-Hellofont-ID-KeSong", 16], ["cn-Hellofont-ID-LeYuanTi", 17], ["cn-Hellofont-ID-Pinocchio", 18], ["cn-Hellofont-ID-QiMiaoTi", 19], ["cn-Hellofont-ID-QingHuaKai", 20], ["cn-Hellofont-ID-QingHuaXingKai", 21], ["cn-Hellofont-ID-ShanShuiXingKai", 22], ["cn-Hellofont-ID-ShouXieQiShu", 23], ["cn-Hellofont-ID-ShouXieTongZhenTi", 24], ["cn-Hellofont-ID-TengLingTi", 25], ["cn-Hellofont-ID-XiaoLiShu", 26], ["cn-Hellofont-ID-XuanZhenSong", 27], ["cn-Hellofont-ID-ZhongLingXingKai", 28], ["cn-HellofontIDJiaoTangTi", 29], ["cn-HellofontIDJiuZhuTi", 30], ["cn-HuXiaoBao-SaoBao", 31], ["cn-HuXiaoBo-NanShen", 32], ["cn-HuXiaoBo-ZhenShuai", 33], ["cn-SourceHanSansSC-Bold", 34], ["cn-SourceHanSansSC-ExtraLight", 35], ["cn-SourceHanSansSC-Heavy", 36], ["cn-SourceHanSansSC-Light", 37], ["cn-SourceHanSansSC-Medium", 38], ["cn-SourceHanSansSC-Normal", 39], ["cn-SourceHanSansSC-Regular", 40], ["cn-SourceHanSerifSC-Bold", 41], ["cn-SourceHanSerifSC-ExtraLight", 42], ["cn-SourceHanSerifSC-Heavy", 43], ["cn-SourceHanSerifSC-Light", 44], ["cn-SourceHanSerifSC-Medium", 45], ["cn-SourceHanSerifSC-Regular", 46], ["cn-SourceHanSerifSC-SemiBold", 47], ["cn-xiaowei", 48], ["cn-AaJianHaoTi", 49], ["cn-AlibabaPuHuiTi-Bold", 50], ["cn-AlibabaPuHuiTi-Heavy", 51], ["cn-AlibabaPuHuiTi-Light", 52], ["cn-AlibabaPuHuiTi-Medium", 53], ["cn-AlibabaPuHuiTi-Regular", 54], ["cn-CanvaAcidBoldSC", 55], ["cn-CanvaBreezeCN", 56], ["cn-CanvaBumperCropSC", 57], ["cn-CanvaCakeShopCN", 58], ["cn-CanvaEndeavorBlackSC", 59], ["cn-CanvaJoyHeiCN", 60], ["cn-CanvaLiCN", 61], ["cn-CanvaOrientalBrushCN", 62], ["cn-CanvaPoster", 63], ["cn-CanvaQinfuCalligraphyCN", 64], ["cn-CanvaSweetHeartCN", 65], ["cn-CanvaSwordLikeDreamCN", 66], ["cn-CanvaTangyuanHandwritingCN", 67], ["cn-CanvaWanderWorldCN", 68], ["cn-CanvaWenCN", 69], ["cn-DianZiChunYi", 70], ["cn-GenSekiGothicTW-H", 71], ["cn-GenWanMinTW-L", 72], ["cn-GenYoMinTW-B", 73], ["cn-GenYoMinTW-EL", 74], ["cn-GenYoMinTW-H", 75], ["cn-GenYoMinTW-M", 76], ["cn-GenYoMinTW-R", 77], ["cn-GenYoMinTW-SB", 78], ["cn-HYQiHei-AZEJ", 79], ["cn-HYQiHei-EES", 80], ["cn-HanaMinA", 81], ["cn-HappyZcool-2016", 82], ["cn-HelloFont ZJ KeKouKeAiTi", 83], ["cn-HelloFont-ID-BoBoTi", 84], ["cn-HelloFont-ID-FuGuHei-25", 85], ["cn-HelloFont-ID-FuGuHei-35", 86], ["cn-HelloFont-ID-FuGuHei-45", 87], ["cn-HelloFont-ID-FuGuHei-55", 88], ["cn-HelloFont-ID-FuGuHei-65", 89], ["cn-HelloFont-ID-FuGuHei-75", 90], ["cn-HelloFont-ID-FuGuHei-85", 91], ["cn-HelloFont-ID-HeiKa", 92], ["cn-HelloFont-ID-HeiTang", 93], ["cn-HelloFont-ID-JianSong-95", 94], ["cn-HelloFont-ID-JueJiangHei-50", 95], ["cn-HelloFont-ID-JueJiangHei-55", 96], ["cn-HelloFont-ID-JueJiangHei-60", 97], ["cn-HelloFont-ID-JueJiangHei-65", 98], ["cn-HelloFont-ID-JueJiangHei-70", 99], ["cn-HelloFont-ID-JueJiangHei-75", 100], ["cn-HelloFont-ID-JueJiangHei-80", 101], ["cn-HelloFont-ID-KuHeiTi", 102], ["cn-HelloFont-ID-LingDongTi", 103], ["cn-HelloFont-ID-LingLiTi", 104], ["cn-HelloFont-ID-MuFengTi", 105], ["cn-HelloFont-ID-NaiNaiJiangTi", 106], ["cn-HelloFont-ID-PangDu", 107], ["cn-HelloFont-ID-ReLieTi", 108], ["cn-HelloFont-ID-RouRun", 109], ["cn-HelloFont-ID-SaShuangShouXieTi", 110], ["cn-HelloFont-ID-WangZheFengFan", 111], ["cn-HelloFont-ID-YouQiTi", 112], ["cn-Hellofont-ID-XiaLeTi", 113], ["cn-Hellofont-ID-XianXiaTi", 114], ["cn-HuXiaoBoKuHei", 115], ["cn-IDDanMoXingKai", 116], ["cn-IDJueJiangHei", 117], ["cn-IDMeiLingTi", 118], ["cn-IDQQSugar", 119], ["cn-LiuJianMaoCao-Regular", 120], ["cn-LongCang-Regular", 121], ["cn-MaShanZheng-Regular", 122], ["cn-PangMenZhengDao-3", 123], ["cn-PangMenZhengDao-Cu", 124], ["cn-PangMenZhengDao", 125], ["cn-SentyCaramel", 126], ["cn-SourceHanSerifSC", 127], ["cn-WenCang-Regular", 128], ["cn-WenQuanYiMicroHei", 129], ["cn-XianErTi", 130], ["cn-YRDZSTJF", 131], ["cn-YS-HelloFont-BangBangTi", 132], ["cn-ZCOOLKuaiLe-Regular", 133], ["cn-ZCOOLQingKeHuangYou-Regular", 134], ["cn-ZCOOLXiaoWei-Regular", 135], ["cn-ZCOOL_KuHei", 136], ["cn-ZhiMangXing-Regular", 137], ["cn-baotuxiaobaiti", 138], ["cn-jiangxizhuokai-Regular", 139], ["cn-zcool-gdh", 140], ["cn-zcoolqingkehuangyouti-Regular", 141], ["cn-zcoolwenyiti", 142]] \ No newline at end of file diff --git a/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/assets/multi_fonts/de.json b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/assets/multi_fonts/de.json new file mode 100644 index 0000000000000000000000000000000000000000..f231d0d5b2f6b95e4455303cec0fc5bf9f1b3d3f --- /dev/null +++ b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/assets/multi_fonts/de.json @@ -0,0 +1 @@ +[["en-Montserrat-Regular", 0], ["en-Poppins-Italic", 1], ["en-GlacialIndifference-Regular", 2], ["en-OpenSans-ExtraBoldItalic", 3], ["en-Montserrat-Bold", 4], ["en-Now-Regular", 5], ["en-Garet-Regular", 6], ["en-LeagueSpartan-Bold", 7], ["en-DMSans-Regular", 8], ["en-OpenSauceOne-Regular", 9], ["en-OpenSans-ExtraBold", 10], ["en-KGPrimaryPenmanship", 11], ["en-Anton-Regular", 12], ["en-Aileron-BlackItalic", 13], ["en-Quicksand-Light", 14], ["en-Roboto-BoldItalic", 15], ["en-TheSeasons-It", 16], ["en-Kollektif", 17], ["en-Inter-BoldItalic", 18], ["en-Poppins-Medium", 19], ["en-Poppins-Light", 20], ["en-RoxboroughCF-RegularItalic", 21], ["en-PlayfairDisplay-SemiBold", 22], ["en-Agrandir-Italic", 23], ["en-Lato-Regular", 24], ["en-MoreSugarRegular", 25], ["en-CanvaSans-RegularItalic", 26], ["en-PublicSans-Italic", 27], ["en-CodePro-NormalLC", 28], ["en-Belleza-Regular", 29], ["en-JosefinSans-Bold", 30], ["en-HKGrotesk-Bold", 31], ["en-Telegraf-Medium", 32], ["en-BrittanySignatureRegular", 33], ["en-Raleway-ExtraBoldItalic", 34], ["en-Mont-RegularItalic", 35], ["en-Arimo-BoldItalic", 36], ["en-Lora-Italic", 37], ["en-ArchivoBlack-Regular", 38], ["en-Poppins", 39], ["en-Barlow-Black", 40], ["en-CormorantGaramond-Bold", 41], ["en-LibreBaskerville-Regular", 42], ["en-LazydogRegular", 45], ["en-FredokaOne-Regular", 46], ["en-Horizon-Bold", 47], ["en-Nourd-Regular", 48], ["en-Hatton-Regular", 49], ["en-Nunito-ExtraBoldItalic", 50], ["en-CerebriSans-Regular", 51], ["en-Montserrat-Light", 52], ["en-TenorSans", 53], ["en-ClearSans-Bold", 55], ["en-Cardo-Regular", 56], ["en-Alice-Regular", 57], ["en-Oswald-Regular", 58], ["en-Muli-Black", 60], ["en-TAN-PEARL-Regular", 61], ["en-CooperHewitt-Book", 62], ["en-Agrandir-Grand", 63], ["en-BlackMango-Thin", 64], ["en-DMSerifDisplay-Regular", 65], ["en-Antonio-Bold", 66], ["en-Sniglet-Regular", 67], ["en-BeVietnam-Regular", 68], ["en-NunitoSans10pt-BlackItalic", 69], ["en-AbhayaLibre-ExtraBold", 70], ["en-Rubik-Regular", 71], ["en-PPNeueMachina-Regular", 72], ["en-TAN - MON CHERI-Regular", 73], ["en-SourceSansPro-BoldItalic", 76], ["en-MoonTime-Regular", 77], ["en-Eczar-ExtraBold", 78], ["en-Gatwick-Regular", 79], ["en-MonumentExtended-Regular", 80], ["en-BarlowSemiCondensed-Regular", 81], ["en-BarlowCondensed-Regular", 82], ["en-Alegreya-Regular", 83], ["en-DreamAvenue", 84], ["en-RobotoCondensed-Italic", 85], ["en-BobbyJones-Regular", 86], ["en-Garet-ExtraBold", 87], ["en-YesevaOne-Regular", 88], ["en-Dosis-ExtraBold", 89], ["en-LeagueGothic-Regular", 90], ["en-OpenSans-Italic", 91], ["en-TANAEGEAN-Regular", 92], ["en-Maharlika-Regular", 93], ["en-Agrandir-Wide", 96], ["en-Chewy-Regular", 97], ["en-BodoniFLF-BoldItalic", 98], ["en-Nunito-BlackItalic", 99], ["en-LilitaOne", 100], ["en-HandyCasualCondensed-Regular", 101], ["en-Ovo", 102], ["en-Livvic-Regular", 103], ["en-Agrandir-Narrow", 104], ["en-CrimsonPro-Italic", 105], ["en-AnonymousPro-Bold", 106], ["en-NF-OneLittleFont-Bold", 107], ["en-RedHatDisplay-BoldItalic", 108], ["en-CodecPro-Regular", 109], ["en-HalimunRegular", 110], ["en-LibreFranklin-Black", 111], ["en-TeXGyreTermes-BoldItalic", 112], ["en-Shrikhand-Regular", 113], ["en-TTNormsPro-Italic", 114], ["en-OpenSans-Bold", 116], ["en-GreatVibes-Regular", 117], ["en-Breathing", 118], ["en-HeroLight-Regular", 119], ["en-KGPrimaryDots", 120], ["en-Quicksand-Bold", 121], ["en-Brice-ExtraLightSemiExpanded", 122], ["en-Lato-BoldItalic", 123], ["en-Fraunces9pt-Italic", 124], ["en-AbrilFatface-Regular", 125], ["en-BerkshireSwash-Regular", 126], ["en-Atma-Bold", 127], ["en-HolidayRegular", 128], ["en-Gistesy", 131], ["en-BDScript-Regular", 132], ["en-Prompt-Black", 134], ["en-TAN MERINGUE", 135], ["en-GentySans-Regular", 137], ["en-NeueEinstellung-Normal", 138], ["en-Garet-Bold", 139], ["en-FiraSans-Black", 140], ["en-BantayogLight", 141], ["en-NotoSerifDisplay-Black", 142], ["en-TTChocolates-Regular", 143], ["en-Ubuntu-Regular", 144], ["en-Assistant-Bold", 145], ["en-ABeeZee-Regular", 146], ["en-LexendDeca-Regular", 147], ["en-KingredSerif", 148], ["en-Radley-Regular", 149], ["en-BrownSugar", 150], ["en-MigraItalic-ExtraboldItalic", 151], ["en-ChildosArabic-Regular", 152], ["en-PeaceSans", 153], ["en-LondrinaSolid-Black", 154], ["en-SpaceMono-BoldItalic", 155], ["en-RobotoMono-Light", 156], ["en-CourierPrime-Regular", 157], ["en-Alata-Regular", 158], ["en-Amsterdam-One", 159], ["en-IreneFlorentina-Regular", 160], ["en-CatchyMager", 161], ["en-Alta_regular", 162], ["en-ArticulatCF-Regular", 163], ["en-Raleway-Regular", 164], ["en-BrasikaDisplay", 165], ["en-TANAngleton-Italic", 166], ["en-NotoSerifDisplay-ExtraCondensedItalic", 167], ["en-Bryndan Write", 168], ["en-TTCommonsPro-It", 169], ["en-AlexBrush-Regular", 170], ["en-Antic-Regular", 171], ["en-TTHoves-Bold", 172], ["en-DroidSerif", 173], ["en-Marcellus-Regular", 175], ["en-Sanchez-Italic", 176], ["en-JosefinSans", 177], ["en-Afrah-Regular", 178], ["en-PinyonScript", 179], ["en-TTInterphases-BoldItalic", 180], ["en-Yellowtail-Regular", 181], ["en-Gliker-Regular", 182], ["en-BobbyJonesSoft-Regular", 183], ["en-IBMPlexSans", 184], ["en-Amsterdam-Three", 185], ["en-Amsterdam-FourSlant", 186], ["en-TTFors-Regular", 187], ["en-Quattrocento", 188], ["en-Sifonn-Basic", 189], ["en-AlegreyaSans-Black", 190], ["en-Daydream", 191], ["en-AristotelicaProTx-Rg", 192], ["en-NotoSerif", 193], ["en-EBGaramond-Italic", 194], ["en-HammersmithOne-Regular", 195], ["en-RobotoSlab-Regular", 196], ["en-KGPrimaryDotsLined", 198], ["en-Blinker-Regular", 199], ["en-TAN NIMBUS", 200], ["en-Blueberry-Regular", 201], ["en-Rosario-Regular", 202], ["en-Forum", 203], ["en-MistrullyRegular", 204], ["en-SourceSerifPro-Regular", 205], ["en-Bugaki-Regular", 206], ["en-CMUSerif-Roman", 207], ["en-GulfsDisplay-NormalItalic", 208], ["en-PTSans-Bold", 209], ["en-Sensei-Medium", 210], ["en-SquadaOne-Regular", 211], ["en-Arapey-Italic", 212], ["en-Parisienne-Regular", 213], ["en-Aleo-Italic", 214], ["en-QuicheDisplay-Italic", 215], ["en-RocaOne-It", 216], ["en-Funtastic-Regular", 217], ["en-PTSerif-BoldItalic", 218], ["en-Muller-RegularItalic", 219], ["en-ArgentCF-Regular", 220], ["en-Brightwall-Italic", 221], ["en-Knewave-Regular", 222], ["en-TYSerif-D", 223], ["en-Agrandir-Tight", 224], ["en-AlfaSlabOne-Regular", 225], ["en-TANTangkiwood-Display", 226], ["en-Kief-Montaser-Regular", 227], ["en-Gotham-Book", 228], ["en-CocoGothic-Italic", 230], ["en-SairaCondensed-Regular", 231], ["en-DellaRespira-Regular", 232], ["en-Questrial-Regular", 233], ["en-BukhariScript-Regular", 234], ["en-HelveticaWorld-Bold", 235], ["en-TANKINDRED-Display", 236], ["en-Vidaloka-Regular", 238], ["en-AlegreyaSansSC-Black", 239], ["en-FeelingPassionate-Regular", 240], ["en-QuincyCF-Regular", 241], ["en-FiraCode-Regular", 242], ["en-Genty-Regular", 243], ["en-Nickainley-Normal", 244], ["en-RubikOne-Regular", 245], ["en-Gidole-Regular", 246], ["en-Gordita-RegularItalic", 248], ["en-Scripter-Regular", 249], ["en-Buffalo-Regular", 250], ["en-KleinText-Regular", 251], ["en-Arvo-Bold", 253], ["en-GabrielSans-NormalItalic", 254], ["en-Heebo-Black", 255], ["en-LexendExa-Regular", 256], ["en-BrixtonSansTC-Regular", 257], ["en-GildaDisplay-Regular", 258], ["en-Amaranth-BoldItalic", 260], ["en-BubbleboddyNeue-Regular", 261], ["en-MavenPro-Bold", 262], ["en-TTDrugs-Italic", 263], ["en-CyGrotesk-KeyRegular", 264], ["en-VarelaRound-Regular", 265], ["en-Ruda-Black", 266], ["en-SafiraMarch", 267], ["en-BloggerSans", 268], ["en-TANHEADLINE-Regular", 269], ["en-SloopScriptPro-Regular", 270], ["en-NeueMontreal-Regular", 271], ["en-Schoolbell-Regular", 272], ["en-InriaSerif-Regular", 274], ["en-JetBrainsMono-Regular", 275], ["en-MADEEvolveSans", 276], ["en-Dekko", 277], ["en-Handyman-Regular", 278], ["en-Aileron-BoldItalic", 279], ["en-Solway-Regular", 281], ["en-Higuen-Regular", 282], ["en-WedgesItalic", 283], ["en-TANASHFORD-BOLD", 284], ["en-IBMPlexMono", 285], ["en-RacingSansOne-Regular", 286], ["en-RegularBrush", 287], ["en-OpenSans-LightItalic", 288], ["en-SpecialElite-Regular", 289], ["en-FuturaLTPro-Medium", 290], ["en-MaragsaDisplay", 291], ["en-BigShouldersDisplay-Regular", 292], ["en-BDSans-Regular", 293], ["en-RasputinRegular", 294], ["en-Yvesyvesdrawing-BoldItalic", 295], ["en-Bitter-Regular", 296], ["en-TTFirsNeue-Italic", 299], ["en-Sunday-Regular", 300], ["en-HKGothic-MediumItalic", 301], ["en-CaveatBrush-Regular", 302], ["en-ArchitectsDaughter-Regular", 304], ["en-Angelina", 305], ["en-Calistoga-Regular", 306], ["en-ArchivoNarrow-Regular", 307], ["en-ObjectSans-MediumSlanted", 308], ["en-Nexa-RegularItalic", 310], ["en-Lustria-Regular", 311], ["en-Amsterdam-TwoSlant", 312], ["en-Virtual-Regular", 313], ["en-NF-Lepetitcochon-Regular", 315], ["en-LeJour-Serif", 317], ["en-Prata-Regular", 318], ["en-PPWoodland-Regular", 319], ["en-PlayfairDisplay-BoldItalic", 320], ["en-AmaticSC-Regular", 321], ["en-Cabin-Regular", 322], ["en-MrDafoe-Regular", 324], ["en-TTRamillas-Italic", 325], ["en-Luckybones-Bold", 326], ["en-DarkerGrotesque-Light", 327], ["en-CormorantSC-Bold", 329], ["en-GochiHand-Regular", 330], ["en-Atteron", 331], ["en-RocaTwo-Lt", 332], ["en-TANSONGBIRD", 334], ["en-HeadingNow-74Regular", 335], ["en-Luthier-BoldItalic", 336], ["en-Oregano-Regular", 337], ["en-AyrTropikaIsland-Int", 338], ["en-Mali-Regular", 339], ["en-DidactGothic-Regular", 340], ["en-Lovelace-Regular", 341], ["en-BakerieSmooth-Regular", 342], ["en-CarterOne", 343], ["en-HussarBd", 344], ["en-OldStandard-Italic", 345], ["en-TAN-ASTORIA-Display", 346], ["en-rugratssans-Regular", 347], ["en-BetterSaturday", 349], ["en-AdigianaToybox", 350], ["en-Sailors", 351], ["en-PlayfairDisplaySC-Italic", 352], ["en-Etna-Regular", 353], ["en-Revive80Signature", 354], ["en-CAGenerated", 355], ["en-Poppins-Regular", 356], ["en-Jonathan-Regular", 357], ["en-Pacifico-Regular", 358], ["en-Saira-Black", 359], ["en-Loubag-Regular", 360], ["en-Decalotype-Black", 361], ["en-Mansalva-Regular", 362], ["en-Allura-Regular", 363], ["en-ProximaNova-Bold", 364], ["en-TANMIGNON-DISPLAY", 365], ["en-ArsenicaAntiqua-Regular", 366], ["en-BreulGroteskA-RegularItalic", 367], ["en-HKModular-Bold", 368], ["en-TANNightingale-Regular", 369], ["en-AristotelicaProCndTxt-Rg", 370], ["en-Aprila-Regular", 371], ["en-Tomorrow-Regular", 372], ["en-AngellaWhite", 373], ["en-KaushanScript-Regular", 374], ["en-NotoSans", 375], ["en-LeJour-Script", 376], ["en-BrixtonTC-Regular", 377], ["en-OleoScript-Regular", 378], ["en-Cakerolli-Regular", 379], ["en-Lobster-Regular", 380], ["en-FrunchySerif-Regular", 381], ["en-PorcelainRegular", 382], ["en-AlojaExtended", 383], ["en-SergioTrendy-Italic", 384], ["en-LovelaceText-Bold", 385], ["en-Anaktoria", 386], ["en-JimmyScript-Light", 387], ["en-IBMPlexSerif", 388], ["en-Marta", 389], ["en-Mango-Regular", 390], ["en-Overpass-Italic", 391], ["en-Hagrid-Regular", 392], ["en-ElikaGorica", 393], ["en-Amiko-Regular", 394], ["en-EFCOBrookshire-Regular", 395], ["en-Caladea-Regular", 396], ["en-Staatliches-Regular", 398], ["en-Helios-Bold", 399], ["en-Satisfy-Regular", 400], ["en-NexaScript-Regular", 401], ["en-Trocchi-Regular", 402], ["en-March", 403], ["en-IbarraRealNova-Regular", 404], ["en-Nectarine-Regular", 405], ["en-Overpass-Light", 406], ["en-TruetypewriterPolyglOTT", 407], ["en-Bangers-Regular", 408], ["en-Lazord-BoldExpandedItalic", 409], ["en-Chloe-Regular", 410], ["en-BaskervilleDisplayPT-Regular", 411], ["en-Bright-Regular", 412], ["en-Vollkorn-Regular", 413], ["en-Harmattan", 414], ["en-SortsMillGoudy-Regular", 415], ["en-Biryani-Bold", 416], ["en-SugoProDisplay-Italic", 417], ["en-Lazord-BoldItalic", 418], ["en-Alike-Regular", 419], ["en-Sacramento-Regular", 421], ["en-HKGroteskPro-Italic", 422], ["en-Aleo-BoldItalic", 423], ["en-TANGARLAND-Regular", 425], ["en-Twister", 426], ["en-Arsenal-Italic", 427], ["en-Bogart-Italic", 428], ["en-BethEllen-Regular", 429], ["en-Caveat-Regular", 430], ["en-BalsamiqSans-Bold", 431], ["en-BreeSerif-Regular", 432], ["en-CodecPro-ExtraBold", 433], ["en-Pierson-Light", 434], ["en-CyGrotesk-WideRegular", 435], ["en-Lumios-Marker", 436], ["en-Comfortaa-Bold", 437], ["en-RTL-AdamScript-Regular", 439], ["en-EastmanGrotesque-Italic", 440], ["en-Kalam-Bold", 441], ["en-ChauPhilomeneOne-Regular", 442], ["en-Coiny-Regular", 443], ["en-Lovera", 444], ["en-Gellatio", 445], ["en-TitilliumWeb-Bold", 446], ["en-OilvareBase-Italic", 447], ["en-Catamaran-Black", 448], ["en-Anteb-Italic", 449], ["en-SueEllenFrancisco", 450], ["en-SweetApricot", 451], ["en-BrightSunshine", 452], ["en-IM_FELL_Double_Pica_Italic", 453], ["en-Granaina-limpia", 454], ["en-TANPARFAIT", 455], ["en-AcherusGrotesque-Regular", 456], ["en-AwesomeLathusca-Italic", 457], ["en-Signika-Bold", 458], ["en-Andasia", 459], ["en-DO-AllCaps-Slanted", 460], ["en-Zenaida-Regular", 461], ["en-Fahkwang-Regular", 462], ["en-Play-Regular", 463], ["en-PlumaThin-Regular", 465], ["en-SportsWorld", 466], ["en-Garet-Black", 467], ["en-CarolloPlayscript-BlackItalic", 468], ["en-SEGO", 470], ["en-BobbyJones-Condensed", 471], ["en-NexaSlab-RegularItalic", 472], ["en-DancingScript-Regular", 473], ["en-Magnolia-Script", 475], ["en-OpunMai-400It", 476], ["en-MadelynFill-Regular", 477], ["en-FingerPaint-Regular", 479], ["en-BostonAngel-Light", 480], ["en-Gliker-RegularExpanded", 481], ["en-Ahsing", 482], ["en-Engagement-Regular", 483], ["en-EyesomeScript", 484], ["en-LibraSerifModern-Regular", 485], ["en-London-Regular", 486], ["en-AtkinsonHyperlegible-Regular", 487], ["en-StadioNow-TextItalic", 488], ["en-Aniyah", 489], ["en-ITCAvantGardePro-Bold", 490], ["en-Comica-Regular", 491], ["en-Coustard-Regular", 492], ["en-Brice-BoldCondensed", 493], ["en-TANNEWYORK-Bold", 494], ["en-TANBUSTER-Bold", 495], ["en-Alatsi-Regular", 496], ["en-TYSerif-Book", 497], ["en-Jingleberry", 498], ["en-Rajdhani-Bold", 499], ["en-LobsterTwo-BoldItalic", 500], ["en-BestLight-Medium", 501], ["en-Hitchcut-Regular", 502], ["en-GermaniaOne-Regular", 503], ["en-Emitha-Script", 504], ["en-LemonTuesday", 505], ["en-MonterchiSerif-Regular", 507], ["en-AllertaStencil-Regular", 508], ["en-RTL-Sondos-Regular", 509], ["en-HomemadeApple-Regular", 510], ["en-CosmicOcto-Medium", 511]] \ No newline at end of file diff --git a/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/assets/multi_fonts/en.json b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/assets/multi_fonts/en.json new file mode 100644 index 0000000000000000000000000000000000000000..41f2a1fb4fee619fc7202de274525e1aeff0f45b --- /dev/null +++ b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/assets/multi_fonts/en.json @@ -0,0 +1 @@ +[["en-Montserrat-Regular", 0], ["en-Poppins-Italic", 1], ["en-GlacialIndifference-Regular", 2], ["en-OpenSans-ExtraBoldItalic", 3], ["en-Montserrat-Bold", 4], ["en-Now-Regular", 5], ["en-Garet-Regular", 6], ["en-LeagueSpartan-Bold", 7], ["en-DMSans-Regular", 8], ["en-OpenSauceOne-Regular", 9], ["en-OpenSans-ExtraBold", 10], ["en-KGPrimaryPenmanship", 11], ["en-Anton-Regular", 12], ["en-Aileron-BlackItalic", 13], ["en-Quicksand-Light", 14], ["en-Roboto-BoldItalic", 15], ["en-TheSeasons-It", 16], ["en-Kollektif", 17], ["en-Inter-BoldItalic", 18], ["en-Poppins-Medium", 19], ["en-Poppins-Light", 20], ["en-RoxboroughCF-RegularItalic", 21], ["en-PlayfairDisplay-SemiBold", 22], ["en-Agrandir-Italic", 23], ["en-Lato-Regular", 24], ["en-MoreSugarRegular", 25], ["en-CanvaSans-RegularItalic", 26], ["en-PublicSans-Italic", 27], ["en-CodePro-NormalLC", 28], ["en-Belleza-Regular", 29], ["en-JosefinSans-Bold", 30], ["en-HKGrotesk-Bold", 31], ["en-Telegraf-Medium", 32], ["en-BrittanySignatureRegular", 33], ["en-Raleway-ExtraBoldItalic", 34], ["en-Mont-RegularItalic", 35], ["en-Arimo-BoldItalic", 36], ["en-Lora-Italic", 37], ["en-ArchivoBlack-Regular", 38], ["en-Poppins", 39], ["en-Barlow-Black", 40], ["en-CormorantGaramond-Bold", 41], ["en-LibreBaskerville-Regular", 42], ["en-CanvaSchoolFontRegular", 43], ["en-BebasNeueBold", 44], ["en-LazydogRegular", 45], ["en-FredokaOne-Regular", 46], ["en-Horizon-Bold", 47], ["en-Nourd-Regular", 48], ["en-Hatton-Regular", 49], ["en-Nunito-ExtraBoldItalic", 50], ["en-CerebriSans-Regular", 51], ["en-Montserrat-Light", 52], ["en-TenorSans", 53], ["en-Norwester-Regular", 54], ["en-ClearSans-Bold", 55], ["en-Cardo-Regular", 56], ["en-Alice-Regular", 57], ["en-Oswald-Regular", 58], ["en-Gaegu-Bold", 59], ["en-Muli-Black", 60], ["en-TAN-PEARL-Regular", 61], ["en-CooperHewitt-Book", 62], ["en-Agrandir-Grand", 63], ["en-BlackMango-Thin", 64], ["en-DMSerifDisplay-Regular", 65], ["en-Antonio-Bold", 66], ["en-Sniglet-Regular", 67], ["en-BeVietnam-Regular", 68], ["en-NunitoSans10pt-BlackItalic", 69], ["en-AbhayaLibre-ExtraBold", 70], ["en-Rubik-Regular", 71], ["en-PPNeueMachina-Regular", 72], ["en-TAN - MON CHERI-Regular", 73], ["en-Jua-Regular", 74], ["en-Playlist-Script", 75], ["en-SourceSansPro-BoldItalic", 76], ["en-MoonTime-Regular", 77], ["en-Eczar-ExtraBold", 78], ["en-Gatwick-Regular", 79], ["en-MonumentExtended-Regular", 80], ["en-BarlowSemiCondensed-Regular", 81], ["en-BarlowCondensed-Regular", 82], ["en-Alegreya-Regular", 83], ["en-DreamAvenue", 84], ["en-RobotoCondensed-Italic", 85], ["en-BobbyJones-Regular", 86], ["en-Garet-ExtraBold", 87], ["en-YesevaOne-Regular", 88], ["en-Dosis-ExtraBold", 89], ["en-LeagueGothic-Regular", 90], ["en-OpenSans-Italic", 91], ["en-TANAEGEAN-Regular", 92], ["en-Maharlika-Regular", 93], ["en-MarykateRegular", 94], ["en-Cinzel-Regular", 95], ["en-Agrandir-Wide", 96], ["en-Chewy-Regular", 97], ["en-BodoniFLF-BoldItalic", 98], ["en-Nunito-BlackItalic", 99], ["en-LilitaOne", 100], ["en-HandyCasualCondensed-Regular", 101], ["en-Ovo", 102], ["en-Livvic-Regular", 103], ["en-Agrandir-Narrow", 104], ["en-CrimsonPro-Italic", 105], ["en-AnonymousPro-Bold", 106], ["en-NF-OneLittleFont-Bold", 107], ["en-RedHatDisplay-BoldItalic", 108], ["en-CodecPro-Regular", 109], ["en-HalimunRegular", 110], ["en-LibreFranklin-Black", 111], ["en-TeXGyreTermes-BoldItalic", 112], ["en-Shrikhand-Regular", 113], ["en-TTNormsPro-Italic", 114], ["en-Gagalin-Regular", 115], ["en-OpenSans-Bold", 116], ["en-GreatVibes-Regular", 117], ["en-Breathing", 118], ["en-HeroLight-Regular", 119], ["en-KGPrimaryDots", 120], ["en-Quicksand-Bold", 121], ["en-Brice-ExtraLightSemiExpanded", 122], ["en-Lato-BoldItalic", 123], ["en-Fraunces9pt-Italic", 124], ["en-AbrilFatface-Regular", 125], ["en-BerkshireSwash-Regular", 126], ["en-Atma-Bold", 127], ["en-HolidayRegular", 128], ["en-BebasNeueCyrillic", 129], ["en-IntroRust-Base", 130], ["en-Gistesy", 131], ["en-BDScript-Regular", 132], ["en-ApricotsRegular", 133], ["en-Prompt-Black", 134], ["en-TAN MERINGUE", 135], ["en-Sukar Regular", 136], ["en-GentySans-Regular", 137], ["en-NeueEinstellung-Normal", 138], ["en-Garet-Bold", 139], ["en-FiraSans-Black", 140], ["en-BantayogLight", 141], ["en-NotoSerifDisplay-Black", 142], ["en-TTChocolates-Regular", 143], ["en-Ubuntu-Regular", 144], ["en-Assistant-Bold", 145], ["en-ABeeZee-Regular", 146], ["en-LexendDeca-Regular", 147], ["en-KingredSerif", 148], ["en-Radley-Regular", 149], ["en-BrownSugar", 150], ["en-MigraItalic-ExtraboldItalic", 151], ["en-ChildosArabic-Regular", 152], ["en-PeaceSans", 153], ["en-LondrinaSolid-Black", 154], ["en-SpaceMono-BoldItalic", 155], ["en-RobotoMono-Light", 156], ["en-CourierPrime-Regular", 157], ["en-Alata-Regular", 158], ["en-Amsterdam-One", 159], ["en-IreneFlorentina-Regular", 160], ["en-CatchyMager", 161], ["en-Alta_regular", 162], ["en-ArticulatCF-Regular", 163], ["en-Raleway-Regular", 164], ["en-BrasikaDisplay", 165], ["en-TANAngleton-Italic", 166], ["en-NotoSerifDisplay-ExtraCondensedItalic", 167], ["en-Bryndan Write", 168], ["en-TTCommonsPro-It", 169], ["en-AlexBrush-Regular", 170], ["en-Antic-Regular", 171], ["en-TTHoves-Bold", 172], ["en-DroidSerif", 173], ["en-AblationRegular", 174], ["en-Marcellus-Regular", 175], ["en-Sanchez-Italic", 176], ["en-JosefinSans", 177], ["en-Afrah-Regular", 178], ["en-PinyonScript", 179], ["en-TTInterphases-BoldItalic", 180], ["en-Yellowtail-Regular", 181], ["en-Gliker-Regular", 182], ["en-BobbyJonesSoft-Regular", 183], ["en-IBMPlexSans", 184], ["en-Amsterdam-Three", 185], ["en-Amsterdam-FourSlant", 186], ["en-TTFors-Regular", 187], ["en-Quattrocento", 188], ["en-Sifonn-Basic", 189], ["en-AlegreyaSans-Black", 190], ["en-Daydream", 191], ["en-AristotelicaProTx-Rg", 192], ["en-NotoSerif", 193], ["en-EBGaramond-Italic", 194], ["en-HammersmithOne-Regular", 195], ["en-RobotoSlab-Regular", 196], ["en-DO-Sans-Regular", 197], ["en-KGPrimaryDotsLined", 198], ["en-Blinker-Regular", 199], ["en-TAN NIMBUS", 200], ["en-Blueberry-Regular", 201], ["en-Rosario-Regular", 202], ["en-Forum", 203], ["en-MistrullyRegular", 204], ["en-SourceSerifPro-Regular", 205], ["en-Bugaki-Regular", 206], ["en-CMUSerif-Roman", 207], ["en-GulfsDisplay-NormalItalic", 208], ["en-PTSans-Bold", 209], ["en-Sensei-Medium", 210], ["en-SquadaOne-Regular", 211], ["en-Arapey-Italic", 212], ["en-Parisienne-Regular", 213], ["en-Aleo-Italic", 214], ["en-QuicheDisplay-Italic", 215], ["en-RocaOne-It", 216], ["en-Funtastic-Regular", 217], ["en-PTSerif-BoldItalic", 218], ["en-Muller-RegularItalic", 219], ["en-ArgentCF-Regular", 220], ["en-Brightwall-Italic", 221], ["en-Knewave-Regular", 222], ["en-TYSerif-D", 223], ["en-Agrandir-Tight", 224], ["en-AlfaSlabOne-Regular", 225], ["en-TANTangkiwood-Display", 226], ["en-Kief-Montaser-Regular", 227], ["en-Gotham-Book", 228], ["en-JuliusSansOne-Regular", 229], ["en-CocoGothic-Italic", 230], ["en-SairaCondensed-Regular", 231], ["en-DellaRespira-Regular", 232], ["en-Questrial-Regular", 233], ["en-BukhariScript-Regular", 234], ["en-HelveticaWorld-Bold", 235], ["en-TANKINDRED-Display", 236], ["en-CinzelDecorative-Regular", 237], ["en-Vidaloka-Regular", 238], ["en-AlegreyaSansSC-Black", 239], ["en-FeelingPassionate-Regular", 240], ["en-QuincyCF-Regular", 241], ["en-FiraCode-Regular", 242], ["en-Genty-Regular", 243], ["en-Nickainley-Normal", 244], ["en-RubikOne-Regular", 245], ["en-Gidole-Regular", 246], ["en-Borsok", 247], ["en-Gordita-RegularItalic", 248], ["en-Scripter-Regular", 249], ["en-Buffalo-Regular", 250], ["en-KleinText-Regular", 251], ["en-Creepster-Regular", 252], ["en-Arvo-Bold", 253], ["en-GabrielSans-NormalItalic", 254], ["en-Heebo-Black", 255], ["en-LexendExa-Regular", 256], ["en-BrixtonSansTC-Regular", 257], ["en-GildaDisplay-Regular", 258], ["en-ChunkFive-Roman", 259], ["en-Amaranth-BoldItalic", 260], ["en-BubbleboddyNeue-Regular", 261], ["en-MavenPro-Bold", 262], ["en-TTDrugs-Italic", 263], ["en-CyGrotesk-KeyRegular", 264], ["en-VarelaRound-Regular", 265], ["en-Ruda-Black", 266], ["en-SafiraMarch", 267], ["en-BloggerSans", 268], ["en-TANHEADLINE-Regular", 269], ["en-SloopScriptPro-Regular", 270], ["en-NeueMontreal-Regular", 271], ["en-Schoolbell-Regular", 272], ["en-SigherRegular", 273], ["en-InriaSerif-Regular", 274], ["en-JetBrainsMono-Regular", 275], ["en-MADEEvolveSans", 276], ["en-Dekko", 277], ["en-Handyman-Regular", 278], ["en-Aileron-BoldItalic", 279], ["en-Bright-Italic", 280], ["en-Solway-Regular", 281], ["en-Higuen-Regular", 282], ["en-WedgesItalic", 283], ["en-TANASHFORD-BOLD", 284], ["en-IBMPlexMono", 285], ["en-RacingSansOne-Regular", 286], ["en-RegularBrush", 287], ["en-OpenSans-LightItalic", 288], ["en-SpecialElite-Regular", 289], ["en-FuturaLTPro-Medium", 290], ["en-MaragsaDisplay", 291], ["en-BigShouldersDisplay-Regular", 292], ["en-BDSans-Regular", 293], ["en-RasputinRegular", 294], ["en-Yvesyvesdrawing-BoldItalic", 295], ["en-Bitter-Regular", 296], ["en-LuckiestGuy-Regular", 297], ["en-CanvaSchoolFontDotted", 298], ["en-TTFirsNeue-Italic", 299], ["en-Sunday-Regular", 300], ["en-HKGothic-MediumItalic", 301], ["en-CaveatBrush-Regular", 302], ["en-HeliosExt", 303], ["en-ArchitectsDaughter-Regular", 304], ["en-Angelina", 305], ["en-Calistoga-Regular", 306], ["en-ArchivoNarrow-Regular", 307], ["en-ObjectSans-MediumSlanted", 308], ["en-AyrLucidityCondensed-Regular", 309], ["en-Nexa-RegularItalic", 310], ["en-Lustria-Regular", 311], ["en-Amsterdam-TwoSlant", 312], ["en-Virtual-Regular", 313], ["en-Brusher-Regular", 314], ["en-NF-Lepetitcochon-Regular", 315], ["en-TANTWINKLE", 316], ["en-LeJour-Serif", 317], ["en-Prata-Regular", 318], ["en-PPWoodland-Regular", 319], ["en-PlayfairDisplay-BoldItalic", 320], ["en-AmaticSC-Regular", 321], ["en-Cabin-Regular", 322], ["en-Manjari-Bold", 323], ["en-MrDafoe-Regular", 324], ["en-TTRamillas-Italic", 325], ["en-Luckybones-Bold", 326], ["en-DarkerGrotesque-Light", 327], ["en-BellabooRegular", 328], ["en-CormorantSC-Bold", 329], ["en-GochiHand-Regular", 330], ["en-Atteron", 331], ["en-RocaTwo-Lt", 332], ["en-ZCOOLXiaoWei-Regular", 333], ["en-TANSONGBIRD", 334], ["en-HeadingNow-74Regular", 335], ["en-Luthier-BoldItalic", 336], ["en-Oregano-Regular", 337], ["en-AyrTropikaIsland-Int", 338], ["en-Mali-Regular", 339], ["en-DidactGothic-Regular", 340], ["en-Lovelace-Regular", 341], ["en-BakerieSmooth-Regular", 342], ["en-CarterOne", 343], ["en-HussarBd", 344], ["en-OldStandard-Italic", 345], ["en-TAN-ASTORIA-Display", 346], ["en-rugratssans-Regular", 347], ["en-BMHANNA", 348], ["en-BetterSaturday", 349], ["en-AdigianaToybox", 350], ["en-Sailors", 351], ["en-PlayfairDisplaySC-Italic", 352], ["en-Etna-Regular", 353], ["en-Revive80Signature", 354], ["en-CAGenerated", 355], ["en-Poppins-Regular", 356], ["en-Jonathan-Regular", 357], ["en-Pacifico-Regular", 358], ["en-Saira-Black", 359], ["en-Loubag-Regular", 360], ["en-Decalotype-Black", 361], ["en-Mansalva-Regular", 362], ["en-Allura-Regular", 363], ["en-ProximaNova-Bold", 364], ["en-TANMIGNON-DISPLAY", 365], ["en-ArsenicaAntiqua-Regular", 366], ["en-BreulGroteskA-RegularItalic", 367], ["en-HKModular-Bold", 368], ["en-TANNightingale-Regular", 369], ["en-AristotelicaProCndTxt-Rg", 370], ["en-Aprila-Regular", 371], ["en-Tomorrow-Regular", 372], ["en-AngellaWhite", 373], ["en-KaushanScript-Regular", 374], ["en-NotoSans", 375], ["en-LeJour-Script", 376], ["en-BrixtonTC-Regular", 377], ["en-OleoScript-Regular", 378], ["en-Cakerolli-Regular", 379], ["en-Lobster-Regular", 380], ["en-FrunchySerif-Regular", 381], ["en-PorcelainRegular", 382], ["en-AlojaExtended", 383], ["en-SergioTrendy-Italic", 384], ["en-LovelaceText-Bold", 385], ["en-Anaktoria", 386], ["en-JimmyScript-Light", 387], ["en-IBMPlexSerif", 388], ["en-Marta", 389], ["en-Mango-Regular", 390], ["en-Overpass-Italic", 391], ["en-Hagrid-Regular", 392], ["en-ElikaGorica", 393], ["en-Amiko-Regular", 394], ["en-EFCOBrookshire-Regular", 395], ["en-Caladea-Regular", 396], ["en-MoonlightBold", 397], ["en-Staatliches-Regular", 398], ["en-Helios-Bold", 399], ["en-Satisfy-Regular", 400], ["en-NexaScript-Regular", 401], ["en-Trocchi-Regular", 402], ["en-March", 403], ["en-IbarraRealNova-Regular", 404], ["en-Nectarine-Regular", 405], ["en-Overpass-Light", 406], ["en-TruetypewriterPolyglOTT", 407], ["en-Bangers-Regular", 408], ["en-Lazord-BoldExpandedItalic", 409], ["en-Chloe-Regular", 410], ["en-BaskervilleDisplayPT-Regular", 411], ["en-Bright-Regular", 412], ["en-Vollkorn-Regular", 413], ["en-Harmattan", 414], ["en-SortsMillGoudy-Regular", 415], ["en-Biryani-Bold", 416], ["en-SugoProDisplay-Italic", 417], ["en-Lazord-BoldItalic", 418], ["en-Alike-Regular", 419], ["en-PermanentMarker-Regular", 420], ["en-Sacramento-Regular", 421], ["en-HKGroteskPro-Italic", 422], ["en-Aleo-BoldItalic", 423], ["en-Noot", 424], ["en-TANGARLAND-Regular", 425], ["en-Twister", 426], ["en-Arsenal-Italic", 427], ["en-Bogart-Italic", 428], ["en-BethEllen-Regular", 429], ["en-Caveat-Regular", 430], ["en-BalsamiqSans-Bold", 431], ["en-BreeSerif-Regular", 432], ["en-CodecPro-ExtraBold", 433], ["en-Pierson-Light", 434], ["en-CyGrotesk-WideRegular", 435], ["en-Lumios-Marker", 436], ["en-Comfortaa-Bold", 437], ["en-TraceFontRegular", 438], ["en-RTL-AdamScript-Regular", 439], ["en-EastmanGrotesque-Italic", 440], ["en-Kalam-Bold", 441], ["en-ChauPhilomeneOne-Regular", 442], ["en-Coiny-Regular", 443], ["en-Lovera", 444], ["en-Gellatio", 445], ["en-TitilliumWeb-Bold", 446], ["en-OilvareBase-Italic", 447], ["en-Catamaran-Black", 448], ["en-Anteb-Italic", 449], ["en-SueEllenFrancisco", 450], ["en-SweetApricot", 451], ["en-BrightSunshine", 452], ["en-IM_FELL_Double_Pica_Italic", 453], ["en-Granaina-limpia", 454], ["en-TANPARFAIT", 455], ["en-AcherusGrotesque-Regular", 456], ["en-AwesomeLathusca-Italic", 457], ["en-Signika-Bold", 458], ["en-Andasia", 459], ["en-DO-AllCaps-Slanted", 460], ["en-Zenaida-Regular", 461], ["en-Fahkwang-Regular", 462], ["en-Play-Regular", 463], ["en-BERNIERRegular-Regular", 464], ["en-PlumaThin-Regular", 465], ["en-SportsWorld", 466], ["en-Garet-Black", 467], ["en-CarolloPlayscript-BlackItalic", 468], ["en-Cheque-Regular", 469], ["en-SEGO", 470], ["en-BobbyJones-Condensed", 471], ["en-NexaSlab-RegularItalic", 472], ["en-DancingScript-Regular", 473], ["en-PaalalabasDisplayWideBETA", 474], ["en-Magnolia-Script", 475], ["en-OpunMai-400It", 476], ["en-MadelynFill-Regular", 477], ["en-ZingRust-Base", 478], ["en-FingerPaint-Regular", 479], ["en-BostonAngel-Light", 480], ["en-Gliker-RegularExpanded", 481], ["en-Ahsing", 482], ["en-Engagement-Regular", 483], ["en-EyesomeScript", 484], ["en-LibraSerifModern-Regular", 485], ["en-London-Regular", 486], ["en-AtkinsonHyperlegible-Regular", 487], ["en-StadioNow-TextItalic", 488], ["en-Aniyah", 489], ["en-ITCAvantGardePro-Bold", 490], ["en-Comica-Regular", 491], ["en-Coustard-Regular", 492], ["en-Brice-BoldCondensed", 493], ["en-TANNEWYORK-Bold", 494], ["en-TANBUSTER-Bold", 495], ["en-Alatsi-Regular", 496], ["en-TYSerif-Book", 497], ["en-Jingleberry", 498], ["en-Rajdhani-Bold", 499], ["en-LobsterTwo-BoldItalic", 500], ["en-BestLight-Medium", 501], ["en-Hitchcut-Regular", 502], ["en-GermaniaOne-Regular", 503], ["en-Emitha-Script", 504], ["en-LemonTuesday", 505], ["en-Cubao_Free_Regular", 506], ["en-MonterchiSerif-Regular", 507], ["en-AllertaStencil-Regular", 508], ["en-RTL-Sondos-Regular", 509], ["en-HomemadeApple-Regular", 510], ["en-CosmicOcto-Medium", 511]] \ No newline at end of file diff --git a/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/assets/multi_fonts/es.json b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/assets/multi_fonts/es.json new file mode 100644 index 0000000000000000000000000000000000000000..499c6177bf09171b4cfa9e0d47a7bd404e627ae1 --- /dev/null +++ b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/assets/multi_fonts/es.json @@ -0,0 +1 @@ +[["en-Montserrat-Regular", 0], ["en-Poppins-Italic", 1], ["en-OpenSans-ExtraBoldItalic", 3], ["en-Montserrat-Bold", 4], ["en-Now-Regular", 5], ["en-Garet-Regular", 6], ["en-LeagueSpartan-Bold", 7], ["en-DMSans-Regular", 8], ["en-OpenSauceOne-Regular", 9], ["en-OpenSans-ExtraBold", 10], ["en-KGPrimaryPenmanship", 11], ["en-Anton-Regular", 12], ["en-Aileron-BlackItalic", 13], ["en-Quicksand-Light", 14], ["en-Roboto-BoldItalic", 15], ["en-TheSeasons-It", 16], ["en-Kollektif", 17], ["en-Inter-BoldItalic", 18], ["en-Poppins-Medium", 19], ["en-Poppins-Light", 20], ["en-RoxboroughCF-RegularItalic", 21], ["en-PlayfairDisplay-SemiBold", 22], ["en-Agrandir-Italic", 23], ["en-Lato-Regular", 24], ["en-CanvaSans-RegularItalic", 26], ["en-PublicSans-Italic", 27], ["en-CodePro-NormalLC", 28], ["en-Belleza-Regular", 29], ["en-JosefinSans-Bold", 30], ["en-HKGrotesk-Bold", 31], ["en-Telegraf-Medium", 32], ["en-BrittanySignatureRegular", 33], ["en-Raleway-ExtraBoldItalic", 34], ["en-Mont-RegularItalic", 35], ["en-Arimo-BoldItalic", 36], ["en-Lora-Italic", 37], ["en-ArchivoBlack-Regular", 38], ["en-Poppins", 39], ["en-Barlow-Black", 40], ["en-CormorantGaramond-Bold", 41], ["en-LibreBaskerville-Regular", 42], ["en-BebasNeueBold", 44], ["en-FredokaOne-Regular", 46], ["en-Horizon-Bold", 47], ["en-Nourd-Regular", 48], ["en-Hatton-Regular", 49], ["en-Nunito-ExtraBoldItalic", 50], ["en-CerebriSans-Regular", 51], ["en-Montserrat-Light", 52], ["en-TenorSans", 53], ["en-ClearSans-Bold", 55], ["en-Cardo-Regular", 56], ["en-Alice-Regular", 57], ["en-Oswald-Regular", 58], ["en-Muli-Black", 60], ["en-TAN-PEARL-Regular", 61], ["en-CooperHewitt-Book", 62], ["en-Agrandir-Grand", 63], ["en-BlackMango-Thin", 64], ["en-DMSerifDisplay-Regular", 65], ["en-Antonio-Bold", 66], ["en-Sniglet-Regular", 67], ["en-BeVietnam-Regular", 68], ["en-NunitoSans10pt-BlackItalic", 69], ["en-AbhayaLibre-ExtraBold", 70], ["en-Rubik-Regular", 71], ["en-PPNeueMachina-Regular", 72], ["en-TAN - MON CHERI-Regular", 73], ["en-SourceSansPro-BoldItalic", 76], ["en-MoonTime-Regular", 77], ["en-Eczar-ExtraBold", 78], ["en-Gatwick-Regular", 79], ["en-MonumentExtended-Regular", 80], ["en-BarlowSemiCondensed-Regular", 81], ["en-BarlowCondensed-Regular", 82], ["en-Alegreya-Regular", 83], ["en-DreamAvenue", 84], ["en-RobotoCondensed-Italic", 85], ["en-Garet-ExtraBold", 87], ["en-YesevaOne-Regular", 88], ["en-Dosis-ExtraBold", 89], ["en-LeagueGothic-Regular", 90], ["en-OpenSans-Italic", 91], ["en-TANAEGEAN-Regular", 92], ["en-Maharlika-Regular", 93], ["en-Cinzel-Regular", 95], ["en-Agrandir-Wide", 96], ["en-Chewy-Regular", 97], ["en-BodoniFLF-BoldItalic", 98], ["en-Nunito-BlackItalic", 99], ["en-LilitaOne", 100], ["en-HandyCasualCondensed-Regular", 101], ["en-Ovo", 102], ["en-Livvic-Regular", 103], ["en-Agrandir-Narrow", 104], ["en-CrimsonPro-Italic", 105], ["en-AnonymousPro-Bold", 106], ["en-NF-OneLittleFont-Bold", 107], ["en-RedHatDisplay-BoldItalic", 108], ["en-CodecPro-Regular", 109], ["en-HalimunRegular", 110], ["en-LibreFranklin-Black", 111], ["en-TeXGyreTermes-BoldItalic", 112], ["en-Shrikhand-Regular", 113], ["en-TTNormsPro-Italic", 114], ["en-Gagalin-Regular", 115], ["en-OpenSans-Bold", 116], ["en-GreatVibes-Regular", 117], ["en-Breathing", 118], ["en-HeroLight-Regular", 119], ["en-KGPrimaryDots", 120], ["en-Quicksand-Bold", 121], ["en-Brice-ExtraLightSemiExpanded", 122], ["en-Lato-BoldItalic", 123], ["en-Fraunces9pt-Italic", 124], ["en-AbrilFatface-Regular", 125], ["en-BerkshireSwash-Regular", 126], ["en-Atma-Bold", 127], ["en-HolidayRegular", 128], ["en-BebasNeueCyrillic", 129], ["en-IntroRust-Base", 130], ["en-Gistesy", 131], ["en-BDScript-Regular", 132], ["en-ApricotsRegular", 133], ["en-Prompt-Black", 134], ["en-TAN MERINGUE", 135], ["en-GentySans-Regular", 137], ["en-NeueEinstellung-Normal", 138], ["en-Garet-Bold", 139], ["en-FiraSans-Black", 140], ["en-BantayogLight", 141], ["en-NotoSerifDisplay-Black", 142], ["en-TTChocolates-Regular", 143], ["en-Ubuntu-Regular", 144], ["en-Assistant-Bold", 145], ["en-ABeeZee-Regular", 146], ["en-LexendDeca-Regular", 147], ["en-KingredSerif", 148], ["en-Radley-Regular", 149], ["en-BrownSugar", 150], ["en-MigraItalic-ExtraboldItalic", 151], ["en-ChildosArabic-Regular", 152], ["en-PeaceSans", 153], ["en-LondrinaSolid-Black", 154], ["en-SpaceMono-BoldItalic", 155], ["en-RobotoMono-Light", 156], ["en-CourierPrime-Regular", 157], ["en-Alata-Regular", 158], ["en-Amsterdam-One", 159], ["en-CatchyMager", 161], ["en-Alta_regular", 162], ["en-ArticulatCF-Regular", 163], ["en-Raleway-Regular", 164], ["en-BrasikaDisplay", 165], ["en-TANAngleton-Italic", 166], ["en-NotoSerifDisplay-ExtraCondensedItalic", 167], ["en-Bryndan Write", 168], ["en-TTCommonsPro-It", 169], ["en-AlexBrush-Regular", 170], ["en-Antic-Regular", 171], ["en-TTHoves-Bold", 172], ["en-DroidSerif", 173], ["en-AblationRegular", 174], ["en-Marcellus-Regular", 175], ["en-Sanchez-Italic", 176], ["en-JosefinSans", 177], ["en-Afrah-Regular", 178], ["en-PinyonScript", 179], ["en-TTInterphases-BoldItalic", 180], ["en-Yellowtail-Regular", 181], ["en-Gliker-Regular", 182], ["en-BobbyJonesSoft-Regular", 183], ["en-IBMPlexSans", 184], ["en-Amsterdam-Three", 185], ["en-Amsterdam-FourSlant", 186], ["en-TTFors-Regular", 187], ["en-Quattrocento", 188], ["en-Sifonn-Basic", 189], ["en-AlegreyaSans-Black", 190], ["en-Daydream", 191], ["en-AristotelicaProTx-Rg", 192], ["en-NotoSerif", 193], ["en-EBGaramond-Italic", 194], ["en-HammersmithOne-Regular", 195], ["en-RobotoSlab-Regular", 196], ["en-DO-Sans-Regular", 197], ["en-KGPrimaryDotsLined", 198], ["en-Blinker-Regular", 199], ["en-TAN NIMBUS", 200], ["en-Rosario-Regular", 202], ["en-Forum", 203], ["en-MistrullyRegular", 204], ["en-SourceSerifPro-Regular", 205], ["en-Bugaki-Regular", 206], ["en-CMUSerif-Roman", 207], ["en-GulfsDisplay-NormalItalic", 208], ["en-PTSans-Bold", 209], ["en-Sensei-Medium", 210], ["en-SquadaOne-Regular", 211], ["en-Arapey-Italic", 212], ["en-Parisienne-Regular", 213], ["en-Aleo-Italic", 214], ["en-QuicheDisplay-Italic", 215], ["en-RocaOne-It", 216], ["en-Funtastic-Regular", 217], ["en-PTSerif-BoldItalic", 218], ["en-Muller-RegularItalic", 219], ["en-ArgentCF-Regular", 220], ["en-Brightwall-Italic", 221], ["en-Knewave-Regular", 222], ["en-Agrandir-Tight", 224], ["en-AlfaSlabOne-Regular", 225], ["en-TANTangkiwood-Display", 226], ["en-Kief-Montaser-Regular", 227], ["en-Gotham-Book", 228], ["en-JuliusSansOne-Regular", 229], ["en-CocoGothic-Italic", 230], ["en-SairaCondensed-Regular", 231], ["en-DellaRespira-Regular", 232], ["en-Questrial-Regular", 233], ["en-BukhariScript-Regular", 234], ["en-HelveticaWorld-Bold", 235], ["en-TANKINDRED-Display", 236], ["en-CinzelDecorative-Regular", 237], ["en-Vidaloka-Regular", 238], ["en-AlegreyaSansSC-Black", 239], ["en-FeelingPassionate-Regular", 240], ["en-QuincyCF-Regular", 241], ["en-FiraCode-Regular", 242], ["en-Genty-Regular", 243], ["en-Nickainley-Normal", 244], ["en-RubikOne-Regular", 245], ["en-Gidole-Regular", 246], ["en-Borsok", 247], ["en-Gordita-RegularItalic", 248], ["en-Scripter-Regular", 249], ["en-Buffalo-Regular", 250], ["en-KleinText-Regular", 251], ["en-Creepster-Regular", 252], ["en-Arvo-Bold", 253], ["en-GabrielSans-NormalItalic", 254], ["en-Heebo-Black", 255], ["en-LexendExa-Regular", 256], ["en-BrixtonSansTC-Regular", 257], ["en-GildaDisplay-Regular", 258], ["en-Amaranth-BoldItalic", 260], ["en-BubbleboddyNeue-Regular", 261], ["en-MavenPro-Bold", 262], ["en-TTDrugs-Italic", 263], ["en-CyGrotesk-KeyRegular", 264], ["en-VarelaRound-Regular", 265], ["en-Ruda-Black", 266], ["en-SafiraMarch", 267], ["en-BloggerSans", 268], ["en-TANHEADLINE-Regular", 269], ["en-SloopScriptPro-Regular", 270], ["en-NeueMontreal-Regular", 271], ["en-Schoolbell-Regular", 272], ["en-InriaSerif-Regular", 274], ["en-JetBrainsMono-Regular", 275], ["en-MADEEvolveSans", 276], ["en-Handyman-Regular", 278], ["en-Aileron-BoldItalic", 279], ["en-Solway-Regular", 281], ["en-Higuen-Regular", 282], ["en-WedgesItalic", 283], ["en-TANASHFORD-BOLD", 284], ["en-IBMPlexMono", 285], ["en-RacingSansOne-Regular", 286], ["en-RegularBrush", 287], ["en-OpenSans-LightItalic", 288], ["en-SpecialElite-Regular", 289], ["en-FuturaLTPro-Medium", 290], ["en-MaragsaDisplay", 291], ["en-BigShouldersDisplay-Regular", 292], ["en-BDSans-Regular", 293], ["en-RasputinRegular", 294], ["en-Yvesyvesdrawing-BoldItalic", 295], ["en-Bitter-Regular", 296], ["en-LuckiestGuy-Regular", 297], ["en-TTFirsNeue-Italic", 299], ["en-Sunday-Regular", 300], ["en-HKGothic-MediumItalic", 301], ["en-CaveatBrush-Regular", 302], ["en-ArchitectsDaughter-Regular", 304], ["en-Calistoga-Regular", 306], ["en-ArchivoNarrow-Regular", 307], ["en-ObjectSans-MediumSlanted", 308], ["en-AyrLucidityCondensed-Regular", 309], ["en-Nexa-RegularItalic", 310], ["en-Lustria-Regular", 311], ["en-Amsterdam-TwoSlant", 312], ["en-Virtual-Regular", 313], ["en-NF-Lepetitcochon-Regular", 315], ["en-TANTWINKLE", 316], ["en-LeJour-Serif", 317], ["en-Prata-Regular", 318], ["en-PPWoodland-Regular", 319], ["en-PlayfairDisplay-BoldItalic", 320], ["en-AmaticSC-Regular", 321], ["en-Cabin-Regular", 322], ["en-MrDafoe-Regular", 324], ["en-TTRamillas-Italic", 325], ["en-Luckybones-Bold", 326], ["en-DarkerGrotesque-Light", 327], ["en-BellabooRegular", 328], ["en-CormorantSC-Bold", 329], ["en-GochiHand-Regular", 330], ["en-Atteron", 331], ["en-RocaTwo-Lt", 332], ["en-TANSONGBIRD", 334], ["en-HeadingNow-74Regular", 335], ["en-Luthier-BoldItalic", 336], ["en-Oregano-Regular", 337], ["en-AyrTropikaIsland-Int", 338], ["en-Mali-Regular", 339], ["en-DidactGothic-Regular", 340], ["en-Lovelace-Regular", 341], ["en-BakerieSmooth-Regular", 342], ["en-CarterOne", 343], ["en-HussarBd", 344], ["en-OldStandard-Italic", 345], ["en-TAN-ASTORIA-Display", 346], ["en-rugratssans-Regular", 347], ["en-BetterSaturday", 349], ["en-AdigianaToybox", 350], ["en-Sailors", 351], ["en-PlayfairDisplaySC-Italic", 352], ["en-CAGenerated", 355], ["en-Poppins-Regular", 356], ["en-Jonathan-Regular", 357], ["en-Pacifico-Regular", 358], ["en-Saira-Black", 359], ["en-Loubag-Regular", 360], ["en-Decalotype-Black", 361], ["en-Mansalva-Regular", 362], ["en-Allura-Regular", 363], ["en-ProximaNova-Bold", 364], ["en-TANMIGNON-DISPLAY", 365], ["en-ArsenicaAntiqua-Regular", 366], ["en-BreulGroteskA-RegularItalic", 367], ["en-HKModular-Bold", 368], ["en-TANNightingale-Regular", 369], ["en-AristotelicaProCndTxt-Rg", 370], ["en-Aprila-Regular", 371], ["en-Tomorrow-Regular", 372], ["en-AngellaWhite", 373], ["en-KaushanScript-Regular", 374], ["en-NotoSans", 375], ["en-BrixtonTC-Regular", 377], ["en-OleoScript-Regular", 378], ["en-Cakerolli-Regular", 379], ["en-Lobster-Regular", 380], ["en-PorcelainRegular", 382], ["en-AlojaExtended", 383], ["en-SergioTrendy-Italic", 384], ["en-LovelaceText-Bold", 385], ["en-Anaktoria", 386], ["en-IBMPlexSerif", 388], ["en-Marta", 389], ["en-Mango-Regular", 390], ["en-Overpass-Italic", 391], ["en-Hagrid-Regular", 392], ["en-ElikaGorica", 393], ["en-Amiko-Regular", 394], ["en-EFCOBrookshire-Regular", 395], ["en-Caladea-Regular", 396], ["en-Staatliches-Regular", 398], ["en-Helios-Bold", 399], ["en-Satisfy-Regular", 400], ["en-NexaScript-Regular", 401], ["en-Trocchi-Regular", 402], ["en-March", 403], ["en-IbarraRealNova-Regular", 404], ["en-Nectarine-Regular", 405], ["en-Overpass-Light", 406], ["en-TruetypewriterPolyglOTT", 407], ["en-Bangers-Regular", 408], ["en-Lazord-BoldExpandedItalic", 409], ["en-Chloe-Regular", 410], ["en-BaskervilleDisplayPT-Regular", 411], ["en-Bright-Regular", 412], ["en-Vollkorn-Regular", 413], ["en-Harmattan", 414], ["en-SortsMillGoudy-Regular", 415], ["en-Biryani-Bold", 416], ["en-SugoProDisplay-Italic", 417], ["en-Lazord-BoldItalic", 418], ["en-Alike-Regular", 419], ["en-PermanentMarker-Regular", 420], ["en-Sacramento-Regular", 421], ["en-HKGroteskPro-Italic", 422], ["en-Aleo-BoldItalic", 423], ["en-TANGARLAND-Regular", 425], ["en-Twister", 426], ["en-Arsenal-Italic", 427], ["en-Bogart-Italic", 428], ["en-BethEllen-Regular", 429], ["en-Caveat-Regular", 430], ["en-BalsamiqSans-Bold", 431], ["en-BreeSerif-Regular", 432], ["en-CodecPro-ExtraBold", 433], ["en-Pierson-Light", 434], ["en-CyGrotesk-WideRegular", 435], ["en-Lumios-Marker", 436], ["en-Comfortaa-Bold", 437], ["en-TraceFontRegular", 438], ["en-RTL-AdamScript-Regular", 439], ["en-EastmanGrotesque-Italic", 440], ["en-Kalam-Bold", 441], ["en-ChauPhilomeneOne-Regular", 442], ["en-Coiny-Regular", 443], ["en-Lovera", 444], ["en-Gellatio", 445], ["en-TitilliumWeb-Bold", 446], ["en-OilvareBase-Italic", 447], ["en-Catamaran-Black", 448], ["en-Anteb-Italic", 449], ["en-SueEllenFrancisco", 450], ["en-SweetApricot", 451], ["en-BrightSunshine", 452], ["en-IM_FELL_Double_Pica_Italic", 453], ["en-Granaina-limpia", 454], ["en-TANPARFAIT", 455], ["en-AcherusGrotesque-Regular", 456], ["en-AwesomeLathusca-Italic", 457], ["en-Signika-Bold", 458], ["en-DO-AllCaps-Slanted", 460], ["en-Zenaida-Regular", 461], ["en-Fahkwang-Regular", 462], ["en-Play-Regular", 463], ["en-BERNIERRegular-Regular", 464], ["en-PlumaThin-Regular", 465], ["en-SportsWorld", 466], ["en-Garet-Black", 467], ["en-CarolloPlayscript-BlackItalic", 468], ["en-SEGO", 470], ["en-BobbyJones-Condensed", 471], ["en-NexaSlab-RegularItalic", 472], ["en-DancingScript-Regular", 473], ["en-Magnolia-Script", 475], ["en-OpunMai-400It", 476], ["en-MadelynFill-Regular", 477], ["en-ZingRust-Base", 478], ["en-FingerPaint-Regular", 479], ["en-BostonAngel-Light", 480], ["en-Gliker-RegularExpanded", 481], ["en-Ahsing", 482], ["en-Engagement-Regular", 483], ["en-EyesomeScript", 484], ["en-LibraSerifModern-Regular", 485], ["en-London-Regular", 486], ["en-AtkinsonHyperlegible-Regular", 487], ["en-StadioNow-TextItalic", 488], ["en-ITCAvantGardePro-Bold", 490], ["en-Comica-Regular", 491], ["en-Coustard-Regular", 492], ["en-Brice-BoldCondensed", 493], ["en-TANNEWYORK-Bold", 494], ["en-TANBUSTER-Bold", 495], ["en-Alatsi-Regular", 496], ["en-Jingleberry", 498], ["en-Rajdhani-Bold", 499], ["en-LobsterTwo-BoldItalic", 500], ["en-Hitchcut-Regular", 502], ["en-GermaniaOne-Regular", 503], ["en-Emitha-Script", 504], ["en-LemonTuesday", 505], ["en-MonterchiSerif-Regular", 507], ["en-AllertaStencil-Regular", 508], ["en-RTL-Sondos-Regular", 509], ["en-HomemadeApple-Regular", 510], ["en-CosmicOcto-Medium", 511]] \ No newline at end of file diff --git a/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/assets/multi_fonts/fr.json b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/assets/multi_fonts/fr.json new file mode 100644 index 0000000000000000000000000000000000000000..57deb3329189aa8bcc4e44fa0fcf30cb542fa514 --- /dev/null +++ b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/assets/multi_fonts/fr.json @@ -0,0 +1 @@ +[["en-Montserrat-Regular", 0], ["en-Poppins-Italic", 1], ["en-GlacialIndifference-Regular", 2], ["en-OpenSans-ExtraBoldItalic", 3], ["en-Montserrat-Bold", 4], ["en-Now-Regular", 5], ["en-Garet-Regular", 6], ["en-LeagueSpartan-Bold", 7], ["en-DMSans-Regular", 8], ["en-OpenSauceOne-Regular", 9], ["en-OpenSans-ExtraBold", 10], ["en-KGPrimaryPenmanship", 11], ["en-Anton-Regular", 12], ["en-Aileron-BlackItalic", 13], ["en-Quicksand-Light", 14], ["en-Roboto-BoldItalic", 15], ["en-TheSeasons-It", 16], ["en-Kollektif", 17], ["en-Inter-BoldItalic", 18], ["en-Poppins-Medium", 19], ["en-Poppins-Light", 20], ["en-RoxboroughCF-RegularItalic", 21], ["en-PlayfairDisplay-SemiBold", 22], ["en-Agrandir-Italic", 23], ["en-Lato-Regular", 24], ["en-MoreSugarRegular", 25], ["en-CanvaSans-RegularItalic", 26], ["en-PublicSans-Italic", 27], ["en-CodePro-NormalLC", 28], ["en-Belleza-Regular", 29], ["en-JosefinSans-Bold", 30], ["en-HKGrotesk-Bold", 31], ["en-Telegraf-Medium", 32], ["en-Raleway-ExtraBoldItalic", 34], ["en-Mont-RegularItalic", 35], ["en-Arimo-BoldItalic", 36], ["en-Lora-Italic", 37], ["en-ArchivoBlack-Regular", 38], ["en-Poppins", 39], ["en-Barlow-Black", 40], ["en-CormorantGaramond-Bold", 41], ["en-LibreBaskerville-Regular", 42], ["en-BebasNeueBold", 44], ["en-LazydogRegular", 45], ["en-FredokaOne-Regular", 46], ["en-Horizon-Bold", 47], ["en-Nourd-Regular", 48], ["en-Hatton-Regular", 49], ["en-Nunito-ExtraBoldItalic", 50], ["en-CerebriSans-Regular", 51], ["en-Montserrat-Light", 52], ["en-TenorSans", 53], ["en-ClearSans-Bold", 55], ["en-Cardo-Regular", 56], ["en-Alice-Regular", 57], ["en-Oswald-Regular", 58], ["en-Muli-Black", 60], ["en-TAN-PEARL-Regular", 61], ["en-CooperHewitt-Book", 62], ["en-Agrandir-Grand", 63], ["en-BlackMango-Thin", 64], ["en-DMSerifDisplay-Regular", 65], ["en-Antonio-Bold", 66], ["en-Sniglet-Regular", 67], ["en-BeVietnam-Regular", 68], ["en-NunitoSans10pt-BlackItalic", 69], ["en-AbhayaLibre-ExtraBold", 70], ["en-Rubik-Regular", 71], ["en-PPNeueMachina-Regular", 72], ["en-TAN - MON CHERI-Regular", 73], ["en-Playlist-Script", 75], ["en-SourceSansPro-BoldItalic", 76], ["en-MoonTime-Regular", 77], ["en-Eczar-ExtraBold", 78], ["en-Gatwick-Regular", 79], ["en-MonumentExtended-Regular", 80], ["en-BarlowSemiCondensed-Regular", 81], ["en-BarlowCondensed-Regular", 82], ["en-Alegreya-Regular", 83], ["en-DreamAvenue", 84], ["en-RobotoCondensed-Italic", 85], ["en-BobbyJones-Regular", 86], ["en-Garet-ExtraBold", 87], ["en-YesevaOne-Regular", 88], ["en-Dosis-ExtraBold", 89], ["en-LeagueGothic-Regular", 90], ["en-OpenSans-Italic", 91], ["en-TANAEGEAN-Regular", 92], ["en-Maharlika-Regular", 93], ["en-MarykateRegular", 94], ["en-Cinzel-Regular", 95], ["en-Agrandir-Wide", 96], ["en-Chewy-Regular", 97], ["en-BodoniFLF-BoldItalic", 98], ["en-Nunito-BlackItalic", 99], ["en-LilitaOne", 100], ["en-HandyCasualCondensed-Regular", 101], ["en-Ovo", 102], ["en-Livvic-Regular", 103], ["en-Agrandir-Narrow", 104], ["en-CrimsonPro-Italic", 105], ["en-AnonymousPro-Bold", 106], ["en-NF-OneLittleFont-Bold", 107], ["en-RedHatDisplay-BoldItalic", 108], ["en-CodecPro-Regular", 109], ["en-HalimunRegular", 110], ["en-LibreFranklin-Black", 111], ["en-TeXGyreTermes-BoldItalic", 112], ["en-Shrikhand-Regular", 113], ["en-TTNormsPro-Italic", 114], ["en-Gagalin-Regular", 115], ["en-OpenSans-Bold", 116], ["en-GreatVibes-Regular", 117], ["en-Breathing", 118], ["en-HeroLight-Regular", 119], ["en-KGPrimaryDots", 120], ["en-Quicksand-Bold", 121], ["en-Brice-ExtraLightSemiExpanded", 122], ["en-Lato-BoldItalic", 123], ["en-Fraunces9pt-Italic", 124], ["en-AbrilFatface-Regular", 125], ["en-BerkshireSwash-Regular", 126], ["en-Atma-Bold", 127], ["en-HolidayRegular", 128], ["en-BebasNeueCyrillic", 129], ["en-IntroRust-Base", 130], ["en-Gistesy", 131], ["en-BDScript-Regular", 132], ["en-ApricotsRegular", 133], ["en-Prompt-Black", 134], ["en-TAN MERINGUE", 135], ["en-GentySans-Regular", 137], ["en-NeueEinstellung-Normal", 138], ["en-Garet-Bold", 139], ["en-FiraSans-Black", 140], ["en-BantayogLight", 141], ["en-NotoSerifDisplay-Black", 142], ["en-TTChocolates-Regular", 143], ["en-Ubuntu-Regular", 144], ["en-Assistant-Bold", 145], ["en-ABeeZee-Regular", 146], ["en-LexendDeca-Regular", 147], ["en-KingredSerif", 148], ["en-Radley-Regular", 149], ["en-BrownSugar", 150], ["en-MigraItalic-ExtraboldItalic", 151], ["en-ChildosArabic-Regular", 152], ["en-PeaceSans", 153], ["en-LondrinaSolid-Black", 154], ["en-SpaceMono-BoldItalic", 155], ["en-RobotoMono-Light", 156], ["en-CourierPrime-Regular", 157], ["en-Alata-Regular", 158], ["en-Amsterdam-One", 159], ["en-IreneFlorentina-Regular", 160], ["en-CatchyMager", 161], ["en-Alta_regular", 162], ["en-ArticulatCF-Regular", 163], ["en-Raleway-Regular", 164], ["en-BrasikaDisplay", 165], ["en-TANAngleton-Italic", 166], ["en-NotoSerifDisplay-ExtraCondensedItalic", 167], ["en-Bryndan Write", 168], ["en-TTCommonsPro-It", 169], ["en-AlexBrush-Regular", 170], ["en-Antic-Regular", 171], ["en-TTHoves-Bold", 172], ["en-DroidSerif", 173], ["en-AblationRegular", 174], ["en-Marcellus-Regular", 175], ["en-Sanchez-Italic", 176], ["en-JosefinSans", 177], ["en-Afrah-Regular", 178], ["en-PinyonScript", 179], ["en-TTInterphases-BoldItalic", 180], ["en-Yellowtail-Regular", 181], ["en-Gliker-Regular", 182], ["en-BobbyJonesSoft-Regular", 183], ["en-IBMPlexSans", 184], ["en-Amsterdam-Three", 185], ["en-Amsterdam-FourSlant", 186], ["en-TTFors-Regular", 187], ["en-Quattrocento", 188], ["en-Sifonn-Basic", 189], ["en-AlegreyaSans-Black", 190], ["en-Daydream", 191], ["en-AristotelicaProTx-Rg", 192], ["en-NotoSerif", 193], ["en-EBGaramond-Italic", 194], ["en-HammersmithOne-Regular", 195], ["en-RobotoSlab-Regular", 196], ["en-DO-Sans-Regular", 197], ["en-KGPrimaryDotsLined", 198], ["en-Blinker-Regular", 199], ["en-TAN NIMBUS", 200], ["en-Blueberry-Regular", 201], ["en-Rosario-Regular", 202], ["en-Forum", 203], ["en-MistrullyRegular", 204], ["en-SourceSerifPro-Regular", 205], ["en-Bugaki-Regular", 206], ["en-CMUSerif-Roman", 207], ["en-GulfsDisplay-NormalItalic", 208], ["en-PTSans-Bold", 209], ["en-Sensei-Medium", 210], ["en-SquadaOne-Regular", 211], ["en-Arapey-Italic", 212], ["en-Parisienne-Regular", 213], ["en-Aleo-Italic", 214], ["en-QuicheDisplay-Italic", 215], ["en-RocaOne-It", 216], ["en-Funtastic-Regular", 217], ["en-PTSerif-BoldItalic", 218], ["en-Muller-RegularItalic", 219], ["en-ArgentCF-Regular", 220], ["en-Brightwall-Italic", 221], ["en-Knewave-Regular", 222], ["en-TYSerif-D", 223], ["en-Agrandir-Tight", 224], ["en-AlfaSlabOne-Regular", 225], ["en-TANTangkiwood-Display", 226], ["en-Kief-Montaser-Regular", 227], ["en-Gotham-Book", 228], ["en-JuliusSansOne-Regular", 229], ["en-CocoGothic-Italic", 230], ["en-SairaCondensed-Regular", 231], ["en-DellaRespira-Regular", 232], ["en-Questrial-Regular", 233], ["en-BukhariScript-Regular", 234], ["en-HelveticaWorld-Bold", 235], ["en-TANKINDRED-Display", 236], ["en-CinzelDecorative-Regular", 237], ["en-Vidaloka-Regular", 238], ["en-AlegreyaSansSC-Black", 239], ["en-FeelingPassionate-Regular", 240], ["en-QuincyCF-Regular", 241], ["en-FiraCode-Regular", 242], ["en-Genty-Regular", 243], ["en-Nickainley-Normal", 244], ["en-RubikOne-Regular", 245], ["en-Gidole-Regular", 246], ["en-Borsok", 247], ["en-Gordita-RegularItalic", 248], ["en-Scripter-Regular", 249], ["en-Buffalo-Regular", 250], ["en-KleinText-Regular", 251], ["en-Creepster-Regular", 252], ["en-Arvo-Bold", 253], ["en-GabrielSans-NormalItalic", 254], ["en-Heebo-Black", 255], ["en-LexendExa-Regular", 256], ["en-BrixtonSansTC-Regular", 257], ["en-GildaDisplay-Regular", 258], ["en-Amaranth-BoldItalic", 260], ["en-BubbleboddyNeue-Regular", 261], ["en-MavenPro-Bold", 262], ["en-TTDrugs-Italic", 263], ["en-CyGrotesk-KeyRegular", 264], ["en-VarelaRound-Regular", 265], ["en-Ruda-Black", 266], ["en-SafiraMarch", 267], ["en-BloggerSans", 268], ["en-TANHEADLINE-Regular", 269], ["en-SloopScriptPro-Regular", 270], ["en-NeueMontreal-Regular", 271], ["en-Schoolbell-Regular", 272], ["en-SigherRegular", 273], ["en-InriaSerif-Regular", 274], ["en-JetBrainsMono-Regular", 275], ["en-MADEEvolveSans", 276], ["en-Dekko", 277], ["en-Handyman-Regular", 278], ["en-Aileron-BoldItalic", 279], ["en-Bright-Italic", 280], ["en-Solway-Regular", 281], ["en-Higuen-Regular", 282], ["en-WedgesItalic", 283], ["en-TANASHFORD-BOLD", 284], ["en-IBMPlexMono", 285], ["en-RacingSansOne-Regular", 286], ["en-RegularBrush", 287], ["en-OpenSans-LightItalic", 288], ["en-SpecialElite-Regular", 289], ["en-FuturaLTPro-Medium", 290], ["en-MaragsaDisplay", 291], ["en-BigShouldersDisplay-Regular", 292], ["en-BDSans-Regular", 293], ["en-RasputinRegular", 294], ["en-Yvesyvesdrawing-BoldItalic", 295], ["en-Bitter-Regular", 296], ["en-LuckiestGuy-Regular", 297], ["en-TTFirsNeue-Italic", 299], ["en-Sunday-Regular", 300], ["en-HKGothic-MediumItalic", 301], ["en-CaveatBrush-Regular", 302], ["en-ArchitectsDaughter-Regular", 304], ["en-Angelina", 305], ["en-Calistoga-Regular", 306], ["en-ArchivoNarrow-Regular", 307], ["en-ObjectSans-MediumSlanted", 308], ["en-AyrLucidityCondensed-Regular", 309], ["en-Nexa-RegularItalic", 310], ["en-Lustria-Regular", 311], ["en-Amsterdam-TwoSlant", 312], ["en-Virtual-Regular", 313], ["en-NF-Lepetitcochon-Regular", 315], ["en-TANTWINKLE", 316], ["en-LeJour-Serif", 317], ["en-Prata-Regular", 318], ["en-PPWoodland-Regular", 319], ["en-PlayfairDisplay-BoldItalic", 320], ["en-AmaticSC-Regular", 321], ["en-Cabin-Regular", 322], ["en-Manjari-Bold", 323], ["en-MrDafoe-Regular", 324], ["en-TTRamillas-Italic", 325], ["en-Luckybones-Bold", 326], ["en-DarkerGrotesque-Light", 327], ["en-BellabooRegular", 328], ["en-CormorantSC-Bold", 329], ["en-GochiHand-Regular", 330], ["en-Atteron", 331], ["en-RocaTwo-Lt", 332], ["en-TANSONGBIRD", 334], ["en-HeadingNow-74Regular", 335], ["en-Luthier-BoldItalic", 336], ["en-Oregano-Regular", 337], ["en-AyrTropikaIsland-Int", 338], ["en-Mali-Regular", 339], ["en-DidactGothic-Regular", 340], ["en-Lovelace-Regular", 341], ["en-BakerieSmooth-Regular", 342], ["en-CarterOne", 343], ["en-HussarBd", 344], ["en-OldStandard-Italic", 345], ["en-TAN-ASTORIA-Display", 346], ["en-rugratssans-Regular", 347], ["en-BetterSaturday", 349], ["en-AdigianaToybox", 350], ["en-Sailors", 351], ["en-PlayfairDisplaySC-Italic", 352], ["en-Etna-Regular", 353], ["en-Revive80Signature", 354], ["en-CAGenerated", 355], ["en-Poppins-Regular", 356], ["en-Jonathan-Regular", 357], ["en-Pacifico-Regular", 358], ["en-Saira-Black", 359], ["en-Loubag-Regular", 360], ["en-Decalotype-Black", 361], ["en-Mansalva-Regular", 362], ["en-ProximaNova-Bold", 364], ["en-TANMIGNON-DISPLAY", 365], ["en-ArsenicaAntiqua-Regular", 366], ["en-BreulGroteskA-RegularItalic", 367], ["en-HKModular-Bold", 368], ["en-TANNightingale-Regular", 369], ["en-AristotelicaProCndTxt-Rg", 370], ["en-Aprila-Regular", 371], ["en-Tomorrow-Regular", 372], ["en-AngellaWhite", 373], ["en-KaushanScript-Regular", 374], ["en-NotoSans", 375], ["en-LeJour-Script", 376], ["en-BrixtonTC-Regular", 377], ["en-OleoScript-Regular", 378], ["en-Cakerolli-Regular", 379], ["en-Lobster-Regular", 380], ["en-FrunchySerif-Regular", 381], ["en-PorcelainRegular", 382], ["en-AlojaExtended", 383], ["en-SergioTrendy-Italic", 384], ["en-LovelaceText-Bold", 385], ["en-Anaktoria", 386], ["en-JimmyScript-Light", 387], ["en-IBMPlexSerif", 388], ["en-Marta", 389], ["en-Mango-Regular", 390], ["en-Overpass-Italic", 391], ["en-Hagrid-Regular", 392], ["en-ElikaGorica", 393], ["en-Amiko-Regular", 394], ["en-EFCOBrookshire-Regular", 395], ["en-Caladea-Regular", 396], ["en-MoonlightBold", 397], ["en-Staatliches-Regular", 398], ["en-Helios-Bold", 399], ["en-Satisfy-Regular", 400], ["en-NexaScript-Regular", 401], ["en-Trocchi-Regular", 402], ["en-March", 403], ["en-IbarraRealNova-Regular", 404], ["en-Nectarine-Regular", 405], ["en-Overpass-Light", 406], ["en-TruetypewriterPolyglOTT", 407], ["en-Bangers-Regular", 408], ["en-Lazord-BoldExpandedItalic", 409], ["en-Chloe-Regular", 410], ["en-BaskervilleDisplayPT-Regular", 411], ["en-Bright-Regular", 412], ["en-Vollkorn-Regular", 413], ["en-Harmattan", 414], ["en-SortsMillGoudy-Regular", 415], ["en-Biryani-Bold", 416], ["en-SugoProDisplay-Italic", 417], ["en-Lazord-BoldItalic", 418], ["en-Alike-Regular", 419], ["en-PermanentMarker-Regular", 420], ["en-Sacramento-Regular", 421], ["en-HKGroteskPro-Italic", 422], ["en-Aleo-BoldItalic", 423], ["en-TANGARLAND-Regular", 425], ["en-Twister", 426], ["en-Arsenal-Italic", 427], ["en-Bogart-Italic", 428], ["en-BethEllen-Regular", 429], ["en-Caveat-Regular", 430], ["en-BalsamiqSans-Bold", 431], ["en-BreeSerif-Regular", 432], ["en-CodecPro-ExtraBold", 433], ["en-Pierson-Light", 434], ["en-CyGrotesk-WideRegular", 435], ["en-Lumios-Marker", 436], ["en-Comfortaa-Bold", 437], ["en-TraceFontRegular", 438], ["en-RTL-AdamScript-Regular", 439], ["en-EastmanGrotesque-Italic", 440], ["en-Kalam-Bold", 441], ["en-ChauPhilomeneOne-Regular", 442], ["en-Coiny-Regular", 443], ["en-Lovera", 444], ["en-Gellatio", 445], ["en-TitilliumWeb-Bold", 446], ["en-OilvareBase-Italic", 447], ["en-Catamaran-Black", 448], ["en-Anteb-Italic", 449], ["en-SueEllenFrancisco", 450], ["en-SweetApricot", 451], ["en-BrightSunshine", 452], ["en-IM_FELL_Double_Pica_Italic", 453], ["en-Granaina-limpia", 454], ["en-TANPARFAIT", 455], ["en-AcherusGrotesque-Regular", 456], ["en-AwesomeLathusca-Italic", 457], ["en-Signika-Bold", 458], ["en-Andasia", 459], ["en-DO-AllCaps-Slanted", 460], ["en-Zenaida-Regular", 461], ["en-Fahkwang-Regular", 462], ["en-Play-Regular", 463], ["en-BERNIERRegular-Regular", 464], ["en-PlumaThin-Regular", 465], ["en-SportsWorld", 466], ["en-Garet-Black", 467], ["en-CarolloPlayscript-BlackItalic", 468], ["en-Cheque-Regular", 469], ["en-SEGO", 470], ["en-BobbyJones-Condensed", 471], ["en-NexaSlab-RegularItalic", 472], ["en-DancingScript-Regular", 473], ["en-Magnolia-Script", 475], ["en-OpunMai-400It", 476], ["en-MadelynFill-Regular", 477], ["en-ZingRust-Base", 478], ["en-FingerPaint-Regular", 479], ["en-BostonAngel-Light", 480], ["en-Gliker-RegularExpanded", 481], ["en-Ahsing", 482], ["en-Engagement-Regular", 483], ["en-EyesomeScript", 484], ["en-LibraSerifModern-Regular", 485], ["en-London-Regular", 486], ["en-AtkinsonHyperlegible-Regular", 487], ["en-StadioNow-TextItalic", 488], ["en-Aniyah", 489], ["en-ITCAvantGardePro-Bold", 490], ["en-Comica-Regular", 491], ["en-Coustard-Regular", 492], ["en-Brice-BoldCondensed", 493], ["en-TANNEWYORK-Bold", 494], ["en-TANBUSTER-Bold", 495], ["en-Alatsi-Regular", 496], ["en-TYSerif-Book", 497], ["en-Jingleberry", 498], ["en-Rajdhani-Bold", 499], ["en-LobsterTwo-BoldItalic", 500], ["en-Hitchcut-Regular", 502], ["en-GermaniaOne-Regular", 503], ["en-Emitha-Script", 504], ["en-LemonTuesday", 505], ["en-MonterchiSerif-Regular", 507], ["en-AllertaStencil-Regular", 508], ["en-RTL-Sondos-Regular", 509], ["en-HomemadeApple-Regular", 510], ["en-CosmicOcto-Medium", 511]] \ No newline at end of file diff --git a/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/assets/multi_fonts/it.json b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/assets/multi_fonts/it.json new file mode 100644 index 0000000000000000000000000000000000000000..43f27d592b19a0667964f97dc19ddec42a6256e3 --- /dev/null +++ b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/assets/multi_fonts/it.json @@ -0,0 +1 @@ +[["en-Montserrat-Regular", 0], ["en-Poppins-Italic", 1], ["en-GlacialIndifference-Regular", 2], ["en-OpenSans-ExtraBoldItalic", 3], ["en-Montserrat-Bold", 4], ["en-Now-Regular", 5], ["en-Garet-Regular", 6], ["en-LeagueSpartan-Bold", 7], ["en-DMSans-Regular", 8], ["en-OpenSauceOne-Regular", 9], ["en-OpenSans-ExtraBold", 10], ["en-KGPrimaryPenmanship", 11], ["en-Anton-Regular", 12], ["en-Aileron-BlackItalic", 13], ["en-Quicksand-Light", 14], ["en-Roboto-BoldItalic", 15], ["en-TheSeasons-It", 16], ["en-Kollektif", 17], ["en-Inter-BoldItalic", 18], ["en-Poppins-Medium", 19], ["en-Poppins-Light", 20], ["en-RoxboroughCF-RegularItalic", 21], ["en-PlayfairDisplay-SemiBold", 22], ["en-Agrandir-Italic", 23], ["en-Lato-Regular", 24], ["en-MoreSugarRegular", 25], ["en-CanvaSans-RegularItalic", 26], ["en-PublicSans-Italic", 27], ["en-CodePro-NormalLC", 28], ["en-Belleza-Regular", 29], ["en-JosefinSans-Bold", 30], ["en-HKGrotesk-Bold", 31], ["en-Telegraf-Medium", 32], ["en-BrittanySignatureRegular", 33], ["en-Raleway-ExtraBoldItalic", 34], ["en-Mont-RegularItalic", 35], ["en-Arimo-BoldItalic", 36], ["en-Lora-Italic", 37], ["en-ArchivoBlack-Regular", 38], ["en-Poppins", 39], ["en-Barlow-Black", 40], ["en-CormorantGaramond-Bold", 41], ["en-LibreBaskerville-Regular", 42], ["en-BebasNeueBold", 44], ["en-LazydogRegular", 45], ["en-FredokaOne-Regular", 46], ["en-Horizon-Bold", 47], ["en-Nourd-Regular", 48], ["en-Hatton-Regular", 49], ["en-Nunito-ExtraBoldItalic", 50], ["en-CerebriSans-Regular", 51], ["en-Montserrat-Light", 52], ["en-TenorSans", 53], ["en-ClearSans-Bold", 55], ["en-Cardo-Regular", 56], ["en-Alice-Regular", 57], ["en-Oswald-Regular", 58], ["en-Muli-Black", 60], ["en-TAN-PEARL-Regular", 61], ["en-CooperHewitt-Book", 62], ["en-Agrandir-Grand", 63], ["en-BlackMango-Thin", 64], ["en-DMSerifDisplay-Regular", 65], ["en-Antonio-Bold", 66], ["en-Sniglet-Regular", 67], ["en-BeVietnam-Regular", 68], ["en-NunitoSans10pt-BlackItalic", 69], ["en-AbhayaLibre-ExtraBold", 70], ["en-Rubik-Regular", 71], ["en-PPNeueMachina-Regular", 72], ["en-TAN - MON CHERI-Regular", 73], ["en-Playlist-Script", 75], ["en-SourceSansPro-BoldItalic", 76], ["en-MoonTime-Regular", 77], ["en-Eczar-ExtraBold", 78], ["en-Gatwick-Regular", 79], ["en-BarlowSemiCondensed-Regular", 81], ["en-BarlowCondensed-Regular", 82], ["en-Alegreya-Regular", 83], ["en-DreamAvenue", 84], ["en-RobotoCondensed-Italic", 85], ["en-BobbyJones-Regular", 86], ["en-Garet-ExtraBold", 87], ["en-YesevaOne-Regular", 88], ["en-Dosis-ExtraBold", 89], ["en-LeagueGothic-Regular", 90], ["en-OpenSans-Italic", 91], ["en-TANAEGEAN-Regular", 92], ["en-Maharlika-Regular", 93], ["en-MarykateRegular", 94], ["en-Cinzel-Regular", 95], ["en-Agrandir-Wide", 96], ["en-Chewy-Regular", 97], ["en-BodoniFLF-BoldItalic", 98], ["en-Nunito-BlackItalic", 99], ["en-LilitaOne", 100], ["en-HandyCasualCondensed-Regular", 101], ["en-Ovo", 102], ["en-Livvic-Regular", 103], ["en-Agrandir-Narrow", 104], ["en-CrimsonPro-Italic", 105], ["en-AnonymousPro-Bold", 106], ["en-NF-OneLittleFont-Bold", 107], ["en-RedHatDisplay-BoldItalic", 108], ["en-CodecPro-Regular", 109], ["en-HalimunRegular", 110], ["en-LibreFranklin-Black", 111], ["en-TeXGyreTermes-BoldItalic", 112], ["en-Shrikhand-Regular", 113], ["en-TTNormsPro-Italic", 114], ["en-Gagalin-Regular", 115], ["en-OpenSans-Bold", 116], ["en-GreatVibes-Regular", 117], ["en-Breathing", 118], ["en-HeroLight-Regular", 119], ["en-KGPrimaryDots", 120], ["en-Quicksand-Bold", 121], ["en-Brice-ExtraLightSemiExpanded", 122], ["en-Lato-BoldItalic", 123], ["en-Fraunces9pt-Italic", 124], ["en-AbrilFatface-Regular", 125], ["en-BerkshireSwash-Regular", 126], ["en-Atma-Bold", 127], ["en-HolidayRegular", 128], ["en-BebasNeueCyrillic", 129], ["en-IntroRust-Base", 130], ["en-Gistesy", 131], ["en-BDScript-Regular", 132], ["en-ApricotsRegular", 133], ["en-Prompt-Black", 134], ["en-TAN MERINGUE", 135], ["en-GentySans-Regular", 137], ["en-NeueEinstellung-Normal", 138], ["en-Garet-Bold", 139], ["en-FiraSans-Black", 140], ["en-BantayogLight", 141], ["en-NotoSerifDisplay-Black", 142], ["en-TTChocolates-Regular", 143], ["en-Ubuntu-Regular", 144], ["en-Assistant-Bold", 145], ["en-ABeeZee-Regular", 146], ["en-LexendDeca-Regular", 147], ["en-KingredSerif", 148], ["en-Radley-Regular", 149], ["en-BrownSugar", 150], ["en-MigraItalic-ExtraboldItalic", 151], ["en-ChildosArabic-Regular", 152], ["en-PeaceSans", 153], ["en-LondrinaSolid-Black", 154], ["en-SpaceMono-BoldItalic", 155], ["en-RobotoMono-Light", 156], ["en-CourierPrime-Regular", 157], ["en-Alata-Regular", 158], ["en-Amsterdam-One", 159], ["en-IreneFlorentina-Regular", 160], ["en-CatchyMager", 161], ["en-Alta_regular", 162], ["en-ArticulatCF-Regular", 163], ["en-Raleway-Regular", 164], ["en-BrasikaDisplay", 165], ["en-TANAngleton-Italic", 166], ["en-NotoSerifDisplay-ExtraCondensedItalic", 167], ["en-Bryndan Write", 168], ["en-TTCommonsPro-It", 169], ["en-AlexBrush-Regular", 170], ["en-Antic-Regular", 171], ["en-TTHoves-Bold", 172], ["en-DroidSerif", 173], ["en-AblationRegular", 174], ["en-Marcellus-Regular", 175], ["en-Sanchez-Italic", 176], ["en-JosefinSans", 177], ["en-Afrah-Regular", 178], ["en-PinyonScript", 179], ["en-TTInterphases-BoldItalic", 180], ["en-Yellowtail-Regular", 181], ["en-Gliker-Regular", 182], ["en-BobbyJonesSoft-Regular", 183], ["en-IBMPlexSans", 184], ["en-Amsterdam-Three", 185], ["en-Amsterdam-FourSlant", 186], ["en-TTFors-Regular", 187], ["en-Quattrocento", 188], ["en-Sifonn-Basic", 189], ["en-AlegreyaSans-Black", 190], ["en-Daydream", 191], ["en-AristotelicaProTx-Rg", 192], ["en-NotoSerif", 193], ["en-EBGaramond-Italic", 194], ["en-HammersmithOne-Regular", 195], ["en-RobotoSlab-Regular", 196], ["en-DO-Sans-Regular", 197], ["en-KGPrimaryDotsLined", 198], ["en-Blinker-Regular", 199], ["en-TAN NIMBUS", 200], ["en-Blueberry-Regular", 201], ["en-Rosario-Regular", 202], ["en-Forum", 203], ["en-MistrullyRegular", 204], ["en-SourceSerifPro-Regular", 205], ["en-Bugaki-Regular", 206], ["en-CMUSerif-Roman", 207], ["en-GulfsDisplay-NormalItalic", 208], ["en-PTSans-Bold", 209], ["en-Sensei-Medium", 210], ["en-SquadaOne-Regular", 211], ["en-Arapey-Italic", 212], ["en-Parisienne-Regular", 213], ["en-Aleo-Italic", 214], ["en-QuicheDisplay-Italic", 215], ["en-RocaOne-It", 216], ["en-Funtastic-Regular", 217], ["en-PTSerif-BoldItalic", 218], ["en-Muller-RegularItalic", 219], ["en-ArgentCF-Regular", 220], ["en-Brightwall-Italic", 221], ["en-Knewave-Regular", 222], ["en-TYSerif-D", 223], ["en-Agrandir-Tight", 224], ["en-AlfaSlabOne-Regular", 225], ["en-TANTangkiwood-Display", 226], ["en-Kief-Montaser-Regular", 227], ["en-Gotham-Book", 228], ["en-JuliusSansOne-Regular", 229], ["en-CocoGothic-Italic", 230], ["en-SairaCondensed-Regular", 231], ["en-DellaRespira-Regular", 232], ["en-Questrial-Regular", 233], ["en-BukhariScript-Regular", 234], ["en-HelveticaWorld-Bold", 235], ["en-TANKINDRED-Display", 236], ["en-CinzelDecorative-Regular", 237], ["en-Vidaloka-Regular", 238], ["en-AlegreyaSansSC-Black", 239], ["en-FeelingPassionate-Regular", 240], ["en-QuincyCF-Regular", 241], ["en-FiraCode-Regular", 242], ["en-Genty-Regular", 243], ["en-Nickainley-Normal", 244], ["en-RubikOne-Regular", 245], ["en-Gidole-Regular", 246], ["en-Borsok", 247], ["en-Gordita-RegularItalic", 248], ["en-Scripter-Regular", 249], ["en-Buffalo-Regular", 250], ["en-KleinText-Regular", 251], ["en-Creepster-Regular", 252], ["en-Arvo-Bold", 253], ["en-GabrielSans-NormalItalic", 254], ["en-Heebo-Black", 255], ["en-LexendExa-Regular", 256], ["en-BrixtonSansTC-Regular", 257], ["en-GildaDisplay-Regular", 258], ["en-ChunkFive-Roman", 259], ["en-Amaranth-BoldItalic", 260], ["en-BubbleboddyNeue-Regular", 261], ["en-MavenPro-Bold", 262], ["en-TTDrugs-Italic", 263], ["en-CyGrotesk-KeyRegular", 264], ["en-VarelaRound-Regular", 265], ["en-Ruda-Black", 266], ["en-SafiraMarch", 267], ["en-BloggerSans", 268], ["en-TANHEADLINE-Regular", 269], ["en-SloopScriptPro-Regular", 270], ["en-NeueMontreal-Regular", 271], ["en-Schoolbell-Regular", 272], ["en-SigherRegular", 273], ["en-InriaSerif-Regular", 274], ["en-JetBrainsMono-Regular", 275], ["en-MADEEvolveSans", 276], ["en-Dekko", 277], ["en-Handyman-Regular", 278], ["en-Aileron-BoldItalic", 279], ["en-Bright-Italic", 280], ["en-Solway-Regular", 281], ["en-Higuen-Regular", 282], ["en-WedgesItalic", 283], ["en-TANASHFORD-BOLD", 284], ["en-IBMPlexMono", 285], ["en-RacingSansOne-Regular", 286], ["en-RegularBrush", 287], ["en-OpenSans-LightItalic", 288], ["en-SpecialElite-Regular", 289], ["en-FuturaLTPro-Medium", 290], ["en-MaragsaDisplay", 291], ["en-BigShouldersDisplay-Regular", 292], ["en-BDSans-Regular", 293], ["en-RasputinRegular", 294], ["en-Yvesyvesdrawing-BoldItalic", 295], ["en-Bitter-Regular", 296], ["en-LuckiestGuy-Regular", 297], ["en-TTFirsNeue-Italic", 299], ["en-Sunday-Regular", 300], ["en-HKGothic-MediumItalic", 301], ["en-CaveatBrush-Regular", 302], ["en-ArchitectsDaughter-Regular", 304], ["en-Angelina", 305], ["en-Calistoga-Regular", 306], ["en-ArchivoNarrow-Regular", 307], ["en-ObjectSans-MediumSlanted", 308], ["en-AyrLucidityCondensed-Regular", 309], ["en-Nexa-RegularItalic", 310], ["en-Lustria-Regular", 311], ["en-Amsterdam-TwoSlant", 312], ["en-Virtual-Regular", 313], ["en-NF-Lepetitcochon-Regular", 315], ["en-TANTWINKLE", 316], ["en-LeJour-Serif", 317], ["en-Prata-Regular", 318], ["en-PPWoodland-Regular", 319], ["en-PlayfairDisplay-BoldItalic", 320], ["en-AmaticSC-Regular", 321], ["en-Cabin-Regular", 322], ["en-Manjari-Bold", 323], ["en-MrDafoe-Regular", 324], ["en-TTRamillas-Italic", 325], ["en-Luckybones-Bold", 326], ["en-DarkerGrotesque-Light", 327], ["en-BellabooRegular", 328], ["en-CormorantSC-Bold", 329], ["en-GochiHand-Regular", 330], ["en-Atteron", 331], ["en-RocaTwo-Lt", 332], ["en-TANSONGBIRD", 334], ["en-HeadingNow-74Regular", 335], ["en-Luthier-BoldItalic", 336], ["en-Oregano-Regular", 337], ["en-AyrTropikaIsland-Int", 338], ["en-Mali-Regular", 339], ["en-DidactGothic-Regular", 340], ["en-Lovelace-Regular", 341], ["en-BakerieSmooth-Regular", 342], ["en-CarterOne", 343], ["en-HussarBd", 344], ["en-OldStandard-Italic", 345], ["en-TAN-ASTORIA-Display", 346], ["en-rugratssans-Regular", 347], ["en-BetterSaturday", 349], ["en-AdigianaToybox", 350], ["en-Sailors", 351], ["en-PlayfairDisplaySC-Italic", 352], ["en-Etna-Regular", 353], ["en-Revive80Signature", 354], ["en-CAGenerated", 355], ["en-Poppins-Regular", 356], ["en-Jonathan-Regular", 357], ["en-Pacifico-Regular", 358], ["en-Saira-Black", 359], ["en-Loubag-Regular", 360], ["en-Decalotype-Black", 361], ["en-Mansalva-Regular", 362], ["en-Allura-Regular", 363], ["en-ProximaNova-Bold", 364], ["en-TANMIGNON-DISPLAY", 365], ["en-ArsenicaAntiqua-Regular", 366], ["en-BreulGroteskA-RegularItalic", 367], ["en-HKModular-Bold", 368], ["en-TANNightingale-Regular", 369], ["en-AristotelicaProCndTxt-Rg", 370], ["en-Aprila-Regular", 371], ["en-Tomorrow-Regular", 372], ["en-AngellaWhite", 373], ["en-KaushanScript-Regular", 374], ["en-NotoSans", 375], ["en-LeJour-Script", 376], ["en-BrixtonTC-Regular", 377], ["en-OleoScript-Regular", 378], ["en-Cakerolli-Regular", 379], ["en-Lobster-Regular", 380], ["en-FrunchySerif-Regular", 381], ["en-PorcelainRegular", 382], ["en-AlojaExtended", 383], ["en-SergioTrendy-Italic", 384], ["en-LovelaceText-Bold", 385], ["en-Anaktoria", 386], ["en-JimmyScript-Light", 387], ["en-IBMPlexSerif", 388], ["en-Marta", 389], ["en-Mango-Regular", 390], ["en-Overpass-Italic", 391], ["en-Hagrid-Regular", 392], ["en-ElikaGorica", 393], ["en-Amiko-Regular", 394], ["en-EFCOBrookshire-Regular", 395], ["en-Caladea-Regular", 396], ["en-MoonlightBold", 397], ["en-Staatliches-Regular", 398], ["en-Helios-Bold", 399], ["en-Satisfy-Regular", 400], ["en-NexaScript-Regular", 401], ["en-Trocchi-Regular", 402], ["en-March", 403], ["en-IbarraRealNova-Regular", 404], ["en-Nectarine-Regular", 405], ["en-Overpass-Light", 406], ["en-TruetypewriterPolyglOTT", 407], ["en-Bangers-Regular", 408], ["en-Lazord-BoldExpandedItalic", 409], ["en-Chloe-Regular", 410], ["en-BaskervilleDisplayPT-Regular", 411], ["en-Bright-Regular", 412], ["en-Vollkorn-Regular", 413], ["en-Harmattan", 414], ["en-SortsMillGoudy-Regular", 415], ["en-Biryani-Bold", 416], ["en-SugoProDisplay-Italic", 417], ["en-Lazord-BoldItalic", 418], ["en-Alike-Regular", 419], ["en-PermanentMarker-Regular", 420], ["en-Sacramento-Regular", 421], ["en-HKGroteskPro-Italic", 422], ["en-Aleo-BoldItalic", 423], ["en-TANGARLAND-Regular", 425], ["en-Twister", 426], ["en-Arsenal-Italic", 427], ["en-Bogart-Italic", 428], ["en-BethEllen-Regular", 429], ["en-Caveat-Regular", 430], ["en-BalsamiqSans-Bold", 431], ["en-BreeSerif-Regular", 432], ["en-CodecPro-ExtraBold", 433], ["en-Pierson-Light", 434], ["en-CyGrotesk-WideRegular", 435], ["en-Lumios-Marker", 436], ["en-Comfortaa-Bold", 437], ["en-TraceFontRegular", 438], ["en-RTL-AdamScript-Regular", 439], ["en-EastmanGrotesque-Italic", 440], ["en-Kalam-Bold", 441], ["en-ChauPhilomeneOne-Regular", 442], ["en-Coiny-Regular", 443], ["en-Lovera", 444], ["en-Gellatio", 445], ["en-TitilliumWeb-Bold", 446], ["en-OilvareBase-Italic", 447], ["en-Catamaran-Black", 448], ["en-Anteb-Italic", 449], ["en-SueEllenFrancisco", 450], ["en-SweetApricot", 451], ["en-BrightSunshine", 452], ["en-IM_FELL_Double_Pica_Italic", 453], ["en-Granaina-limpia", 454], ["en-TANPARFAIT", 455], ["en-AcherusGrotesque-Regular", 456], ["en-AwesomeLathusca-Italic", 457], ["en-Signika-Bold", 458], ["en-Andasia", 459], ["en-DO-AllCaps-Slanted", 460], ["en-Zenaida-Regular", 461], ["en-Fahkwang-Regular", 462], ["en-Play-Regular", 463], ["en-BERNIERRegular-Regular", 464], ["en-PlumaThin-Regular", 465], ["en-SportsWorld", 466], ["en-Garet-Black", 467], ["en-CarolloPlayscript-BlackItalic", 468], ["en-Cheque-Regular", 469], ["en-SEGO", 470], ["en-BobbyJones-Condensed", 471], ["en-NexaSlab-RegularItalic", 472], ["en-DancingScript-Regular", 473], ["en-Magnolia-Script", 475], ["en-OpunMai-400It", 476], ["en-MadelynFill-Regular", 477], ["en-ZingRust-Base", 478], ["en-FingerPaint-Regular", 479], ["en-BostonAngel-Light", 480], ["en-Gliker-RegularExpanded", 481], ["en-Ahsing", 482], ["en-Engagement-Regular", 483], ["en-EyesomeScript", 484], ["en-LibraSerifModern-Regular", 485], ["en-London-Regular", 486], ["en-AtkinsonHyperlegible-Regular", 487], ["en-StadioNow-TextItalic", 488], ["en-Aniyah", 489], ["en-ITCAvantGardePro-Bold", 490], ["en-Comica-Regular", 491], ["en-Coustard-Regular", 492], ["en-Brice-BoldCondensed", 493], ["en-TANNEWYORK-Bold", 494], ["en-TANBUSTER-Bold", 495], ["en-Alatsi-Regular", 496], ["en-TYSerif-Book", 497], ["en-Jingleberry", 498], ["en-Rajdhani-Bold", 499], ["en-LobsterTwo-BoldItalic", 500], ["en-BestLight-Medium", 501], ["en-Hitchcut-Regular", 502], ["en-GermaniaOne-Regular", 503], ["en-Emitha-Script", 504], ["en-LemonTuesday", 505], ["en-MonterchiSerif-Regular", 507], ["en-AllertaStencil-Regular", 508], ["en-RTL-Sondos-Regular", 509], ["en-HomemadeApple-Regular", 510], ["en-CosmicOcto-Medium", 511]] \ No newline at end of file diff --git a/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/assets/multi_fonts/jp.json b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/assets/multi_fonts/jp.json new file mode 100644 index 0000000000000000000000000000000000000000..7a94ec5373069c0e5ef2e8b1820092ece251703e --- /dev/null +++ b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/assets/multi_fonts/jp.json @@ -0,0 +1 @@ +[["jp-04KanjyukuGothic", 0], ["jp-07LightNovelPOP", 1], ["jp-07NikumaruFont", 2], ["jp-07YasashisaAntique", 3], ["jp-07YasashisaGothic", 4], ["jp-BokutachinoGothic2Bold", 5], ["jp-BokutachinoGothic2Regular", 6], ["jp-CHI_SpeedyRight_full_211128-Regular", 7], ["jp-CHI_SpeedyRight_italic_full_211127-Regular", 8], ["jp-CP-Font", 9], ["jp-Canva_CezanneProN-B", 10], ["jp-Canva_CezanneProN-M", 11], ["jp-Canva_ChiaroStd-B", 12], ["jp-Canva_CometStd-B", 13], ["jp-Canva_DotMincho16Std-M", 14], ["jp-Canva_GrecoStd-B", 15], ["jp-Canva_GrecoStd-M", 16], ["jp-Canva_LyraStd-DB", 17], ["jp-Canva_MatisseHatsuhiPro-B", 18], ["jp-Canva_MatisseHatsuhiPro-M", 19], ["jp-Canva_ModeMinAStd-B", 20], ["jp-Canva_NewCezanneProN-B", 21], ["jp-Canva_NewCezanneProN-M", 22], ["jp-Canva_PearlStd-L", 23], ["jp-Canva_RaglanStd-UB", 24], ["jp-Canva_RailwayStd-B", 25], ["jp-Canva_ReggaeStd-B", 26], ["jp-Canva_RocknRollStd-DB", 27], ["jp-Canva_RodinCattleyaPro-B", 28], ["jp-Canva_RodinCattleyaPro-M", 29], ["jp-Canva_RodinCattleyaPro-UB", 30], ["jp-Canva_RodinHimawariPro-B", 31], ["jp-Canva_RodinHimawariPro-M", 32], ["jp-Canva_RodinMariaPro-B", 33], ["jp-Canva_RodinMariaPro-DB", 34], ["jp-Canva_RodinProN-M", 35], ["jp-Canva_ShadowTLStd-B", 36], ["jp-Canva_StickStd-B", 37], ["jp-Canva_TsukuAOldMinPr6N-B", 38], ["jp-Canva_TsukuAOldMinPr6N-R", 39], ["jp-Canva_UtrilloPro-DB", 40], ["jp-Canva_UtrilloPro-M", 41], ["jp-Canva_YurukaStd-UB", 42], ["jp-FGUIGEN", 43], ["jp-GlowSansJ-Condensed-Heavy", 44], ["jp-GlowSansJ-Condensed-Light", 45], ["jp-GlowSansJ-Normal-Bold", 46], ["jp-GlowSansJ-Normal-Light", 47], ["jp-HannariMincho", 48], ["jp-HarenosoraMincho", 49], ["jp-Jiyucho", 50], ["jp-Kaiso-Makina-B", 51], ["jp-Kaisotai-Next-UP-B", 52], ["jp-KokoroMinchoutai", 53], ["jp-Mamelon-3-Hi-Regular", 54], ["jp-MotoyaAnemoneStd-W1", 55], ["jp-MotoyaAnemoneStd-W5", 56], ["jp-MotoyaAnticPro-W3", 57], ["jp-MotoyaCedarStd-W3", 58], ["jp-MotoyaCedarStd-W5", 59], ["jp-MotoyaGochikaStd-W4", 60], ["jp-MotoyaGochikaStd-W8", 61], ["jp-MotoyaGothicMiyabiStd-W6", 62], ["jp-MotoyaGothicStd-W3", 63], ["jp-MotoyaGothicStd-W5", 64], ["jp-MotoyaKoinStd-W3", 65], ["jp-MotoyaKyotaiStd-W2", 66], ["jp-MotoyaKyotaiStd-W4", 67], ["jp-MotoyaMaruStd-W3", 68], ["jp-MotoyaMaruStd-W5", 69], ["jp-MotoyaMinchoMiyabiStd-W4", 70], ["jp-MotoyaMinchoMiyabiStd-W6", 71], ["jp-MotoyaMinchoModernStd-W4", 72], ["jp-MotoyaMinchoModernStd-W6", 73], ["jp-MotoyaMinchoStd-W3", 74], ["jp-MotoyaMinchoStd-W5", 75], ["jp-MotoyaReisyoStd-W2", 76], ["jp-MotoyaReisyoStd-W6", 77], ["jp-MotoyaTohitsuStd-W4", 78], ["jp-MotoyaTohitsuStd-W6", 79], ["jp-MtySousyokuEmBcJis-W6", 80], ["jp-MtySousyokuLiBcJis-W6", 81], ["jp-Mushin", 82], ["jp-NotoSansJP-Bold", 83], ["jp-NotoSansJP-Regular", 84], ["jp-NudMotoyaAporoStd-W3", 85], ["jp-NudMotoyaAporoStd-W5", 86], ["jp-NudMotoyaCedarStd-W3", 87], ["jp-NudMotoyaCedarStd-W5", 88], ["jp-NudMotoyaMaruStd-W3", 89], ["jp-NudMotoyaMaruStd-W5", 90], ["jp-NudMotoyaMinchoStd-W5", 91], ["jp-Ounen-mouhitsu", 92], ["jp-Ronde-B-Square", 93], ["jp-SMotoyaGyosyoStd-W5", 94], ["jp-SMotoyaSinkaiStd-W3", 95], ["jp-SMotoyaSinkaiStd-W5", 96], ["jp-SourceHanSansJP-Bold", 97], ["jp-SourceHanSansJP-Regular", 98], ["jp-SourceHanSerifJP-Bold", 99], ["jp-SourceHanSerifJP-Regular", 100], ["jp-TazuganeGothicStdN-Bold", 101], ["jp-TazuganeGothicStdN-Regular", 102], ["jp-TelopMinProN-B", 103], ["jp-Togalite-Bold", 104], ["jp-Togalite-Regular", 105], ["jp-TsukuMinPr6N-E", 106], ["jp-TsukuMinPr6N-M", 107], ["jp-mikachan_o", 108], ["jp-nagayama_kai", 109], ["jp-07LogoTypeGothic7", 110], ["jp-07TetsubinGothic", 111], ["jp-851CHIKARA-DZUYOKU-KANA-A", 112], ["jp-ARMinchoJIS-Light", 113], ["jp-ARMinchoJIS-Ultra", 114], ["jp-ARPCrystalMinchoJIS-Medium", 115], ["jp-ARPCrystalRGothicJIS-Medium", 116], ["jp-ARShounanShinpitsuGyosyoJIS-Medium", 117], ["jp-AozoraMincho-bold", 118], ["jp-AozoraMinchoRegular", 119], ["jp-ArialUnicodeMS-Bold", 120], ["jp-ArialUnicodeMS", 121], ["jp-CanvaBreezeJP", 122], ["jp-CanvaLiCN", 123], ["jp-CanvaLiJP", 124], ["jp-CanvaOrientalBrushCN", 125], ["jp-CanvaQinfuCalligraphyJP", 126], ["jp-CanvaSweetHeartJP", 127], ["jp-CanvaWenJP", 128], ["jp-Corporate-Logo-Bold", 129], ["jp-DelaGothicOne-Regular", 130], ["jp-GN-Kin-iro_SansSerif", 131], ["jp-GN-Koharuiro_Sunray", 132], ["jp-GenEiGothicM-B", 133], ["jp-GenEiGothicM-R", 134], ["jp-GenJyuuGothic-Bold", 135], ["jp-GenRyuMinTW-B", 136], ["jp-GenRyuMinTW-R", 137], ["jp-GenSekiGothicTW-B", 138], ["jp-GenSekiGothicTW-R", 139], ["jp-GenSenRoundedTW-B", 140], ["jp-GenSenRoundedTW-R", 141], ["jp-GenShinGothic-Bold", 142], ["jp-GenShinGothic-Normal", 143], ["jp-GenWanMinTW-L", 144], ["jp-GenYoGothicTW-B", 145], ["jp-GenYoGothicTW-R", 146], ["jp-GenYoMinTW-B", 147], ["jp-GenYoMinTW-R", 148], ["jp-HGBouquet", 149], ["jp-HanaMinA", 150], ["jp-HanazomeFont", 151], ["jp-HinaMincho-Regular", 152], ["jp-Honoka-Antique-Maru", 153], ["jp-Honoka-Mincho", 154], ["jp-HuiFontP", 155], ["jp-IPAexMincho", 156], ["jp-JK-Gothic-L", 157], ["jp-JK-Gothic-M", 158], ["jp-JackeyFont", 159], ["jp-KaiseiTokumin-Bold", 160], ["jp-KaiseiTokumin-Regular", 161], ["jp-Keifont", 162], ["jp-KiwiMaru-Regular", 163], ["jp-Koku-Mincho-Regular", 164], ["jp-MotoyaLMaru-W3-90ms-RKSJ-H", 165], ["jp-NewTegomin-Regular", 166], ["jp-NicoKaku", 167], ["jp-NicoMoji+", 168], ["jp-Otsutome_font-Bold", 169], ["jp-PottaOne-Regular", 170], ["jp-RampartOne-Regular", 171], ["jp-Senobi-Gothic-Bold", 172], ["jp-Senobi-Gothic-Regular", 173], ["jp-SmartFontUI-Proportional", 174], ["jp-SoukouMincho", 175], ["jp-TEST_Klee-DB", 176], ["jp-TEST_Klee-M", 177], ["jp-TEST_UDMincho-B", 178], ["jp-TEST_UDMincho-L", 179], ["jp-TT_Akakane-EB", 180], ["jp-Tanuki-Permanent-Marker", 181], ["jp-TrainOne-Regular", 182], ["jp-TsunagiGothic-Black", 183], ["jp-Ume-Hy-Gothic", 184], ["jp-Ume-P-Mincho", 185], ["jp-WenQuanYiMicroHei", 186], ["jp-XANO-mincho-U32", 187], ["jp-YOzFontM90-Regular", 188], ["jp-Yomogi-Regular", 189], ["jp-YujiBoku-Regular", 190], ["jp-YujiSyuku-Regular", 191], ["jp-ZenKakuGothicNew-Bold", 192], ["jp-ZenKakuGothicNew-Regular", 193], ["jp-ZenKurenaido-Regular", 194], ["jp-ZenMaruGothic-Bold", 195], ["jp-ZenMaruGothic-Regular", 196], ["jp-darts-font", 197], ["jp-irohakakuC-Bold", 198], ["jp-irohakakuC-Medium", 199], ["jp-irohakakuC-Regular", 200], ["jp-katyou", 201], ["jp-mplus-1m-bold", 202], ["jp-mplus-1m-regular", 203], ["jp-mplus-1p-bold", 204], ["jp-mplus-1p-regular", 205], ["jp-rounded-mplus-1p-bold", 206], ["jp-rounded-mplus-1p-regular", 207], ["jp-timemachine-wa", 208], ["jp-ttf-GenEiLateMin-Medium", 209], ["jp-uzura_font", 210]] \ No newline at end of file diff --git a/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/assets/multi_fonts/kr.json b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/assets/multi_fonts/kr.json new file mode 100644 index 0000000000000000000000000000000000000000..778c949347801d04958d600f7334f60f01bd181f --- /dev/null +++ b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/assets/multi_fonts/kr.json @@ -0,0 +1 @@ +[["kr-Arita-buri-Bold_OTF", 0], ["kr-Arita-buri-HairLine_OTF", 1], ["kr-Arita-buri-Light_OTF", 2], ["kr-Arita-buri-Medium_OTF", 3], ["kr-Arita-buri-SemiBold_OTF", 4], ["kr-Canva_YDSunshineL", 5], ["kr-Canva_YDSunshineM", 6], ["kr-Canva_YoonGulimPro710", 7], ["kr-Canva_YoonGulimPro730", 8], ["kr-Canva_YoonGulimPro740", 9], ["kr-Canva_YoonGulimPro760", 10], ["kr-Canva_YoonGulimPro770", 11], ["kr-Canva_YoonGulimPro790", 12], ["kr-CreHappB", 13], ["kr-CreHappL", 14], ["kr-CreHappM", 15], ["kr-CreHappS", 16], ["kr-OTAuroraB", 17], ["kr-OTAuroraL", 18], ["kr-OTAuroraR", 19], ["kr-OTDoldamgilB", 20], ["kr-OTDoldamgilL", 21], ["kr-OTDoldamgilR", 22], ["kr-OTHamsterB", 23], ["kr-OTHamsterL", 24], ["kr-OTHamsterR", 25], ["kr-OTHapchangdanB", 26], ["kr-OTHapchangdanL", 27], ["kr-OTHapchangdanR", 28], ["kr-OTSupersizeBkBOX", 29], ["kr-SourceHanSansKR-Bold", 30], ["kr-SourceHanSansKR-ExtraLight", 31], ["kr-SourceHanSansKR-Heavy", 32], ["kr-SourceHanSansKR-Light", 33], ["kr-SourceHanSansKR-Medium", 34], ["kr-SourceHanSansKR-Normal", 35], ["kr-SourceHanSansKR-Regular", 36], ["kr-SourceHanSansSC-Bold", 37], ["kr-SourceHanSansSC-ExtraLight", 38], ["kr-SourceHanSansSC-Heavy", 39], ["kr-SourceHanSansSC-Light", 40], ["kr-SourceHanSansSC-Medium", 41], ["kr-SourceHanSansSC-Normal", 42], ["kr-SourceHanSansSC-Regular", 43], ["kr-SourceHanSerifSC-Bold", 44], ["kr-SourceHanSerifSC-SemiBold", 45], ["kr-TDTDBubbleBubbleOTF", 46], ["kr-TDTDConfusionOTF", 47], ["kr-TDTDCuteAndCuteOTF", 48], ["kr-TDTDEggTakOTF", 49], ["kr-TDTDEmotionalLetterOTF", 50], ["kr-TDTDGalapagosOTF", 51], ["kr-TDTDHappyHourOTF", 52], ["kr-TDTDLatteOTF", 53], ["kr-TDTDMoonLightOTF", 54], ["kr-TDTDParkForestOTF", 55], ["kr-TDTDPencilOTF", 56], ["kr-TDTDSmileOTF", 57], ["kr-TDTDSproutOTF", 58], ["kr-TDTDSunshineOTF", 59], ["kr-TDTDWaferOTF", 60], ["kr-777Chyaochyureu", 61], ["kr-ArialUnicodeMS-Bold", 62], ["kr-ArialUnicodeMS", 63], ["kr-BMHANNA", 64], ["kr-Baekmuk-Dotum", 65], ["kr-BagelFatOne-Regular", 66], ["kr-CoreBandi", 67], ["kr-CoreBandiFace", 68], ["kr-CoreBori", 69], ["kr-DoHyeon-Regular", 70], ["kr-Dokdo-Regular", 71], ["kr-Gaegu-Bold", 72], ["kr-Gaegu-Light", 73], ["kr-Gaegu-Regular", 74], ["kr-GamjaFlower-Regular", 75], ["kr-GasoekOne-Regular", 76], ["kr-GothicA1-Black", 77], ["kr-GothicA1-Bold", 78], ["kr-GothicA1-ExtraBold", 79], ["kr-GothicA1-ExtraLight", 80], ["kr-GothicA1-Light", 81], ["kr-GothicA1-Medium", 82], ["kr-GothicA1-Regular", 83], ["kr-GothicA1-SemiBold", 84], ["kr-GothicA1-Thin", 85], ["kr-Gugi-Regular", 86], ["kr-HiMelody-Regular", 87], ["kr-Jua-Regular", 88], ["kr-KirangHaerang-Regular", 89], ["kr-NanumBrush", 90], ["kr-NanumPen", 91], ["kr-NanumSquareRoundB", 92], ["kr-NanumSquareRoundEB", 93], ["kr-NanumSquareRoundL", 94], ["kr-NanumSquareRoundR", 95], ["kr-SeH-CB", 96], ["kr-SeH-CBL", 97], ["kr-SeH-CEB", 98], ["kr-SeH-CL", 99], ["kr-SeH-CM", 100], ["kr-SeN-CB", 101], ["kr-SeN-CBL", 102], ["kr-SeN-CEB", 103], ["kr-SeN-CL", 104], ["kr-SeN-CM", 105], ["kr-Sunflower-Bold", 106], ["kr-Sunflower-Light", 107], ["kr-Sunflower-Medium", 108], ["kr-TTClaytoyR", 109], ["kr-TTDalpangiR", 110], ["kr-TTMamablockR", 111], ["kr-TTNauidongmuR", 112], ["kr-TTOktapbangR", 113], ["kr-UhBeeMiMi", 114], ["kr-UhBeeMiMiBold", 115], ["kr-UhBeeSe_hyun", 116], ["kr-UhBeeSe_hyunBold", 117], ["kr-UhBeenamsoyoung", 118], ["kr-UhBeenamsoyoungBold", 119], ["kr-WenQuanYiMicroHei", 120], ["kr-YeonSung-Regular", 121]] \ No newline at end of file diff --git a/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/assets/multi_fonts/pt.json b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/assets/multi_fonts/pt.json new file mode 100644 index 0000000000000000000000000000000000000000..00962ab5bbcf087c9ae8d691a99b21f5797275b7 --- /dev/null +++ b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/assets/multi_fonts/pt.json @@ -0,0 +1 @@ +[["en-Montserrat-Regular", 0], ["en-Poppins-Italic", 1], ["en-GlacialIndifference-Regular", 2], ["en-OpenSans-ExtraBoldItalic", 3], ["en-Montserrat-Bold", 4], ["en-Now-Regular", 5], ["en-Garet-Regular", 6], ["en-LeagueSpartan-Bold", 7], ["en-DMSans-Regular", 8], ["en-OpenSauceOne-Regular", 9], ["en-OpenSans-ExtraBold", 10], ["en-KGPrimaryPenmanship", 11], ["en-Anton-Regular", 12], ["en-Aileron-BlackItalic", 13], ["en-Quicksand-Light", 14], ["en-Roboto-BoldItalic", 15], ["en-TheSeasons-It", 16], ["en-Kollektif", 17], ["en-Inter-BoldItalic", 18], ["en-Poppins-Medium", 19], ["en-Poppins-Light", 20], ["en-RoxboroughCF-RegularItalic", 21], ["en-PlayfairDisplay-SemiBold", 22], ["en-Agrandir-Italic", 23], ["en-Lato-Regular", 24], ["en-MoreSugarRegular", 25], ["en-CanvaSans-RegularItalic", 26], ["en-PublicSans-Italic", 27], ["en-CodePro-NormalLC", 28], ["en-Belleza-Regular", 29], ["en-JosefinSans-Bold", 30], ["en-HKGrotesk-Bold", 31], ["en-Telegraf-Medium", 32], ["en-BrittanySignatureRegular", 33], ["en-Raleway-ExtraBoldItalic", 34], ["en-Mont-RegularItalic", 35], ["en-Arimo-BoldItalic", 36], ["en-Lora-Italic", 37], ["en-ArchivoBlack-Regular", 38], ["en-Poppins", 39], ["en-Barlow-Black", 40], ["en-CormorantGaramond-Bold", 41], ["en-LibreBaskerville-Regular", 42], ["en-BebasNeueBold", 44], ["en-LazydogRegular", 45], ["en-FredokaOne-Regular", 46], ["en-Horizon-Bold", 47], ["en-Nourd-Regular", 48], ["en-Hatton-Regular", 49], ["en-Nunito-ExtraBoldItalic", 50], ["en-CerebriSans-Regular", 51], ["en-Montserrat-Light", 52], ["en-TenorSans", 53], ["en-ClearSans-Bold", 55], ["en-Cardo-Regular", 56], ["en-Alice-Regular", 57], ["en-Oswald-Regular", 58], ["en-Muli-Black", 60], ["en-TAN-PEARL-Regular", 61], ["en-CooperHewitt-Book", 62], ["en-Agrandir-Grand", 63], ["en-BlackMango-Thin", 64], ["en-DMSerifDisplay-Regular", 65], ["en-Antonio-Bold", 66], ["en-Sniglet-Regular", 67], ["en-BeVietnam-Regular", 68], ["en-NunitoSans10pt-BlackItalic", 69], ["en-AbhayaLibre-ExtraBold", 70], ["en-Rubik-Regular", 71], ["en-PPNeueMachina-Regular", 72], ["en-TAN - MON CHERI-Regular", 73], ["en-Playlist-Script", 75], ["en-SourceSansPro-BoldItalic", 76], ["en-MoonTime-Regular", 77], ["en-Eczar-ExtraBold", 78], ["en-Gatwick-Regular", 79], ["en-MonumentExtended-Regular", 80], ["en-BarlowSemiCondensed-Regular", 81], ["en-BarlowCondensed-Regular", 82], ["en-Alegreya-Regular", 83], ["en-DreamAvenue", 84], ["en-RobotoCondensed-Italic", 85], ["en-BobbyJones-Regular", 86], ["en-Garet-ExtraBold", 87], ["en-YesevaOne-Regular", 88], ["en-Dosis-ExtraBold", 89], ["en-LeagueGothic-Regular", 90], ["en-OpenSans-Italic", 91], ["en-TANAEGEAN-Regular", 92], ["en-Maharlika-Regular", 93], ["en-MarykateRegular", 94], ["en-Cinzel-Regular", 95], ["en-Agrandir-Wide", 96], ["en-Chewy-Regular", 97], ["en-BodoniFLF-BoldItalic", 98], ["en-Nunito-BlackItalic", 99], ["en-LilitaOne", 100], ["en-HandyCasualCondensed-Regular", 101], ["en-Ovo", 102], ["en-Livvic-Regular", 103], ["en-Agrandir-Narrow", 104], ["en-CrimsonPro-Italic", 105], ["en-AnonymousPro-Bold", 106], ["en-NF-OneLittleFont-Bold", 107], ["en-RedHatDisplay-BoldItalic", 108], ["en-CodecPro-Regular", 109], ["en-HalimunRegular", 110], ["en-LibreFranklin-Black", 111], ["en-TeXGyreTermes-BoldItalic", 112], ["en-Shrikhand-Regular", 113], ["en-TTNormsPro-Italic", 114], ["en-Gagalin-Regular", 115], ["en-OpenSans-Bold", 116], ["en-GreatVibes-Regular", 117], ["en-Breathing", 118], ["en-HeroLight-Regular", 119], ["en-KGPrimaryDots", 120], ["en-Quicksand-Bold", 121], ["en-Brice-ExtraLightSemiExpanded", 122], ["en-Lato-BoldItalic", 123], ["en-Fraunces9pt-Italic", 124], ["en-AbrilFatface-Regular", 125], ["en-BerkshireSwash-Regular", 126], ["en-Atma-Bold", 127], ["en-HolidayRegular", 128], ["en-BebasNeueCyrillic", 129], ["en-IntroRust-Base", 130], ["en-Gistesy", 131], ["en-BDScript-Regular", 132], ["en-ApricotsRegular", 133], ["en-Prompt-Black", 134], ["en-TAN MERINGUE", 135], ["en-GentySans-Regular", 137], ["en-NeueEinstellung-Normal", 138], ["en-Garet-Bold", 139], ["en-FiraSans-Black", 140], ["en-BantayogLight", 141], ["en-NotoSerifDisplay-Black", 142], ["en-TTChocolates-Regular", 143], ["en-Ubuntu-Regular", 144], ["en-Assistant-Bold", 145], ["en-ABeeZee-Regular", 146], ["en-LexendDeca-Regular", 147], ["en-KingredSerif", 148], ["en-Radley-Regular", 149], ["en-BrownSugar", 150], ["en-MigraItalic-ExtraboldItalic", 151], ["en-ChildosArabic-Regular", 152], ["en-PeaceSans", 153], ["en-LondrinaSolid-Black", 154], ["en-SpaceMono-BoldItalic", 155], ["en-RobotoMono-Light", 156], ["en-CourierPrime-Regular", 157], ["en-Alata-Regular", 158], ["en-Amsterdam-One", 159], ["en-IreneFlorentina-Regular", 160], ["en-CatchyMager", 161], ["en-Alta_regular", 162], ["en-ArticulatCF-Regular", 163], ["en-Raleway-Regular", 164], ["en-BrasikaDisplay", 165], ["en-TANAngleton-Italic", 166], ["en-NotoSerifDisplay-ExtraCondensedItalic", 167], ["en-Bryndan Write", 168], ["en-TTCommonsPro-It", 169], ["en-AlexBrush-Regular", 170], ["en-Antic-Regular", 171], ["en-TTHoves-Bold", 172], ["en-DroidSerif", 173], ["en-AblationRegular", 174], ["en-Marcellus-Regular", 175], ["en-Sanchez-Italic", 176], ["en-JosefinSans", 177], ["en-Afrah-Regular", 178], ["en-PinyonScript", 179], ["en-TTInterphases-BoldItalic", 180], ["en-Yellowtail-Regular", 181], ["en-Gliker-Regular", 182], ["en-BobbyJonesSoft-Regular", 183], ["en-IBMPlexSans", 184], ["en-Amsterdam-Three", 185], ["en-Amsterdam-FourSlant", 186], ["en-TTFors-Regular", 187], ["en-Quattrocento", 188], ["en-Sifonn-Basic", 189], ["en-AlegreyaSans-Black", 190], ["en-Daydream", 191], ["en-AristotelicaProTx-Rg", 192], ["en-NotoSerif", 193], ["en-EBGaramond-Italic", 194], ["en-HammersmithOne-Regular", 195], ["en-RobotoSlab-Regular", 196], ["en-DO-Sans-Regular", 197], ["en-KGPrimaryDotsLined", 198], ["en-Blinker-Regular", 199], ["en-TAN NIMBUS", 200], ["en-Blueberry-Regular", 201], ["en-Rosario-Regular", 202], ["en-Forum", 203], ["en-MistrullyRegular", 204], ["en-SourceSerifPro-Regular", 205], ["en-Bugaki-Regular", 206], ["en-CMUSerif-Roman", 207], ["en-GulfsDisplay-NormalItalic", 208], ["en-PTSans-Bold", 209], ["en-SquadaOne-Regular", 211], ["en-Arapey-Italic", 212], ["en-Parisienne-Regular", 213], ["en-Aleo-Italic", 214], ["en-QuicheDisplay-Italic", 215], ["en-RocaOne-It", 216], ["en-Funtastic-Regular", 217], ["en-PTSerif-BoldItalic", 218], ["en-Muller-RegularItalic", 219], ["en-ArgentCF-Regular", 220], ["en-Brightwall-Italic", 221], ["en-Knewave-Regular", 222], ["en-TYSerif-D", 223], ["en-Agrandir-Tight", 224], ["en-AlfaSlabOne-Regular", 225], ["en-TANTangkiwood-Display", 226], ["en-Kief-Montaser-Regular", 227], ["en-Gotham-Book", 228], ["en-JuliusSansOne-Regular", 229], ["en-CocoGothic-Italic", 230], ["en-SairaCondensed-Regular", 231], ["en-DellaRespira-Regular", 232], ["en-Questrial-Regular", 233], ["en-BukhariScript-Regular", 234], ["en-HelveticaWorld-Bold", 235], ["en-TANKINDRED-Display", 236], ["en-CinzelDecorative-Regular", 237], ["en-Vidaloka-Regular", 238], ["en-AlegreyaSansSC-Black", 239], ["en-FeelingPassionate-Regular", 240], ["en-QuincyCF-Regular", 241], ["en-FiraCode-Regular", 242], ["en-Genty-Regular", 243], ["en-Nickainley-Normal", 244], ["en-RubikOne-Regular", 245], ["en-Gidole-Regular", 246], ["en-Borsok", 247], ["en-Gordita-RegularItalic", 248], ["en-Scripter-Regular", 249], ["en-Buffalo-Regular", 250], ["en-KleinText-Regular", 251], ["en-Creepster-Regular", 252], ["en-Arvo-Bold", 253], ["en-GabrielSans-NormalItalic", 254], ["en-Heebo-Black", 255], ["en-LexendExa-Regular", 256], ["en-BrixtonSansTC-Regular", 257], ["en-GildaDisplay-Regular", 258], ["en-ChunkFive-Roman", 259], ["en-Amaranth-BoldItalic", 260], ["en-BubbleboddyNeue-Regular", 261], ["en-MavenPro-Bold", 262], ["en-TTDrugs-Italic", 263], ["en-CyGrotesk-KeyRegular", 264], ["en-VarelaRound-Regular", 265], ["en-Ruda-Black", 266], ["en-SafiraMarch", 267], ["en-BloggerSans", 268], ["en-TANHEADLINE-Regular", 269], ["en-SloopScriptPro-Regular", 270], ["en-NeueMontreal-Regular", 271], ["en-Schoolbell-Regular", 272], ["en-SigherRegular", 273], ["en-InriaSerif-Regular", 274], ["en-JetBrainsMono-Regular", 275], ["en-MADEEvolveSans", 276], ["en-Dekko", 277], ["en-Handyman-Regular", 278], ["en-Aileron-BoldItalic", 279], ["en-Bright-Italic", 280], ["en-Solway-Regular", 281], ["en-Higuen-Regular", 282], ["en-WedgesItalic", 283], ["en-TANASHFORD-BOLD", 284], ["en-IBMPlexMono", 285], ["en-RacingSansOne-Regular", 286], ["en-RegularBrush", 287], ["en-OpenSans-LightItalic", 288], ["en-SpecialElite-Regular", 289], ["en-FuturaLTPro-Medium", 290], ["en-MaragsaDisplay", 291], ["en-BigShouldersDisplay-Regular", 292], ["en-BDSans-Regular", 293], ["en-RasputinRegular", 294], ["en-Yvesyvesdrawing-BoldItalic", 295], ["en-Bitter-Regular", 296], ["en-LuckiestGuy-Regular", 297], ["en-TTFirsNeue-Italic", 299], ["en-Sunday-Regular", 300], ["en-HKGothic-MediumItalic", 301], ["en-CaveatBrush-Regular", 302], ["en-ArchitectsDaughter-Regular", 304], ["en-Angelina", 305], ["en-Calistoga-Regular", 306], ["en-ArchivoNarrow-Regular", 307], ["en-ObjectSans-MediumSlanted", 308], ["en-AyrLucidityCondensed-Regular", 309], ["en-Nexa-RegularItalic", 310], ["en-Lustria-Regular", 311], ["en-Amsterdam-TwoSlant", 312], ["en-Virtual-Regular", 313], ["en-NF-Lepetitcochon-Regular", 315], ["en-TANTWINKLE", 316], ["en-LeJour-Serif", 317], ["en-Prata-Regular", 318], ["en-PPWoodland-Regular", 319], ["en-PlayfairDisplay-BoldItalic", 320], ["en-AmaticSC-Regular", 321], ["en-Cabin-Regular", 322], ["en-Manjari-Bold", 323], ["en-MrDafoe-Regular", 324], ["en-TTRamillas-Italic", 325], ["en-Luckybones-Bold", 326], ["en-DarkerGrotesque-Light", 327], ["en-BellabooRegular", 328], ["en-CormorantSC-Bold", 329], ["en-GochiHand-Regular", 330], ["en-Atteron", 331], ["en-RocaTwo-Lt", 332], ["en-TANSONGBIRD", 334], ["en-HeadingNow-74Regular", 335], ["en-Luthier-BoldItalic", 336], ["en-Oregano-Regular", 337], ["en-AyrTropikaIsland-Int", 338], ["en-Mali-Regular", 339], ["en-DidactGothic-Regular", 340], ["en-Lovelace-Regular", 341], ["en-BakerieSmooth-Regular", 342], ["en-CarterOne", 343], ["en-HussarBd", 344], ["en-OldStandard-Italic", 345], ["en-TAN-ASTORIA-Display", 346], ["en-rugratssans-Regular", 347], ["en-BetterSaturday", 349], ["en-AdigianaToybox", 350], ["en-Sailors", 351], ["en-PlayfairDisplaySC-Italic", 352], ["en-Etna-Regular", 353], ["en-Revive80Signature", 354], ["en-CAGenerated", 355], ["en-Poppins-Regular", 356], ["en-Jonathan-Regular", 357], ["en-Pacifico-Regular", 358], ["en-Saira-Black", 359], ["en-Loubag-Regular", 360], ["en-Decalotype-Black", 361], ["en-Mansalva-Regular", 362], ["en-Allura-Regular", 363], ["en-ProximaNova-Bold", 364], ["en-TANMIGNON-DISPLAY", 365], ["en-ArsenicaAntiqua-Regular", 366], ["en-BreulGroteskA-RegularItalic", 367], ["en-HKModular-Bold", 368], ["en-TANNightingale-Regular", 369], ["en-AristotelicaProCndTxt-Rg", 370], ["en-Aprila-Regular", 371], ["en-Tomorrow-Regular", 372], ["en-AngellaWhite", 373], ["en-KaushanScript-Regular", 374], ["en-NotoSans", 375], ["en-LeJour-Script", 376], ["en-BrixtonTC-Regular", 377], ["en-OleoScript-Regular", 378], ["en-Cakerolli-Regular", 379], ["en-Lobster-Regular", 380], ["en-FrunchySerif-Regular", 381], ["en-PorcelainRegular", 382], ["en-AlojaExtended", 383], ["en-SergioTrendy-Italic", 384], ["en-LovelaceText-Bold", 385], ["en-Anaktoria", 386], ["en-JimmyScript-Light", 387], ["en-IBMPlexSerif", 388], ["en-Marta", 389], ["en-Mango-Regular", 390], ["en-Overpass-Italic", 391], ["en-Hagrid-Regular", 392], ["en-ElikaGorica", 393], ["en-Amiko-Regular", 394], ["en-EFCOBrookshire-Regular", 395], ["en-Caladea-Regular", 396], ["en-MoonlightBold", 397], ["en-Staatliches-Regular", 398], ["en-Helios-Bold", 399], ["en-Satisfy-Regular", 400], ["en-NexaScript-Regular", 401], ["en-Trocchi-Regular", 402], ["en-March", 403], ["en-IbarraRealNova-Regular", 404], ["en-Nectarine-Regular", 405], ["en-Overpass-Light", 406], ["en-TruetypewriterPolyglOTT", 407], ["en-Bangers-Regular", 408], ["en-Lazord-BoldExpandedItalic", 409], ["en-Chloe-Regular", 410], ["en-BaskervilleDisplayPT-Regular", 411], ["en-Bright-Regular", 412], ["en-Vollkorn-Regular", 413], ["en-Harmattan", 414], ["en-SortsMillGoudy-Regular", 415], ["en-Biryani-Bold", 416], ["en-SugoProDisplay-Italic", 417], ["en-Lazord-BoldItalic", 418], ["en-Alike-Regular", 419], ["en-PermanentMarker-Regular", 420], ["en-Sacramento-Regular", 421], ["en-HKGroteskPro-Italic", 422], ["en-Aleo-BoldItalic", 423], ["en-TANGARLAND-Regular", 425], ["en-Twister", 426], ["en-Arsenal-Italic", 427], ["en-Bogart-Italic", 428], ["en-BethEllen-Regular", 429], ["en-Caveat-Regular", 430], ["en-BalsamiqSans-Bold", 431], ["en-BreeSerif-Regular", 432], ["en-CodecPro-ExtraBold", 433], ["en-Pierson-Light", 434], ["en-CyGrotesk-WideRegular", 435], ["en-Lumios-Marker", 436], ["en-Comfortaa-Bold", 437], ["en-TraceFontRegular", 438], ["en-RTL-AdamScript-Regular", 439], ["en-EastmanGrotesque-Italic", 440], ["en-Kalam-Bold", 441], ["en-ChauPhilomeneOne-Regular", 442], ["en-Coiny-Regular", 443], ["en-Lovera", 444], ["en-Gellatio", 445], ["en-TitilliumWeb-Bold", 446], ["en-OilvareBase-Italic", 447], ["en-Catamaran-Black", 448], ["en-Anteb-Italic", 449], ["en-SueEllenFrancisco", 450], ["en-SweetApricot", 451], ["en-BrightSunshine", 452], ["en-IM_FELL_Double_Pica_Italic", 453], ["en-Granaina-limpia", 454], ["en-TANPARFAIT", 455], ["en-AcherusGrotesque-Regular", 456], ["en-AwesomeLathusca-Italic", 457], ["en-Signika-Bold", 458], ["en-Andasia", 459], ["en-DO-AllCaps-Slanted", 460], ["en-Zenaida-Regular", 461], ["en-Fahkwang-Regular", 462], ["en-Play-Regular", 463], ["en-BERNIERRegular-Regular", 464], ["en-PlumaThin-Regular", 465], ["en-SportsWorld", 466], ["en-Garet-Black", 467], ["en-CarolloPlayscript-BlackItalic", 468], ["en-Cheque-Regular", 469], ["en-SEGO", 470], ["en-BobbyJones-Condensed", 471], ["en-NexaSlab-RegularItalic", 472], ["en-DancingScript-Regular", 473], ["en-Magnolia-Script", 475], ["en-OpunMai-400It", 476], ["en-MadelynFill-Regular", 477], ["en-ZingRust-Base", 478], ["en-FingerPaint-Regular", 479], ["en-BostonAngel-Light", 480], ["en-Gliker-RegularExpanded", 481], ["en-Ahsing", 482], ["en-Engagement-Regular", 483], ["en-EyesomeScript", 484], ["en-LibraSerifModern-Regular", 485], ["en-London-Regular", 486], ["en-AtkinsonHyperlegible-Regular", 487], ["en-StadioNow-TextItalic", 488], ["en-Aniyah", 489], ["en-ITCAvantGardePro-Bold", 490], ["en-Comica-Regular", 491], ["en-Coustard-Regular", 492], ["en-Brice-BoldCondensed", 493], ["en-TANNEWYORK-Bold", 494], ["en-TANBUSTER-Bold", 495], ["en-Alatsi-Regular", 496], ["en-TYSerif-Book", 497], ["en-Jingleberry", 498], ["en-Rajdhani-Bold", 499], ["en-LobsterTwo-BoldItalic", 500], ["en-Hitchcut-Regular", 502], ["en-GermaniaOne-Regular", 503], ["en-Emitha-Script", 504], ["en-LemonTuesday", 505], ["en-MonterchiSerif-Regular", 507], ["en-AllertaStencil-Regular", 508], ["en-RTL-Sondos-Regular", 509], ["en-HomemadeApple-Regular", 510], ["en-CosmicOcto-Medium", 511]] \ No newline at end of file diff --git a/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/assets/multi_fonts/ru.json b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/assets/multi_fonts/ru.json new file mode 100644 index 0000000000000000000000000000000000000000..49aec4f46efc8d51ab933e9d516891afc7de7e6c --- /dev/null +++ b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/assets/multi_fonts/ru.json @@ -0,0 +1 @@ +[["en-OpenSans-ExtraBold", 10], ["en-AnonymousPro-Bold", 106], ["en-CodecPro-Regular", 109], ["en-TTNormsPro-Italic", 114], ["en-Gagalin-Regular", 115], ["en-OpenSans-Bold", 116], ["en-HeroLight-Regular", 119], ["en-Lato-BoldItalic", 123], ["en-BebasNeueCyrillic", 129], ["en-IntroRust-Base", 130], ["en-Garet-Bold", 139], ["en-FiraSans-Black", 140], ["en-NotoSerifDisplay-Black", 142], ["en-TTChocolates-Regular", 143], ["en-Ubuntu-Regular", 144], ["en-Roboto-BoldItalic", 15], ["en-PeaceSans", 153], ["en-RobotoMono-Light", 156], ["en-ArticulatCF-Regular", 163], ["en-Raleway-Regular", 164], ["en-NotoSerifDisplay-ExtraCondensedItalic", 167], ["en-Bryndan Write", 168], ["en-TTCommonsPro-It", 169], ["en-TTHoves-Bold", 172], ["en-DroidSerif", 173], ["en-AblationRegular", 174], ["en-Inter-BoldItalic", 18], ["en-TTInterphases-BoldItalic", 180], ["en-IBMPlexSans", 184], ["en-TTFors-Regular", 187], ["en-AlegreyaSans-Black", 190], ["en-AristotelicaProTx-Rg", 192], ["en-NotoSerif", 193], ["en-EBGaramond-Italic", 194], ["en-RobotoSlab-Regular", 196], ["en-DO-Sans-Regular", 197], ["en-Forum", 203], ["en-CMUSerif-Roman", 207], ["en-PTSans-Bold", 209], ["en-Sensei-Medium", 210], ["en-RocaOne-It", 216], ["en-Funtastic-Regular", 217], ["en-PTSerif-BoldItalic", 218], ["en-Muller-RegularItalic", 219], ["en-PlayfairDisplay-SemiBold", 22], ["en-ArgentCF-Regular", 220], ["en-TYSerif-D", 223], ["en-Gotham-Book", 228], ["en-CocoGothic-Italic", 230], ["en-BukhariScript-Regular", 234], ["en-HelveticaWorld-Bold", 235], ["en-AlegreyaSansSC-Black", 239], ["en-Lato-Regular", 24], ["en-QuincyCF-Regular", 241], ["en-FiraCode-Regular", 242], ["en-Nickainley-Normal", 244], ["en-RubikOne-Regular", 245], ["en-Borsok", 247], ["en-Gordita-RegularItalic", 248], ["en-Scripter-Regular", 249], ["en-KleinText-Regular", 251], ["en-BubbleboddyNeue-Regular", 261], ["en-TTDrugs-Italic", 263], ["en-CyGrotesk-KeyRegular", 264], ["en-BloggerSans", 268], ["en-SloopScriptPro-Regular", 270], ["en-JetBrainsMono-Regular", 275], ["en-MADEEvolveSans", 276], ["en-Handyman-Regular", 278], ["en-CodePro-NormalLC", 28], ["en-IBMPlexMono", 285], ["en-OpenSans-LightItalic", 288], ["en-TTFirsNeue-Italic", 299], ["en-OpenSans-ExtraBoldItalic", 3], ["en-Sunday-Regular", 300], ["en-HeliosExt", 303], ["en-ObjectSans-MediumSlanted", 308], ["en-HKGrotesk-Bold", 31], ["en-Brusher-Regular", 314], ["en-Prata-Regular", 318], ["en-PlayfairDisplay-BoldItalic", 320], ["en-AmaticSC-Regular", 321], ["en-TTRamillas-Italic", 325], ["en-CormorantSC-Bold", 329], ["en-Atteron", 331], ["en-RocaTwo-Lt", 332], ["en-HeadingNow-74Regular", 335], ["en-Raleway-ExtraBoldItalic", 34], ["en-DidactGothic-Regular", 340], ["en-Lovelace-Regular", 341], ["en-HussarBd", 344], ["en-OldStandard-Italic", 345], ["en-Mont-RegularItalic", 35], ["en-PlayfairDisplaySC-Italic", 352], ["en-Etna-Regular", 353], ["en-Arimo-BoldItalic", 36], ["en-ProximaNova-Bold", 364], ["en-ArsenicaAntiqua-Regular", 366], ["en-Lora-Italic", 37], ["en-AristotelicaProCndTxt-Rg", 370], ["en-NotoSans", 375], ["en-Lobster-Regular", 380], ["en-LovelaceText-Bold", 385], ["en-Anaktoria", 386], ["en-IBMPlexSerif", 388], ["en-Marta", 389], ["en-Mango-Regular", 390], ["en-Hagrid-Regular", 392], ["en-Helios-Bold", 399], ["en-NexaScript-Regular", 401], ["en-TruetypewriterPolyglOTT", 407], ["en-CormorantGaramond-Bold", 41], ["en-BaskervilleDisplayPT-Regular", 411], ["en-Vollkorn-Regular", 413], ["en-SugoProDisplay-Italic", 417], ["en-Arsenal-Italic", 427], ["en-Bogart-Italic", 428], ["en-Caveat-Regular", 430], ["en-BalsamiqSans-Bold", 431], ["en-CodecPro-ExtraBold", 433], ["en-CyGrotesk-WideRegular", 435], ["en-Lumios-Marker", 436], ["en-Comfortaa-Bold", 437], ["en-RTL-AdamScript-Regular", 439], ["en-BebasNeueBold", 44], ["en-EastmanGrotesque-Italic", 440], ["en-LazydogRegular", 45], ["en-DO-AllCaps-Slanted", 460], ["en-Play-Regular", 463], ["en-BERNIERRegular-Regular", 464], ["en-SportsWorld", 466], ["en-Garet-Black", 467], ["en-CarolloPlayscript-BlackItalic", 468], ["en-Cheque-Regular", 469], ["en-Magnolia-Script", 475], ["en-MadelynFill-Regular", 477], ["en-ZingRust-Base", 478], ["en-LibraSerifModern-Regular", 485], ["en-StadioNow-TextItalic", 488], ["en-Comica-Regular", 491], ["en-TYSerif-Book", 497], ["en-Jingleberry", 498], ["en-Nunito-ExtraBoldItalic", 50], ["en-LemonTuesday", 505], ["en-MonterchiSerif-Regular", 507], ["en-Montserrat-Light", 52], ["en-TenorSans", 53], ["en-ClearSans-Bold", 55], ["en-Alice-Regular", 57], ["en-Oswald-Regular", 58], ["en-Garet-Regular", 6], ["en-NunitoSans10pt-BlackItalic", 69], ["en-Rubik-Regular", 71], ["en-PPNeueMachina-Regular", 72], ["en-Alegreya-Regular", 83], ["en-RobotoCondensed-Italic", 85], ["en-Garet-ExtraBold", 87], ["en-YesevaOne-Regular", 88], ["en-OpenSans-Italic", 91], ["en-Nunito-BlackItalic", 99]] \ No newline at end of file diff --git a/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/assets/multilingual_10-lang_idx.json b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/assets/multilingual_10-lang_idx.json new file mode 100644 index 0000000000000000000000000000000000000000..ccce7212f98923c4862d3c0c1236cea7bc04f8e2 --- /dev/null +++ b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/assets/multilingual_10-lang_idx.json @@ -0,0 +1 @@ +{"en-Montserrat-Regular": 0, "en-Poppins-Italic": 1, "en-GlacialIndifference-Regular": 2, "en-OpenSans-ExtraBoldItalic": 3, "en-Montserrat-Bold": 4, "en-Now-Regular": 5, "en-Garet-Regular": 6, "en-LeagueSpartan-Bold": 7, "en-DMSans-Regular": 8, "en-OpenSauceOne-Regular": 9, "en-OpenSans-ExtraBold": 10, "en-KGPrimaryPenmanship": 11, "en-Anton-Regular": 12, "en-Aileron-BlackItalic": 13, "en-Quicksand-Light": 14, "en-Roboto-BoldItalic": 15, "en-TheSeasons-It": 16, "en-Kollektif": 17, "en-Inter-BoldItalic": 18, "en-Poppins-Medium": 19, "en-Poppins-Light": 20, "en-RoxboroughCF-RegularItalic": 21, "en-PlayfairDisplay-SemiBold": 22, "en-Agrandir-Italic": 23, "en-Lato-Regular": 24, "en-MoreSugarRegular": 25, "en-CanvaSans-RegularItalic": 26, "en-PublicSans-Italic": 27, "en-CodePro-NormalLC": 28, "en-Belleza-Regular": 29, "en-JosefinSans-Bold": 30, "en-HKGrotesk-Bold": 31, "en-Telegraf-Medium": 32, "en-BrittanySignatureRegular": 33, "en-Raleway-ExtraBoldItalic": 34, "en-Mont-RegularItalic": 35, "en-Arimo-BoldItalic": 36, "en-Lora-Italic": 37, "en-ArchivoBlack-Regular": 38, "en-Poppins": 39, "en-Barlow-Black": 40, "en-CormorantGaramond-Bold": 41, "en-LibreBaskerville-Regular": 42, "en-CanvaSchoolFontRegular": 43, "en-BebasNeueBold": 44, "en-LazydogRegular": 45, "en-FredokaOne-Regular": 46, "en-Horizon-Bold": 47, "en-Nourd-Regular": 48, "en-Hatton-Regular": 49, "en-Nunito-ExtraBoldItalic": 50, "en-CerebriSans-Regular": 51, "en-Montserrat-Light": 52, "en-TenorSans": 53, "en-Norwester-Regular": 54, "en-ClearSans-Bold": 55, "en-Cardo-Regular": 56, "en-Alice-Regular": 57, "en-Oswald-Regular": 58, "en-Gaegu-Bold": 59, "en-Muli-Black": 60, "en-TAN-PEARL-Regular": 61, "en-CooperHewitt-Book": 62, "en-Agrandir-Grand": 63, "en-BlackMango-Thin": 64, "en-DMSerifDisplay-Regular": 65, "en-Antonio-Bold": 66, "en-Sniglet-Regular": 67, "en-BeVietnam-Regular": 68, "en-NunitoSans10pt-BlackItalic": 69, "en-AbhayaLibre-ExtraBold": 70, "en-Rubik-Regular": 71, "en-PPNeueMachina-Regular": 72, "en-TAN - MON CHERI-Regular": 73, "en-Jua-Regular": 74, "en-Playlist-Script": 75, "en-SourceSansPro-BoldItalic": 76, "en-MoonTime-Regular": 77, "en-Eczar-ExtraBold": 78, "en-Gatwick-Regular": 79, "en-MonumentExtended-Regular": 80, "en-BarlowSemiCondensed-Regular": 81, "en-BarlowCondensed-Regular": 82, "en-Alegreya-Regular": 83, "en-DreamAvenue": 84, "en-RobotoCondensed-Italic": 85, "en-BobbyJones-Regular": 86, "en-Garet-ExtraBold": 87, "en-YesevaOne-Regular": 88, "en-Dosis-ExtraBold": 89, "en-LeagueGothic-Regular": 90, "en-OpenSans-Italic": 91, "en-TANAEGEAN-Regular": 92, "en-Maharlika-Regular": 93, "en-MarykateRegular": 94, "en-Cinzel-Regular": 95, "en-Agrandir-Wide": 96, "en-Chewy-Regular": 97, "en-BodoniFLF-BoldItalic": 98, "en-Nunito-BlackItalic": 99, "en-LilitaOne": 100, "en-HandyCasualCondensed-Regular": 101, "en-Ovo": 102, "en-Livvic-Regular": 103, "en-Agrandir-Narrow": 104, "en-CrimsonPro-Italic": 105, "en-AnonymousPro-Bold": 106, "en-NF-OneLittleFont-Bold": 107, "en-RedHatDisplay-BoldItalic": 108, "en-CodecPro-Regular": 109, "en-HalimunRegular": 110, "en-LibreFranklin-Black": 111, "en-TeXGyreTermes-BoldItalic": 112, "en-Shrikhand-Regular": 113, "en-TTNormsPro-Italic": 114, "en-Gagalin-Regular": 115, "en-OpenSans-Bold": 116, "en-GreatVibes-Regular": 117, "en-Breathing": 118, "en-HeroLight-Regular": 119, "en-KGPrimaryDots": 120, "en-Quicksand-Bold": 121, "en-Brice-ExtraLightSemiExpanded": 122, "en-Lato-BoldItalic": 123, "en-Fraunces9pt-Italic": 124, "en-AbrilFatface-Regular": 125, "en-BerkshireSwash-Regular": 126, "en-Atma-Bold": 127, "en-HolidayRegular": 128, "en-BebasNeueCyrillic": 129, "en-IntroRust-Base": 130, "en-Gistesy": 131, "en-BDScript-Regular": 132, "en-ApricotsRegular": 133, "en-Prompt-Black": 134, "en-TAN MERINGUE": 135, "en-Sukar Regular": 136, "en-GentySans-Regular": 137, "en-NeueEinstellung-Normal": 138, "en-Garet-Bold": 139, "en-FiraSans-Black": 140, "en-BantayogLight": 141, "en-NotoSerifDisplay-Black": 142, "en-TTChocolates-Regular": 143, "en-Ubuntu-Regular": 144, "en-Assistant-Bold": 145, "en-ABeeZee-Regular": 146, "en-LexendDeca-Regular": 147, "en-KingredSerif": 148, "en-Radley-Regular": 149, "en-BrownSugar": 150, "en-MigraItalic-ExtraboldItalic": 151, "en-ChildosArabic-Regular": 152, "en-PeaceSans": 153, "en-LondrinaSolid-Black": 154, "en-SpaceMono-BoldItalic": 155, "en-RobotoMono-Light": 156, "en-CourierPrime-Regular": 157, "en-Alata-Regular": 158, "en-Amsterdam-One": 159, "en-IreneFlorentina-Regular": 160, "en-CatchyMager": 161, "en-Alta_regular": 162, "en-ArticulatCF-Regular": 163, "en-Raleway-Regular": 164, "en-BrasikaDisplay": 165, "en-TANAngleton-Italic": 166, "en-NotoSerifDisplay-ExtraCondensedItalic": 167, "en-Bryndan Write": 168, "en-TTCommonsPro-It": 169, "en-AlexBrush-Regular": 170, "en-Antic-Regular": 171, "en-TTHoves-Bold": 172, "en-DroidSerif": 173, "en-AblationRegular": 174, "en-Marcellus-Regular": 175, "en-Sanchez-Italic": 176, "en-JosefinSans": 177, "en-Afrah-Regular": 178, "en-PinyonScript": 179, "en-TTInterphases-BoldItalic": 180, "en-Yellowtail-Regular": 181, "en-Gliker-Regular": 182, "en-BobbyJonesSoft-Regular": 183, "en-IBMPlexSans": 184, "en-Amsterdam-Three": 185, "en-Amsterdam-FourSlant": 186, "en-TTFors-Regular": 187, "en-Quattrocento": 188, "en-Sifonn-Basic": 189, "en-AlegreyaSans-Black": 190, "en-Daydream": 191, "en-AristotelicaProTx-Rg": 192, "en-NotoSerif": 193, "en-EBGaramond-Italic": 194, "en-HammersmithOne-Regular": 195, "en-RobotoSlab-Regular": 196, "en-DO-Sans-Regular": 197, "en-KGPrimaryDotsLined": 198, "en-Blinker-Regular": 199, "en-TAN NIMBUS": 200, "en-Blueberry-Regular": 201, "en-Rosario-Regular": 202, "en-Forum": 203, "en-MistrullyRegular": 204, "en-SourceSerifPro-Regular": 205, "en-Bugaki-Regular": 206, "en-CMUSerif-Roman": 207, "en-GulfsDisplay-NormalItalic": 208, "en-PTSans-Bold": 209, "en-Sensei-Medium": 210, "en-SquadaOne-Regular": 211, "en-Arapey-Italic": 212, "en-Parisienne-Regular": 213, "en-Aleo-Italic": 214, "en-QuicheDisplay-Italic": 215, "en-RocaOne-It": 216, "en-Funtastic-Regular": 217, "en-PTSerif-BoldItalic": 218, "en-Muller-RegularItalic": 219, "en-ArgentCF-Regular": 220, "en-Brightwall-Italic": 221, "en-Knewave-Regular": 222, "en-TYSerif-D": 223, "en-Agrandir-Tight": 224, "en-AlfaSlabOne-Regular": 225, "en-TANTangkiwood-Display": 226, "en-Kief-Montaser-Regular": 227, "en-Gotham-Book": 228, "en-JuliusSansOne-Regular": 229, "en-CocoGothic-Italic": 230, "en-SairaCondensed-Regular": 231, "en-DellaRespira-Regular": 232, "en-Questrial-Regular": 233, "en-BukhariScript-Regular": 234, "en-HelveticaWorld-Bold": 235, "en-TANKINDRED-Display": 236, "en-CinzelDecorative-Regular": 237, "en-Vidaloka-Regular": 238, "en-AlegreyaSansSC-Black": 239, "en-FeelingPassionate-Regular": 240, "en-QuincyCF-Regular": 241, "en-FiraCode-Regular": 242, "en-Genty-Regular": 243, "en-Nickainley-Normal": 244, "en-RubikOne-Regular": 245, "en-Gidole-Regular": 246, "en-Borsok": 247, "en-Gordita-RegularItalic": 248, "en-Scripter-Regular": 249, "en-Buffalo-Regular": 250, "en-KleinText-Regular": 251, "en-Creepster-Regular": 252, "en-Arvo-Bold": 253, "en-GabrielSans-NormalItalic": 254, "en-Heebo-Black": 255, "en-LexendExa-Regular": 256, "en-BrixtonSansTC-Regular": 257, "en-GildaDisplay-Regular": 258, "en-ChunkFive-Roman": 259, "en-Amaranth-BoldItalic": 260, "en-BubbleboddyNeue-Regular": 261, "en-MavenPro-Bold": 262, "en-TTDrugs-Italic": 263, "en-CyGrotesk-KeyRegular": 264, "en-VarelaRound-Regular": 265, "en-Ruda-Black": 266, "en-SafiraMarch": 267, "en-BloggerSans": 268, "en-TANHEADLINE-Regular": 269, "en-SloopScriptPro-Regular": 270, "en-NeueMontreal-Regular": 271, "en-Schoolbell-Regular": 272, "en-SigherRegular": 273, "en-InriaSerif-Regular": 274, "en-JetBrainsMono-Regular": 275, "en-MADEEvolveSans": 276, "en-Dekko": 277, "en-Handyman-Regular": 278, "en-Aileron-BoldItalic": 279, "en-Bright-Italic": 280, "en-Solway-Regular": 281, "en-Higuen-Regular": 282, "en-WedgesItalic": 283, "en-TANASHFORD-BOLD": 284, "en-IBMPlexMono": 285, "en-RacingSansOne-Regular": 286, "en-RegularBrush": 287, "en-OpenSans-LightItalic": 288, "en-SpecialElite-Regular": 289, "en-FuturaLTPro-Medium": 290, "en-MaragsaDisplay": 291, "en-BigShouldersDisplay-Regular": 292, "en-BDSans-Regular": 293, "en-RasputinRegular": 294, "en-Yvesyvesdrawing-BoldItalic": 295, "en-Bitter-Regular": 296, "en-LuckiestGuy-Regular": 297, "en-CanvaSchoolFontDotted": 298, "en-TTFirsNeue-Italic": 299, "en-Sunday-Regular": 300, "en-HKGothic-MediumItalic": 301, "en-CaveatBrush-Regular": 302, "en-HeliosExt": 303, "en-ArchitectsDaughter-Regular": 304, "en-Angelina": 305, "en-Calistoga-Regular": 306, "en-ArchivoNarrow-Regular": 307, "en-ObjectSans-MediumSlanted": 308, "en-AyrLucidityCondensed-Regular": 309, "en-Nexa-RegularItalic": 310, "en-Lustria-Regular": 311, "en-Amsterdam-TwoSlant": 312, "en-Virtual-Regular": 313, "en-Brusher-Regular": 314, "en-NF-Lepetitcochon-Regular": 315, "en-TANTWINKLE": 316, "en-LeJour-Serif": 317, "en-Prata-Regular": 318, "en-PPWoodland-Regular": 319, "en-PlayfairDisplay-BoldItalic": 320, "en-AmaticSC-Regular": 321, "en-Cabin-Regular": 322, "en-Manjari-Bold": 323, "en-MrDafoe-Regular": 324, "en-TTRamillas-Italic": 325, "en-Luckybones-Bold": 326, "en-DarkerGrotesque-Light": 327, "en-BellabooRegular": 328, "en-CormorantSC-Bold": 329, "en-GochiHand-Regular": 330, "en-Atteron": 331, "en-RocaTwo-Lt": 332, "en-ZCOOLXiaoWei-Regular": 333, "en-TANSONGBIRD": 334, "en-HeadingNow-74Regular": 335, "en-Luthier-BoldItalic": 336, "en-Oregano-Regular": 337, "en-AyrTropikaIsland-Int": 338, "en-Mali-Regular": 339, "en-DidactGothic-Regular": 340, "en-Lovelace-Regular": 341, "en-BakerieSmooth-Regular": 342, "en-CarterOne": 343, "en-HussarBd": 344, "en-OldStandard-Italic": 345, "en-TAN-ASTORIA-Display": 346, "en-rugratssans-Regular": 347, "en-BMHANNA": 348, "en-BetterSaturday": 349, "en-AdigianaToybox": 350, "en-Sailors": 351, "en-PlayfairDisplaySC-Italic": 352, "en-Etna-Regular": 353, "en-Revive80Signature": 354, "en-CAGenerated": 355, "en-Poppins-Regular": 356, "en-Jonathan-Regular": 357, "en-Pacifico-Regular": 358, "en-Saira-Black": 359, "en-Loubag-Regular": 360, "en-Decalotype-Black": 361, "en-Mansalva-Regular": 362, "en-Allura-Regular": 363, "en-ProximaNova-Bold": 364, "en-TANMIGNON-DISPLAY": 365, "en-ArsenicaAntiqua-Regular": 366, "en-BreulGroteskA-RegularItalic": 367, "en-HKModular-Bold": 368, "en-TANNightingale-Regular": 369, "en-AristotelicaProCndTxt-Rg": 370, "en-Aprila-Regular": 371, "en-Tomorrow-Regular": 372, "en-AngellaWhite": 373, "en-KaushanScript-Regular": 374, "en-NotoSans": 375, "en-LeJour-Script": 376, "en-BrixtonTC-Regular": 377, "en-OleoScript-Regular": 378, "en-Cakerolli-Regular": 379, "en-Lobster-Regular": 380, "en-FrunchySerif-Regular": 381, "en-PorcelainRegular": 382, "en-AlojaExtended": 383, "en-SergioTrendy-Italic": 384, "en-LovelaceText-Bold": 385, "en-Anaktoria": 386, "en-JimmyScript-Light": 387, "en-IBMPlexSerif": 388, "en-Marta": 389, "en-Mango-Regular": 390, "en-Overpass-Italic": 391, "en-Hagrid-Regular": 392, "en-ElikaGorica": 393, "en-Amiko-Regular": 394, "en-EFCOBrookshire-Regular": 395, "en-Caladea-Regular": 396, "en-MoonlightBold": 397, "en-Staatliches-Regular": 398, "en-Helios-Bold": 399, "en-Satisfy-Regular": 400, "en-NexaScript-Regular": 401, "en-Trocchi-Regular": 402, "en-March": 403, "en-IbarraRealNova-Regular": 404, "en-Nectarine-Regular": 405, "en-Overpass-Light": 406, "en-TruetypewriterPolyglOTT": 407, "en-Bangers-Regular": 408, "en-Lazord-BoldExpandedItalic": 409, "en-Chloe-Regular": 410, "en-BaskervilleDisplayPT-Regular": 411, "en-Bright-Regular": 412, "en-Vollkorn-Regular": 413, "en-Harmattan": 414, "en-SortsMillGoudy-Regular": 415, "en-Biryani-Bold": 416, "en-SugoProDisplay-Italic": 417, "en-Lazord-BoldItalic": 418, "en-Alike-Regular": 419, "en-PermanentMarker-Regular": 420, "en-Sacramento-Regular": 421, "en-HKGroteskPro-Italic": 422, "en-Aleo-BoldItalic": 423, "en-Noot": 424, "en-TANGARLAND-Regular": 425, "en-Twister": 426, "en-Arsenal-Italic": 427, "en-Bogart-Italic": 428, "en-BethEllen-Regular": 429, "en-Caveat-Regular": 430, "en-BalsamiqSans-Bold": 431, "en-BreeSerif-Regular": 432, "en-CodecPro-ExtraBold": 433, "en-Pierson-Light": 434, "en-CyGrotesk-WideRegular": 435, "en-Lumios-Marker": 436, "en-Comfortaa-Bold": 437, "en-TraceFontRegular": 438, "en-RTL-AdamScript-Regular": 439, "en-EastmanGrotesque-Italic": 440, "en-Kalam-Bold": 441, "en-ChauPhilomeneOne-Regular": 442, "en-Coiny-Regular": 443, "en-Lovera": 444, "en-Gellatio": 445, "en-TitilliumWeb-Bold": 446, "en-OilvareBase-Italic": 447, "en-Catamaran-Black": 448, "en-Anteb-Italic": 449, "en-SueEllenFrancisco": 450, "en-SweetApricot": 451, "en-BrightSunshine": 452, "en-IM_FELL_Double_Pica_Italic": 453, "en-Granaina-limpia": 454, "en-TANPARFAIT": 455, "en-AcherusGrotesque-Regular": 456, "en-AwesomeLathusca-Italic": 457, "en-Signika-Bold": 458, "en-Andasia": 459, "en-DO-AllCaps-Slanted": 460, "en-Zenaida-Regular": 461, "en-Fahkwang-Regular": 462, "en-Play-Regular": 463, "en-BERNIERRegular-Regular": 464, "en-PlumaThin-Regular": 465, "en-SportsWorld": 466, "en-Garet-Black": 467, "en-CarolloPlayscript-BlackItalic": 468, "en-Cheque-Regular": 469, "en-SEGO": 470, "en-BobbyJones-Condensed": 471, "en-NexaSlab-RegularItalic": 472, "en-DancingScript-Regular": 473, "en-PaalalabasDisplayWideBETA": 474, "en-Magnolia-Script": 475, "en-OpunMai-400It": 476, "en-MadelynFill-Regular": 477, "en-ZingRust-Base": 478, "en-FingerPaint-Regular": 479, "en-BostonAngel-Light": 480, "en-Gliker-RegularExpanded": 481, "en-Ahsing": 482, "en-Engagement-Regular": 483, "en-EyesomeScript": 484, "en-LibraSerifModern-Regular": 485, "en-London-Regular": 486, "en-AtkinsonHyperlegible-Regular": 487, "en-StadioNow-TextItalic": 488, "en-Aniyah": 489, "en-ITCAvantGardePro-Bold": 490, "en-Comica-Regular": 491, "en-Coustard-Regular": 492, "en-Brice-BoldCondensed": 493, "en-TANNEWYORK-Bold": 494, "en-TANBUSTER-Bold": 495, "en-Alatsi-Regular": 496, "en-TYSerif-Book": 497, "en-Jingleberry": 498, "en-Rajdhani-Bold": 499, "en-LobsterTwo-BoldItalic": 500, "en-BestLight-Medium": 501, "en-Hitchcut-Regular": 502, "en-GermaniaOne-Regular": 503, "en-Emitha-Script": 504, "en-LemonTuesday": 505, "en-Cubao_Free_Regular": 506, "en-MonterchiSerif-Regular": 507, "en-AllertaStencil-Regular": 508, "en-RTL-Sondos-Regular": 509, "en-HomemadeApple-Regular": 510, "en-CosmicOcto-Medium": 511, "cn-HelloFont-FangHuaTi": 0, "cn-HelloFont-ID-DianFangSong-Bold": 1, "cn-HelloFont-ID-DianFangSong": 2, "cn-HelloFont-ID-DianHei-CEJ": 3, "cn-HelloFont-ID-DianHei-DEJ": 4, "cn-HelloFont-ID-DianHei-EEJ": 5, "cn-HelloFont-ID-DianHei-FEJ": 6, "cn-HelloFont-ID-DianHei-GEJ": 7, "cn-HelloFont-ID-DianKai-Bold": 8, "cn-HelloFont-ID-DianKai": 9, "cn-HelloFont-WenYiHei": 10, "cn-Hellofont-ID-ChenYanXingKai": 11, "cn-Hellofont-ID-DaZiBao": 12, "cn-Hellofont-ID-DaoCaoRen": 13, "cn-Hellofont-ID-JianSong": 14, "cn-Hellofont-ID-JiangHuZhaoPaiHei": 15, "cn-Hellofont-ID-KeSong": 16, "cn-Hellofont-ID-LeYuanTi": 17, "cn-Hellofont-ID-Pinocchio": 18, "cn-Hellofont-ID-QiMiaoTi": 19, "cn-Hellofont-ID-QingHuaKai": 20, "cn-Hellofont-ID-QingHuaXingKai": 21, "cn-Hellofont-ID-ShanShuiXingKai": 22, "cn-Hellofont-ID-ShouXieQiShu": 23, "cn-Hellofont-ID-ShouXieTongZhenTi": 24, "cn-Hellofont-ID-TengLingTi": 25, "cn-Hellofont-ID-XiaoLiShu": 26, "cn-Hellofont-ID-XuanZhenSong": 27, "cn-Hellofont-ID-ZhongLingXingKai": 28, "cn-HellofontIDJiaoTangTi": 29, "cn-HellofontIDJiuZhuTi": 30, "cn-HuXiaoBao-SaoBao": 31, "cn-HuXiaoBo-NanShen": 32, "cn-HuXiaoBo-ZhenShuai": 33, "cn-SourceHanSansSC-Bold": 34, "cn-SourceHanSansSC-ExtraLight": 35, "cn-SourceHanSansSC-Heavy": 36, "cn-SourceHanSansSC-Light": 37, "cn-SourceHanSansSC-Medium": 38, "cn-SourceHanSansSC-Normal": 39, "cn-SourceHanSansSC-Regular": 40, "cn-SourceHanSerifSC-Bold": 41, "cn-SourceHanSerifSC-ExtraLight": 42, "cn-SourceHanSerifSC-Heavy": 43, "cn-SourceHanSerifSC-Light": 44, "cn-SourceHanSerifSC-Medium": 45, "cn-SourceHanSerifSC-Regular": 46, "cn-SourceHanSerifSC-SemiBold": 47, "cn-xiaowei": 48, "cn-AaJianHaoTi": 49, "cn-AlibabaPuHuiTi-Bold": 50, "cn-AlibabaPuHuiTi-Heavy": 51, "cn-AlibabaPuHuiTi-Light": 52, "cn-AlibabaPuHuiTi-Medium": 53, "cn-AlibabaPuHuiTi-Regular": 54, "cn-CanvaAcidBoldSC": 55, "cn-CanvaBreezeCN": 56, "cn-CanvaBumperCropSC": 57, "cn-CanvaCakeShopCN": 58, "cn-CanvaEndeavorBlackSC": 59, "cn-CanvaJoyHeiCN": 60, "cn-CanvaLiCN": 61, "cn-CanvaOrientalBrushCN": 62, "cn-CanvaPoster": 63, "cn-CanvaQinfuCalligraphyCN": 64, "cn-CanvaSweetHeartCN": 65, "cn-CanvaSwordLikeDreamCN": 66, "cn-CanvaTangyuanHandwritingCN": 67, "cn-CanvaWanderWorldCN": 68, "cn-CanvaWenCN": 69, "cn-DianZiChunYi": 70, "cn-GenSekiGothicTW-H": 71, "cn-GenWanMinTW-L": 72, "cn-GenYoMinTW-B": 73, "cn-GenYoMinTW-EL": 74, "cn-GenYoMinTW-H": 75, "cn-GenYoMinTW-M": 76, "cn-GenYoMinTW-R": 77, "cn-GenYoMinTW-SB": 78, "cn-HYQiHei-AZEJ": 79, "cn-HYQiHei-EES": 80, "cn-HanaMinA": 81, "cn-HappyZcool-2016": 82, "cn-HelloFont ZJ KeKouKeAiTi": 83, "cn-HelloFont-ID-BoBoTi": 84, "cn-HelloFont-ID-FuGuHei-25": 85, "cn-HelloFont-ID-FuGuHei-35": 86, "cn-HelloFont-ID-FuGuHei-45": 87, "cn-HelloFont-ID-FuGuHei-55": 88, "cn-HelloFont-ID-FuGuHei-65": 89, "cn-HelloFont-ID-FuGuHei-75": 90, "cn-HelloFont-ID-FuGuHei-85": 91, "cn-HelloFont-ID-HeiKa": 92, "cn-HelloFont-ID-HeiTang": 93, "cn-HelloFont-ID-JianSong-95": 94, "cn-HelloFont-ID-JueJiangHei-50": 95, "cn-HelloFont-ID-JueJiangHei-55": 96, "cn-HelloFont-ID-JueJiangHei-60": 97, "cn-HelloFont-ID-JueJiangHei-65": 98, "cn-HelloFont-ID-JueJiangHei-70": 99, "cn-HelloFont-ID-JueJiangHei-75": 100, "cn-HelloFont-ID-JueJiangHei-80": 101, "cn-HelloFont-ID-KuHeiTi": 102, "cn-HelloFont-ID-LingDongTi": 103, "cn-HelloFont-ID-LingLiTi": 104, "cn-HelloFont-ID-MuFengTi": 105, "cn-HelloFont-ID-NaiNaiJiangTi": 106, "cn-HelloFont-ID-PangDu": 107, "cn-HelloFont-ID-ReLieTi": 108, "cn-HelloFont-ID-RouRun": 109, "cn-HelloFont-ID-SaShuangShouXieTi": 110, "cn-HelloFont-ID-WangZheFengFan": 111, "cn-HelloFont-ID-YouQiTi": 112, "cn-Hellofont-ID-XiaLeTi": 113, "cn-Hellofont-ID-XianXiaTi": 114, "cn-HuXiaoBoKuHei": 115, "cn-IDDanMoXingKai": 116, "cn-IDJueJiangHei": 117, "cn-IDMeiLingTi": 118, "cn-IDQQSugar": 119, "cn-LiuJianMaoCao-Regular": 120, "cn-LongCang-Regular": 121, "cn-MaShanZheng-Regular": 122, "cn-PangMenZhengDao-3": 123, "cn-PangMenZhengDao-Cu": 124, "cn-PangMenZhengDao": 125, "cn-SentyCaramel": 126, "cn-SourceHanSerifSC": 127, "cn-WenCang-Regular": 128, "cn-WenQuanYiMicroHei": 129, "cn-XianErTi": 130, "cn-YRDZSTJF": 131, "cn-YS-HelloFont-BangBangTi": 132, "cn-ZCOOLKuaiLe-Regular": 133, "cn-ZCOOLQingKeHuangYou-Regular": 134, "cn-ZCOOLXiaoWei-Regular": 135, "cn-ZCOOL_KuHei": 136, "cn-ZhiMangXing-Regular": 137, "cn-baotuxiaobaiti": 138, "cn-jiangxizhuokai-Regular": 139, "cn-zcool-gdh": 140, "cn-zcoolqingkehuangyouti-Regular": 141, "cn-zcoolwenyiti": 142, "jp-04KanjyukuGothic": 0, "jp-07LightNovelPOP": 1, "jp-07NikumaruFont": 2, "jp-07YasashisaAntique": 3, "jp-07YasashisaGothic": 4, "jp-BokutachinoGothic2Bold": 5, "jp-BokutachinoGothic2Regular": 6, "jp-CHI_SpeedyRight_full_211128-Regular": 7, "jp-CHI_SpeedyRight_italic_full_211127-Regular": 8, "jp-CP-Font": 9, "jp-Canva_CezanneProN-B": 10, "jp-Canva_CezanneProN-M": 11, "jp-Canva_ChiaroStd-B": 12, "jp-Canva_CometStd-B": 13, "jp-Canva_DotMincho16Std-M": 14, "jp-Canva_GrecoStd-B": 15, "jp-Canva_GrecoStd-M": 16, "jp-Canva_LyraStd-DB": 17, "jp-Canva_MatisseHatsuhiPro-B": 18, "jp-Canva_MatisseHatsuhiPro-M": 19, "jp-Canva_ModeMinAStd-B": 20, "jp-Canva_NewCezanneProN-B": 21, "jp-Canva_NewCezanneProN-M": 22, "jp-Canva_PearlStd-L": 23, "jp-Canva_RaglanStd-UB": 24, "jp-Canva_RailwayStd-B": 25, "jp-Canva_ReggaeStd-B": 26, "jp-Canva_RocknRollStd-DB": 27, "jp-Canva_RodinCattleyaPro-B": 28, "jp-Canva_RodinCattleyaPro-M": 29, "jp-Canva_RodinCattleyaPro-UB": 30, "jp-Canva_RodinHimawariPro-B": 31, "jp-Canva_RodinHimawariPro-M": 32, "jp-Canva_RodinMariaPro-B": 33, "jp-Canva_RodinMariaPro-DB": 34, "jp-Canva_RodinProN-M": 35, "jp-Canva_ShadowTLStd-B": 36, "jp-Canva_StickStd-B": 37, "jp-Canva_TsukuAOldMinPr6N-B": 38, "jp-Canva_TsukuAOldMinPr6N-R": 39, "jp-Canva_UtrilloPro-DB": 40, "jp-Canva_UtrilloPro-M": 41, "jp-Canva_YurukaStd-UB": 42, "jp-FGUIGEN": 43, "jp-GlowSansJ-Condensed-Heavy": 44, "jp-GlowSansJ-Condensed-Light": 45, "jp-GlowSansJ-Normal-Bold": 46, "jp-GlowSansJ-Normal-Light": 47, "jp-HannariMincho": 48, "jp-HarenosoraMincho": 49, "jp-Jiyucho": 50, "jp-Kaiso-Makina-B": 51, "jp-Kaisotai-Next-UP-B": 52, "jp-KokoroMinchoutai": 53, "jp-Mamelon-3-Hi-Regular": 54, "jp-MotoyaAnemoneStd-W1": 55, "jp-MotoyaAnemoneStd-W5": 56, "jp-MotoyaAnticPro-W3": 57, "jp-MotoyaCedarStd-W3": 58, "jp-MotoyaCedarStd-W5": 59, "jp-MotoyaGochikaStd-W4": 60, "jp-MotoyaGochikaStd-W8": 61, "jp-MotoyaGothicMiyabiStd-W6": 62, "jp-MotoyaGothicStd-W3": 63, "jp-MotoyaGothicStd-W5": 64, "jp-MotoyaKoinStd-W3": 65, "jp-MotoyaKyotaiStd-W2": 66, "jp-MotoyaKyotaiStd-W4": 67, "jp-MotoyaMaruStd-W3": 68, "jp-MotoyaMaruStd-W5": 69, "jp-MotoyaMinchoMiyabiStd-W4": 70, "jp-MotoyaMinchoMiyabiStd-W6": 71, "jp-MotoyaMinchoModernStd-W4": 72, "jp-MotoyaMinchoModernStd-W6": 73, "jp-MotoyaMinchoStd-W3": 74, "jp-MotoyaMinchoStd-W5": 75, "jp-MotoyaReisyoStd-W2": 76, "jp-MotoyaReisyoStd-W6": 77, "jp-MotoyaTohitsuStd-W4": 78, "jp-MotoyaTohitsuStd-W6": 79, "jp-MtySousyokuEmBcJis-W6": 80, "jp-MtySousyokuLiBcJis-W6": 81, "jp-Mushin": 82, "jp-NotoSansJP-Bold": 83, "jp-NotoSansJP-Regular": 84, "jp-NudMotoyaAporoStd-W3": 85, "jp-NudMotoyaAporoStd-W5": 86, "jp-NudMotoyaCedarStd-W3": 87, "jp-NudMotoyaCedarStd-W5": 88, "jp-NudMotoyaMaruStd-W3": 89, "jp-NudMotoyaMaruStd-W5": 90, "jp-NudMotoyaMinchoStd-W5": 91, "jp-Ounen-mouhitsu": 92, "jp-Ronde-B-Square": 93, "jp-SMotoyaGyosyoStd-W5": 94, "jp-SMotoyaSinkaiStd-W3": 95, "jp-SMotoyaSinkaiStd-W5": 96, "jp-SourceHanSansJP-Bold": 97, "jp-SourceHanSansJP-Regular": 98, "jp-SourceHanSerifJP-Bold": 99, "jp-SourceHanSerifJP-Regular": 100, "jp-TazuganeGothicStdN-Bold": 101, "jp-TazuganeGothicStdN-Regular": 102, "jp-TelopMinProN-B": 103, "jp-Togalite-Bold": 104, "jp-Togalite-Regular": 105, "jp-TsukuMinPr6N-E": 106, "jp-TsukuMinPr6N-M": 107, "jp-mikachan_o": 108, "jp-nagayama_kai": 109, "jp-07LogoTypeGothic7": 110, "jp-07TetsubinGothic": 111, "jp-851CHIKARA-DZUYOKU-KANA-A": 112, "jp-ARMinchoJIS-Light": 113, "jp-ARMinchoJIS-Ultra": 114, "jp-ARPCrystalMinchoJIS-Medium": 115, "jp-ARPCrystalRGothicJIS-Medium": 116, "jp-ARShounanShinpitsuGyosyoJIS-Medium": 117, "jp-AozoraMincho-bold": 118, "jp-AozoraMinchoRegular": 119, "jp-ArialUnicodeMS-Bold": 120, "jp-ArialUnicodeMS": 121, "jp-CanvaBreezeJP": 122, "jp-CanvaLiCN": 123, "jp-CanvaLiJP": 124, "jp-CanvaOrientalBrushCN": 125, "jp-CanvaQinfuCalligraphyJP": 126, "jp-CanvaSweetHeartJP": 127, "jp-CanvaWenJP": 128, "jp-Corporate-Logo-Bold": 129, "jp-DelaGothicOne-Regular": 130, "jp-GN-Kin-iro_SansSerif": 131, "jp-GN-Koharuiro_Sunray": 132, "jp-GenEiGothicM-B": 133, "jp-GenEiGothicM-R": 134, "jp-GenJyuuGothic-Bold": 135, "jp-GenRyuMinTW-B": 136, "jp-GenRyuMinTW-R": 137, "jp-GenSekiGothicTW-B": 138, "jp-GenSekiGothicTW-R": 139, "jp-GenSenRoundedTW-B": 140, "jp-GenSenRoundedTW-R": 141, "jp-GenShinGothic-Bold": 142, "jp-GenShinGothic-Normal": 143, "jp-GenWanMinTW-L": 144, "jp-GenYoGothicTW-B": 145, "jp-GenYoGothicTW-R": 146, "jp-GenYoMinTW-B": 147, "jp-GenYoMinTW-R": 148, "jp-HGBouquet": 149, "jp-HanaMinA": 150, "jp-HanazomeFont": 151, "jp-HinaMincho-Regular": 152, "jp-Honoka-Antique-Maru": 153, "jp-Honoka-Mincho": 154, "jp-HuiFontP": 155, "jp-IPAexMincho": 156, "jp-JK-Gothic-L": 157, "jp-JK-Gothic-M": 158, "jp-JackeyFont": 159, "jp-KaiseiTokumin-Bold": 160, "jp-KaiseiTokumin-Regular": 161, "jp-Keifont": 162, "jp-KiwiMaru-Regular": 163, "jp-Koku-Mincho-Regular": 164, "jp-MotoyaLMaru-W3-90ms-RKSJ-H": 165, "jp-NewTegomin-Regular": 166, "jp-NicoKaku": 167, "jp-NicoMoji+": 168, "jp-Otsutome_font-Bold": 169, "jp-PottaOne-Regular": 170, "jp-RampartOne-Regular": 171, "jp-Senobi-Gothic-Bold": 172, "jp-Senobi-Gothic-Regular": 173, "jp-SmartFontUI-Proportional": 174, "jp-SoukouMincho": 175, "jp-TEST_Klee-DB": 176, "jp-TEST_Klee-M": 177, "jp-TEST_UDMincho-B": 178, "jp-TEST_UDMincho-L": 179, "jp-TT_Akakane-EB": 180, "jp-Tanuki-Permanent-Marker": 181, "jp-TrainOne-Regular": 182, "jp-TsunagiGothic-Black": 183, "jp-Ume-Hy-Gothic": 184, "jp-Ume-P-Mincho": 185, "jp-WenQuanYiMicroHei": 186, "jp-XANO-mincho-U32": 187, "jp-YOzFontM90-Regular": 188, "jp-Yomogi-Regular": 189, "jp-YujiBoku-Regular": 190, "jp-YujiSyuku-Regular": 191, "jp-ZenKakuGothicNew-Bold": 192, "jp-ZenKakuGothicNew-Regular": 193, "jp-ZenKurenaido-Regular": 194, "jp-ZenMaruGothic-Bold": 195, "jp-ZenMaruGothic-Regular": 196, "jp-darts-font": 197, "jp-irohakakuC-Bold": 198, "jp-irohakakuC-Medium": 199, "jp-irohakakuC-Regular": 200, "jp-katyou": 201, "jp-mplus-1m-bold": 202, "jp-mplus-1m-regular": 203, "jp-mplus-1p-bold": 204, "jp-mplus-1p-regular": 205, "jp-rounded-mplus-1p-bold": 206, "jp-rounded-mplus-1p-regular": 207, "jp-timemachine-wa": 208, "jp-ttf-GenEiLateMin-Medium": 209, "jp-uzura_font": 210, "kr-Arita-buri-Bold_OTF": 0, "kr-Arita-buri-HairLine_OTF": 1, "kr-Arita-buri-Light_OTF": 2, "kr-Arita-buri-Medium_OTF": 3, "kr-Arita-buri-SemiBold_OTF": 4, "kr-Canva_YDSunshineL": 5, "kr-Canva_YDSunshineM": 6, "kr-Canva_YoonGulimPro710": 7, "kr-Canva_YoonGulimPro730": 8, "kr-Canva_YoonGulimPro740": 9, "kr-Canva_YoonGulimPro760": 10, "kr-Canva_YoonGulimPro770": 11, "kr-Canva_YoonGulimPro790": 12, "kr-CreHappB": 13, "kr-CreHappL": 14, "kr-CreHappM": 15, "kr-CreHappS": 16, "kr-OTAuroraB": 17, "kr-OTAuroraL": 18, "kr-OTAuroraR": 19, "kr-OTDoldamgilB": 20, "kr-OTDoldamgilL": 21, "kr-OTDoldamgilR": 22, "kr-OTHamsterB": 23, "kr-OTHamsterL": 24, "kr-OTHamsterR": 25, "kr-OTHapchangdanB": 26, "kr-OTHapchangdanL": 27, "kr-OTHapchangdanR": 28, "kr-OTSupersizeBkBOX": 29, "kr-SourceHanSansKR-Bold": 30, "kr-SourceHanSansKR-ExtraLight": 31, "kr-SourceHanSansKR-Heavy": 32, "kr-SourceHanSansKR-Light": 33, "kr-SourceHanSansKR-Medium": 34, "kr-SourceHanSansKR-Normal": 35, "kr-SourceHanSansKR-Regular": 36, "kr-SourceHanSansSC-Bold": 37, "kr-SourceHanSansSC-ExtraLight": 38, "kr-SourceHanSansSC-Heavy": 39, "kr-SourceHanSansSC-Light": 40, "kr-SourceHanSansSC-Medium": 41, "kr-SourceHanSansSC-Normal": 42, "kr-SourceHanSansSC-Regular": 43, "kr-SourceHanSerifSC-Bold": 44, "kr-SourceHanSerifSC-SemiBold": 45, "kr-TDTDBubbleBubbleOTF": 46, "kr-TDTDConfusionOTF": 47, "kr-TDTDCuteAndCuteOTF": 48, "kr-TDTDEggTakOTF": 49, "kr-TDTDEmotionalLetterOTF": 50, "kr-TDTDGalapagosOTF": 51, "kr-TDTDHappyHourOTF": 52, "kr-TDTDLatteOTF": 53, "kr-TDTDMoonLightOTF": 54, "kr-TDTDParkForestOTF": 55, "kr-TDTDPencilOTF": 56, "kr-TDTDSmileOTF": 57, "kr-TDTDSproutOTF": 58, "kr-TDTDSunshineOTF": 59, "kr-TDTDWaferOTF": 60, "kr-777Chyaochyureu": 61, "kr-ArialUnicodeMS-Bold": 62, "kr-ArialUnicodeMS": 63, "kr-BMHANNA": 64, "kr-Baekmuk-Dotum": 65, "kr-BagelFatOne-Regular": 66, "kr-CoreBandi": 67, "kr-CoreBandiFace": 68, "kr-CoreBori": 69, "kr-DoHyeon-Regular": 70, "kr-Dokdo-Regular": 71, "kr-Gaegu-Bold": 72, "kr-Gaegu-Light": 73, "kr-Gaegu-Regular": 74, "kr-GamjaFlower-Regular": 75, "kr-GasoekOne-Regular": 76, "kr-GothicA1-Black": 77, "kr-GothicA1-Bold": 78, "kr-GothicA1-ExtraBold": 79, "kr-GothicA1-ExtraLight": 80, "kr-GothicA1-Light": 81, "kr-GothicA1-Medium": 82, "kr-GothicA1-Regular": 83, "kr-GothicA1-SemiBold": 84, "kr-GothicA1-Thin": 85, "kr-Gugi-Regular": 86, "kr-HiMelody-Regular": 87, "kr-Jua-Regular": 88, "kr-KirangHaerang-Regular": 89, "kr-NanumBrush": 90, "kr-NanumPen": 91, "kr-NanumSquareRoundB": 92, "kr-NanumSquareRoundEB": 93, "kr-NanumSquareRoundL": 94, "kr-NanumSquareRoundR": 95, "kr-SeH-CB": 96, "kr-SeH-CBL": 97, "kr-SeH-CEB": 98, "kr-SeH-CL": 99, "kr-SeH-CM": 100, "kr-SeN-CB": 101, "kr-SeN-CBL": 102, "kr-SeN-CEB": 103, "kr-SeN-CL": 104, "kr-SeN-CM": 105, "kr-Sunflower-Bold": 106, "kr-Sunflower-Light": 107, "kr-Sunflower-Medium": 108, "kr-TTClaytoyR": 109, "kr-TTDalpangiR": 110, "kr-TTMamablockR": 111, "kr-TTNauidongmuR": 112, "kr-TTOktapbangR": 113, "kr-UhBeeMiMi": 114, "kr-UhBeeMiMiBold": 115, "kr-UhBeeSe_hyun": 116, "kr-UhBeeSe_hyunBold": 117, "kr-UhBeenamsoyoung": 118, "kr-UhBeenamsoyoungBold": 119, "kr-WenQuanYiMicroHei": 120, "kr-YeonSung-Regular": 121} \ No newline at end of file diff --git a/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/assets/teaser/teaser_multilingual_1.webp b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/assets/teaser/teaser_multilingual_1.webp new file mode 100644 index 0000000000000000000000000000000000000000..909b3e8be5452ae637f1e0e1b59eb73a1cd6393e Binary files /dev/null and b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/assets/teaser/teaser_multilingual_1.webp differ diff --git a/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/assets/teaser/teaser_multilingual_2.webp b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/assets/teaser/teaser_multilingual_2.webp new file mode 100644 index 0000000000000000000000000000000000000000..ab48fd87a16df9634f84518339829f9d65e15b5f Binary files /dev/null and b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/assets/teaser/teaser_multilingual_2.webp differ diff --git a/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/assets/teaser/teaser_multilingual_3.webp b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/assets/teaser/teaser_multilingual_3.webp new file mode 100644 index 0000000000000000000000000000000000000000..24e0592c44b57004b5402ba46dddb6057f472877 Binary files /dev/null and b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/assets/teaser/teaser_multilingual_3.webp differ diff --git a/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/assets/teaser/teaser_multilingual_4.webp b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/assets/teaser/teaser_multilingual_4.webp new file mode 100644 index 0000000000000000000000000000000000000000..7a96ae0b74bfe2d94222cc202fadc32f4add880b Binary files /dev/null and b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/assets/teaser/teaser_multilingual_4.webp differ diff --git a/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/checkpoints/byt5_mapper.pt b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/checkpoints/byt5_mapper.pt new file mode 100644 index 0000000000000000000000000000000000000000..7f2231e72af0ea65c02a47d4eff2c43c8e137cce --- /dev/null +++ b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/checkpoints/byt5_mapper.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d5911cf07328d949eff29cc08ca7637dc4fe5312a8fa351ca4bec07d357b1c5 +size 301553807 diff --git a/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/checkpoints/byt5_model.pt b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/checkpoints/byt5_model.pt new file mode 100644 index 0000000000000000000000000000000000000000..b75471ebeb3c18d4fb673501aa73e30ad431df3d --- /dev/null +++ b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/checkpoints/byt5_model.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca8c97c89136f767d4534449bbf3f25296d390574e0af1cc16f09774a901d6db +size 877308845 diff --git a/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/checkpoints/unet_inserted_attn.pt b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/checkpoints/unet_inserted_attn.pt new file mode 100644 index 0000000000000000000000000000000000000000..9a0fa4873f6b2f36220807261c193f17df58c5c2 --- /dev/null +++ b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/checkpoints/unet_inserted_attn.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5b6af4376281be262f3b52ca0b16b0244099161693f65a7db352f53878481767 +size 908 diff --git a/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/checkpoints/unet_lora.pt b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/checkpoints/unet_lora.pt new file mode 100644 index 0000000000000000000000000000000000000000..b2151650fd973486e89471bea239de0e663f110c --- /dev/null +++ b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/checkpoints/unet_lora.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:47ae2328a9c4892a24c4a66f25780ab61a55cbd8eb693a1966cc99e674e832be +size 743590514 diff --git a/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/configs/glyph_sdxl_multilingual_albedo.py b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/configs/glyph_sdxl_multilingual_albedo.py new file mode 100644 index 0000000000000000000000000000000000000000..82706d92909f4cf10f07c9edd90c6d2da9c77502 --- /dev/null +++ b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/configs/glyph_sdxl_multilingual_albedo.py @@ -0,0 +1,96 @@ +#### Model Setting +pretrained_model_name_or_path = 'stablediffusionapi/albedobase-xl-20' +pretrained_vae_model_name_or_path = 'madebyollin/sdxl-vae-fp16-fix' +revision = None + +byt5_max_length = 512 +byt5_mapper_type = 'T5EncoderBlockByT5Mapper' +byt5_mapper_config = dict( + num_layers=4, + sdxl_channels=2048, +) +byt5_config = dict( + byt5_name='google/byt5-small', + special_token=True, + color_special_token=True, + font_special_token=True, + color_ann_path='assets/color_idx.json', + font_ann_path='assets/multilingual_10-lang_idx.json', + multilingual=True, +) + +attn_block_to_modify = [ + "down_blocks.1.attentions.0.transformer_blocks.0", + "down_blocks.1.attentions.0.transformer_blocks.1", + "down_blocks.1.attentions.1.transformer_blocks.0", + "down_blocks.1.attentions.1.transformer_blocks.1", + "down_blocks.2.attentions.0.transformer_blocks.0", + "down_blocks.2.attentions.0.transformer_blocks.1", + "down_blocks.2.attentions.0.transformer_blocks.2", + "down_blocks.2.attentions.0.transformer_blocks.3", + "down_blocks.2.attentions.0.transformer_blocks.4", + "down_blocks.2.attentions.0.transformer_blocks.5", + "down_blocks.2.attentions.0.transformer_blocks.6", + "down_blocks.2.attentions.0.transformer_blocks.7", + "down_blocks.2.attentions.0.transformer_blocks.8", + "down_blocks.2.attentions.0.transformer_blocks.9", + "down_blocks.2.attentions.1.transformer_blocks.0", + "down_blocks.2.attentions.1.transformer_blocks.1", + "down_blocks.2.attentions.1.transformer_blocks.2", + "down_blocks.2.attentions.1.transformer_blocks.3", + "down_blocks.2.attentions.1.transformer_blocks.4", + "down_blocks.2.attentions.1.transformer_blocks.5", + "down_blocks.2.attentions.1.transformer_blocks.6", + "down_blocks.2.attentions.1.transformer_blocks.7", + "down_blocks.2.attentions.1.transformer_blocks.8", + "down_blocks.2.attentions.1.transformer_blocks.9", + "up_blocks.0.attentions.0.transformer_blocks.0", + "up_blocks.0.attentions.0.transformer_blocks.1", + "up_blocks.0.attentions.0.transformer_blocks.2", + "up_blocks.0.attentions.0.transformer_blocks.3", + "up_blocks.0.attentions.0.transformer_blocks.4", + "up_blocks.0.attentions.0.transformer_blocks.5", + "up_blocks.0.attentions.0.transformer_blocks.6", + "up_blocks.0.attentions.0.transformer_blocks.7", + "up_blocks.0.attentions.0.transformer_blocks.8", + "up_blocks.0.attentions.0.transformer_blocks.9", + "up_blocks.0.attentions.1.transformer_blocks.0", + "up_blocks.0.attentions.1.transformer_blocks.1", + "up_blocks.0.attentions.1.transformer_blocks.2", + "up_blocks.0.attentions.1.transformer_blocks.3", + "up_blocks.0.attentions.1.transformer_blocks.4", + "up_blocks.0.attentions.1.transformer_blocks.5", + "up_blocks.0.attentions.1.transformer_blocks.6", + "up_blocks.0.attentions.1.transformer_blocks.7", + "up_blocks.0.attentions.1.transformer_blocks.8", + "up_blocks.0.attentions.1.transformer_blocks.9", + "up_blocks.0.attentions.2.transformer_blocks.0", + "up_blocks.0.attentions.2.transformer_blocks.1", + "up_blocks.0.attentions.2.transformer_blocks.2", + "up_blocks.0.attentions.2.transformer_blocks.3", + "up_blocks.0.attentions.2.transformer_blocks.4", + "up_blocks.0.attentions.2.transformer_blocks.5", + "up_blocks.0.attentions.2.transformer_blocks.6", + "up_blocks.0.attentions.2.transformer_blocks.7", + "up_blocks.0.attentions.2.transformer_blocks.8", + "up_blocks.0.attentions.2.transformer_blocks.9", + "up_blocks.1.attentions.0.transformer_blocks.0", + "up_blocks.1.attentions.0.transformer_blocks.1", + "up_blocks.1.attentions.1.transformer_blocks.0", + "up_blocks.1.attentions.1.transformer_blocks.1", + "up_blocks.1.attentions.2.transformer_blocks.0", + "up_blocks.1.attentions.2.transformer_blocks.1", + "mid_block.attentions.0.transformer_blocks.0", + "mid_block.attentions.0.transformer_blocks.1", + "mid_block.attentions.0.transformer_blocks.2", + "mid_block.attentions.0.transformer_blocks.3", + "mid_block.attentions.0.transformer_blocks.4", + "mid_block.attentions.0.transformer_blocks.5", + "mid_block.attentions.0.transformer_blocks.6", + "mid_block.attentions.0.transformer_blocks.7", + "mid_block.attentions.0.transformer_blocks.8", + "mid_block.attentions.0.transformer_blocks.9", +] + +unet_lora_rank = 128 +inference_dtype = 'fp16' diff --git a/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/configuration.json b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/configuration.json new file mode 100644 index 0000000000000000000000000000000000000000..7600045091e74defbb184808c1b4482398a7def8 --- /dev/null +++ b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/configuration.json @@ -0,0 +1 @@ +{"framework": "pytorch", "task": "text-image", "allow_remote": true} \ No newline at end of file diff --git a/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/examples/xiaoman.json b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/examples/xiaoman.json new file mode 100644 index 0000000000000000000000000000000000000000..36197355190ad747327c0c6940797258835dafa7 --- /dev/null +++ b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/examples/xiaoman.json @@ -0,0 +1,65 @@ +{ + "texts": [ + "小满是二十四节气之一，夏季的第二个节气。该节气是指夏熟作物的籽粒开始灌浆饱满，但还未成熟，只是小满，还未大满。", + "2022.5.21", + "饱满的灵魂 无畏的生长 二十四节气之一", + "今日小满", + "Grain Buds" + ], + "styles": [ + { + "color": "#427227", + "font-family": "cn-HYQiHei-AZEJ" + }, + { + "font-family": "en-TAN MERINGUE", + "color": "#f89b2b" + }, + { + "color": "#ffffff", + "font-family": "cn-SourceHanSansSC-ExtraLight" + }, + { + "color": "#427227", + "font-family": "cn-AlibabaPuHuiTi-Bold" + }, + { + "color": "#427227", + "font-family": "en-SairaCondensed-Regular" + } + ], + "bbox": [ + [ + 0.09969604863221884, + 0.4370820668693009, + 0.31124620060790276, + 0.2072948328267477 + ], + [ + 0.10455927051671733, + 0.09908814589665653, + 0.22127659574468084, + 0.034650455927051675 + ], + [ + 0.09969604863221884, + 0.9398176291793313, + 0.7993920972644377, + 0.026747720364741642 + ], + [ + 0.09787234042553192, + 0.17142857142857143, + 0.4231003039513678, + 0.10577507598784194 + ], + [ + 0.10091185410334347, + 0.3100303951367781, + 0.2772036474164134, + 0.053495440729483285 + ] + ], + "bg_prompt": "The image portrays a young girl sitting on a large green leaf. The leaf is part of a plant with other green leaves. The girl is wearing a yellow dress and a straw hat. She is holding a small yellow flower in her hand. The background of the image is a light blue sky with a few clouds. The overall style of the image is a colorful, cartoon-like illustration.", + "seed": 0 +} \ No newline at end of file diff --git a/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/glyph_sdxl/custom_diffusers/__init__.py b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/glyph_sdxl/custom_diffusers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..cb8e293067659717f27792b10c14413fa72b4f24 --- /dev/null +++ b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/glyph_sdxl/custom_diffusers/__init__.py @@ -0,0 +1,2 @@ +from .pipelines import * +from .models import * diff --git a/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/glyph_sdxl/custom_diffusers/models/__init__.py b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/glyph_sdxl/custom_diffusers/models/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..648dae2e74c678c47c86f631522e24d07d2e8569 --- /dev/null +++ b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/glyph_sdxl/custom_diffusers/models/__init__.py @@ -0,0 +1,3 @@ +from .cross_attn_insert_transformer_blocks import CrossAttnInsertBasicTransformerBlock + +__all__ = ['CrossAttnInsertBasicTransformerBlock'] \ No newline at end of file diff --git a/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/glyph_sdxl/custom_diffusers/models/cross_attn_insert_transformer_blocks.py b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/glyph_sdxl/custom_diffusers/models/cross_attn_insert_transformer_blocks.py new file mode 100644 index 0000000000000000000000000000000000000000..8df97ea5a3baa47f8aedef66cd225f4530ef6002 --- /dev/null +++ b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/glyph_sdxl/custom_diffusers/models/cross_attn_insert_transformer_blocks.py @@ -0,0 +1,377 @@ +from typing import Optional, Dict, Any +import copy + +import torch +import torch.nn as nn + +from diffusers.models.attention import ( + BasicTransformerBlock, + SinusoidalPositionalEmbedding, + AdaLayerNorm, + AdaLayerNormZero, + AdaLayerNormContinuous, + Attention, + FeedForward, + GatedSelfAttentionDense, + GELU, + GEGLU, + ApproximateGELU, + _chunked_feed_forward, +) + +class CrossAttnInsertBasicTransformerBlock(BasicTransformerBlock): + def __init__( + self, + dim: int, + num_attention_heads: int, + attention_head_dim: int, + dropout=0.0, + cross_attention_dim: Optional[int] = None, + glyph_cross_attention_dim: Optional[int] = None, + activation_fn: str = "geglu", + num_embeds_ada_norm: Optional[int] = None, + attention_bias: bool = False, + only_cross_attention: bool = False, + double_self_attention: bool = False, + upcast_attention: bool = False, + norm_elementwise_affine: bool = True, + norm_type: str = "layer_norm", # 'layer_norm', 'ada_norm', 'ada_norm_zero', 'ada_norm_single', 'layer_norm_i2vgen' + norm_eps: float = 1e-5, + final_dropout: bool = False, + attention_type: str = "default", + positional_embeddings: Optional[str] = None, + num_positional_embeddings: Optional[int] = None, + ada_norm_continous_conditioning_embedding_dim: Optional[int] = None, + ada_norm_bias: Optional[int] = None, + ff_inner_dim: Optional[int] = None, + ff_bias: bool = True, + attention_out_bias: bool = True, + ): + super(BasicTransformerBlock, self).__init__() + self.only_cross_attention = only_cross_attention + + if norm_type in ("ada_norm", "ada_norm_zero") and num_embeds_ada_norm is None: + raise ValueError( + f"`norm_type` is set to {norm_type}, but `num_embeds_ada_norm` is not defined. Please make sure to" + f" define `num_embeds_ada_norm` if setting `norm_type` to {norm_type}." + ) + + self.norm_type = norm_type + self.num_embeds_ada_norm = num_embeds_ada_norm + + if positional_embeddings and (num_positional_embeddings is None): + raise ValueError( + "If `positional_embedding` type is defined, `num_positition_embeddings` must also be defined." + ) + + if positional_embeddings == "sinusoidal": + self.pos_embed = SinusoidalPositionalEmbedding(dim, max_seq_length=num_positional_embeddings) + else: + self.pos_embed = None + + # Define 3 blocks. Each block has its own normalization layer. + # 1. Self-Attn + if norm_type == "ada_norm": + self.norm1 = AdaLayerNorm(dim, num_embeds_ada_norm) + elif norm_type == "ada_norm_zero": + self.norm1 = AdaLayerNormZero(dim, num_embeds_ada_norm) + elif norm_type == "ada_norm_continuous": + self.norm1 = AdaLayerNormContinuous( + dim, + ada_norm_continous_conditioning_embedding_dim, + norm_elementwise_affine, + norm_eps, + ada_norm_bias, + "rms_norm", + ) + else: + self.norm1 = nn.LayerNorm(dim, elementwise_affine=norm_elementwise_affine, eps=norm_eps) + + self.attn1 = Attention( + query_dim=dim, + heads=num_attention_heads, + dim_head=attention_head_dim, + dropout=dropout, + bias=attention_bias, + cross_attention_dim=cross_attention_dim if only_cross_attention else None, + upcast_attention=upcast_attention, + out_bias=attention_out_bias, + ) + + # 2. Cross-Attn + if cross_attention_dim is not None or double_self_attention: + # We currently only use AdaLayerNormZero for self attention where there will only be one attention block. + # I.e. the number of returned modulation chunks from AdaLayerZero would not make sense if returned during + # the second cross attention block. + if norm_type == "ada_norm": + self.norm2 = AdaLayerNorm(dim, num_embeds_ada_norm) + elif norm_type == "ada_norm_continuous": + self.norm2 = AdaLayerNormContinuous( + dim, + ada_norm_continous_conditioning_embedding_dim, + norm_elementwise_affine, + norm_eps, + ada_norm_bias, + "rms_norm", + ) + else: + self.norm2 = nn.LayerNorm(dim, norm_eps, norm_elementwise_affine) + + self.attn2 = Attention( + query_dim=dim, + cross_attention_dim=cross_attention_dim if not double_self_attention else None, + heads=num_attention_heads, + dim_head=attention_head_dim, + dropout=dropout, + bias=attention_bias, + upcast_attention=upcast_attention, + out_bias=attention_out_bias, + ) # is self-attn if encoder_hidden_states is none + else: + self.norm2 = None + self.attn2 = None + + # 3. Feed-forward + if norm_type == "ada_norm_continuous": + self.norm3 = AdaLayerNormContinuous( + dim, + ada_norm_continous_conditioning_embedding_dim, + norm_elementwise_affine, + norm_eps, + ada_norm_bias, + "layer_norm", + ) + + elif norm_type in ["ada_norm_zero", "ada_norm", "layer_norm", "ada_norm_continuous"]: + self.norm3 = nn.LayerNorm(dim, norm_eps, norm_elementwise_affine) + elif norm_type == "layer_norm_i2vgen": + self.norm3 = None + + self.ff = FeedForward( + dim, + dropout=dropout, + activation_fn=activation_fn, + final_dropout=final_dropout, + inner_dim=ff_inner_dim, + bias=ff_bias, + ) + + # 4. Fuser + if attention_type == "gated" or attention_type == "gated-text-image": + self.fuser = GatedSelfAttentionDense(dim, cross_attention_dim, num_attention_heads, attention_head_dim) + + # 5. Scale-shift for PixArt-Alpha. + if norm_type == "ada_norm_single": + self.scale_shift_table = nn.Parameter(torch.randn(6, dim) / dim**0.5) + + # let chunk size default to None + self._chunk_size = None + self._chunk_dim = 0 + + def get_inserted_modules(self): + return () + + def get_inserted_modules_names(self): + return () + + def get_origin_modules(self): + inserted_modules = self.get_inserted_modules() + origin_modules = [] + for module in self.children(): + if module not in inserted_modules: + origin_modules.append(module) + return tuple(origin_modules) + + + @classmethod + def from_transformer_block( + cls, + transformer_block, + glyph_cross_attention_dim, + ): + inner_dim = transformer_block.attn1.query_dim + num_attention_heads = transformer_block.attn1.heads + attention_head_dim = transformer_block.attn1.inner_dim // num_attention_heads + dropout = transformer_block.attn1.dropout + cross_attention_dim = transformer_block.attn2.cross_attention_dim + if isinstance(transformer_block.ff.net[0], GELU): + if transformer_block.ff.net[0].approximate == "tanh": + activation_fn = "gelu-approximate" + else: + activation_fn = "gelu" + elif isinstance(transformer_block.ff.net[0], GEGLU): + activation_fn = "geglu" + elif isinstance(transformer_block.ff.net[0], ApproximateGELU): + activation_fn = "geglu-approximate" + num_embeds_ada_norm = transformer_block.num_embeds_ada_norm + attention_bias = transformer_block.attn1.to_q.bias is not None + only_cross_attention = transformer_block.only_cross_attention + double_self_attention = transformer_block.attn2.cross_attention_dim is None + upcast_attention = transformer_block.attn1.upcast_attention + norm_type = transformer_block.norm_type + assert isinstance(transformer_block.norm1, nn.LayerNorm) + norm_elementwise_affine = transformer_block.norm1.elementwise_affine + norm_eps = transformer_block.norm1.eps + assert getattr(transformer_block, 'fuser', None) is None + attention_type = "default" + model = cls( + inner_dim, + num_attention_heads, + attention_head_dim, + dropout=dropout, + cross_attention_dim=cross_attention_dim, + glyph_cross_attention_dim=glyph_cross_attention_dim, + activation_fn=activation_fn, + num_embeds_ada_norm=num_embeds_ada_norm, + attention_bias=attention_bias, + only_cross_attention=only_cross_attention, + double_self_attention=double_self_attention, + upcast_attention=upcast_attention, + norm_type=norm_type, + norm_elementwise_affine=norm_elementwise_affine, + norm_eps=norm_eps, + attention_type=attention_type, + ) + missing_keys, unexpected_keys = model.load_state_dict( + transformer_block.state_dict(), + strict=False, + ) + assert len(unexpected_keys) == 0 + assert all(i.startswith('glyph') for i in missing_keys) + + return model + + def forward( + self, + hidden_states: torch.FloatTensor, + attention_mask: Optional[torch.FloatTensor] = None, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + encoder_attention_mask: Optional[torch.FloatTensor] = None, + timestep: Optional[torch.LongTensor] = None, + cross_attention_kwargs: Dict[str, Any] = None, + class_labels: Optional[torch.LongTensor] = None, + added_cond_kwargs: Optional[Dict[str, torch.Tensor]] = None, + ) -> torch.FloatTensor: + # Notice that normalization is always applied before the real computation in the following blocks. + # 0. Self-Attention + batch_size = hidden_states.shape[0] + + if self.norm_type == "ada_norm": + norm_hidden_states = self.norm1(hidden_states, timestep) + elif self.norm_type == "ada_norm_zero": + norm_hidden_states, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.norm1( + hidden_states, timestep, class_labels, hidden_dtype=hidden_states.dtype + ) + elif self.norm_type in ["layer_norm", "layer_norm_i2vgen"]: + norm_hidden_states = self.norm1(hidden_states) + elif self.norm_type == "ada_norm_continuous": + norm_hidden_states = self.norm1(hidden_states, added_cond_kwargs["pooled_text_emb"]) + elif self.norm_type == "ada_norm_single": + shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = ( + self.scale_shift_table[None] + timestep.reshape(batch_size, 6, -1) + ).chunk(6, dim=1) + norm_hidden_states = self.norm1(hidden_states) + norm_hidden_states = norm_hidden_states * (1 + scale_msa) + shift_msa + norm_hidden_states = norm_hidden_states.squeeze(1) + else: + raise ValueError("Incorrect norm used") + + if self.pos_embed is not None: + norm_hidden_states = self.pos_embed(norm_hidden_states) + + # 1. Retrieve lora scale. + lora_scale = cross_attention_kwargs.get("scale", 1.0) if cross_attention_kwargs is not None else 1.0 + + # 2. Prepare GLIGEN inputs + cross_attention_kwargs = cross_attention_kwargs.copy() if cross_attention_kwargs is not None else {} + gligen_kwargs = cross_attention_kwargs.pop("gligen", None) + + glyph_encoder_hidden_states = cross_attention_kwargs.pop("glyph_encoder_hidden_states", None) + # a dict. visual_feat_len: tensor(b, visual_feat_len，text—_feat_len) + glyph_attn_mask = cross_attention_kwargs.pop("glyph_attn_masks_dict", None) + bg_attn_mask = cross_attention_kwargs.pop("bg_attn_masks_dict", None) + if glyph_attn_mask is not None: + glyph_attn_mask = glyph_attn_mask[hidden_states.shape[1]] + if bg_attn_mask is not None: + bg_attn_mask = bg_attn_mask[hidden_states.shape[1]] + assert encoder_attention_mask is None, "encoder_attention_mask is not supported in this block." + + attn_output = self.attn1( + norm_hidden_states, + encoder_hidden_states=encoder_hidden_states if self.only_cross_attention else None, + attention_mask=attention_mask, + **cross_attention_kwargs, + ) + if self.norm_type == "ada_norm_zero": + attn_output = gate_msa.unsqueeze(1) * attn_output + elif self.norm_type == "ada_norm_single": + attn_output = gate_msa * attn_output + + hidden_states = attn_output + hidden_states + if hidden_states.ndim == 4: + hidden_states = hidden_states.squeeze(1) + + # 2.5 GLIGEN Control + if gligen_kwargs is not None: + hidden_states = self.fuser(hidden_states, gligen_kwargs["objs"]) + + # 3. Cross-Attention + if self.attn2 is not None: + if self.norm_type == "ada_norm": + norm_hidden_states = self.norm2(hidden_states, timestep) + elif self.norm_type in ["ada_norm_zero", "layer_norm", "layer_norm_i2vgen"]: + norm_hidden_states = self.norm2(hidden_states) + elif self.norm_type == "ada_norm_single": + # For PixArt norm2 isn't applied here: + # https://github.com/PixArt-alpha/PixArt-alpha/blob/0f55e922376d8b797edd44d25d0e7464b260dcab/diffusion/model/nets/PixArtMS.py#L70C1-L76C103 + norm_hidden_states = hidden_states + elif self.norm_type == "ada_norm_continuous": + norm_hidden_states = self.norm2(hidden_states, added_cond_kwargs["pooled_text_emb"]) + else: + raise ValueError("Incorrect norm") + + if self.pos_embed is not None and self.norm_type != "ada_norm_single": + norm_hidden_states = self.pos_embed(norm_hidden_states) + + attn_output = self.attn2( + norm_hidden_states, + encoder_hidden_states=torch.cat([encoder_hidden_states, glyph_encoder_hidden_states], dim=1), + attention_mask=torch.cat([bg_attn_mask, glyph_attn_mask], dim=-1), + **cross_attention_kwargs, + ) + + hidden_states = attn_output + hidden_states + + # 4. Feed-forward + # i2vgen doesn't have this norm 🤷‍♂️ + if self.norm_type == "ada_norm_continuous": + norm_hidden_states = self.norm3(hidden_states, added_cond_kwargs["pooled_text_emb"]) + elif not self.norm_type == "ada_norm_single": + norm_hidden_states = self.norm3(hidden_states) + + if self.norm_type == "ada_norm_zero": + norm_hidden_states = norm_hidden_states * (1 + scale_mlp[:, None]) + shift_mlp[:, None] + + if self.norm_type == "ada_norm_single": + norm_hidden_states = self.norm2(hidden_states) + norm_hidden_states = norm_hidden_states * (1 + scale_mlp) + shift_mlp + + if self._chunk_size is not None: + # "feed_forward_chunk_size" can be used to save memory + ff_output = _chunked_feed_forward( + self.ff, norm_hidden_states, self._chunk_dim, self._chunk_size, lora_scale=lora_scale + ) + else: + ff_output = self.ff(norm_hidden_states, scale=lora_scale) + + if self.norm_type == "ada_norm_zero": + ff_output = gate_mlp.unsqueeze(1) * ff_output + elif self.norm_type == "ada_norm_single": + ff_output = gate_mlp * ff_output + + hidden_states = ff_output + hidden_states + if hidden_states.ndim == 4: + hidden_states = hidden_states.squeeze(1) + + return hidden_states + diff --git a/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/glyph_sdxl/custom_diffusers/pipelines/__init__.py b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/glyph_sdxl/custom_diffusers/pipelines/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e594d844fea0345c966b02699048fe28f11d9c70 --- /dev/null +++ b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/glyph_sdxl/custom_diffusers/pipelines/__init__.py @@ -0,0 +1,5 @@ +from .pipeline_stable_diffusion_glyph_xl import StableDiffusionGlyphXLPipeline + +__all__ = [ + 'StableDiffusionGlyphXLPipeline', +] \ No newline at end of file diff --git a/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/glyph_sdxl/custom_diffusers/pipelines/pipeline_stable_diffusion_glyph_xl.py b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/glyph_sdxl/custom_diffusers/pipelines/pipeline_stable_diffusion_glyph_xl.py new file mode 100644 index 0000000000000000000000000000000000000000..53603fe7978811e504d95543eff92f458ae5c45d --- /dev/null +++ b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/glyph_sdxl/custom_diffusers/pipelines/pipeline_stable_diffusion_glyph_xl.py @@ -0,0 +1,922 @@ + +from typing import Optional, List, Union, Dict, Tuple, Callable, Any +import torch + +from transformers import T5EncoderModel, T5Tokenizer +import torch.nn.functional as F + +from diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl import ( + StableDiffusionXLPipeline, + AutoencoderKL, + CLIPTextModel, + CLIPTextModelWithProjection, + CLIPTokenizer, + UNet2DConditionModel, + KarrasDiffusionSchedulers, + CLIPVisionModelWithProjection, + CLIPImageProcessor, + VaeImageProcessor, + is_invisible_watermark_available, + StableDiffusionXLLoraLoaderMixin, + PipelineImageInput, + adjust_lora_scale_text_encoder, + scale_lora_layers, + unscale_lora_layers, + USE_PEFT_BACKEND, + StableDiffusionXLPipelineOutput, + ImageProjection, + logging, + rescale_noise_cfg, + retrieve_timesteps, + deprecate, +) +import numpy as np +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +from diffusers.pipelines.stable_diffusion_xl.watermark import StableDiffusionXLWatermarker + +class StableDiffusionGlyphXLPipeline(StableDiffusionXLPipeline): + model_cpu_offload_seq = "text_encoder->text_encoder_2->byt5_text_encoder->image_encoder->unet->byt5_mapper->vae" + _optional_components = [ + "tokenizer", + "tokenizer_2", + "byt5_tokenizer", + "text_encoder", + "text_encoder_2", + "byt5_text_encoder", + "byt5_mapper", + "image_encoder", + "feature_extractor", + ] + _callback_tensor_inputs = [ + "latents", + "prompt_embeds", + "negative_prompt_embeds", + "add_text_embeds", + "add_time_ids", + "negative_pooled_prompt_embeds", + "negative_add_time_ids", + ] + def __init__( + self, + vae: AutoencoderKL, + text_encoder: CLIPTextModel, + text_encoder_2: CLIPTextModelWithProjection, + byt5_text_encoder: T5EncoderModel, + tokenizer: CLIPTokenizer, + tokenizer_2: CLIPTokenizer, + byt5_tokenizer: T5Tokenizer, + byt5_mapper, + unet: UNet2DConditionModel, + scheduler: KarrasDiffusionSchedulers, + byt5_max_length: int = 512, + image_encoder: CLIPVisionModelWithProjection = None, + feature_extractor: CLIPImageProcessor = None, + force_zeros_for_empty_prompt: bool = True, + add_watermarker: Optional[bool] = None, + ): + super(StableDiffusionXLPipeline, self).__init__() + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + text_encoder_2=text_encoder_2, + byt5_text_encoder=byt5_text_encoder, + tokenizer=tokenizer, + tokenizer_2=tokenizer_2, + byt5_tokenizer=byt5_tokenizer, + byt5_mapper=byt5_mapper, + unet=unet, + scheduler=scheduler, + image_encoder=image_encoder, + feature_extractor=feature_extractor, + ) + self.register_to_config(force_zeros_for_empty_prompt=force_zeros_for_empty_prompt) + self.register_to_config(byt5_max_length=byt5_max_length) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor) + self.byt5_max_length = byt5_max_length + + self.default_sample_size = self.unet.config.sample_size + + add_watermarker = add_watermarker if add_watermarker is not None else is_invisible_watermark_available() + + if add_watermarker: + self.watermark = StableDiffusionXLWatermarker() + else: + self.watermark = None + + def encode_prompt( + self, + prompt: str, + prompt_2: Optional[str] = None, + text_prompt = None, + device: Optional[torch.device] = None, + num_images_per_prompt: int = 1, + do_classifier_free_guidance: bool = True, + negative_prompt: Optional[str] = None, + negative_prompt_2: Optional[str] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + pooled_prompt_embeds: Optional[torch.FloatTensor] = None, + negative_pooled_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + clip_skip: Optional[int] = None, + text_attn_mask: Optional[torch.LongTensor] = None, + byt5_prompt_embeds: Optional[torch.FloatTensor] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is + used in both text-encoders + device: (`torch.device`): + torch device + num_images_per_prompt (`int`): + number of images that should be generated per prompt + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + negative_prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and + `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. + If not provided, pooled text embeddings will be generated from `prompt` input argument. + negative_pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt` + input argument. + lora_scale (`float`, *optional*): + A lora scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + """ + device = device or self._execution_device + + # set lora scale so that monkey patched LoRA + # function of text encoder can correctly access it + if lora_scale is not None and isinstance(self, StableDiffusionXLLoraLoaderMixin): + self._lora_scale = lora_scale + + # dynamically adjust the LoRA scale + if self.text_encoder is not None: + if not USE_PEFT_BACKEND: + adjust_lora_scale_text_encoder(self.text_encoder, lora_scale) + else: + scale_lora_layers(self.text_encoder, lora_scale) + + if self.text_encoder_2 is not None: + if not USE_PEFT_BACKEND: + adjust_lora_scale_text_encoder(self.text_encoder_2, lora_scale) + else: + scale_lora_layers(self.text_encoder_2, lora_scale) + + prompt = [prompt] if isinstance(prompt, str) else prompt + + if prompt is not None: + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + # Define tokenizers and text encoders + tokenizers = [self.tokenizer, self.tokenizer_2] if self.tokenizer is not None else [self.tokenizer_2] + text_encoders = ( + [self.text_encoder, self.text_encoder_2] if self.text_encoder is not None else [self.text_encoder_2] + ) + + if prompt_embeds is None: + assert len(prompt) == 1 + prompt_2 = prompt_2 or prompt + prompt_2 = [prompt_2] if isinstance(prompt_2, str) else prompt_2 + + text_prompt = [text_prompt] if isinstance(text_prompt, str) else text_prompt + + # textual inversion: procecss multi-vector tokens if necessary + prompt_embeds_list = [] + prompts = [prompt, prompt_2] + text_input_id_batchs = [] + for prompt, tokenizer in zip(prompts, tokenizers): + pad_token = tokenizer.pad_token_id + total_tokens = tokenizer(prompt, truncation=False)['input_ids'][0] + bos = total_tokens[0] + eos = total_tokens[-1] + total_tokens = total_tokens[1:-1] + new_total_tokens = [] + empty_flag = True + while len(total_tokens) >= 75: + head_75_tokens = [total_tokens.pop(0) for _ in range(75)] + temp_77_token_ids = [bos] + head_75_tokens + [eos] + new_total_tokens.append(temp_77_token_ids) + empty_flag = False + if len(total_tokens) > 0 or empty_flag: + padding_len = 75 - len(total_tokens) + temp_77_token_ids = [bos] + total_tokens + [eos] + [pad_token] * padding_len + new_total_tokens.append(temp_77_token_ids) + # 1,segment_len, 77 + new_total_tokens = torch.tensor(new_total_tokens, dtype=torch.long).unsqueeze(0) + text_input_id_batchs.append(new_total_tokens) + if text_input_id_batchs[0].shape[1] > text_input_id_batchs[1].shape[1]: + tokenizer = tokenizers[1] + pad_token = tokenizer.pad_token_id + bos = tokenizer.bos_token_id + eos = tokenizer.eos_token_id + padding_len = text_input_id_batchs[0].shape[1] - text_input_id_batchs[1].shape[1] + # padding_len, 77 + padding_part = torch.tensor([[bos] + [eos] + [pad_token] * 75 for _ in range(padding_len)]) + # 1, padding_len, 77 + padding_part = padding_part.unsqueeze(0) + text_input_id_batchs[1] = torch.cat((text_input_id_batchs[1],padding_part), dim=1) + elif text_input_id_batchs[0].shape[1] < text_input_id_batchs[1].shape[1]: + tokenizer = tokenizers[0] + pad_token = tokenizer.pad_token_id + bos = tokenizer.bos_token_id + eos = tokenizer.eos_token_id + padding_len = text_input_id_batchs[1].shape[1] - text_input_id_batchs[0].shape[1] + # padding_len, 77 + padding_part = torch.tensor([[bos] + [eos] + [pad_token] * 75 for _ in range(padding_len)]) + # 1, padding_len, 77 + padding_part = padding_part.unsqueeze(0) + text_input_id_batchs[0] = torch.cat((text_input_id_batchs[0],padding_part), dim=1) + + embeddings = [] + for segment_idx in range(text_input_id_batchs[0].shape[1]): + prompt_embeds_list = [] + for i, text_encoder in enumerate(text_encoders): + # 1, segment_len, sequence_len + text_input_ids = text_input_id_batchs[i].to(text_encoder.device) + # 1, sequence_len, dim + prompt_embeds = text_encoder( + text_input_ids[:, segment_idx], + output_hidden_states=True, + ) + + # We are only ALWAYS interested in the pooled output of the final text encoder + temp_pooled_prompt_embeds = prompt_embeds[0] + if clip_skip is None: + prompt_embeds = prompt_embeds.hidden_states[-2] + else: + prompt_embeds = prompt_embeds.hidden_states[-(clip_skip + 2)] + bs_embed, seq_len, _ = prompt_embeds.shape + prompt_embeds = prompt_embeds.view(bs_embed, seq_len, -1) + prompt_embeds_list.append(prompt_embeds) + # b, sequence_len, dim + prompt_embeds = torch.concat(prompt_embeds_list, dim=-1) + embeddings.append(prompt_embeds) + if segment_idx == 0: + # use the first segment's pooled prompt embeddings as + # the pooled prompt embeddings + # b, dim->b, dim + pooled_prompt_embeds = temp_pooled_prompt_embeds.view(bs_embed, -1) + # b, segment_len * sequence_len, dim + prompt_embeds = torch.cat(embeddings, dim=1) + + if byt5_prompt_embeds is None: + byt5_text_inputs = self.byt5_tokenizer( + text_prompt, + padding="max_length", + max_length=self.byt5_max_length, + truncation=True, + add_special_tokens=True, + return_tensors="pt", + ) + byt5_text_input_ids = byt5_text_inputs.input_ids + byt5_attention_mask = byt5_text_inputs.attention_mask.to(self.byt5_text_encoder.device) if text_attn_mask is None else text_attn_mask.to(self.byt5_text_encoder.device, dtype=byt5_text_inputs.attention_mask.dtype) + with torch.cuda.amp.autocast(enabled=False): + byt5_prompt_embeds = self.byt5_text_encoder( + byt5_text_input_ids.to(self.byt5_text_encoder.device), + attention_mask=byt5_attention_mask.float(), + ) + byt5_prompt_embeds = byt5_prompt_embeds[0] + byt5_prompt_embeds = self.byt5_mapper(byt5_prompt_embeds, byt5_attention_mask) + + # get unconditional embeddings for classifier free guidance + zero_out_negative_prompt = negative_prompt is None and self.config.force_zeros_for_empty_prompt + if do_classifier_free_guidance and negative_prompt_embeds is None and zero_out_negative_prompt: + negative_prompt_embeds = torch.zeros_like(prompt_embeds) + negative_byt5_prompt_embeds = torch.zeros_like(byt5_prompt_embeds) + negative_pooled_prompt_embeds = torch.zeros_like(pooled_prompt_embeds) + elif do_classifier_free_guidance and negative_prompt_embeds is None: + raise NotImplementedError + + if self.text_encoder_2 is not None: + prompt_embeds = prompt_embeds.to(dtype=self.text_encoder_2.dtype, device=device) + else: + prompt_embeds = prompt_embeds.to(dtype=self.unet.dtype, device=device) + + bs_embed, seq_len, _ = prompt_embeds.shape + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1) + + if do_classifier_free_guidance: + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + byt5_seq_len = negative_byt5_prompt_embeds.shape[1] + + if self.text_encoder_2 is not None: + negative_prompt_embeds = negative_prompt_embeds.to(dtype=self.text_encoder_2.dtype, device=device) + else: + negative_prompt_embeds = negative_prompt_embeds.to(dtype=self.unet.dtype, device=device) + negative_byt5_prompt_embeds = negative_byt5_prompt_embeds.to(dtype=self.byt5_text_encoder.dtype, device=device) + + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + negative_byt5_prompt_embeds = negative_byt5_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_byt5_prompt_embeds = negative_byt5_prompt_embeds.view(batch_size * num_images_per_prompt, byt5_seq_len, -1) + + pooled_prompt_embeds = pooled_prompt_embeds.repeat(1, num_images_per_prompt).view( + bs_embed * num_images_per_prompt, -1 + ) + if do_classifier_free_guidance: + negative_pooled_prompt_embeds = negative_pooled_prompt_embeds.repeat(1, num_images_per_prompt).view( + bs_embed * num_images_per_prompt, -1 + ) + + if self.text_encoder is not None: + if isinstance(self, StableDiffusionXLLoraLoaderMixin) and USE_PEFT_BACKEND: + # Retrieve the original scale by scaling back the LoRA layers + unscale_lora_layers(self.text_encoder, lora_scale) + + if self.text_encoder_2 is not None: + if isinstance(self, StableDiffusionXLLoraLoaderMixin) and USE_PEFT_BACKEND: + # Retrieve the original scale by scaling back the LoRA layers + unscale_lora_layers(self.text_encoder_2, lora_scale) + + return ( + prompt_embeds, + negative_prompt_embeds, + pooled_prompt_embeds, + negative_pooled_prompt_embeds, + byt5_prompt_embeds, + negative_byt5_prompt_embeds, + ) + + @torch.no_grad() + def __call__( + self, + prompt: Union[str, List[str]] = None, + prompt_2: Optional[Union[str, List[str]]] = None, + text_prompt = None, + texts = None, + bboxes = None, + height: Optional[int] = None, + width: Optional[int] = None, + num_inference_steps: int = 50, + timesteps: List[int] = None, + denoising_end: Optional[float] = None, + guidance_scale: float = 5.0, + negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt_2: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + pooled_prompt_embeds: Optional[torch.FloatTensor] = None, + negative_pooled_prompt_embeds: Optional[torch.FloatTensor] = None, + ip_adapter_image: Optional[PipelineImageInput] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + guidance_rescale: float = 0.0, + original_size: Optional[Tuple[int, int]] = None, + crops_coords_top_left: Tuple[int, int] = (0, 0), + target_size: Optional[Tuple[int, int]] = None, + negative_original_size: Optional[Tuple[int, int]] = None, + negative_crops_coords_top_left: Tuple[int, int] = (0, 0), + negative_target_size: Optional[Tuple[int, int]] = None, + clip_skip: Optional[int] = None, + callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + text_attn_mask: torch.LongTensor = None, + denoising_start: Optional[float] = None, + byt5_prompt_embeds: Optional[torch.FloatTensor] = None, + **kwargs, + ): + r""" + Function invoked when calling the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. + instead. + prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is + used in both text-encoders + height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): + The height in pixels of the generated image. This is set to 1024 by default for the best results. + Anything below 512 pixels won't work well for + [stabilityai/stable-diffusion-xl-base-1.0](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0) + and checkpoints that are not specifically fine-tuned on low resolutions. + width (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): + The width in pixels of the generated image. This is set to 1024 by default for the best results. + Anything below 512 pixels won't work well for + [stabilityai/stable-diffusion-xl-base-1.0](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0) + and checkpoints that are not specifically fine-tuned on low resolutions. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + timesteps (`List[int]`, *optional*): + Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument + in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is + passed will be used. Must be in descending order. + denoising_end (`float`, *optional*): + When specified, determines the fraction (between 0.0 and 1.0) of the total denoising process to be + completed before it is intentionally prematurely terminated. As a result, the returned sample will + still retain a substantial amount of noise as determined by the discrete timesteps selected by the + scheduler. The denoising_end parameter should ideally be utilized when this pipeline forms a part of a + "Mixture of Denoisers" multi-pipeline setup, as elaborated in [**Refining the Image + Output**](https://huggingface.co/docs/diffusers/api/pipelines/stable_diffusion/stable_diffusion_xl#refining-the-image-output) + guidance_scale (`float`, *optional*, defaults to 5.0): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + negative_prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and + `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to + [`schedulers.DDIMScheduler`], will be ignored for others. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) + to make generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor will ge generated by sampling using the supplied random `generator`. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. + If not provided, pooled text embeddings will be generated from `prompt` input argument. + negative_pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt` + input argument. + ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between + [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion_xl.StableDiffusionXLPipelineOutput`] instead + of a plain tuple. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under + `self.processor` in + [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + guidance_rescale (`float`, *optional*, defaults to 0.0): + Guidance rescale factor proposed by [Common Diffusion Noise Schedules and Sample Steps are + Flawed](https://arxiv.org/pdf/2305.08891.pdf) `guidance_scale` is defined as `φ` in equation 16. of + [Common Diffusion Noise Schedules and Sample Steps are Flawed](https://arxiv.org/pdf/2305.08891.pdf). + Guidance rescale factor should fix overexposure when using zero terminal SNR. + original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + If `original_size` is not the same as `target_size` the image will appear to be down- or upsampled. + `original_size` defaults to `(height, width)` if not specified. Part of SDXL's micro-conditioning as + explained in section 2.2 of + [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). + crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + `crops_coords_top_left` can be used to generate an image that appears to be "cropped" from the position + `crops_coords_top_left` downwards. Favorable, well-centered images are usually achieved by setting + `crops_coords_top_left` to (0, 0). Part of SDXL's micro-conditioning as explained in section 2.2 of + [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). + target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + For most cases, `target_size` should be set to the desired height and width of the generated image. If + not specified it will default to `(height, width)`. Part of SDXL's micro-conditioning as explained in + section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). + negative_original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + To negatively condition the generation process based on a specific image resolution. Part of SDXL's + micro-conditioning as explained in section 2.2 of + [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more + information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. + negative_crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + To negatively condition the generation process based on a specific crop coordinates. Part of SDXL's + micro-conditioning as explained in section 2.2 of + [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more + information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. + negative_target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + To negatively condition the generation process based on a target image resolution. It should be as same + as the `target_size` for most cases. Part of SDXL's micro-conditioning as explained in section 2.2 of + [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more + information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. + callback_on_step_end (`Callable`, *optional*): + A function that calls at the end of each denoising steps during the inference. The function is called + with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, + callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by + `callback_on_step_end_tensor_inputs`. + callback_on_step_end_tensor_inputs (`List`, *optional*): + The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list + will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the + `._callback_tensor_inputs` attribute of your pipeline class. + + Examples: + + Returns: + [`~pipelines.stable_diffusion_xl.StableDiffusionXLPipelineOutput`] or `tuple`: + [`~pipelines.stable_diffusion_xl.StableDiffusionXLPipelineOutput`] if `return_dict` is True, otherwise a + `tuple`. When returning a tuple, the first element is a list with the generated images. + """ + + callback = kwargs.pop("callback", None) + callback_steps = kwargs.pop("callback_steps", None) + + if callback is not None: + deprecate( + "callback", + "1.0.0", + "Passing `callback` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`", + ) + if callback_steps is not None: + deprecate( + "callback_steps", + "1.0.0", + "Passing `callback_steps` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`", + ) + + # 0. Default height and width to unet + height = height or self.default_sample_size * self.vae_scale_factor + width = width or self.default_sample_size * self.vae_scale_factor + + original_size = original_size or (height, width) + target_size = target_size or (height, width) + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, + prompt_2, + height, + width, + callback_steps, + negative_prompt, + negative_prompt_2, + prompt_embeds, + negative_prompt_embeds, + pooled_prompt_embeds, + negative_pooled_prompt_embeds, + callback_on_step_end_tensor_inputs, + ) + + self._guidance_scale = guidance_scale + self._guidance_rescale = guidance_rescale + self._clip_skip = clip_skip + self._cross_attention_kwargs = cross_attention_kwargs + self._denoising_end = denoising_end + self._interrupt = False + + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + + # 3. Encode input prompt + lora_scale = ( + self.cross_attention_kwargs.get("scale", None) if self.cross_attention_kwargs is not None else None + ) + + ( + prompt_embeds, + negative_prompt_embeds, + pooled_prompt_embeds, + negative_pooled_prompt_embeds, + byt5_prompt_embeds, + negative_byt5_prompt_embeds, + ) = self.encode_prompt( + prompt=prompt, + prompt_2=prompt_2, + text_prompt=text_prompt, + device=device, + num_images_per_prompt=num_images_per_prompt, + do_classifier_free_guidance=self.do_classifier_free_guidance, + negative_prompt=negative_prompt, + negative_prompt_2=negative_prompt_2, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + pooled_prompt_embeds=pooled_prompt_embeds, + negative_pooled_prompt_embeds=negative_pooled_prompt_embeds, + lora_scale=lora_scale, + clip_skip=self.clip_skip, + text_attn_mask=text_attn_mask, + byt5_prompt_embeds=byt5_prompt_embeds, + ) + + # 4. Prepare timesteps + timesteps, num_inference_steps = retrieve_timesteps(self.scheduler, num_inference_steps, device, timesteps) + + # 5. Prepare latent variables + num_channels_latents = self.unet.config.in_channels + latents = self.prepare_latents( + batch_size * num_images_per_prompt, + num_channels_latents, + height, + width, + prompt_embeds.dtype, + device, + generator, + latents, + ) + + # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + # 7. Prepare added time ids & embeddings + add_text_embeds = pooled_prompt_embeds + if self.text_encoder_2 is None: + text_encoder_projection_dim = int(pooled_prompt_embeds.shape[-1]) + else: + text_encoder_projection_dim = self.text_encoder_2.config.projection_dim + + add_time_ids = self._get_add_time_ids( + original_size, + crops_coords_top_left, + target_size, + dtype=prompt_embeds.dtype, + text_encoder_projection_dim=text_encoder_projection_dim, + ) + if negative_original_size is not None and negative_target_size is not None: + negative_add_time_ids = self._get_add_time_ids( + negative_original_size, + negative_crops_coords_top_left, + negative_target_size, + dtype=prompt_embeds.dtype, + text_encoder_projection_dim=text_encoder_projection_dim, + ) + else: + negative_add_time_ids = add_time_ids + + if self.do_classifier_free_guidance: + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds], dim=0) + byt5_prompt_embeds = torch.cat([negative_byt5_prompt_embeds, byt5_prompt_embeds], dim=0) + + add_text_embeds = torch.cat([negative_pooled_prompt_embeds, add_text_embeds], dim=0) + add_time_ids = torch.cat([negative_add_time_ids, add_time_ids], dim=0) + + prompt_embeds = prompt_embeds.to(device) + byt5_prompt_embeds = byt5_prompt_embeds.to(device) + add_text_embeds = add_text_embeds.to(device) + add_time_ids = add_time_ids.to(device).repeat(batch_size * num_images_per_prompt, 1) + + if ip_adapter_image is not None: + output_hidden_state = False if isinstance(self.unet.encoder_hid_proj, ImageProjection) else True + image_embeds, negative_image_embeds = self.encode_image( + ip_adapter_image, device, num_images_per_prompt, output_hidden_state + ) + if self.do_classifier_free_guidance: + image_embeds = torch.cat([negative_image_embeds, image_embeds]) + image_embeds = image_embeds.to(device) + + # 8. Denoising loop + num_warmup_steps = max(len(timesteps) - num_inference_steps * self.scheduler.order, 0) + + # 8.1 Apply denoising_end + if ( + self.denoising_end is not None + and isinstance(self.denoising_end, float) + and self.denoising_end > 0 + and self.denoising_end < 1 + ): + discrete_timestep_cutoff = int( + round( + self.scheduler.config.num_train_timesteps + - (self.denoising_end * self.scheduler.config.num_train_timesteps) + ) + ) + num_inference_steps = len(list(filter(lambda ts: ts >= discrete_timestep_cutoff, timesteps))) + timesteps = timesteps[:num_inference_steps] + + # 9. Optionally get Guidance Scale Embedding + timestep_cond = None + if self.unet.config.time_cond_proj_dim is not None: + guidance_scale_tensor = torch.tensor(self.guidance_scale - 1).repeat(batch_size * num_images_per_prompt) + timestep_cond = self.get_guidance_scale_embedding( + guidance_scale_tensor, embedding_dim=self.unet.config.time_cond_proj_dim + ).to(device=device, dtype=latents.dtype) + + assert batch_size == 1, "batch_size > 1 is not supported" + if texts is not None: + glyph_attn_mask = self.get_glyph_attn_mask(texts, bboxes) + # h,w + bg_attn_mask = glyph_attn_mask.sum(-1) == 0 + # 1,h,w,byt5_max_len + glyph_attn_masks = glyph_attn_mask.unsqueeze(0).to(device) + # 1,h,w + bg_attn_masks = bg_attn_mask.unsqueeze(0).to(glyph_attn_masks.dtype).to(device) + + # b, h, w, text_feat_len + glyph_attn_masks = (1 - glyph_attn_masks) * -10000.0 + # b, h, w + bg_attn_masks = (1 - bg_attn_masks) * -10000.0 + num_down_sample = sum(1 if i == 'CrossAttnDownBlock2D' else 0 for i in self.unet.config['down_block_types']) - 1 + initial_resolution = self.default_sample_size + initial_resolution = initial_resolution // 2**sum(1 if i == 'DownBlock2D' else 0 for i in self.unet.config['down_block_types']) + resolution_list = [initial_resolution] + [initial_resolution // 2**i for i in range(1, num_down_sample + 1)] + glyph_attn_masks_dict = dict() + bg_attn_masks_dict = dict() + # b, text_fet_len, h, w + glyph_attn_masks = glyph_attn_masks.permute(0, 3, 1, 2) + # b, 1, h, w + bg_attn_masks = bg_attn_masks.unsqueeze(1) + for mask_resolution in resolution_list: + down_scaled_glyph_attn_masks = F.interpolate( + glyph_attn_masks, size=(mask_resolution, mask_resolution), mode='nearest', + ) + # b, text_fet_len, h, w->b, h, w, text_fet_len->b, h*w, text_fet_len + down_scaled_glyph_attn_masks = down_scaled_glyph_attn_masks.permute(0, 2, 3, 1).flatten(1, 2) + glyph_attn_masks_dict[mask_resolution * mask_resolution] = down_scaled_glyph_attn_masks + + down_scaled_bg_attn_masks = F.interpolate( + bg_attn_masks, size=(mask_resolution, mask_resolution), mode='nearest', + ) + # b,1,h,w->b,h,w->b,h,w,1->b,h*w,1->b,h*w,clip_feat_len + down_scaled_bg_attn_masks = down_scaled_bg_attn_masks.squeeze(1).unsqueeze(-1) + down_scaled_bg_attn_masks = down_scaled_bg_attn_masks.flatten(1, 2) + down_scaled_bg_attn_masks = down_scaled_bg_attn_masks.repeat(1, 1, prompt_embeds.shape[1]) + bg_attn_masks_dict[mask_resolution * mask_resolution] = down_scaled_bg_attn_masks + if self.do_classifier_free_guidance: + for key in glyph_attn_masks_dict: + glyph_attn_masks_dict[key] = torch.cat([ + torch.zeros_like(glyph_attn_masks_dict[key]), + glyph_attn_masks_dict[key]], + dim=0) + for key in bg_attn_masks_dict: + bg_attn_masks_dict[key] = torch.cat([ + torch.zeros_like(bg_attn_masks_dict[key]), + bg_attn_masks_dict[key]], + dim=0) + else: + glyph_attn_masks_dict = None + bg_attn_masks_dict = None + + self._num_timesteps = len(timesteps) + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + if self.interrupt: + continue + + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if self.do_classifier_free_guidance else latents + + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + # predict the noise residual + added_cond_kwargs = {"text_embeds": add_text_embeds, "time_ids": add_time_ids} + if ip_adapter_image is not None: + added_cond_kwargs["image_embeds"] = image_embeds + if self.cross_attention_kwargs is None: + cross_attention_kwargs = {} + else: + cross_attention_kwargs = self.cross_attention_kwargs + cross_attention_kwargs['glyph_encoder_hidden_states'] = byt5_prompt_embeds + cross_attention_kwargs['glyph_attn_masks_dict'] = glyph_attn_masks_dict + cross_attention_kwargs['bg_attn_masks_dict'] = bg_attn_masks_dict + + noise_pred = self.unet( + latent_model_input, + t, + encoder_hidden_states=prompt_embeds, + timestep_cond=timestep_cond, + cross_attention_kwargs=cross_attention_kwargs, + added_cond_kwargs=added_cond_kwargs, + return_dict=False, + )[0] + + # perform guidance + if self.do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + self.guidance_scale * (noise_pred_text - noise_pred_uncond) + + if self.do_classifier_free_guidance and self.guidance_rescale > 0.0: + # Based on 3.4. in https://arxiv.org/pdf/2305.08891.pdf + noise_pred = rescale_noise_cfg(noise_pred, noise_pred_text, guidance_rescale=self.guidance_rescale) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0] + + if callback_on_step_end is not None: + callback_kwargs = {} + for k in callback_on_step_end_tensor_inputs: + callback_kwargs[k] = locals()[k] + callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) + + latents = callback_outputs.pop("latents", latents) + prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds) + negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds) + add_text_embeds = callback_outputs.pop("add_text_embeds", add_text_embeds) + negative_pooled_prompt_embeds = callback_outputs.pop( + "negative_pooled_prompt_embeds", negative_pooled_prompt_embeds + ) + add_time_ids = callback_outputs.pop("add_time_ids", add_time_ids) + negative_add_time_ids = callback_outputs.pop("negative_add_time_ids", negative_add_time_ids) + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + + if not output_type == "latent": + # make sure the VAE is in float32 mode, as it overflows in float16 + needs_upcasting = self.vae.dtype == torch.float16 and self.vae.config.force_upcast + + if needs_upcasting: + self.upcast_vae() + latents = latents.to(next(iter(self.vae.post_quant_conv.parameters())).dtype) + + image = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False)[0] + + # cast back to fp16 if needed + if needs_upcasting: + self.vae.to(dtype=torch.float16) + else: + image = latents + + if not output_type == "latent": + # apply watermark if available + if self.watermark is not None: + image = self.watermark.apply_watermark(image) + + image = self.image_processor.postprocess(image, output_type=output_type) + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + return (image,) + + return StableDiffusionXLPipelineOutput(images=image) + + def get_glyph_attn_mask(self, texts, bboxes): + resolution = self.default_sample_size + text_idx_list = self.get_text_start_pos(texts) + mask_tensor = torch.zeros( + resolution, resolution, self.byt5_max_length, + ) + for idx, bbox in enumerate(bboxes): + # box is in [x, y, w, h] format + # area of [y:y+h, x:x+w] + bbox = [int(v * resolution + 0.5) for v in bbox] + bbox[2] = max(bbox[2], 1) + bbox[3] = max(bbox[3], 1) + bbox[0: 2] = np.clip(bbox[0: 2], 0, resolution - 1).tolist() + bbox[2: 4] = np.clip(bbox[2: 4], 1, resolution).tolist() + mask_tensor[ + bbox[1]: bbox[1] + bbox[3], + bbox[0]: bbox[0] + bbox[2], + text_idx_list[idx]: text_idx_list[idx + 1] + ] = 1 + return mask_tensor + + def get_text_start_pos(self, texts): + prompt = "".encode('utf-8') + ''' + Text "{text}" in {color}, {type}. + ''' + pos_list = [] + for text in texts: + pos_list.append(len(prompt)) + text_prompt = f'Text "{text}"' + + attr_list = ['0', '1'] + + attr_suffix = ", ".join(attr_list) + text_prompt += " in " + attr_suffix + text_prompt += ". " + text_prompt = text_prompt.encode('utf-8') + + prompt = prompt + text_prompt + pos_list.append(len(prompt)) + return pos_list diff --git a/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/glyph_sdxl/modules/__init__.py b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/glyph_sdxl/modules/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6f8b7e221eb0e6184b014cf3dcfff92342841727 --- /dev/null +++ b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/glyph_sdxl/modules/__init__.py @@ -0,0 +1,7 @@ +from .simple_byt5_mapper import ByT5Mapper +from .byt5_block_byt5_mapper import T5EncoderBlockByT5Mapper + +__all__ = [ + 'ByT5Mapper', + 'T5EncoderBlockByT5Mapper', +] \ No newline at end of file diff --git a/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/glyph_sdxl/modules/byt5_block_byt5_mapper.py b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/glyph_sdxl/modules/byt5_block_byt5_mapper.py new file mode 100644 index 0000000000000000000000000000000000000000..574d05077ce0d296e006e1e6862955a9f477bad8 --- /dev/null +++ b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/glyph_sdxl/modules/byt5_block_byt5_mapper.py @@ -0,0 +1,151 @@ +import torch +import torch.nn as nn +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + +import warnings + +import logging +from torch import Tensor +from diffusers import ModelMixin +from transformers.models.t5.modeling_t5 import T5LayerSelfAttention, T5LayerFF, T5LayerNorm + +logger = logging.getLogger(__name__) + +class T5EncoderBlock(nn.Module): + def __init__(self, config, has_relative_attention_bias=False): + super().__init__() + self.layer = nn.ModuleList() + self.layer.append(T5LayerSelfAttention(config, has_relative_attention_bias=has_relative_attention_bias)) + self.layer.append(T5LayerFF(config)) + + def forward( + self, + hidden_states, + attention_mask=None, + position_bias=None, + layer_head_mask=None, + output_attentions=False, + ): + self_attn_past_key_value, cross_attn_past_key_value = None, None + + self_attention_outputs = self.layer[0]( + hidden_states, + attention_mask=attention_mask, + position_bias=position_bias, + layer_head_mask=layer_head_mask, + past_key_value=self_attn_past_key_value, + use_cache=False, + output_attentions=output_attentions, + ) + hidden_states, present_key_value_state = self_attention_outputs[:2] + attention_outputs = self_attention_outputs[2:] # Keep self-attention outputs and relative position weights + + # clamp inf values to enable fp16 training + if hidden_states.dtype == torch.float16: + clamp_value = torch.where( + torch.isinf(hidden_states).any(), + torch.finfo(hidden_states.dtype).max - 1000, + torch.finfo(hidden_states.dtype).max, + ) + hidden_states = torch.clamp(hidden_states, min=-clamp_value, max=clamp_value) + + # Apply Feed Forward layer + hidden_states = self.layer[-1](hidden_states) + + # clamp inf values to enable fp16 training + if hidden_states.dtype == torch.float16: + clamp_value = torch.where( + torch.isinf(hidden_states).any(), + torch.finfo(hidden_states.dtype).max - 1000, + torch.finfo(hidden_states.dtype).max, + ) + hidden_states = torch.clamp(hidden_states, min=-clamp_value, max=clamp_value) + + outputs = (hidden_states,) + attention_outputs + + return outputs # hidden-states, present_key_value_states, (self-attention position bias), (self-attention weights), (cross-attention position bias), (cross-attention weights) + +class T5EncoderBlockByT5Mapper(ModelMixin): + def __init__(self, byt5_config, num_layers, sdxl_channels=None): + super().__init__() + if num_layers > 0: + self.blocks = nn.ModuleList( + [ + T5EncoderBlock( + byt5_config, + has_relative_attention_bias=bool(i == 0)) + for i in range(num_layers) + ] + ) + else: + self.blocks = None + self.layer_norm = T5LayerNorm(byt5_config.d_model, eps=byt5_config.layer_norm_epsilon) + if sdxl_channels is not None: + self.channel_mapper = nn.Linear(byt5_config.d_model, sdxl_channels) + self.final_layer_norm = T5LayerNorm(sdxl_channels, eps=byt5_config.layer_norm_epsilon) + else: + self.channel_mapper = None + self.final_layer_norm = None + + def get_extended_attention_mask( + self, attention_mask: Tensor, input_shape: Tuple[int], device: torch.device = None, dtype: torch.float = None + ) -> Tensor: + """ + Makes broadcastable attention and causal masks so that future and masked tokens are ignored. + + Arguments: + attention_mask (`torch.Tensor`): + Mask with ones indicating tokens to attend to, zeros for tokens to ignore. + input_shape (`Tuple[int]`): + The shape of the input to the model. + + Returns: + `torch.Tensor` The extended attention mask, with a the same dtype as `attention_mask.dtype`. + """ + if dtype is None: + dtype = self.dtype + + # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length] + # ourselves in which case we just need to make it broadcastable to all heads. + if attention_mask.dim() == 3: + extended_attention_mask = attention_mask[:, None, :, :] + elif attention_mask.dim() == 2: + # Provided a padding mask of dimensions [batch_size, seq_length] + # - if the model is a decoder, apply a causal mask in addition to the padding mask + # - if the model is an encoder, make the mask broadcastable to [batch_size, num_heads, seq_length, seq_length] + extended_attention_mask = attention_mask[:, None, None, :] + else: + raise ValueError( + f"Wrong shape for input_ids (shape {input_shape}) or attention_mask (shape {attention_mask.shape})" + ) + + # Since attention_mask is 1.0 for positions we want to attend and 0.0 for + # masked positions, this operation will create a tensor which is 0.0 for + # positions we want to attend and the dtype's smallest value for masked positions. + # Since we are adding it to the raw scores before the softmax, this is + # effectively the same as removing these entirely. + extended_attention_mask = extended_attention_mask.to(dtype=dtype) # fp16 compatibility + extended_attention_mask = (1.0 - extended_attention_mask) * torch.finfo(dtype).min + return extended_attention_mask + + + def forward(self, inputs_embeds, attention_mask): + input_shape = inputs_embeds.size()[:-1] + extended_attention_mask = self.get_extended_attention_mask(attention_mask, input_shape) + + hidden_states = inputs_embeds + position_bias = None + + if self.blocks is not None: + for layer_module in self.blocks: + layer_outputs = layer_module( + hidden_states, + attention_mask=extended_attention_mask, + position_bias=position_bias, + ) + hidden_states, position_bias = layer_outputs + hidden_states = self.layer_norm(hidden_states) + if self.channel_mapper is not None: + hidden_states = self.channel_mapper(hidden_states) + hidden_states = self.final_layer_norm(hidden_states) + return hidden_states diff --git a/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/glyph_sdxl/modules/simple_byt5_mapper.py b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/glyph_sdxl/modules/simple_byt5_mapper.py new file mode 100644 index 0000000000000000000000000000000000000000..ed31fe06f04cc4ae2d2862901dacf84c8bc0f549 --- /dev/null +++ b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/glyph_sdxl/modules/simple_byt5_mapper.py @@ -0,0 +1,16 @@ +from diffusers import ModelMixin +import torch.nn as nn + +class ByT5Mapper(ModelMixin): + def __init__(self, byt5_output_dim, sdxl_text_dim): + super().__init__() + self.mapper = nn.Sequential( + nn.LayerNorm(byt5_output_dim), + nn.Linear(byt5_output_dim, sdxl_text_dim), + nn.ReLU(), + nn.Linear(sdxl_text_dim, sdxl_text_dim) + ) + + def forward(self, byt5_embedding): + return self.mapper(byt5_embedding) + \ No newline at end of file diff --git a/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/glyph_sdxl/utils/__init__.py b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/glyph_sdxl/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..267a07065978dcbdb1a16c3f21ca3017e1a9cd7a --- /dev/null +++ b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/glyph_sdxl/utils/__init__.py @@ -0,0 +1,23 @@ +from .parse_config import parse_config +from .constants import ( + UNET_CKPT_NAME, + BYT5_CKPT_NAME, + BYT5_MAPPER_CKPT_NAME, + INSERTED_ATTN_CKPT_NAME, + huggingface_cache_dir, +) +from .load_pretrained_byt5 import load_byt5_and_byt5_tokenizer +from .format_prompt import PromptFormat, MultilingualPromptFormat + +__all__ = [ + 'parse_config', + 'UNET_CKPT_NAME', + 'BYT5_CKPT_NAME', + 'BYT5_MAPPER_CKPT_NAME', + 'huggingface_cache_dir', + 'load_byt5_and_byt5_tokenizer', + 'INSERTED_ATTN_CKPT_NAME', + 'PromptFormat', + 'MultilingualPromptFormat', +] + diff --git a/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/glyph_sdxl/utils/constants.py b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/glyph_sdxl/utils/constants.py new file mode 100644 index 0000000000000000000000000000000000000000..5ed6b7f0082350d101e19183e267ed2c5976148d --- /dev/null +++ b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/glyph_sdxl/utils/constants.py @@ -0,0 +1,5 @@ +UNET_CKPT_NAME = "unet_lora.pt" +INSERTED_ATTN_CKPT_NAME = "unet_inserted_attn.pt" +BYT5_CKPT_NAME = "byt5_model.pt" +BYT5_MAPPER_CKPT_NAME = "byt5_mapper.pt" +huggingface_cache_dir = None diff --git a/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/glyph_sdxl/utils/format_prompt.py b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/glyph_sdxl/utils/format_prompt.py new file mode 100644 index 0000000000000000000000000000000000000000..692762cf548ba6cb51204da526cc96fd1defe283 --- /dev/null +++ b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/glyph_sdxl/utils/format_prompt.py @@ -0,0 +1,113 @@ +import json +import webcolors + + +def closest_color(requested_color): + min_colors = {} + for key, name in webcolors.CSS3_HEX_TO_NAMES.items(): + r_c, g_c, b_c = webcolors.hex_to_rgb(key) + rd = (r_c - requested_color[0]) ** 2 + gd = (g_c - requested_color[1]) ** 2 + bd = (b_c - requested_color[2]) ** 2 + min_colors[(rd + gd + bd)] = name + return min_colors[min(min_colors.keys())] + +def convert_rgb_to_names(rgb_tuple): + try: + color_name = webcolors.rgb_to_name(rgb_tuple) + except ValueError: + color_name = closest_color(rgb_tuple) + return color_name + +class PromptFormat(): + def __init__( + self, + font_path: str = 'assets/font_idx_512.json', + color_path: str = 'assets/color_idx.json', + ): + with open(font_path, 'r') as f: + self.font_dict = json.load(f) + with open(color_path, 'r') as f: + self.color_dict = json.load(f) + + def format_checker(self, texts, styles): + assert len(texts) == len(styles), 'length of texts must be equal to length of styles' + for style in styles: + assert style['font-family'] in self.font_dict, f"invalid font-family: {style['font-family']}" + rgb_color = webcolors.hex_to_rgb(style['color']) + color_name = convert_rgb_to_names(rgb_color) + assert color_name in self.color_dict, f"invalid color hex {color_name}" + + def format_prompt(self, texts, styles): + self.format_checker(texts, styles) + + prompt = "" + ''' + Text "{text}" in {color}, {type}. + ''' + for text, style in zip(texts, styles): + text_prompt = f'Text "{text}"' + + attr_list = [] + + # format color + hex_color = style["color"] + rgb_color = webcolors.hex_to_rgb(hex_color) + color_name = convert_rgb_to_names(rgb_color) + attr_list.append(f"") + + # format font + attr_list.append(f"") + attr_suffix = ", ".join(attr_list) + text_prompt += " in " + attr_suffix + text_prompt += ". " + + prompt = prompt + text_prompt + return prompt + + +class MultilingualPromptFormat(): + def __init__( + self, + font_path: str = 'assets/multilingual_10-lang_idx.json', + color_path: str = 'assets/color_idx.json', + ): + with open(font_path, 'r') as f: + self.font_dict = json.load(f) + with open(color_path, 'r') as f: + self.color_dict = json.load(f) + + def format_checker(self, texts, styles): + assert len(texts) == len(styles), 'length of texts must be equal to length of styles' + for style in styles: + assert style['font-family'] in self.font_dict, f"invalid font-family: {style['font-family']}" + rgb_color = webcolors.hex_to_rgb(style['color']) + color_name = convert_rgb_to_names(rgb_color) + assert color_name in self.color_dict, f"invalid color hex {color_name}" + + def format_prompt(self, texts, styles): + self.format_checker(texts, styles) + + prompt = "" + ''' + Text "{text}" in {color}, {type}. + ''' + for text, style in zip(texts, styles): + text_prompt = f'Text "{text}"' + + attr_list = [] + + # format color + hex_color = style["color"] + rgb_color = webcolors.hex_to_rgb(hex_color) + color_name = convert_rgb_to_names(rgb_color) + attr_list.append(f"") + + # format font + attr_list.append(f"<{style['font-family'][:2]}-font-{self.font_dict[style['font-family']]}>") + attr_suffix = ", ".join(attr_list) + text_prompt += " in " + attr_suffix + text_prompt += ". " + + prompt = prompt + text_prompt + return prompt diff --git a/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/glyph_sdxl/utils/load_pretrained_byt5.py b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/glyph_sdxl/utils/load_pretrained_byt5.py new file mode 100644 index 0000000000000000000000000000000000000000..f74661aaf27eee6f2fd75b14e9a1f7fa45840255 --- /dev/null +++ b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/glyph_sdxl/utils/load_pretrained_byt5.py @@ -0,0 +1,60 @@ +import json + +from transformers import AutoTokenizer, T5ForConditionalGeneration +from diffusers.utils import logging + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +def add_special_token(tokenizer, text_encoder, add_color, add_font, color_ann_path, font_ann_path, multilingual=False): + with open(font_ann_path, 'r') as f: + idx_font_dict = json.load(f) + with open(color_ann_path, 'r') as f: + idx_color_dict = json.load(f) + + if multilingual: + font_token = [] + for font_code in idx_font_dict: + prefix = font_code[:2] + font_token.append(f'<{prefix}-font-{idx_font_dict[font_code]}>') + else: + font_token = [f'' for i in range(len(idx_font_dict))] + color_token = [f'' for i in range(len(idx_color_dict))] + additional_special_tokens = [] + if add_color: + additional_special_tokens += color_token + if add_font: + additional_special_tokens += font_token + tokenizer.add_tokens(additional_special_tokens, special_tokens=True) + text_encoder.resize_token_embeddings(len(tokenizer)) + +def load_byt5_and_byt5_tokenizer( + byt5_name='google/byt5-small', + special_token=False, + color_special_token=False, + font_special_token=False, + color_ann_path='assets/color_idx.json', + font_ann_path='assets/font_idx_512.json', + huggingface_cache_dir=None, + multilingual=False, +): + byt5_tokenizer = AutoTokenizer.from_pretrained( + byt5_name, cache_dir=huggingface_cache_dir, + ) + byt5_text_encoder = T5ForConditionalGeneration.from_pretrained( + byt5_name, cache_dir=huggingface_cache_dir, + ).get_encoder() + + if special_token: + add_special_token( + byt5_tokenizer, + byt5_text_encoder, + add_color=color_special_token, + add_font=font_special_token, + color_ann_path=color_ann_path, + font_ann_path=font_ann_path, + multilingual=multilingual, + ) + + logger.info(f'Loaded original byt5 weight') + + return byt5_text_encoder, byt5_tokenizer \ No newline at end of file diff --git a/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/glyph_sdxl/utils/parse_config.py b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/glyph_sdxl/utils/parse_config.py new file mode 100644 index 0000000000000000000000000000000000000000..1bfa29d19c098ceb69a2e4f315209e56b454f0bd --- /dev/null +++ b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/glyph_sdxl/utils/parse_config.py @@ -0,0 +1,17 @@ +import argparse +import os +import os.path as osp +from mmengine.config import Config + + +def parse_config(path=None): + if path is None: + parser = argparse.ArgumentParser() + parser.add_argument('config_dir', type=str) + args = parser.parse_args() + path = args.config_dir + config = Config.fromfile(path) + + config.config_dir = path + + return config diff --git a/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/inference_multilingual.py b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/inference_multilingual.py new file mode 100644 index 0000000000000000000000000000000000000000..51c875916a682b7e35730bb98d14b076e12d3f73 --- /dev/null +++ b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/inference_multilingual.py @@ -0,0 +1,236 @@ +import argparse +import os +import json +import copy +import os.path as osp + +import torch +from diffusers import UNet2DConditionModel, AutoencoderKL +from diffusers.models.attention import BasicTransformerBlock +from peft import LoraConfig +from peft.utils import set_peft_model_state_dict +from transformers import PretrainedConfig + +from diffusers import DPMSolverMultistepScheduler + +from glyph_sdxl.utils import ( + parse_config, + UNET_CKPT_NAME, + huggingface_cache_dir, + load_byt5_and_byt5_tokenizer, + BYT5_MAPPER_CKPT_NAME, + INSERTED_ATTN_CKPT_NAME, + BYT5_CKPT_NAME, + MultilingualPromptFormat, +) +from glyph_sdxl.custom_diffusers import ( + StableDiffusionGlyphXLPipeline, + CrossAttnInsertBasicTransformerBlock, +) +from glyph_sdxl.modules import T5EncoderBlockByT5Mapper + +byt5_mapper_dict = [T5EncoderBlockByT5Mapper] +byt5_mapper_dict = {mapper.__name__: mapper for mapper in byt5_mapper_dict} + + +def import_model_class_from_model_name_or_path( + pretrained_model_name_or_path: str, revision: str, subfolder: str = "text_encoder", +): + text_encoder_config = PretrainedConfig.from_pretrained( + pretrained_model_name_or_path, + subfolder=subfolder, + revision=revision, + ) + model_class = text_encoder_config.architectures[0] + + if model_class == "CLIPTextModel": + from transformers import CLIPTextModel + + return CLIPTextModel + elif model_class == "CLIPTextModelWithProjection": + from transformers import CLIPTextModelWithProjection + + return CLIPTextModelWithProjection + else: + raise ValueError(f"{model_class} is not supported.") + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("config_dir", type=str) + parser.add_argument("ckpt_dir", type=str) + parser.add_argument("ann_path", type=str, default='examples/shower.json') + parser.add_argument("--out_folder", type=str, default='None') + parser.add_argument("--device", type=str, default='cuda') + parser.add_argument("--sampler", type=str, choices=['euler', 'dpm']) + parser.add_argument("--cfg", type=float, default=5.0) + args = parser.parse_args() + + config = parse_config(args.config_dir) + + text_encoder_cls_one = import_model_class_from_model_name_or_path( + config.pretrained_model_name_or_path, config.revision, + ) + text_encoder_cls_two = import_model_class_from_model_name_or_path( + config.pretrained_model_name_or_path, config.revision, subfolder="text_encoder_2", + ) + text_encoder_one = text_encoder_cls_one.from_pretrained( + config.pretrained_model_name_or_path, subfolder="text_encoder", revision=config.revision, + cache_dir=huggingface_cache_dir, + ) + text_encoder_two = text_encoder_cls_two.from_pretrained( + config.pretrained_model_name_or_path, subfolder="text_encoder_2", revision=config.revision, + cache_dir=huggingface_cache_dir, + ) + + unet = UNet2DConditionModel.from_pretrained( + config.pretrained_model_name_or_path, + subfolder="unet", + revision=config.revision, + cache_dir=huggingface_cache_dir, + ) + + vae_path = ( + config.pretrained_model_name_or_path + if config.pretrained_vae_model_name_or_path is None + else config.pretrained_vae_model_name_or_path + ) + vae = AutoencoderKL.from_pretrained( + vae_path, subfolder="vae" if config.pretrained_vae_model_name_or_path is None else None, + revision=config.revision, + cache_dir=huggingface_cache_dir, + ) + + byt5_model, byt5_tokenizer = load_byt5_and_byt5_tokenizer( + **config.byt5_config, + huggingface_cache_dir=huggingface_cache_dir, + ) + + inference_dtype = torch.float32 + if config.inference_dtype == "fp16": + inference_dtype = torch.float16 + elif config.inference_dtype == "bf16": + inference_dtype = torch.bfloat16 + + if config.pretrained_vae_model_name_or_path is None: + vae.to(args.device, dtype=torch.float32) + else: + vae.to(args.device, dtype=inference_dtype) + text_encoder_one.to(args.device, dtype=inference_dtype) + text_encoder_two.to(args.device, dtype=inference_dtype) + byt5_model.to(args.device) + unet.to(args.device, dtype=inference_dtype) + + inserted_new_modules_para_set = set() + for name, module in unet.named_modules(): + if isinstance(module, BasicTransformerBlock) and name in config.attn_block_to_modify: + parent_module = unet + for n in name.split(".")[:-1]: + parent_module = getattr(parent_module, n) + new_block = CrossAttnInsertBasicTransformerBlock.from_transformer_block( + module, + byt5_model.config.d_model if config.byt5_mapper_config.sdxl_channels is None else config.byt5_mapper_config.sdxl_channels, + ) + new_block.requires_grad_(False) + for inserted_module_name, inserted_module in zip( + new_block.get_inserted_modules_names(), + new_block.get_inserted_modules() + ): + inserted_module.requires_grad_(True) + for para_name, para in inserted_module.named_parameters(): + para_key = name + '.' + inserted_module_name + '.' + para_name + assert para_key not in inserted_new_modules_para_set + inserted_new_modules_para_set.add(para_key) + for origin_module in new_block.get_origin_modules(): + origin_module.to(args.device, dtype=inference_dtype) + parent_module.register_module(name.split(".")[-1], new_block) + print(f"inserted cross attn block to {name}") + + byt5_mapper = byt5_mapper_dict[config.byt5_mapper_type]( + byt5_model.config, + **config.byt5_mapper_config, + ) + + unet_lora_target_modules = [ + "attn1.to_k", "attn1.to_q", "attn1.to_v", "attn1.to_out.0", + "attn2.to_k", "attn2.to_q", "attn2.to_v", "attn2.to_out.0", + ] + unet_lora_config = LoraConfig( + r=config.unet_lora_rank, + lora_alpha=config.unet_lora_rank, + init_lora_weights="gaussian", + target_modules=unet_lora_target_modules, + ) + unet.add_adapter(unet_lora_config) + + unet_lora_layers_para = torch.load(osp.join(args.ckpt_dir, UNET_CKPT_NAME), map_location='cpu') + incompatible_keys = set_peft_model_state_dict(unet, unet_lora_layers_para, adapter_name="default") + if getattr(incompatible_keys, 'unexpected_keys', []) == []: + print(f"loaded unet_lora_layers_para") + else: + print(f"unet_lora_layers has unexpected_keys: {getattr(incompatible_keys, 'unexpected_keys', None)}") + + inserted_attn_module_paras = torch.load(osp.join(args.ckpt_dir, INSERTED_ATTN_CKPT_NAME), map_location='cpu') + missing_keys, unexpected_keys = unet.load_state_dict(inserted_attn_module_paras, strict=False) + assert len(unexpected_keys) == 0, unexpected_keys + + byt5_mapper_para = torch.load(osp.join(args.ckpt_dir, BYT5_MAPPER_CKPT_NAME), map_location='cpu') + byt5_mapper.load_state_dict(byt5_mapper_para) + + byt5_model_para = torch.load(osp.join(args.ckpt_dir, BYT5_CKPT_NAME), map_location='cpu') + byt5_model.load_state_dict(byt5_model_para) + + pipeline = StableDiffusionGlyphXLPipeline.from_pretrained( + config.pretrained_model_name_or_path, + vae=vae, + text_encoder=text_encoder_one, + text_encoder_2=text_encoder_two, + byt5_text_encoder=byt5_model, + byt5_tokenizer=byt5_tokenizer, + byt5_mapper=byt5_mapper, + unet=unet, + byt5_max_length=config.byt5_max_length, + revision=config.revision, + torch_dtype=inference_dtype, + safety_checker=None, + cache_dir=huggingface_cache_dir, + ) + + if args.sampler == 'dpm': + pipeline.scheduler = DPMSolverMultistepScheduler.from_pretrained( + config.pretrained_model_name_or_path, + subfolder="scheduler", + use_karras_sigmas=True, + ) + + pipeline = pipeline.to(args.device) + + with open(args.ann_path, 'r') as f: + ann = json.load(f) + + os.makedirs(args.out_folder, exist_ok=True) + + prompt_format = MultilingualPromptFormat() + + texts = copy.deepcopy(ann['texts']) + bboxes = copy.deepcopy(ann['bbox']) + styles = copy.deepcopy(ann['styles']) + + text_prompt = prompt_format.format_prompt(texts, styles) + + if 'seed' not in ann: + generator = torch.Generator(device=args.device) + else: + generator = torch.Generator(device=args.device).manual_seed(ann['seed']) + + with torch.cuda.amp.autocast(): + image = pipeline( + prompt=ann['bg_prompt'], + text_prompt=text_prompt, + texts=texts, + bboxes=bboxes, + num_inference_steps=50, + generator=generator, + text_attn_mask=None, + guidance_scale=args.cfg, + ).images[0] + image.save(f'{args.out_folder}/result.png') diff --git a/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/requirements.txt b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..ff971fa8ff7a7b62aa50006d0a77e8e7034e3566 --- /dev/null +++ b/outputs/hf_snapshot_repaired_9b49404/text_encoder/Glyph-SDXL-v2/requirements.txt @@ -0,0 +1,10 @@ +transformers==4.36.2 +diffusers==0.26.1 +mmengine +accelerate +torch==2.2.0 +torchvision==0.17.0 +deepspeed +peft +webcolors +gradio \ No newline at end of file diff --git a/outputs/hf_snapshot_repaired_9b49404/text_encoder/byt5-small/.gitattributes b/outputs/hf_snapshot_repaired_9b49404/text_encoder/byt5-small/.gitattributes new file mode 100644 index 0000000000000000000000000000000000000000..d699711d884c16db686245dffb53739ddf85119c --- /dev/null +++ b/outputs/hf_snapshot_repaired_9b49404/text_encoder/byt5-small/.gitattributes @@ -0,0 +1,17 @@ +*.bin.* filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tar.gz filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text diff --git a/outputs/hf_snapshot_repaired_9b49404/text_encoder/byt5-small/README.md b/outputs/hf_snapshot_repaired_9b49404/text_encoder/byt5-small/README.md new file mode 100644 index 0000000000000000000000000000000000000000..d5bcdbff691101114c9e68dd5aed60e5aca833ca --- /dev/null +++ b/outputs/hf_snapshot_repaired_9b49404/text_encoder/byt5-small/README.md @@ -0,0 +1,158 @@ +--- +language: +- multilingual +- af +- am +- ar +- az +- be +- bg +- bn +- ca +- ceb +- co +- cs +- cy +- da +- de +- el +- en +- eo +- es +- et +- eu +- fa +- fi +- fil +- fr +- fy +- ga +- gd +- gl +- gu +- ha +- haw +- hi +- hmn +- ht +- hu +- hy +- ig +- is +- it +- iw +- ja +- jv +- ka +- kk +- km +- kn +- ko +- ku +- ky +- la +- lb +- lo +- lt +- lv +- mg +- mi +- mk +- ml +- mn +- mr +- ms +- mt +- my +- ne +- nl +- no +- ny +- pa +- pl +- ps +- pt +- ro +- ru +- sd +- si +- sk +- sl +- sm +- sn +- so +- sq +- sr +- st +- su +- sv +- sw +- ta +- te +- tg +- th +- tr +- uk +- und +- ur +- uz +- vi +- xh +- yi +- yo +- zh +- zu +datasets: +- mc4 + +license: apache-2.0 +--- + +# ByT5 - Small + +ByT5 is a tokenizer-free version of [Google's T5](https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html) and generally follows the architecture of [MT5](https://huggingface.co/google/mt5-small). + +ByT5 was only pre-trained on [mC4](https://www.tensorflow.org/datasets/catalog/c4#c4multilingual) excluding any supervised training with an average span-mask of 20 UTF-8 characters. Therefore, this model has to be fine-tuned before it is useable on a downstream task. + +ByT5 works especially well on noisy text data,*e.g.*, `google/byt5-small` significantly outperforms [mt5-small](https://huggingface.co/google/mt5-small) on [TweetQA](https://arxiv.org/abs/1907.06292). + +Paper: [ByT5: Towards a token-free future with pre-trained byte-to-byte models](https://arxiv.org/abs/2105.13626) + +Authors: *Linting Xue, Aditya Barua, Noah Constant, Rami Al-Rfou, Sharan Narang, Mihir Kale, Adam Roberts, Colin Raffel* + +## Example Inference + +ByT5 works on raw UTF-8 bytes and can be used without a tokenizer: + +```python +from transformers import T5ForConditionalGeneration +import torch + +model = T5ForConditionalGeneration.from_pretrained('google/byt5-small') + +input_ids = torch.tensor([list("Life is like a box of chocolates.".encode("utf-8"))]) + 3 # add 3 for special tokens +labels = torch.tensor([list("La vie est comme une boîte de chocolat.".encode("utf-8"))]) + 3 # add 3 for special tokens + +loss = model(input_ids, labels=labels).loss # forward pass +``` + +For batched inference & training it is however recommended using a tokenizer class for padding: + +```python +from transformers import T5ForConditionalGeneration, AutoTokenizer + +model = T5ForConditionalGeneration.from_pretrained('google/byt5-small') +tokenizer = AutoTokenizer.from_pretrained('google/byt5-small') + +model_inputs = tokenizer(["Life is like a box of chocolates.", "Today is Monday."], padding="longest", return_tensors="pt") +labels = tokenizer(["La vie est comme une boîte de chocolat.", "Aujourd'hui c'est lundi."], padding="longest", return_tensors="pt").input_ids + +loss = model(**model_inputs, labels=labels).loss # forward pass +``` + +## Abstract + +Most widely-used pre-trained language models operate on sequences of tokens corresponding to word or subword units. Encoding text as a sequence of tokens requires a tokenizer, which is typically created as an independent artifact from the model. Token-free models that instead operate directly on raw text (bytes or characters) have many benefits: they can process text in any language out of the box, they are more robust to noise, and they minimize technical debt by removing complex and error-prone text preprocessing pipelines. Since byte or character sequences are longer than token sequences, past work on token-free models has often introduced new model architectures designed to amortize the cost of operating directly on raw text. In this paper, we show that a standard Transformer architecture can be used with minimal modifications to process byte sequences. We carefully characterize the trade-offs in terms of parameter count, training FLOPs, and inference speed, and show that byte-level models are competitive with their token-level counterparts. We also demonstrate that byte-level models are significantly more robust to noise and perform better on tasks that are sensitive to spelling and pronunciation. As part of our contribution, we release a new set of pre-trained byte-level Transformer models based on the T5 architecture, as well as all code and data used in our experiments. + +![model image](https://raw.githubusercontent.com/patrickvonplaten/scientific_images/master/ByT5.png) + diff --git a/outputs/hf_snapshot_repaired_9b49404/text_encoder/byt5-small/config.json b/outputs/hf_snapshot_repaired_9b49404/text_encoder/byt5-small/config.json new file mode 100644 index 0000000000000000000000000000000000000000..1e1b57007120fd114ec64194d658c6abd74a4fb3 --- /dev/null +++ b/outputs/hf_snapshot_repaired_9b49404/text_encoder/byt5-small/config.json @@ -0,0 +1,28 @@ +{ + "_name_or_path": "/home/patrick/t5/byt5-small", + "architectures": [ + "T5ForConditionalGeneration" + ], + "d_ff": 3584, + "d_kv": 64, + "d_model": 1472, + "decoder_start_token_id": 0, + "dropout_rate": 0.1, + "eos_token_id": 1, + "feed_forward_proj": "gated-gelu", + "gradient_checkpointing": false, + "initializer_factor": 1.0, + "is_encoder_decoder": true, + "layer_norm_epsilon": 1e-06, + "model_type": "t5", + "num_decoder_layers": 4, + "num_heads": 6, + "num_layers": 12, + "pad_token_id": 0, + "relative_attention_num_buckets": 32, + "tie_word_embeddings": false, + "tokenizer_class": "ByT5Tokenizer", + "transformers_version": "4.7.0.dev0", + "use_cache": true, + "vocab_size": 384 +} diff --git a/outputs/hf_snapshot_repaired_9b49404/text_encoder/byt5-small/flax_model.msgpack b/outputs/hf_snapshot_repaired_9b49404/text_encoder/byt5-small/flax_model.msgpack new file mode 100644 index 0000000000000000000000000000000000000000..533dee8f7f6b75816dcef5dca39a5c8f68980dbd --- /dev/null +++ b/outputs/hf_snapshot_repaired_9b49404/text_encoder/byt5-small/flax_model.msgpack @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b3aafee96d60e98aa18b3c7f73a2c5a2360f1f2f6df79361190a4c9e05c5ab21 +size 1198558445 diff --git a/outputs/hf_snapshot_repaired_9b49404/text_encoder/byt5-small/generation_config.json b/outputs/hf_snapshot_repaired_9b49404/text_encoder/byt5-small/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..d52815623b46b7db1c4b957b5a83a8ad30b0146a --- /dev/null +++ b/outputs/hf_snapshot_repaired_9b49404/text_encoder/byt5-small/generation_config.json @@ -0,0 +1,7 @@ +{ + "_from_model_config": true, + "decoder_start_token_id": 0, + "eos_token_id": 1, + "pad_token_id": 0, + "transformers_version": "4.27.0.dev0" +} diff --git a/outputs/hf_snapshot_repaired_9b49404/text_encoder/byt5-small/model.safetensors b/outputs/hf_snapshot_repaired_9b49404/text_encoder/byt5-small/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..578ecca59e2878af4ec89ea943aa56b62022e49e --- /dev/null +++ b/outputs/hf_snapshot_repaired_9b49404/text_encoder/byt5-small/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b8ea1904b143bc206f9801009b7653b9ff0ecef1de2911c10294d05117ac64de +size 1203093688