diff --git a/README.md b/README.md index 6875829f1b2991eaf1d84821caedebf619905692..8186af87d2414059a7f82277b98ac66d4f6b6bcf 100644 --- a/README.md +++ b/README.md @@ -29,3 +29,7 @@ Inference checkpoints, original configurations and model provenance. | profiles | [latency-sensitive-bench/profiles](https://huggingface.co/datasets/latency-sensitive-bench/profiles) | [Benchmark release inventory](reports/benchmark-release/README.md) + +## Artifact retention + +HAIC and Extreme Parkour releases are retired. Model bundles retain the published evaluation checkpoint, or the latest checkpoint when no evaluation selection exists. Optimizer and trainer recovery state are not release assets. Identical dataset copies use the canonical task paths. Original configurations and experiment evidence remain source records. diff --git a/archive/task-transfer/openvla_bridge_cross_mixed_all_024_5env200ep_deadly1000ep_exp1/artifact.json b/archive/task-transfer/openvla_bridge_cross_mixed_all_024_5env200ep_deadly1000ep_exp1/artifact.json index c4cb5274285e32df608c43721021c4ecf5187b9c..0d849286c504fea67ed7024c2c173f88d067f64f 100644 --- a/archive/task-transfer/openvla_bridge_cross_mixed_all_024_5env200ep_deadly1000ep_exp1/artifact.json +++ b/archive/task-transfer/openvla_bridge_cross_mixed_all_024_5env200ep_deadly1000ep_exp1/artifact.json @@ -482,10 +482,10 @@ "path": "archive/task-transfer/openvla_bridge_cross_mixed_all_024_5env200ep_deadly1000ep_exp1/.gitattributes" }, { - "repo": "latency-sensitive-bench/benchmark-datasets", - "repo_type": "dataset", - "revision": "d6f0ad0995f6e4b095f315920e8de18e04297048", - "path": "archive/task-transfer/openvla_bridge_cross_mixed_all_024_5env200ep_deadly1000ep_exp1/checkpoints/_initialization/StarVLA_Qwen3VL-OFT-Bridge-RT-1/checkpoints/steps_5000_pytorch_model.pt" + "repo": "StarVLA/Qwen3VL-OFT-Bridge-RT-1", + "repo_type": "model", + "revision": "1c814bc494cd95c844535fffed2a890838651dd6", + "path": "checkpoints/steps_5000_pytorch_model.pt" }, { "repo": "latency-sensitive-bench/benchmark-models", @@ -493,18 +493,6 @@ "revision": "671e2a7451fda7f7dba5606c6d11183ebd804b6d", "path": "archive/task-transfer/openvla_bridge_cross_mixed_all_024_5env200ep_deadly1000ep_exp1/checkpoints/steps_15000_model.safetensors" }, - { - "repo": "latency-sensitive-bench/benchmark-datasets", - "repo_type": "dataset", - "revision": "d6f0ad0995f6e4b095f315920e8de18e04297048", - "path": "archive/task-transfer/openvla_bridge_cross_mixed_all_024_5env200ep_deadly1000ep_exp1/checkpoints/steps_15000_state/optimizer.bin" - }, - { - "repo": "latency-sensitive-bench/benchmark-datasets", - "repo_type": "dataset", - "revision": "d6f0ad0995f6e4b095f315920e8de18e04297048", - "path": "archive/task-transfer/openvla_bridge_cross_mixed_all_024_5env200ep_deadly1000ep_exp1/checkpoints/steps_15000_state/random_states_0.pkl" - }, { "repo": "latency-sensitive-bench/benchmark-models", "repo_type": "model", @@ -607,12 +595,6 @@ "revision": "d6f0ad0995f6e4b095f315920e8de18e04297048", "path": "archive/task-transfer/openvla_bridge_cross_mixed_all_024_5env200ep_deadly1000ep_exp1/wandb/wandb/run-20260923_064644-r8358sud/logs/debug.log" }, - { - "repo": "latency-sensitive-bench/benchmark-datasets", - "repo_type": "dataset", - "revision": "d6f0ad0995f6e4b095f315920e8de18e04297048", - "path": "archive/task-transfer/openvla_bridge_cross_mixed_all_024_5env200ep_deadly1000ep_exp1/wandb/wandb/run-20260923_064644-r8358sud/run-r8358sud.wandb" - }, { "repo": "latency-sensitive-bench/benchmark-datasets", "repo_type": "dataset", @@ -661,12 +643,6 @@ "revision": "d6f0ad0995f6e4b095f315920e8de18e04297048", "path": "archive/task-transfer/openvla_bridge_cross_mixed_all_024_5env200ep_deadly1000ep_exp1/wandb/wandb/run-20260923_065340-67ykhdhp/logs/debug.log" }, - { - "repo": "latency-sensitive-bench/benchmark-datasets", - "repo_type": "dataset", - "revision": "d6f0ad0995f6e4b095f315920e8de18e04297048", - "path": "archive/task-transfer/openvla_bridge_cross_mixed_all_024_5env200ep_deadly1000ep_exp1/wandb/wandb/run-20260923_065340-67ykhdhp/run-67ykhdhp.wandb" - }, { "repo": "latency-sensitive-bench/benchmark-datasets", "repo_type": "dataset", @@ -714,12 +690,6 @@ "repo_type": "dataset", "revision": "d6f0ad0995f6e4b095f315920e8de18e04297048", "path": "archive/task-transfer/openvla_bridge_cross_mixed_all_024_5env200ep_deadly1000ep_exp1/wandb/wandb/run-20260923_105406-7ujo2tgu/logs/debug.log" - }, - { - "repo": "latency-sensitive-bench/benchmark-datasets", - "repo_type": "dataset", - "revision": "d6f0ad0995f6e4b095f315920e8de18e04297048", - "path": "archive/task-transfer/openvla_bridge_cross_mixed_all_024_5env200ep_deadly1000ep_exp1/wandb/wandb/run-20260923_105406-7ujo2tgu/run-7ujo2tgu.wandb" } ], "original_training_bindings": [ diff --git a/latency-aware/deadly-corridor/small-policy/factor_sweep:deadly_corridor:observation_stride:fixed_l4:fs4:obs5p83333:stride6:seed12/checkpoint_p0/checkpoint_000048836_25004032.pth b/latency-aware/deadly-corridor/small-policy/factor_sweep:deadly_corridor:observation_stride:fixed_l4:fs4:obs5p83333:stride6:seed12/checkpoint_p0/checkpoint_000048836_25004032.pth deleted file mode 100644 index a9aedde1de016792b07ba8724b154923779b1319..0000000000000000000000000000000000000000 --- a/latency-aware/deadly-corridor/small-policy/factor_sweep:deadly_corridor:observation_stride:fixed_l4:fs4:obs5p83333:stride6:seed12/checkpoint_p0/checkpoint_000048836_25004032.pth +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:6e990229272411cf177d56e5c97cfe85341497828a3848cace8c6055498b1a72 -size 13353289 diff --git a/latency-aware/deadly-corridor/small-policy/factor_sweep:deadly_corridor:observation_stride:fixed_l4:fs4:obs7:stride5:seed12/checkpoint_p0/checkpoint_000048836_25004032.pth b/latency-aware/deadly-corridor/small-policy/factor_sweep:deadly_corridor:observation_stride:fixed_l4:fs4:obs7:stride5:seed12/checkpoint_p0/checkpoint_000048836_25004032.pth deleted file mode 100644 index 8ea52cd659aaccacd48dce504eeb71a20e2ee321..0000000000000000000000000000000000000000 --- a/latency-aware/deadly-corridor/small-policy/factor_sweep:deadly_corridor:observation_stride:fixed_l4:fs4:obs7:stride5:seed12/checkpoint_p0/checkpoint_000048836_25004032.pth +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:318304e07d049d916671413cc0f4d83581b8fcc99535392b3ebc63d64c79fda1 -size 13353289 diff --git a/latency-aware/deadly-corridor/small-policy/factor_sweep:deadly_corridor:observation_stride:uniform_u2_6:fs4:obs11p6667:stride3:seed11/checkpoint_p0/checkpoint_000048836_25004032.pth b/latency-aware/deadly-corridor/small-policy/factor_sweep:deadly_corridor:observation_stride:uniform_u2_6:fs4:obs11p6667:stride3:seed11/checkpoint_p0/checkpoint_000048836_25004032.pth deleted file mode 100644 index 7f48fa9d3c608b7a169ea58938ee4115f5a57470..0000000000000000000000000000000000000000 --- a/latency-aware/deadly-corridor/small-policy/factor_sweep:deadly_corridor:observation_stride:uniform_u2_6:fs4:obs11p6667:stride3:seed11/checkpoint_p0/checkpoint_000048836_25004032.pth +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:973d2b152476dde80b0ab1be9d66ed11a6e44a37529e8a8a8aa7d0b5bd2877b3 -size 13353289 diff --git a/latency-aware/deadly-corridor/small-policy/factor_sweep:deadly_corridor:observation_stride:uniform_u2_6:fs4:obs11p6667:stride3:seed13/checkpoint_p0/checkpoint_000048836_25004032.pth b/latency-aware/deadly-corridor/small-policy/factor_sweep:deadly_corridor:observation_stride:uniform_u2_6:fs4:obs11p6667:stride3:seed13/checkpoint_p0/checkpoint_000048836_25004032.pth deleted file mode 100644 index 75e275c3afd0f828e72288fe726c7113bafa2085..0000000000000000000000000000000000000000 --- a/latency-aware/deadly-corridor/small-policy/factor_sweep:deadly_corridor:observation_stride:uniform_u2_6:fs4:obs11p6667:stride3:seed13/checkpoint_p0/checkpoint_000048836_25004032.pth +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:380ee41d7dcb3407473049516e7e13514d3bddd981a164065767dc1ea31f660a -size 13353289 diff --git a/latency-aware/deadly-corridor/small-policy/factor_sweep:deadly_corridor:observation_stride:uniform_u2_6:fs4:obs11p6667:stride3:seed14/checkpoint_p0/checkpoint_000048836_25004032.pth b/latency-aware/deadly-corridor/small-policy/factor_sweep:deadly_corridor:observation_stride:uniform_u2_6:fs4:obs11p6667:stride3:seed14/checkpoint_p0/checkpoint_000048836_25004032.pth deleted file mode 100644 index dc6dd78a4097eb553e9bca509ae74efb5c5fa702..0000000000000000000000000000000000000000 --- a/latency-aware/deadly-corridor/small-policy/factor_sweep:deadly_corridor:observation_stride:uniform_u2_6:fs4:obs11p6667:stride3:seed14/checkpoint_p0/checkpoint_000048836_25004032.pth +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:345c4b62af029741aa4815321434e353537482184c2cc4540fd17e343b07413a -size 13353289 diff --git a/latency-aware/deadly-corridor/small-policy/factor_sweep:deadly_corridor:observation_stride:uniform_u2_6:fs4:obs17p5:stride2:seed10/checkpoint_p0/checkpoint_000048836_25004032.pth b/latency-aware/deadly-corridor/small-policy/factor_sweep:deadly_corridor:observation_stride:uniform_u2_6:fs4:obs17p5:stride2:seed10/checkpoint_p0/checkpoint_000048836_25004032.pth deleted file mode 100644 index 93e3233ae251e3db294c2ef0d82b17f9d79809e7..0000000000000000000000000000000000000000 --- a/latency-aware/deadly-corridor/small-policy/factor_sweep:deadly_corridor:observation_stride:uniform_u2_6:fs4:obs17p5:stride2:seed10/checkpoint_p0/checkpoint_000048836_25004032.pth +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:c7c054e7d5a7f05fd6d4fe310eac21d1f7e4f137807af7eadc8cb7f2626425e5 -size 13353289 diff --git a/latency-aware/deadly-corridor/small-policy/factor_sweep:deadly_corridor:observation_stride:uniform_u2_6:fs4:obs5p83333:stride6:seed12/checkpoint_p0/checkpoint_000048836_25004032.pth b/latency-aware/deadly-corridor/small-policy/factor_sweep:deadly_corridor:observation_stride:uniform_u2_6:fs4:obs5p83333:stride6:seed12/checkpoint_p0/checkpoint_000048836_25004032.pth deleted file mode 100644 index 764f5708fa5d9ffaddb9d1b712f7f0d1e2a80661..0000000000000000000000000000000000000000 --- a/latency-aware/deadly-corridor/small-policy/factor_sweep:deadly_corridor:observation_stride:uniform_u2_6:fs4:obs5p83333:stride6:seed12/checkpoint_p0/checkpoint_000048836_25004032.pth +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:d9ee24783c3136de50dcf7e259e05e51331de8692287f647acccb26d710ba651 -size 13353289 diff --git a/latency-aware/deadly-corridor/small-policy/factor_sweep:deadly_corridor:observation_stride:uniform_u2_6:fs4:obs5p83333:stride6:seed13/checkpoint_p0/checkpoint_000048836_25004032.pth b/latency-aware/deadly-corridor/small-policy/factor_sweep:deadly_corridor:observation_stride:uniform_u2_6:fs4:obs5p83333:stride6:seed13/checkpoint_p0/checkpoint_000048836_25004032.pth deleted file mode 100644 index 9002d1e5c548ee17412ff2e2b55daf1048a29240..0000000000000000000000000000000000000000 --- a/latency-aware/deadly-corridor/small-policy/factor_sweep:deadly_corridor:observation_stride:uniform_u2_6:fs4:obs5p83333:stride6:seed13/checkpoint_p0/checkpoint_000048836_25004032.pth +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:0bc5c89dd97295a9c816c93db075e96b3ddd261bf340b3ada6369f366e472001 -size 13353289 diff --git a/latency-aware/deadly-corridor/small-policy/factor_sweep:deadly_corridor:observation_stride:uniform_u2_6:fs4:obs7:stride5:seed13/checkpoint_p0/checkpoint_000048836_25004032.pth b/latency-aware/deadly-corridor/small-policy/factor_sweep:deadly_corridor:observation_stride:uniform_u2_6:fs4:obs7:stride5:seed13/checkpoint_p0/checkpoint_000048836_25004032.pth deleted file mode 100644 index f4d15e45b7938d34eef3e68f979082e557938962..0000000000000000000000000000000000000000 --- a/latency-aware/deadly-corridor/small-policy/factor_sweep:deadly_corridor:observation_stride:uniform_u2_6:fs4:obs7:stride5:seed13/checkpoint_p0/checkpoint_000048836_25004032.pth +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:73be58dc658781d947739a61fea15385bcd078f71510c1d1302db69589f70cf1 -size 13353289 diff --git a/latency-aware/deadly-corridor/vla/starvla-qwenoft-h1/openvla_deadly_corridor_fix_latency_2/artifact.json b/latency-aware/deadly-corridor/vla/starvla-qwenoft-h1/openvla_deadly_corridor_fix_latency_2/artifact.json index 45b651979fc7732074d5f46e37d50401e5fb2f6e..5489745d3d41bfa58c783cfc470a60a8f85956b4 100644 --- a/latency-aware/deadly-corridor/vla/starvla-qwenoft-h1/openvla_deadly_corridor_fix_latency_2/artifact.json +++ b/latency-aware/deadly-corridor/vla/starvla-qwenoft-h1/openvla_deadly_corridor_fix_latency_2/artifact.json @@ -5,11 +5,6 @@ "path": "historical/openvla_deadly_corridor_fix_latency_2" }, "checkpoint": [ - { - "repo": "latency-sensitive-bench/benchmark-models", - "revision": "46266ba5a184ac622f7fa3386ac689d8d755d64c", - "path": "latency-aware/deadly-corridor/vla/starvla-qwenoft-h1/openvla_deadly_corridor_fix_latency_2/checkpoints/best_model.safetensors" - }, { "repo": "latency-sensitive-bench/benchmark-models", "revision": "46266ba5a184ac622f7fa3386ac689d8d755d64c", diff --git a/latency-aware/deadly-corridor/vla/starvla-qwenoft-h1/openvla_deadly_corridor_fix_latency_2/checkpoints/best_model.safetensors b/latency-aware/deadly-corridor/vla/starvla-qwenoft-h1/openvla_deadly_corridor_fix_latency_2/checkpoints/best_model.safetensors deleted file mode 100644 index c2bb001d61d70fa6b682eb94361690332795d828..0000000000000000000000000000000000000000 --- a/latency-aware/deadly-corridor/vla/starvla-qwenoft-h1/openvla_deadly_corridor_fix_latency_2/checkpoints/best_model.safetensors +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:630d55bc258d986e0de4b04d9223f418855bb94c2d310c13ae808912553d6c7f -size 9138230516 diff --git a/latency-aware/demon-attack/vla/starvla-qwenoft-h1/openvla_bridge_demon_attack_latency_mixed_024_exp1/artifact.json b/latency-aware/demon-attack/vla/starvla-qwenoft-h1/openvla_bridge_demon_attack_latency_mixed_024_exp1/artifact.json index 5deae5e3a98c85c95c9b2ec21c5f9173db6abfe1..b981dfc682cd964b033785951f8e8a6a8c5a157c 100644 --- a/latency-aware/demon-attack/vla/starvla-qwenoft-h1/openvla_bridge_demon_attack_latency_mixed_024_exp1/artifact.json +++ b/latency-aware/demon-attack/vla/starvla-qwenoft-h1/openvla_bridge_demon_attack_latency_mixed_024_exp1/artifact.json @@ -38,18 +38,6 @@ "repo_type": "model", "revision": "671e2a7451fda7f7dba5606c6d11183ebd804b6d", "path": "latency-aware/demon-attack/vla/starvla-qwenoft-h1/openvla_bridge_demon_attack_latency_mixed_024_exp1/checkpoints/steps_5000_model.safetensors" - }, - { - "repo": "latency-sensitive-bench/benchmark-datasets", - "repo_type": "dataset", - "revision": "d6f0ad0995f6e4b095f315920e8de18e04297048", - "path": "archive/task-transfer/openvla_bridge_demon_attack_latency_mixed_024_exp1/steps_5000_state/optimizer.bin" - }, - { - "repo": "latency-sensitive-bench/benchmark-datasets", - "repo_type": "dataset", - "revision": "d6f0ad0995f6e4b095f315920e8de18e04297048", - "path": "archive/task-transfer/openvla_bridge_demon_attack_latency_mixed_024_exp1/steps_5000_state/random_states_0.pkl" } ], "interpretation": "Formal classification is the release decision. Exact identity binding to the cross-task figure remains unverified." diff --git a/latency-aware/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_1_small/artifact.json b/latency-aware/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_1_small/artifact.json index 23f4a2e0d9ae0ccd2eb354d5fc5541011afe2f65..5e89f4bd4ef6ad42228d9f5ed362da3dfd293a44 100644 --- a/latency-aware/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_1_small/artifact.json +++ b/latency-aware/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_1_small/artifact.json @@ -5,11 +5,6 @@ "path": "historical/openvla_flappy_fix_latency_1_small" }, "checkpoint": [ - { - "repo": "latency-sensitive-bench/benchmark-models", - "revision": "46266ba5a184ac622f7fa3386ac689d8d755d64c", - "path": "latency-aware/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_1_small/checkpoints/best_model.safetensors" - }, { "repo": "latency-sensitive-bench/benchmark-models", "revision": "46266ba5a184ac622f7fa3386ac689d8d755d64c", diff --git a/latency-aware/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_1_small/checkpoints/best_model.safetensors b/latency-aware/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_1_small/checkpoints/best_model.safetensors deleted file mode 100644 index 438b05c05fbd1b7192d0a56de824fb496fcc2e54..0000000000000000000000000000000000000000 --- a/latency-aware/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_1_small/checkpoints/best_model.safetensors +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:758b4ccf870860c583af4da492b2d6782333e847759afd7314ba4547610fb04e -size 9138230516 diff --git a/latency-aware/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_1_small_tmp/artifact.json b/latency-aware/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_1_small_tmp/artifact.json index 0a917220655f2013a23173435758aefc9917e7d0..cec8588044b546fa0d087f1381994bfdf50f505a 100644 --- a/latency-aware/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_1_small_tmp/artifact.json +++ b/latency-aware/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_1_small_tmp/artifact.json @@ -5,11 +5,6 @@ "path": "historical/openvla_flappy_fix_latency_1_small_tmp" }, "checkpoint": [ - { - "repo": "latency-sensitive-bench/benchmark-models", - "revision": "46266ba5a184ac622f7fa3386ac689d8d755d64c", - "path": "latency-aware/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_1_small_tmp/checkpoints/best_model.safetensors" - }, { "repo": "latency-sensitive-bench/benchmark-models", "revision": "46266ba5a184ac622f7fa3386ac689d8d755d64c", diff --git a/latency-aware/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_1_small_tmp/checkpoints/best_model.safetensors b/latency-aware/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_1_small_tmp/checkpoints/best_model.safetensors deleted file mode 100644 index 2845503555aac40c333c9c69b21a3e7c7c3cf17f..0000000000000000000000000000000000000000 --- a/latency-aware/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_1_small_tmp/checkpoints/best_model.safetensors +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:6abc545e6b1abb272b7829b5676f412eebf33cfe0f21ba56368eb619c4610d83 -size 9138230516 diff --git a/latency-aware/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_2/artifact.json b/latency-aware/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_2/artifact.json index 0ed3c39ff0b0c8ba44014e9892e237d5cee21c4d..f16a48d89cc2b8022dd09a48f947b65372ad5e7a 100644 --- a/latency-aware/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_2/artifact.json +++ b/latency-aware/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_2/artifact.json @@ -5,11 +5,6 @@ "path": "historical/openvla_flappy_fix_latency_2" }, "checkpoint": [ - { - "repo": "latency-sensitive-bench/benchmark-models", - "revision": "46266ba5a184ac622f7fa3386ac689d8d755d64c", - "path": "latency-aware/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_2/checkpoints/steps_5000_model.safetensors" - }, { "repo": "latency-sensitive-bench/benchmark-models", "revision": "46266ba5a184ac622f7fa3386ac689d8d755d64c", diff --git a/latency-aware/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_2/checkpoints/steps_5000_model.safetensors b/latency-aware/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_2/checkpoints/steps_5000_model.safetensors deleted file mode 100644 index 8f98df5e12f1af4045bd5c98aa5584609ad8ed42..0000000000000000000000000000000000000000 --- a/latency-aware/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_2/checkpoints/steps_5000_model.safetensors +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:0fc5f0a0cd58f4f9726d53639e193c48efd5afd512fb0d0f644d239eb4079668 -size 9138230516 diff --git a/latency-aware/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_2_small/artifact.json b/latency-aware/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_2_small/artifact.json index 8ef7621b66634067113ca72758db84e3debad58f..0d05be9e5e018b6be2fd9c807d0203f5f651fd52 100644 --- a/latency-aware/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_2_small/artifact.json +++ b/latency-aware/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_2_small/artifact.json @@ -5,11 +5,6 @@ "path": "historical/openvla_flappy_fix_latency_2_small" }, "checkpoint": [ - { - "repo": "latency-sensitive-bench/benchmark-models", - "revision": "46266ba5a184ac622f7fa3386ac689d8d755d64c", - "path": "latency-aware/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_2_small/checkpoints/best_model.safetensors" - }, { "repo": "latency-sensitive-bench/benchmark-models", "revision": "46266ba5a184ac622f7fa3386ac689d8d755d64c", diff --git a/latency-aware/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_2_small/checkpoints/best_model.safetensors b/latency-aware/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_2_small/checkpoints/best_model.safetensors deleted file mode 100644 index cfb8a6d5175113161301ffc8f84c7bdbd26df328..0000000000000000000000000000000000000000 --- a/latency-aware/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_2_small/checkpoints/best_model.safetensors +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:ee25ed102ecbf335b536f8260de7ae96b84cfa7158024c19e6dc51d32cd9ed27 -size 9138230516 diff --git a/latency-aware/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_3_small/artifact.json b/latency-aware/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_3_small/artifact.json index 7153e2d7f0b6693b70ad3563027e91013c483359..9990004a76b1ef6be877109e27f5bd4e4f6c6a8c 100644 --- a/latency-aware/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_3_small/artifact.json +++ b/latency-aware/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_3_small/artifact.json @@ -5,11 +5,6 @@ "path": "historical/openvla_flappy_fix_latency_3_small" }, "checkpoint": [ - { - "repo": "latency-sensitive-bench/benchmark-models", - "revision": "46266ba5a184ac622f7fa3386ac689d8d755d64c", - "path": "latency-aware/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_3_small/checkpoints/best_model.safetensors" - }, { "repo": "latency-sensitive-bench/benchmark-models", "revision": "46266ba5a184ac622f7fa3386ac689d8d755d64c", diff --git a/latency-aware/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_3_small/checkpoints/best_model.safetensors b/latency-aware/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_3_small/checkpoints/best_model.safetensors deleted file mode 100644 index fa5a35978d4e3cc35a28c9ced3156a60aa655aaa..0000000000000000000000000000000000000000 --- a/latency-aware/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_3_small/checkpoints/best_model.safetensors +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:dfd6f77b285dff5a628b8a2e8aee12d2c8f11c2196464ed2505f7e138ec2a828 -size 9138230516 diff --git a/latency-aware/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_4_exp/artifact.json b/latency-aware/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_4_exp/artifact.json index a8f2076d73eec1a23b0c4284759337e10b6ab546..71443ebdea849b79f1326816898667af4310ea1e 100644 --- a/latency-aware/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_4_exp/artifact.json +++ b/latency-aware/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_4_exp/artifact.json @@ -5,11 +5,6 @@ "path": "historical/openvla_flappy_fix_latency_4_exp" }, "checkpoint": [ - { - "repo": "latency-sensitive-bench/benchmark-models", - "revision": "46266ba5a184ac622f7fa3386ac689d8d755d64c", - "path": "latency-aware/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_4_exp/checkpoints/steps_2500_model.safetensors" - }, { "repo": "latency-sensitive-bench/benchmark-models", "revision": "46266ba5a184ac622f7fa3386ac689d8d755d64c", diff --git a/latency-aware/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_4_exp/checkpoints/steps_2500_model.safetensors b/latency-aware/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_4_exp/checkpoints/steps_2500_model.safetensors deleted file mode 100644 index 2420a9f1fd3ded8907c96892853aaac92a2f4c20..0000000000000000000000000000000000000000 --- a/latency-aware/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_4_exp/checkpoints/steps_2500_model.safetensors +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:b4a6e3ade6d47371998034284d7a0ccbeb1f92b9713a657eb6041604cdbe9235 -size 9138230516 diff --git a/latency-aware/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_4_small/artifact.json b/latency-aware/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_4_small/artifact.json index ad1d31a36084dbabaa65ea37e22fcb659b57a2ce..646e0d8f4001ec11f080e9061914de4cc4217ca1 100644 --- a/latency-aware/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_4_small/artifact.json +++ b/latency-aware/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_4_small/artifact.json @@ -5,11 +5,6 @@ "path": "historical/openvla_flappy_fix_latency_4_small" }, "checkpoint": [ - { - "repo": "latency-sensitive-bench/benchmark-models", - "revision": "46266ba5a184ac622f7fa3386ac689d8d755d64c", - "path": "latency-aware/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_4_small/checkpoints/best_model.safetensors" - }, { "repo": "latency-sensitive-bench/benchmark-models", "revision": "46266ba5a184ac622f7fa3386ac689d8d755d64c", diff --git a/latency-aware/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_4_small/checkpoints/best_model.safetensors b/latency-aware/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_4_small/checkpoints/best_model.safetensors deleted file mode 100644 index 6687baf1b5e283a975603dc315a0cda7dd89c620..0000000000000000000000000000000000000000 --- a/latency-aware/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_4_small/checkpoints/best_model.safetensors +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:f893f7feb006ca5eee1dbc84ac222cf50c7b0dddcb7d572cf6d52135605bb0b9 -size 9138230516 diff --git a/latency-aware/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_6/artifact.json b/latency-aware/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_6/artifact.json index 8dd461fb66e88e8f28007621fe475452c5c08033..a13e4530c427ce65336994a9864a99f0fa0b3880 100644 --- a/latency-aware/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_6/artifact.json +++ b/latency-aware/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_6/artifact.json @@ -5,11 +5,6 @@ "path": "historical/openvla_flappy_fix_latency_6" }, "checkpoint": [ - { - "repo": "latency-sensitive-bench/benchmark-models", - "revision": "46266ba5a184ac622f7fa3386ac689d8d755d64c", - "path": "latency-aware/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_6/checkpoints/steps_5000_model.safetensors" - }, { "repo": "latency-sensitive-bench/benchmark-models", "revision": "46266ba5a184ac622f7fa3386ac689d8d755d64c", diff --git a/latency-aware/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_6/checkpoints/steps_5000_model.safetensors b/latency-aware/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_6/checkpoints/steps_5000_model.safetensors deleted file mode 100644 index ba8abf65fe9fb65ed31470b7b516766fdbb03ae9..0000000000000000000000000000000000000000 --- a/latency-aware/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_6/checkpoints/steps_5000_model.safetensors +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:a24aac79d3d3fd72e0a8e782c0fd1f5b3294ee74eca4bbce1cbff73128b72122 -size 9138230516 diff --git a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwengr00t-h1/mikasa_qwengr00t_latency_3090_2xh200_20260922/README.md b/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwengr00t-h1/mikasa_qwengr00t_latency_3090_2xh200_20260922/README.md index f49932fc8dd49986efa89c1503299b47969ca218..7ed4a84e642c16050d3f744136bdb7d3cbe0c840 100644 --- a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwengr00t-h1/mikasa_qwengr00t_latency_3090_2xh200_20260922/README.md +++ b/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwengr00t-h1/mikasa_qwengr00t_latency_3090_2xh200_20260922/README.md @@ -1,3 +1,5 @@ # qwengr00t — MIKASA InterceptGrabFast H1 Latency-aware SFT with the matched RTX 3090 temporal profile. See `eval_profile.json` and `provenance.json` for the 200-episode protocol and sources. + +The byte-identical weight is stored once in the corrected release: [checkpoint](https://huggingface.co/latency-sensitive-bench/benchmark-models/blob/6b07ea8e3f768a920e7d97ffbab4a5bedfa712ed/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwengr00t-h1/mikasa_qwengr00t_latency_3090_2xh200_20260922_evalfix_20260923/checkpoints/model.pt). This directory retains the original run metadata. diff --git a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwengr00t-h1/mikasa_qwengr00t_latency_3090_2xh200_20260922/checkpoints/model.pt b/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwengr00t-h1/mikasa_qwengr00t_latency_3090_2xh200_20260922/checkpoints/model.pt deleted file mode 100644 index 5087af7557285aa10f98830eda33da970cb10b6d..0000000000000000000000000000000000000000 --- a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwengr00t-h1/mikasa_qwengr00t_latency_3090_2xh200_20260922/checkpoints/model.pt +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:26e7217bde9e107c5005a2d0c06dfb223ca467e25cbb23d11a2e3b703bb325b9 -size 9976844067 diff --git a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwengr00t-h1/mikasa_qwengr00t_latency_3090_2xh200_20260922/provenance.json b/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwengr00t-h1/mikasa_qwengr00t_latency_3090_2xh200_20260922/provenance.json index 6827dc92be05c2581be1c5f3f657c8305b6e7a4f..8db507ff975f6d8b7e29ce5ca766709d1f26eb13 100644 --- a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwengr00t-h1/mikasa_qwengr00t_latency_3090_2xh200_20260922/provenance.json +++ b/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwengr00t-h1/mikasa_qwengr00t_latency_3090_2xh200_20260922/provenance.json @@ -1126,5 +1126,11 @@ "worker_capacity": 1, "parallel_envs": 10, "max_transitions": 60 + }, + "checkpoint_reference": { + "repo_id": "latency-sensitive-bench/benchmark-models", + "repo_type": "model", + "revision": "6b07ea8e3f768a920e7d97ffbab4a5bedfa712ed", + "path": "latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwengr00t-h1/mikasa_qwengr00t_latency_3090_2xh200_20260922_evalfix_20260923/checkpoints/model.pt" } } diff --git a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwengr00t-h8/h8-conditioned-dagger-20260923-profile/checkpoints/latest_model.pt b/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwengr00t-h8/h8-conditioned-dagger-20260923-profile/checkpoints/latest_model.pt deleted file mode 100644 index af92db29ff0a87664ddc4fa4c9ed8772da0ac4b4..0000000000000000000000000000000000000000 --- a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwengr00t-h8/h8-conditioned-dagger-20260923-profile/checkpoints/latest_model.pt +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:6ec5b32115c3f3e5e48fdc37aa6396cc07b492e72224747c890585d9f70441a9 -size 9976975139 diff --git a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwengr00t-h8/h8-conditioned-dagger-20260923-profile/provenance.json b/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwengr00t-h8/h8-conditioned-dagger-20260923-profile/provenance.json index 9383a1ab66e2d6337125f014f1f77185988ecd05..c5ff1d673daa4675f3f5053212dc40d88efc8f67 100644 --- a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwengr00t-h8/h8-conditioned-dagger-20260923-profile/provenance.json +++ b/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwengr00t-h8/h8-conditioned-dagger-20260923-profile/provenance.json @@ -15,6 +15,5 @@ 4242424542, 4242424741 ], - "checkpoint": "checkpoints/model.pt", - "latest_checkpoint": "checkpoints/latest_model.pt" + "checkpoint": "checkpoints/model.pt" } diff --git a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h1/mikasa_qwenoft_latency_3090_2xh200_20260922/README.md b/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h1/mikasa_qwenoft_latency_3090_2xh200_20260922/README.md index a573efbf8c400bff557290f8f62893413a5e6ddc..71b43183aee7b954de4e2a68880be9c5be9cf18f 100644 --- a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h1/mikasa_qwenoft_latency_3090_2xh200_20260922/README.md +++ b/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h1/mikasa_qwenoft_latency_3090_2xh200_20260922/README.md @@ -1,3 +1,5 @@ # qwenoft — MIKASA InterceptGrabFast H1 Latency-aware SFT with the matched RTX 3090 temporal profile. See `eval_profile.json` and `provenance.json` for the 200-episode protocol and sources. + +The byte-identical weight is stored once in the corrected release: [checkpoint](https://huggingface.co/latency-sensitive-bench/benchmark-models/blob/6b07ea8e3f768a920e7d97ffbab4a5bedfa712ed/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h1/mikasa_qwenoft_latency_3090_2xh200_20260922_evalfix_20260923/checkpoints/model.pt). This directory retains the original run metadata. diff --git a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h1/mikasa_qwenoft_latency_3090_2xh200_20260922/checkpoints/model.pt b/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h1/mikasa_qwenoft_latency_3090_2xh200_20260922/checkpoints/model.pt deleted file mode 100644 index 830a92f8553b24301cb822838fbf930746716422..0000000000000000000000000000000000000000 --- a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h1/mikasa_qwenoft_latency_3090_2xh200_20260922/checkpoints/model.pt +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:c98a7895a68788343533ddf2e09a3a4aa5bc29207abee10e096cc1cd9dbad890 -size 9785101729 diff --git a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h1/mikasa_qwenoft_latency_3090_2xh200_20260922/provenance.json b/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h1/mikasa_qwenoft_latency_3090_2xh200_20260922/provenance.json index 4dffc15297b70ee5e1e4026d7249dc49ddd99a51..02eabea9faaddc0ac7e77f1202954233d0d35b8f 100644 --- a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h1/mikasa_qwenoft_latency_3090_2xh200_20260922/provenance.json +++ b/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h1/mikasa_qwenoft_latency_3090_2xh200_20260922/provenance.json @@ -1126,5 +1126,11 @@ "worker_capacity": 1, "parallel_envs": 10, "max_transitions": 60 + }, + "checkpoint_reference": { + "repo_id": "latency-sensitive-bench/benchmark-models", + "repo_type": "model", + "revision": "6b07ea8e3f768a920e7d97ffbab4a5bedfa712ed", + "path": "latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h1/mikasa_qwenoft_latency_3090_2xh200_20260922_evalfix_20260923/checkpoints/model.pt" } } diff --git a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed1-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/custom_checkpoint_0.pkl b/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed1-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/custom_checkpoint_0.pkl deleted file mode 100644 index 934d1829fe2c9fb79c035b298e9af9b7fd45867b..0000000000000000000000000000000000000000 --- a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed1-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/custom_checkpoint_0.pkl +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:4f2f99b7145922f6a1aa58ef156069bdb6ea34e0b3d24750bc0752f5b38c7fd1 -size 1525 diff --git a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed1-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/latest b/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed1-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/latest deleted file mode 100644 index 7b2c8602be034ae63f23b293f8d037fa7afa0c54..0000000000000000000000000000000000000000 --- a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed1-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/latest +++ /dev/null @@ -1 +0,0 @@ -pytorch_model \ No newline at end of file diff --git a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed1-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/pytorch_model/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed1-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/pytorch_model/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt deleted file mode 100644 index a9f41d96d8955ae9365acf3cc1929b6239a68da2..0000000000000000000000000000000000000000 --- a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed1-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/pytorch_model/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:5ad2e398c06a33b5ee2e9f8b9a2a235c0c52326109d6d8fb456540f8dfed6d8d -size 27021673085 diff --git a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed1-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/pytorch_model/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed1-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/pytorch_model/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt deleted file mode 100644 index d8bad3a9906271653bc41518dcecb7911aab7a67..0000000000000000000000000000000000000000 --- a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed1-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/pytorch_model/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:27016fcbc41fa11bbe39289974a4496201f0074459eb22a24687fbbab8325c12 -size 27021651453 diff --git a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed1-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/pytorch_model/mp_rank_00_model_states.pt b/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed1-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/pytorch_model/mp_rank_00_model_states.pt deleted file mode 100644 index d9ec85d5440484d5a071befdbba0e0198b5fd528..0000000000000000000000000000000000000000 --- a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed1-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/pytorch_model/mp_rank_00_model_states.pt +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:19857129428b9dce92b13120514aca480bd77310b03608910fe951f8df6b62ee -size 9007481061 diff --git a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed1-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/random_states_0.pkl b/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed1-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/random_states_0.pkl deleted file mode 100644 index 643be72278b559711045cb44c18891b117b6cb94..0000000000000000000000000000000000000000 --- a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed1-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/random_states_0.pkl +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:d941a9657caba00045418ce4bd1c0a25f918c543af57e5825de2b1178cf047e4 -size 14953 diff --git a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed1-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/random_states_1.pkl b/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed1-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/random_states_1.pkl deleted file mode 100644 index 0e2d3e19d7258149472f70d502ea5b8ae5ed3990..0000000000000000000000000000000000000000 --- a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed1-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/random_states_1.pkl +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:e381c3041ee50113fb4c9114764f363c4699b73e6177124121e43238fec57bb7 -size 15017 diff --git a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed1-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/zero_to_fp32.py b/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed1-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/zero_to_fp32.py deleted file mode 100644 index 0e759146cadd92ddfefab3680146c2bd6a2b5c04..0000000000000000000000000000000000000000 --- a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed1-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/zero_to_fp32.py +++ /dev/null @@ -1,760 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) Microsoft Corporation. -# SPDX-License-Identifier: Apache-2.0 - -# DeepSpeed Team - -# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets -# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in -# the future. Once extracted, the weights don't require DeepSpeed and can be used in any -# application. -# -# example: -# python zero_to_fp32.py . output_dir/ -# or -# python zero_to_fp32.py . output_dir/ --safe_serialization - -import argparse -import torch -import glob -import math -import os -import re -import gc -import json -import numpy as np -from tqdm import tqdm -from collections import OrderedDict -from dataclasses import dataclass - -# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with -# DeepSpeed data structures it has to be available in the current python environment. -from deepspeed.utils import logger -from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, - FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, - FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) - - -@dataclass -class zero_model_state: - buffers: dict() - param_shapes: dict() - shared_params: list - ds_version: int - frozen_param_shapes: dict() - frozen_param_fragments: dict() - - -debug = 0 - -# load to cpu -device = torch.device('cpu') - - -def atoi(text): - return int(text) if text.isdigit() else text - - -def natural_keys(text): - ''' - alist.sort(key=natural_keys) sorts in human order - http://nedbatchelder.com/blog/200712/human_sorting.html - (See Toothy's implementation in the comments) - ''' - return [atoi(c) for c in re.split(r'(\d+)', text)] - - -def get_model_state_file(checkpoint_dir, zero_stage): - if not os.path.isdir(checkpoint_dir): - raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") - - # there should be only one file - if zero_stage <= 2: - file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") - elif zero_stage == 3: - file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") - - if not os.path.exists(file): - raise FileNotFoundError(f"can't find model states file at '{file}'") - - return file - - -def get_checkpoint_files(checkpoint_dir, glob_pattern): - # XXX: need to test that this simple glob rule works for multi-node setup too - ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) - - if len(ckpt_files) == 0: - raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") - - return ckpt_files - - -def get_optim_files(checkpoint_dir): - return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") - - -def get_model_state_files(checkpoint_dir): - return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") - - -def parse_model_states(files): - zero_model_states = [] - for file in files: - state_dict = torch.load(file, map_location=device, weights_only=False) - - if BUFFER_NAMES not in state_dict: - raise ValueError(f"{file} is not a model state checkpoint") - buffer_names = state_dict[BUFFER_NAMES] - if debug: - print("Found buffers:", buffer_names) - - # recover just the buffers while restoring them to fp32 if they were saved in fp16 - buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} - param_shapes = state_dict[PARAM_SHAPES] - - # collect parameters that are included in param_shapes - param_names = [] - for s in param_shapes: - for name in s.keys(): - param_names.append(name) - - # update with frozen parameters - frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) - if frozen_param_shapes is not None: - if debug: - print(f"Found frozen_param_shapes: {frozen_param_shapes}") - param_names += list(frozen_param_shapes.keys()) - - # handle shared params - shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] - - ds_version = state_dict.get(DS_VERSION, None) - - frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) - - z_model_state = zero_model_state(buffers=buffers, - param_shapes=param_shapes, - shared_params=shared_params, - ds_version=ds_version, - frozen_param_shapes=frozen_param_shapes, - frozen_param_fragments=frozen_param_fragments) - zero_model_states.append(z_model_state) - - return zero_model_states - - -def parse_optim_states(files, ds_checkpoint_dir): - total_files = len(files) - state_dicts = [] - for f in tqdm(files, desc='Loading checkpoint shards'): - state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False) - # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights - # and also handle the case where it was already removed by another helper script - state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) - state_dicts.append(state_dict) - - if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]: - raise ValueError(f"{files[0]} is not a zero checkpoint") - zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] - world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] - - # For ZeRO-2 each param group can have different partition_count as data parallelism for expert - # parameters can be different from data parallelism for non-expert parameters. So we can just - # use the max of the partition_count to get the dp world_size. - - if type(world_size) is list: - world_size = max(world_size) - - if world_size != total_files: - raise ValueError( - f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " - "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." - ) - - # the groups are named differently in each stage - if zero_stage <= 2: - fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS - elif zero_stage == 3: - fp32_groups_key = FP32_FLAT_GROUPS - else: - raise ValueError(f"unknown zero stage {zero_stage}") - - fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] - return zero_stage, world_size, fp32_flat_groups - - -def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): - """ - Returns fp32 state_dict reconstructed from ds checkpoint - - Args: - - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) - - """ - print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") - - optim_files = get_optim_files(ds_checkpoint_dir) - zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) - print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") - - model_files = get_model_state_files(ds_checkpoint_dir) - - zero_model_states = parse_model_states(model_files) - print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') - - if zero_stage <= 2: - return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, - exclude_frozen_parameters) - elif zero_stage == 3: - return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, - exclude_frozen_parameters) - - -def _zero2_merge_frozen_params(state_dict, zero_model_states): - if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: - return - - frozen_param_shapes = zero_model_states[0].frozen_param_shapes - frozen_param_fragments = zero_model_states[0].frozen_param_fragments - - if debug: - num_elem = sum(s.numel() for s in frozen_param_shapes.values()) - print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') - - wanted_params = len(frozen_param_shapes) - wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) - avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) - print(f'Frozen params: Have {avail_numel} numels to process.') - print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') - - total_params = 0 - total_numel = 0 - for name, shape in frozen_param_shapes.items(): - total_params += 1 - unpartitioned_numel = shape.numel() - total_numel += unpartitioned_numel - - state_dict[name] = frozen_param_fragments[name] - - if debug: - print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") - - print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") - - -def _has_callable(obj, fn): - attr = getattr(obj, fn, None) - return callable(attr) - - -def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): - param_shapes = zero_model_states[0].param_shapes - - # Reconstruction protocol: - # - # XXX: document this - - if debug: - for i in range(world_size): - for j in range(len(fp32_flat_groups[0])): - print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") - - # XXX: memory usage doubles here (zero2) - num_param_groups = len(fp32_flat_groups[0]) - merged_single_partition_of_fp32_groups = [] - for i in range(num_param_groups): - merged_partitions = [sd[i] for sd in fp32_flat_groups] - full_single_fp32_vector = torch.cat(merged_partitions, 0) - merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) - avail_numel = sum( - [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) - - if debug: - wanted_params = sum([len(shapes) for shapes in param_shapes]) - wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) - # not asserting if there is a mismatch due to possible padding - print(f"Have {avail_numel} numels to process.") - print(f"Need {wanted_numel} numels in {wanted_params} params.") - - # params - # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support - # out-of-core computing solution - total_numel = 0 - total_params = 0 - for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): - offset = 0 - avail_numel = full_single_fp32_vector.numel() - for name, shape in shapes.items(): - - unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) - total_numel += unpartitioned_numel - total_params += 1 - - if debug: - print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") - state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) - offset += unpartitioned_numel - - # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and - # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex - # paddings performed in the code it's almost impossible to predict the exact numbers w/o the - # live optimizer object, so we are checking that the numbers are within the right range - align_to = 2 * world_size - - def zero2_align(x): - return align_to * math.ceil(x / align_to) - - if debug: - print(f"original offset={offset}, avail_numel={avail_numel}") - - offset = zero2_align(offset) - avail_numel = zero2_align(avail_numel) - - if debug: - print(f"aligned offset={offset}, avail_numel={avail_numel}") - - # Sanity check - if offset != avail_numel: - raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") - - print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") - - -def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, - exclude_frozen_parameters): - state_dict = OrderedDict() - - # buffers - buffers = zero_model_states[0].buffers - state_dict.update(buffers) - if debug: - print(f"added {len(buffers)} buffers") - - if not exclude_frozen_parameters: - _zero2_merge_frozen_params(state_dict, zero_model_states) - - _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) - - # recover shared parameters - for pair in zero_model_states[0].shared_params: - if pair[1] in state_dict: - state_dict[pair[0]] = state_dict[pair[1]] - - return state_dict - - -def zero3_partitioned_param_info(unpartitioned_numel, world_size): - remainder = unpartitioned_numel % world_size - padding_numel = (world_size - remainder) if remainder else 0 - partitioned_numel = math.ceil(unpartitioned_numel / world_size) - return partitioned_numel, padding_numel - - -def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): - if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: - return - - if debug: - for i in range(world_size): - num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) - print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') - - frozen_param_shapes = zero_model_states[0].frozen_param_shapes - wanted_params = len(frozen_param_shapes) - wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) - avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size - print(f'Frozen params: Have {avail_numel} numels to process.') - print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') - - total_params = 0 - total_numel = 0 - for name, shape in zero_model_states[0].frozen_param_shapes.items(): - total_params += 1 - unpartitioned_numel = shape.numel() - total_numel += unpartitioned_numel - - param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) - state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) - - partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) - - if debug: - print( - f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" - ) - - print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") - - -class GatheredTensor: - """ - A pseudo tensor that collects partitioned weights. - It is more memory efficient when there are multiple groups. - """ - - def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape): - self.flat_groups = flat_groups - self.flat_groups_offset = flat_groups_offset - self.offset = offset - self.partitioned_numel = partitioned_numel - self.shape = shape - self.dtype = self.flat_groups[0][0].dtype - - def contiguous(self): - """ - Merge partitioned weights from flat_groups into a single tensor. - """ - end_idx = self.offset + self.partitioned_numel - world_size = len(self.flat_groups) - pad_flat_param_chunks = [] - - for rank_i in range(world_size): - # for each rank, we need to collect weights from related group/groups - flat_groups_at_rank_i = self.flat_groups[rank_i] - start_group_id = None - end_group_id = None - for group_id in range(len(self.flat_groups_offset)): - if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]: - start_group_id = group_id - if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]: - end_group_id = group_id - break - # collect weights from related group/groups - for group_id in range(start_group_id, end_group_id + 1): - flat_tensor = flat_groups_at_rank_i[group_id] - start_offset = self.offset - self.flat_groups_offset[group_id] - end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id] - pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset]) - - # collect weights from all ranks - pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0) - param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous() - return param - - -def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): - param_shapes = zero_model_states[0].param_shapes - avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size - - # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each - # param, re-consolidating each param, while dealing with padding if any - - # merge list of dicts, preserving order - param_shapes = {k: v for d in param_shapes for k, v in d.items()} - - if debug: - for i in range(world_size): - print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") - - wanted_params = len(param_shapes) - wanted_numel = sum(shape.numel() for shape in param_shapes.values()) - # not asserting if there is a mismatch due to possible padding - avail_numel = fp32_flat_groups[0].numel() * world_size - print(f"Trainable params: Have {avail_numel} numels to process.") - print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") - - # params - # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support - # out-of-core computing solution - offset = 0 - total_numel = 0 - total_params = 0 - flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]])) - for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'): - unpartitioned_numel = shape.numel() - total_numel += unpartitioned_numel - total_params += 1 - partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) - - if debug: - print( - f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" - ) - - # memory efficient tensor - tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape) - state_dict[name] = tensor - offset += partitioned_numel - - offset *= world_size - - # Sanity check - if offset != avail_numel: - raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") - - print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") - - -def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, - exclude_frozen_parameters): - state_dict = OrderedDict() - - # buffers - buffers = zero_model_states[0].buffers - state_dict.update(buffers) - if debug: - print(f"added {len(buffers)} buffers") - - if not exclude_frozen_parameters: - _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) - - _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) - - # recover shared parameters - for pair in zero_model_states[0].shared_params: - if pair[1] in state_dict: - state_dict[pair[0]] = state_dict[pair[1]] - - return state_dict - - -def to_torch_tensor(state_dict, return_empty_tensor=False): - """ - Convert state_dict of GatheredTensor to torch tensor - """ - torch_state_dict = {} - converted_tensors = {} - for name, tensor in state_dict.items(): - tensor_id = id(tensor) - if tensor_id in converted_tensors: # shared tensors - shared_tensor = torch_state_dict[converted_tensors[tensor_id]] - torch_state_dict[name] = shared_tensor - else: - converted_tensors[tensor_id] = name - if return_empty_tensor: - torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype) - else: - torch_state_dict[name] = tensor.contiguous() - return torch_state_dict - - -def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, - tag=None, - exclude_frozen_parameters=False, - lazy_mode=False): - """ - Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with - ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example - via a model hub. - - Args: - - ``checkpoint_dir``: path to the desired checkpoint folder - - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` - - ``exclude_frozen_parameters``: exclude frozen parameters - - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient. - Convert the pesduo tensor to torch tensor by ``.contiguous()`` - - Returns: - - pytorch ``state_dict`` - - A typical usage might be :: - - from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint - # do the training and checkpoint saving - state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu - model = model.cpu() # move to cpu - model.load_state_dict(state_dict) - # submit to model hub or save the model to share with others - - In this example the ``model`` will no longer be usable in the deepspeed context of the same - application. i.e. you will need to re-initialize the deepspeed engine, since - ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. - - If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. - - Note: the above usage may not work if your application doesn't have sufficient free CPU memory. - You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with - the checkpoint. Or you can load state_dict in lazy mode :: - - from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint - state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu - for name, lazy_tensor in state_dict.item(): - tensor = lazy_tensor.contiguous() # to cpu - print(name, tensor) - # del tensor to release memory if it no longer in use - """ - if tag is None: - latest_path = os.path.join(checkpoint_dir, 'latest') - if os.path.isfile(latest_path): - with open(latest_path, 'r') as fd: - tag = fd.read().strip() - else: - raise ValueError(f"Unable to find 'latest' file at {latest_path}") - - ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) - - if not os.path.isdir(ds_checkpoint_dir): - raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") - - state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) - if lazy_mode: - return state_dict - else: - return to_torch_tensor(state_dict) - - -def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, - output_dir, - max_shard_size="5GB", - safe_serialization=False, - tag=None, - exclude_frozen_parameters=False): - """ - Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be - loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. - - Args: - - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) - - ``output_dir``: directory to the pytorch fp32 state_dict output files - - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB - - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). - - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` - - ``exclude_frozen_parameters``: exclude frozen parameters - """ - - # Dependency pre-check - if safe_serialization: - try: - from safetensors.torch import save_file - except ImportError: - print('If you want to use `safe_serialization`, please `pip install safetensors`') - raise - if max_shard_size is not None: - try: - from huggingface_hub import split_torch_state_dict_into_shards - except ImportError: - print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') - raise - - # Convert zero checkpoint to state_dict - state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, - tag, - exclude_frozen_parameters, - lazy_mode=True) - - # Shard the model if it is too big. - weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" - if max_shard_size is not None: - filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") - # an memory-efficient approach for sharding - empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True) - state_dict_split = split_torch_state_dict_into_shards(empty_state_dict, - filename_pattern=filename_pattern, - max_shard_size=max_shard_size) - else: - from collections import namedtuple - StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) - state_dict_split = StateDictSplit(is_sharded=False, - filename_to_tensors={weights_name: list(state_dict.keys())}) - - # Save the model by shard - os.makedirs(output_dir, exist_ok=True) - filename_to_tensors = state_dict_split.filename_to_tensors.items() - for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): - shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors} - shard_state_dict = to_torch_tensor(shard_state_dict) - output_path = os.path.join(output_dir, shard_file) - if safe_serialization: - save_file(shard_state_dict, output_path, metadata={"format": "pt"}) - else: - torch.save(shard_state_dict, output_path) - # release the memory of current shard - for tensor_name in list(shard_state_dict.keys()): - del state_dict[tensor_name] - del shard_state_dict[tensor_name] - del shard_state_dict - gc.collect() - - # Save index if sharded - if state_dict_split.is_sharded: - index = { - "metadata": state_dict_split.metadata, - "weight_map": state_dict_split.tensor_to_filename, - } - save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" - save_index_file = os.path.join(output_dir, save_index_file) - with open(save_index_file, "w", encoding="utf-8") as f: - content = json.dumps(index, indent=2, sort_keys=True) + "\n" - f.write(content) - - -def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): - """ - 1. Put the provided model to cpu - 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` - 3. Load it into the provided model - - Args: - - ``model``: the model object to update - - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) - - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` - - Returns: - - ``model`: modified model - - Make sure you have plenty of CPU memory available before you call this function. If you don't - have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it - conveniently placed for you in the checkpoint folder. - - A typical usage might be :: - - from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint - model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) - # submit to model hub or save the model to share with others - - Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context - of the same application. i.e. you will need to re-initialize the deepspeed engine, since - ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. - - """ - logger.info(f"Extracting fp32 weights") - state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) - - logger.info(f"Overwriting model with fp32 weights") - model = model.cpu() - model.load_state_dict(state_dict, strict=False) - - return model - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("checkpoint_dir", - type=str, - help="path to the desired checkpoint folder, e.g., path/checkpoint-12") - parser.add_argument("output_dir", - type=str, - help="directory to the pytorch fp32 state_dict output files" - "(e.g. path/checkpoint-12-output/)") - parser.add_argument( - "--max_shard_size", - type=str, - default="5GB", - help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" - "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" - "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" - "without CPU OOM issues.") - parser.add_argument( - "--safe_serialization", - default=False, - action='store_true', - help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") - parser.add_argument("-t", - "--tag", - type=str, - default=None, - help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") - parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") - parser.add_argument("-d", "--debug", action='store_true', help="enable debug") - args = parser.parse_args() - - debug = args.debug - - convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, - args.output_dir, - max_shard_size=args.max_shard_size, - safe_serialization=args.safe_serialization, - tag=args.tag, - exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed2-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/custom_checkpoint_0.pkl b/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed2-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/custom_checkpoint_0.pkl deleted file mode 100644 index 934d1829fe2c9fb79c035b298e9af9b7fd45867b..0000000000000000000000000000000000000000 --- a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed2-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/custom_checkpoint_0.pkl +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:4f2f99b7145922f6a1aa58ef156069bdb6ea34e0b3d24750bc0752f5b38c7fd1 -size 1525 diff --git a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed2-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/latest b/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed2-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/latest deleted file mode 100644 index 7b2c8602be034ae63f23b293f8d037fa7afa0c54..0000000000000000000000000000000000000000 --- a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed2-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/latest +++ /dev/null @@ -1 +0,0 @@ -pytorch_model \ No newline at end of file diff --git a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed2-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/pytorch_model/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed2-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/pytorch_model/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt deleted file mode 100644 index 755f3b5bcf5bf73e6c15bde8b5a8ce6a24fb8813..0000000000000000000000000000000000000000 --- a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed2-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/pytorch_model/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:d984f25958afd98d3f43a8adf892495bc3358f36d3fbd28e60929698e102a508 -size 27021673085 diff --git a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed2-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/pytorch_model/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed2-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/pytorch_model/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt deleted file mode 100644 index 9f657d6df1e886819e39987cb2e304b7053af2f2..0000000000000000000000000000000000000000 --- a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed2-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/pytorch_model/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:568ac8349f73e5977cf9769d5a50fe1723327634066feda30fbf3b6e5af19670 -size 27021651453 diff --git a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed2-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/pytorch_model/mp_rank_00_model_states.pt b/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed2-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/pytorch_model/mp_rank_00_model_states.pt deleted file mode 100644 index 6233b05200a3af47e70331638591503913301efc..0000000000000000000000000000000000000000 --- a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed2-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/pytorch_model/mp_rank_00_model_states.pt +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:d43569e2d988629f487ec2412b3bb6e2bbb018fb413ef5fb3d4de7996bcfaffa -size 9007481061 diff --git a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed2-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/random_states_0.pkl b/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed2-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/random_states_0.pkl deleted file mode 100644 index 643be72278b559711045cb44c18891b117b6cb94..0000000000000000000000000000000000000000 --- a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed2-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/random_states_0.pkl +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:d941a9657caba00045418ce4bd1c0a25f918c543af57e5825de2b1178cf047e4 -size 14953 diff --git a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed2-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/random_states_1.pkl b/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed2-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/random_states_1.pkl deleted file mode 100644 index 0e2d3e19d7258149472f70d502ea5b8ae5ed3990..0000000000000000000000000000000000000000 --- a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed2-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/random_states_1.pkl +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:e381c3041ee50113fb4c9114764f363c4699b73e6177124121e43238fec57bb7 -size 15017 diff --git a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed2-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/zero_to_fp32.py b/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed2-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/zero_to_fp32.py deleted file mode 100644 index 0e759146cadd92ddfefab3680146c2bd6a2b5c04..0000000000000000000000000000000000000000 --- a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed2-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/zero_to_fp32.py +++ /dev/null @@ -1,760 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) Microsoft Corporation. -# SPDX-License-Identifier: Apache-2.0 - -# DeepSpeed Team - -# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets -# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in -# the future. Once extracted, the weights don't require DeepSpeed and can be used in any -# application. -# -# example: -# python zero_to_fp32.py . output_dir/ -# or -# python zero_to_fp32.py . output_dir/ --safe_serialization - -import argparse -import torch -import glob -import math -import os -import re -import gc -import json -import numpy as np -from tqdm import tqdm -from collections import OrderedDict -from dataclasses import dataclass - -# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with -# DeepSpeed data structures it has to be available in the current python environment. -from deepspeed.utils import logger -from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, - FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, - FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) - - -@dataclass -class zero_model_state: - buffers: dict() - param_shapes: dict() - shared_params: list - ds_version: int - frozen_param_shapes: dict() - frozen_param_fragments: dict() - - -debug = 0 - -# load to cpu -device = torch.device('cpu') - - -def atoi(text): - return int(text) if text.isdigit() else text - - -def natural_keys(text): - ''' - alist.sort(key=natural_keys) sorts in human order - http://nedbatchelder.com/blog/200712/human_sorting.html - (See Toothy's implementation in the comments) - ''' - return [atoi(c) for c in re.split(r'(\d+)', text)] - - -def get_model_state_file(checkpoint_dir, zero_stage): - if not os.path.isdir(checkpoint_dir): - raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") - - # there should be only one file - if zero_stage <= 2: - file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") - elif zero_stage == 3: - file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") - - if not os.path.exists(file): - raise FileNotFoundError(f"can't find model states file at '{file}'") - - return file - - -def get_checkpoint_files(checkpoint_dir, glob_pattern): - # XXX: need to test that this simple glob rule works for multi-node setup too - ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) - - if len(ckpt_files) == 0: - raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") - - return ckpt_files - - -def get_optim_files(checkpoint_dir): - return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") - - -def get_model_state_files(checkpoint_dir): - return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") - - -def parse_model_states(files): - zero_model_states = [] - for file in files: - state_dict = torch.load(file, map_location=device, weights_only=False) - - if BUFFER_NAMES not in state_dict: - raise ValueError(f"{file} is not a model state checkpoint") - buffer_names = state_dict[BUFFER_NAMES] - if debug: - print("Found buffers:", buffer_names) - - # recover just the buffers while restoring them to fp32 if they were saved in fp16 - buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} - param_shapes = state_dict[PARAM_SHAPES] - - # collect parameters that are included in param_shapes - param_names = [] - for s in param_shapes: - for name in s.keys(): - param_names.append(name) - - # update with frozen parameters - frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) - if frozen_param_shapes is not None: - if debug: - print(f"Found frozen_param_shapes: {frozen_param_shapes}") - param_names += list(frozen_param_shapes.keys()) - - # handle shared params - shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] - - ds_version = state_dict.get(DS_VERSION, None) - - frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) - - z_model_state = zero_model_state(buffers=buffers, - param_shapes=param_shapes, - shared_params=shared_params, - ds_version=ds_version, - frozen_param_shapes=frozen_param_shapes, - frozen_param_fragments=frozen_param_fragments) - zero_model_states.append(z_model_state) - - return zero_model_states - - -def parse_optim_states(files, ds_checkpoint_dir): - total_files = len(files) - state_dicts = [] - for f in tqdm(files, desc='Loading checkpoint shards'): - state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False) - # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights - # and also handle the case where it was already removed by another helper script - state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) - state_dicts.append(state_dict) - - if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]: - raise ValueError(f"{files[0]} is not a zero checkpoint") - zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] - world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] - - # For ZeRO-2 each param group can have different partition_count as data parallelism for expert - # parameters can be different from data parallelism for non-expert parameters. So we can just - # use the max of the partition_count to get the dp world_size. - - if type(world_size) is list: - world_size = max(world_size) - - if world_size != total_files: - raise ValueError( - f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " - "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." - ) - - # the groups are named differently in each stage - if zero_stage <= 2: - fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS - elif zero_stage == 3: - fp32_groups_key = FP32_FLAT_GROUPS - else: - raise ValueError(f"unknown zero stage {zero_stage}") - - fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] - return zero_stage, world_size, fp32_flat_groups - - -def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): - """ - Returns fp32 state_dict reconstructed from ds checkpoint - - Args: - - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) - - """ - print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") - - optim_files = get_optim_files(ds_checkpoint_dir) - zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) - print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") - - model_files = get_model_state_files(ds_checkpoint_dir) - - zero_model_states = parse_model_states(model_files) - print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') - - if zero_stage <= 2: - return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, - exclude_frozen_parameters) - elif zero_stage == 3: - return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, - exclude_frozen_parameters) - - -def _zero2_merge_frozen_params(state_dict, zero_model_states): - if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: - return - - frozen_param_shapes = zero_model_states[0].frozen_param_shapes - frozen_param_fragments = zero_model_states[0].frozen_param_fragments - - if debug: - num_elem = sum(s.numel() for s in frozen_param_shapes.values()) - print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') - - wanted_params = len(frozen_param_shapes) - wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) - avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) - print(f'Frozen params: Have {avail_numel} numels to process.') - print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') - - total_params = 0 - total_numel = 0 - for name, shape in frozen_param_shapes.items(): - total_params += 1 - unpartitioned_numel = shape.numel() - total_numel += unpartitioned_numel - - state_dict[name] = frozen_param_fragments[name] - - if debug: - print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") - - print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") - - -def _has_callable(obj, fn): - attr = getattr(obj, fn, None) - return callable(attr) - - -def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): - param_shapes = zero_model_states[0].param_shapes - - # Reconstruction protocol: - # - # XXX: document this - - if debug: - for i in range(world_size): - for j in range(len(fp32_flat_groups[0])): - print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") - - # XXX: memory usage doubles here (zero2) - num_param_groups = len(fp32_flat_groups[0]) - merged_single_partition_of_fp32_groups = [] - for i in range(num_param_groups): - merged_partitions = [sd[i] for sd in fp32_flat_groups] - full_single_fp32_vector = torch.cat(merged_partitions, 0) - merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) - avail_numel = sum( - [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) - - if debug: - wanted_params = sum([len(shapes) for shapes in param_shapes]) - wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) - # not asserting if there is a mismatch due to possible padding - print(f"Have {avail_numel} numels to process.") - print(f"Need {wanted_numel} numels in {wanted_params} params.") - - # params - # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support - # out-of-core computing solution - total_numel = 0 - total_params = 0 - for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): - offset = 0 - avail_numel = full_single_fp32_vector.numel() - for name, shape in shapes.items(): - - unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) - total_numel += unpartitioned_numel - total_params += 1 - - if debug: - print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") - state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) - offset += unpartitioned_numel - - # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and - # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex - # paddings performed in the code it's almost impossible to predict the exact numbers w/o the - # live optimizer object, so we are checking that the numbers are within the right range - align_to = 2 * world_size - - def zero2_align(x): - return align_to * math.ceil(x / align_to) - - if debug: - print(f"original offset={offset}, avail_numel={avail_numel}") - - offset = zero2_align(offset) - avail_numel = zero2_align(avail_numel) - - if debug: - print(f"aligned offset={offset}, avail_numel={avail_numel}") - - # Sanity check - if offset != avail_numel: - raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") - - print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") - - -def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, - exclude_frozen_parameters): - state_dict = OrderedDict() - - # buffers - buffers = zero_model_states[0].buffers - state_dict.update(buffers) - if debug: - print(f"added {len(buffers)} buffers") - - if not exclude_frozen_parameters: - _zero2_merge_frozen_params(state_dict, zero_model_states) - - _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) - - # recover shared parameters - for pair in zero_model_states[0].shared_params: - if pair[1] in state_dict: - state_dict[pair[0]] = state_dict[pair[1]] - - return state_dict - - -def zero3_partitioned_param_info(unpartitioned_numel, world_size): - remainder = unpartitioned_numel % world_size - padding_numel = (world_size - remainder) if remainder else 0 - partitioned_numel = math.ceil(unpartitioned_numel / world_size) - return partitioned_numel, padding_numel - - -def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): - if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: - return - - if debug: - for i in range(world_size): - num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) - print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') - - frozen_param_shapes = zero_model_states[0].frozen_param_shapes - wanted_params = len(frozen_param_shapes) - wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) - avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size - print(f'Frozen params: Have {avail_numel} numels to process.') - print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') - - total_params = 0 - total_numel = 0 - for name, shape in zero_model_states[0].frozen_param_shapes.items(): - total_params += 1 - unpartitioned_numel = shape.numel() - total_numel += unpartitioned_numel - - param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) - state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) - - partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) - - if debug: - print( - f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" - ) - - print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") - - -class GatheredTensor: - """ - A pseudo tensor that collects partitioned weights. - It is more memory efficient when there are multiple groups. - """ - - def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape): - self.flat_groups = flat_groups - self.flat_groups_offset = flat_groups_offset - self.offset = offset - self.partitioned_numel = partitioned_numel - self.shape = shape - self.dtype = self.flat_groups[0][0].dtype - - def contiguous(self): - """ - Merge partitioned weights from flat_groups into a single tensor. - """ - end_idx = self.offset + self.partitioned_numel - world_size = len(self.flat_groups) - pad_flat_param_chunks = [] - - for rank_i in range(world_size): - # for each rank, we need to collect weights from related group/groups - flat_groups_at_rank_i = self.flat_groups[rank_i] - start_group_id = None - end_group_id = None - for group_id in range(len(self.flat_groups_offset)): - if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]: - start_group_id = group_id - if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]: - end_group_id = group_id - break - # collect weights from related group/groups - for group_id in range(start_group_id, end_group_id + 1): - flat_tensor = flat_groups_at_rank_i[group_id] - start_offset = self.offset - self.flat_groups_offset[group_id] - end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id] - pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset]) - - # collect weights from all ranks - pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0) - param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous() - return param - - -def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): - param_shapes = zero_model_states[0].param_shapes - avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size - - # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each - # param, re-consolidating each param, while dealing with padding if any - - # merge list of dicts, preserving order - param_shapes = {k: v for d in param_shapes for k, v in d.items()} - - if debug: - for i in range(world_size): - print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") - - wanted_params = len(param_shapes) - wanted_numel = sum(shape.numel() for shape in param_shapes.values()) - # not asserting if there is a mismatch due to possible padding - avail_numel = fp32_flat_groups[0].numel() * world_size - print(f"Trainable params: Have {avail_numel} numels to process.") - print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") - - # params - # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support - # out-of-core computing solution - offset = 0 - total_numel = 0 - total_params = 0 - flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]])) - for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'): - unpartitioned_numel = shape.numel() - total_numel += unpartitioned_numel - total_params += 1 - partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) - - if debug: - print( - f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" - ) - - # memory efficient tensor - tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape) - state_dict[name] = tensor - offset += partitioned_numel - - offset *= world_size - - # Sanity check - if offset != avail_numel: - raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") - - print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") - - -def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, - exclude_frozen_parameters): - state_dict = OrderedDict() - - # buffers - buffers = zero_model_states[0].buffers - state_dict.update(buffers) - if debug: - print(f"added {len(buffers)} buffers") - - if not exclude_frozen_parameters: - _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) - - _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) - - # recover shared parameters - for pair in zero_model_states[0].shared_params: - if pair[1] in state_dict: - state_dict[pair[0]] = state_dict[pair[1]] - - return state_dict - - -def to_torch_tensor(state_dict, return_empty_tensor=False): - """ - Convert state_dict of GatheredTensor to torch tensor - """ - torch_state_dict = {} - converted_tensors = {} - for name, tensor in state_dict.items(): - tensor_id = id(tensor) - if tensor_id in converted_tensors: # shared tensors - shared_tensor = torch_state_dict[converted_tensors[tensor_id]] - torch_state_dict[name] = shared_tensor - else: - converted_tensors[tensor_id] = name - if return_empty_tensor: - torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype) - else: - torch_state_dict[name] = tensor.contiguous() - return torch_state_dict - - -def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, - tag=None, - exclude_frozen_parameters=False, - lazy_mode=False): - """ - Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with - ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example - via a model hub. - - Args: - - ``checkpoint_dir``: path to the desired checkpoint folder - - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` - - ``exclude_frozen_parameters``: exclude frozen parameters - - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient. - Convert the pesduo tensor to torch tensor by ``.contiguous()`` - - Returns: - - pytorch ``state_dict`` - - A typical usage might be :: - - from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint - # do the training and checkpoint saving - state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu - model = model.cpu() # move to cpu - model.load_state_dict(state_dict) - # submit to model hub or save the model to share with others - - In this example the ``model`` will no longer be usable in the deepspeed context of the same - application. i.e. you will need to re-initialize the deepspeed engine, since - ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. - - If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. - - Note: the above usage may not work if your application doesn't have sufficient free CPU memory. - You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with - the checkpoint. Or you can load state_dict in lazy mode :: - - from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint - state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu - for name, lazy_tensor in state_dict.item(): - tensor = lazy_tensor.contiguous() # to cpu - print(name, tensor) - # del tensor to release memory if it no longer in use - """ - if tag is None: - latest_path = os.path.join(checkpoint_dir, 'latest') - if os.path.isfile(latest_path): - with open(latest_path, 'r') as fd: - tag = fd.read().strip() - else: - raise ValueError(f"Unable to find 'latest' file at {latest_path}") - - ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) - - if not os.path.isdir(ds_checkpoint_dir): - raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") - - state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) - if lazy_mode: - return state_dict - else: - return to_torch_tensor(state_dict) - - -def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, - output_dir, - max_shard_size="5GB", - safe_serialization=False, - tag=None, - exclude_frozen_parameters=False): - """ - Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be - loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. - - Args: - - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) - - ``output_dir``: directory to the pytorch fp32 state_dict output files - - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB - - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). - - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` - - ``exclude_frozen_parameters``: exclude frozen parameters - """ - - # Dependency pre-check - if safe_serialization: - try: - from safetensors.torch import save_file - except ImportError: - print('If you want to use `safe_serialization`, please `pip install safetensors`') - raise - if max_shard_size is not None: - try: - from huggingface_hub import split_torch_state_dict_into_shards - except ImportError: - print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') - raise - - # Convert zero checkpoint to state_dict - state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, - tag, - exclude_frozen_parameters, - lazy_mode=True) - - # Shard the model if it is too big. - weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" - if max_shard_size is not None: - filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") - # an memory-efficient approach for sharding - empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True) - state_dict_split = split_torch_state_dict_into_shards(empty_state_dict, - filename_pattern=filename_pattern, - max_shard_size=max_shard_size) - else: - from collections import namedtuple - StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) - state_dict_split = StateDictSplit(is_sharded=False, - filename_to_tensors={weights_name: list(state_dict.keys())}) - - # Save the model by shard - os.makedirs(output_dir, exist_ok=True) - filename_to_tensors = state_dict_split.filename_to_tensors.items() - for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): - shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors} - shard_state_dict = to_torch_tensor(shard_state_dict) - output_path = os.path.join(output_dir, shard_file) - if safe_serialization: - save_file(shard_state_dict, output_path, metadata={"format": "pt"}) - else: - torch.save(shard_state_dict, output_path) - # release the memory of current shard - for tensor_name in list(shard_state_dict.keys()): - del state_dict[tensor_name] - del shard_state_dict[tensor_name] - del shard_state_dict - gc.collect() - - # Save index if sharded - if state_dict_split.is_sharded: - index = { - "metadata": state_dict_split.metadata, - "weight_map": state_dict_split.tensor_to_filename, - } - save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" - save_index_file = os.path.join(output_dir, save_index_file) - with open(save_index_file, "w", encoding="utf-8") as f: - content = json.dumps(index, indent=2, sort_keys=True) + "\n" - f.write(content) - - -def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): - """ - 1. Put the provided model to cpu - 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` - 3. Load it into the provided model - - Args: - - ``model``: the model object to update - - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) - - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` - - Returns: - - ``model`: modified model - - Make sure you have plenty of CPU memory available before you call this function. If you don't - have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it - conveniently placed for you in the checkpoint folder. - - A typical usage might be :: - - from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint - model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) - # submit to model hub or save the model to share with others - - Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context - of the same application. i.e. you will need to re-initialize the deepspeed engine, since - ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. - - """ - logger.info(f"Extracting fp32 weights") - state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) - - logger.info(f"Overwriting model with fp32 weights") - model = model.cpu() - model.load_state_dict(state_dict, strict=False) - - return model - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("checkpoint_dir", - type=str, - help="path to the desired checkpoint folder, e.g., path/checkpoint-12") - parser.add_argument("output_dir", - type=str, - help="directory to the pytorch fp32 state_dict output files" - "(e.g. path/checkpoint-12-output/)") - parser.add_argument( - "--max_shard_size", - type=str, - default="5GB", - help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" - "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" - "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" - "without CPU OOM issues.") - parser.add_argument( - "--safe_serialization", - default=False, - action='store_true', - help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") - parser.add_argument("-t", - "--tag", - type=str, - default=None, - help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") - parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") - parser.add_argument("-d", "--debug", action='store_true', help="enable debug") - args = parser.parse_args() - - debug = args.debug - - convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, - args.output_dir, - max_shard_size=args.max_shard_size, - safe_serialization=args.safe_serialization, - tag=args.tag, - exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed3-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/custom_checkpoint_0.pkl b/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed3-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/custom_checkpoint_0.pkl deleted file mode 100644 index 934d1829fe2c9fb79c035b298e9af9b7fd45867b..0000000000000000000000000000000000000000 --- a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed3-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/custom_checkpoint_0.pkl +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:4f2f99b7145922f6a1aa58ef156069bdb6ea34e0b3d24750bc0752f5b38c7fd1 -size 1525 diff --git a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed3-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/latest b/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed3-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/latest deleted file mode 100644 index 7b2c8602be034ae63f23b293f8d037fa7afa0c54..0000000000000000000000000000000000000000 --- a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed3-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/latest +++ /dev/null @@ -1 +0,0 @@ -pytorch_model \ No newline at end of file diff --git a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed3-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/pytorch_model/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed3-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/pytorch_model/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt deleted file mode 100644 index 4230f318cd8b87fd22c0af61efb5aaf2b20707e4..0000000000000000000000000000000000000000 --- a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed3-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/pytorch_model/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:ce138a9e8568d1eedefe7474db17d4be74a46775aa2fb97235ddefd635d761d6 -size 27021673085 diff --git a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed3-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/pytorch_model/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed3-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/pytorch_model/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt deleted file mode 100644 index f9298995ebed84adeb24093a559054302d1dbc8f..0000000000000000000000000000000000000000 --- a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed3-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/pytorch_model/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:0e631dd5398bbd06b45f94793c17740faa5d0e29146857dc9a10d8b40ad41be9 -size 27021651453 diff --git a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed3-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/pytorch_model/mp_rank_00_model_states.pt b/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed3-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/pytorch_model/mp_rank_00_model_states.pt deleted file mode 100644 index 10e1b609c9542a4256aa5bba4d0fa9b6302f9e1a..0000000000000000000000000000000000000000 --- a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed3-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/pytorch_model/mp_rank_00_model_states.pt +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:2a68cccdd401db583ae4f89b081b2e1990c975d94d75b7fae3736ffb08d3e508 -size 9007481061 diff --git a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed3-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/random_states_0.pkl b/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed3-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/random_states_0.pkl deleted file mode 100644 index 643be72278b559711045cb44c18891b117b6cb94..0000000000000000000000000000000000000000 --- a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed3-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/random_states_0.pkl +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:d941a9657caba00045418ce4bd1c0a25f918c543af57e5825de2b1178cf047e4 -size 14953 diff --git a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed3-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/random_states_1.pkl b/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed3-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/random_states_1.pkl deleted file mode 100644 index 0e2d3e19d7258149472f70d502ea5b8ae5ed3990..0000000000000000000000000000000000000000 --- a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed3-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/random_states_1.pkl +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:e381c3041ee50113fb4c9114764f363c4699b73e6177124121e43238fec57bb7 -size 15017 diff --git a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed3-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/zero_to_fp32.py b/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed3-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/zero_to_fp32.py deleted file mode 100644 index 0e759146cadd92ddfefab3680146c2bd6a2b5c04..0000000000000000000000000000000000000000 --- a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed3-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/zero_to_fp32.py +++ /dev/null @@ -1,760 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) Microsoft Corporation. -# SPDX-License-Identifier: Apache-2.0 - -# DeepSpeed Team - -# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets -# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in -# the future. Once extracted, the weights don't require DeepSpeed and can be used in any -# application. -# -# example: -# python zero_to_fp32.py . output_dir/ -# or -# python zero_to_fp32.py . output_dir/ --safe_serialization - -import argparse -import torch -import glob -import math -import os -import re -import gc -import json -import numpy as np -from tqdm import tqdm -from collections import OrderedDict -from dataclasses import dataclass - -# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with -# DeepSpeed data structures it has to be available in the current python environment. -from deepspeed.utils import logger -from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, - FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, - FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) - - -@dataclass -class zero_model_state: - buffers: dict() - param_shapes: dict() - shared_params: list - ds_version: int - frozen_param_shapes: dict() - frozen_param_fragments: dict() - - -debug = 0 - -# load to cpu -device = torch.device('cpu') - - -def atoi(text): - return int(text) if text.isdigit() else text - - -def natural_keys(text): - ''' - alist.sort(key=natural_keys) sorts in human order - http://nedbatchelder.com/blog/200712/human_sorting.html - (See Toothy's implementation in the comments) - ''' - return [atoi(c) for c in re.split(r'(\d+)', text)] - - -def get_model_state_file(checkpoint_dir, zero_stage): - if not os.path.isdir(checkpoint_dir): - raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") - - # there should be only one file - if zero_stage <= 2: - file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") - elif zero_stage == 3: - file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") - - if not os.path.exists(file): - raise FileNotFoundError(f"can't find model states file at '{file}'") - - return file - - -def get_checkpoint_files(checkpoint_dir, glob_pattern): - # XXX: need to test that this simple glob rule works for multi-node setup too - ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) - - if len(ckpt_files) == 0: - raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") - - return ckpt_files - - -def get_optim_files(checkpoint_dir): - return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") - - -def get_model_state_files(checkpoint_dir): - return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") - - -def parse_model_states(files): - zero_model_states = [] - for file in files: - state_dict = torch.load(file, map_location=device, weights_only=False) - - if BUFFER_NAMES not in state_dict: - raise ValueError(f"{file} is not a model state checkpoint") - buffer_names = state_dict[BUFFER_NAMES] - if debug: - print("Found buffers:", buffer_names) - - # recover just the buffers while restoring them to fp32 if they were saved in fp16 - buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} - param_shapes = state_dict[PARAM_SHAPES] - - # collect parameters that are included in param_shapes - param_names = [] - for s in param_shapes: - for name in s.keys(): - param_names.append(name) - - # update with frozen parameters - frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) - if frozen_param_shapes is not None: - if debug: - print(f"Found frozen_param_shapes: {frozen_param_shapes}") - param_names += list(frozen_param_shapes.keys()) - - # handle shared params - shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] - - ds_version = state_dict.get(DS_VERSION, None) - - frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) - - z_model_state = zero_model_state(buffers=buffers, - param_shapes=param_shapes, - shared_params=shared_params, - ds_version=ds_version, - frozen_param_shapes=frozen_param_shapes, - frozen_param_fragments=frozen_param_fragments) - zero_model_states.append(z_model_state) - - return zero_model_states - - -def parse_optim_states(files, ds_checkpoint_dir): - total_files = len(files) - state_dicts = [] - for f in tqdm(files, desc='Loading checkpoint shards'): - state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False) - # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights - # and also handle the case where it was already removed by another helper script - state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) - state_dicts.append(state_dict) - - if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]: - raise ValueError(f"{files[0]} is not a zero checkpoint") - zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] - world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] - - # For ZeRO-2 each param group can have different partition_count as data parallelism for expert - # parameters can be different from data parallelism for non-expert parameters. So we can just - # use the max of the partition_count to get the dp world_size. - - if type(world_size) is list: - world_size = max(world_size) - - if world_size != total_files: - raise ValueError( - f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " - "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." - ) - - # the groups are named differently in each stage - if zero_stage <= 2: - fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS - elif zero_stage == 3: - fp32_groups_key = FP32_FLAT_GROUPS - else: - raise ValueError(f"unknown zero stage {zero_stage}") - - fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] - return zero_stage, world_size, fp32_flat_groups - - -def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): - """ - Returns fp32 state_dict reconstructed from ds checkpoint - - Args: - - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) - - """ - print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") - - optim_files = get_optim_files(ds_checkpoint_dir) - zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) - print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") - - model_files = get_model_state_files(ds_checkpoint_dir) - - zero_model_states = parse_model_states(model_files) - print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') - - if zero_stage <= 2: - return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, - exclude_frozen_parameters) - elif zero_stage == 3: - return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, - exclude_frozen_parameters) - - -def _zero2_merge_frozen_params(state_dict, zero_model_states): - if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: - return - - frozen_param_shapes = zero_model_states[0].frozen_param_shapes - frozen_param_fragments = zero_model_states[0].frozen_param_fragments - - if debug: - num_elem = sum(s.numel() for s in frozen_param_shapes.values()) - print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') - - wanted_params = len(frozen_param_shapes) - wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) - avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) - print(f'Frozen params: Have {avail_numel} numels to process.') - print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') - - total_params = 0 - total_numel = 0 - for name, shape in frozen_param_shapes.items(): - total_params += 1 - unpartitioned_numel = shape.numel() - total_numel += unpartitioned_numel - - state_dict[name] = frozen_param_fragments[name] - - if debug: - print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") - - print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") - - -def _has_callable(obj, fn): - attr = getattr(obj, fn, None) - return callable(attr) - - -def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): - param_shapes = zero_model_states[0].param_shapes - - # Reconstruction protocol: - # - # XXX: document this - - if debug: - for i in range(world_size): - for j in range(len(fp32_flat_groups[0])): - print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") - - # XXX: memory usage doubles here (zero2) - num_param_groups = len(fp32_flat_groups[0]) - merged_single_partition_of_fp32_groups = [] - for i in range(num_param_groups): - merged_partitions = [sd[i] for sd in fp32_flat_groups] - full_single_fp32_vector = torch.cat(merged_partitions, 0) - merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) - avail_numel = sum( - [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) - - if debug: - wanted_params = sum([len(shapes) for shapes in param_shapes]) - wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) - # not asserting if there is a mismatch due to possible padding - print(f"Have {avail_numel} numels to process.") - print(f"Need {wanted_numel} numels in {wanted_params} params.") - - # params - # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support - # out-of-core computing solution - total_numel = 0 - total_params = 0 - for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): - offset = 0 - avail_numel = full_single_fp32_vector.numel() - for name, shape in shapes.items(): - - unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) - total_numel += unpartitioned_numel - total_params += 1 - - if debug: - print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") - state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) - offset += unpartitioned_numel - - # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and - # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex - # paddings performed in the code it's almost impossible to predict the exact numbers w/o the - # live optimizer object, so we are checking that the numbers are within the right range - align_to = 2 * world_size - - def zero2_align(x): - return align_to * math.ceil(x / align_to) - - if debug: - print(f"original offset={offset}, avail_numel={avail_numel}") - - offset = zero2_align(offset) - avail_numel = zero2_align(avail_numel) - - if debug: - print(f"aligned offset={offset}, avail_numel={avail_numel}") - - # Sanity check - if offset != avail_numel: - raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") - - print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") - - -def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, - exclude_frozen_parameters): - state_dict = OrderedDict() - - # buffers - buffers = zero_model_states[0].buffers - state_dict.update(buffers) - if debug: - print(f"added {len(buffers)} buffers") - - if not exclude_frozen_parameters: - _zero2_merge_frozen_params(state_dict, zero_model_states) - - _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) - - # recover shared parameters - for pair in zero_model_states[0].shared_params: - if pair[1] in state_dict: - state_dict[pair[0]] = state_dict[pair[1]] - - return state_dict - - -def zero3_partitioned_param_info(unpartitioned_numel, world_size): - remainder = unpartitioned_numel % world_size - padding_numel = (world_size - remainder) if remainder else 0 - partitioned_numel = math.ceil(unpartitioned_numel / world_size) - return partitioned_numel, padding_numel - - -def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): - if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: - return - - if debug: - for i in range(world_size): - num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) - print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') - - frozen_param_shapes = zero_model_states[0].frozen_param_shapes - wanted_params = len(frozen_param_shapes) - wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) - avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size - print(f'Frozen params: Have {avail_numel} numels to process.') - print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') - - total_params = 0 - total_numel = 0 - for name, shape in zero_model_states[0].frozen_param_shapes.items(): - total_params += 1 - unpartitioned_numel = shape.numel() - total_numel += unpartitioned_numel - - param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) - state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) - - partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) - - if debug: - print( - f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" - ) - - print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") - - -class GatheredTensor: - """ - A pseudo tensor that collects partitioned weights. - It is more memory efficient when there are multiple groups. - """ - - def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape): - self.flat_groups = flat_groups - self.flat_groups_offset = flat_groups_offset - self.offset = offset - self.partitioned_numel = partitioned_numel - self.shape = shape - self.dtype = self.flat_groups[0][0].dtype - - def contiguous(self): - """ - Merge partitioned weights from flat_groups into a single tensor. - """ - end_idx = self.offset + self.partitioned_numel - world_size = len(self.flat_groups) - pad_flat_param_chunks = [] - - for rank_i in range(world_size): - # for each rank, we need to collect weights from related group/groups - flat_groups_at_rank_i = self.flat_groups[rank_i] - start_group_id = None - end_group_id = None - for group_id in range(len(self.flat_groups_offset)): - if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]: - start_group_id = group_id - if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]: - end_group_id = group_id - break - # collect weights from related group/groups - for group_id in range(start_group_id, end_group_id + 1): - flat_tensor = flat_groups_at_rank_i[group_id] - start_offset = self.offset - self.flat_groups_offset[group_id] - end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id] - pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset]) - - # collect weights from all ranks - pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0) - param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous() - return param - - -def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): - param_shapes = zero_model_states[0].param_shapes - avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size - - # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each - # param, re-consolidating each param, while dealing with padding if any - - # merge list of dicts, preserving order - param_shapes = {k: v for d in param_shapes for k, v in d.items()} - - if debug: - for i in range(world_size): - print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") - - wanted_params = len(param_shapes) - wanted_numel = sum(shape.numel() for shape in param_shapes.values()) - # not asserting if there is a mismatch due to possible padding - avail_numel = fp32_flat_groups[0].numel() * world_size - print(f"Trainable params: Have {avail_numel} numels to process.") - print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") - - # params - # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support - # out-of-core computing solution - offset = 0 - total_numel = 0 - total_params = 0 - flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]])) - for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'): - unpartitioned_numel = shape.numel() - total_numel += unpartitioned_numel - total_params += 1 - partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) - - if debug: - print( - f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" - ) - - # memory efficient tensor - tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape) - state_dict[name] = tensor - offset += partitioned_numel - - offset *= world_size - - # Sanity check - if offset != avail_numel: - raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") - - print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") - - -def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, - exclude_frozen_parameters): - state_dict = OrderedDict() - - # buffers - buffers = zero_model_states[0].buffers - state_dict.update(buffers) - if debug: - print(f"added {len(buffers)} buffers") - - if not exclude_frozen_parameters: - _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) - - _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) - - # recover shared parameters - for pair in zero_model_states[0].shared_params: - if pair[1] in state_dict: - state_dict[pair[0]] = state_dict[pair[1]] - - return state_dict - - -def to_torch_tensor(state_dict, return_empty_tensor=False): - """ - Convert state_dict of GatheredTensor to torch tensor - """ - torch_state_dict = {} - converted_tensors = {} - for name, tensor in state_dict.items(): - tensor_id = id(tensor) - if tensor_id in converted_tensors: # shared tensors - shared_tensor = torch_state_dict[converted_tensors[tensor_id]] - torch_state_dict[name] = shared_tensor - else: - converted_tensors[tensor_id] = name - if return_empty_tensor: - torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype) - else: - torch_state_dict[name] = tensor.contiguous() - return torch_state_dict - - -def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, - tag=None, - exclude_frozen_parameters=False, - lazy_mode=False): - """ - Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with - ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example - via a model hub. - - Args: - - ``checkpoint_dir``: path to the desired checkpoint folder - - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` - - ``exclude_frozen_parameters``: exclude frozen parameters - - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient. - Convert the pesduo tensor to torch tensor by ``.contiguous()`` - - Returns: - - pytorch ``state_dict`` - - A typical usage might be :: - - from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint - # do the training and checkpoint saving - state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu - model = model.cpu() # move to cpu - model.load_state_dict(state_dict) - # submit to model hub or save the model to share with others - - In this example the ``model`` will no longer be usable in the deepspeed context of the same - application. i.e. you will need to re-initialize the deepspeed engine, since - ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. - - If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. - - Note: the above usage may not work if your application doesn't have sufficient free CPU memory. - You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with - the checkpoint. Or you can load state_dict in lazy mode :: - - from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint - state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu - for name, lazy_tensor in state_dict.item(): - tensor = lazy_tensor.contiguous() # to cpu - print(name, tensor) - # del tensor to release memory if it no longer in use - """ - if tag is None: - latest_path = os.path.join(checkpoint_dir, 'latest') - if os.path.isfile(latest_path): - with open(latest_path, 'r') as fd: - tag = fd.read().strip() - else: - raise ValueError(f"Unable to find 'latest' file at {latest_path}") - - ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) - - if not os.path.isdir(ds_checkpoint_dir): - raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") - - state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) - if lazy_mode: - return state_dict - else: - return to_torch_tensor(state_dict) - - -def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, - output_dir, - max_shard_size="5GB", - safe_serialization=False, - tag=None, - exclude_frozen_parameters=False): - """ - Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be - loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. - - Args: - - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) - - ``output_dir``: directory to the pytorch fp32 state_dict output files - - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB - - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). - - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` - - ``exclude_frozen_parameters``: exclude frozen parameters - """ - - # Dependency pre-check - if safe_serialization: - try: - from safetensors.torch import save_file - except ImportError: - print('If you want to use `safe_serialization`, please `pip install safetensors`') - raise - if max_shard_size is not None: - try: - from huggingface_hub import split_torch_state_dict_into_shards - except ImportError: - print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') - raise - - # Convert zero checkpoint to state_dict - state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, - tag, - exclude_frozen_parameters, - lazy_mode=True) - - # Shard the model if it is too big. - weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" - if max_shard_size is not None: - filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") - # an memory-efficient approach for sharding - empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True) - state_dict_split = split_torch_state_dict_into_shards(empty_state_dict, - filename_pattern=filename_pattern, - max_shard_size=max_shard_size) - else: - from collections import namedtuple - StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) - state_dict_split = StateDictSplit(is_sharded=False, - filename_to_tensors={weights_name: list(state_dict.keys())}) - - # Save the model by shard - os.makedirs(output_dir, exist_ok=True) - filename_to_tensors = state_dict_split.filename_to_tensors.items() - for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): - shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors} - shard_state_dict = to_torch_tensor(shard_state_dict) - output_path = os.path.join(output_dir, shard_file) - if safe_serialization: - save_file(shard_state_dict, output_path, metadata={"format": "pt"}) - else: - torch.save(shard_state_dict, output_path) - # release the memory of current shard - for tensor_name in list(shard_state_dict.keys()): - del state_dict[tensor_name] - del shard_state_dict[tensor_name] - del shard_state_dict - gc.collect() - - # Save index if sharded - if state_dict_split.is_sharded: - index = { - "metadata": state_dict_split.metadata, - "weight_map": state_dict_split.tensor_to_filename, - } - save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" - save_index_file = os.path.join(output_dir, save_index_file) - with open(save_index_file, "w", encoding="utf-8") as f: - content = json.dumps(index, indent=2, sort_keys=True) + "\n" - f.write(content) - - -def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): - """ - 1. Put the provided model to cpu - 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` - 3. Load it into the provided model - - Args: - - ``model``: the model object to update - - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) - - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` - - Returns: - - ``model`: modified model - - Make sure you have plenty of CPU memory available before you call this function. If you don't - have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it - conveniently placed for you in the checkpoint folder. - - A typical usage might be :: - - from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint - model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) - # submit to model hub or save the model to share with others - - Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context - of the same application. i.e. you will need to re-initialize the deepspeed engine, since - ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. - - """ - logger.info(f"Extracting fp32 weights") - state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) - - logger.info(f"Overwriting model with fp32 weights") - model = model.cpu() - model.load_state_dict(state_dict, strict=False) - - return model - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("checkpoint_dir", - type=str, - help="path to the desired checkpoint folder, e.g., path/checkpoint-12") - parser.add_argument("output_dir", - type=str, - help="directory to the pytorch fp32 state_dict output files" - "(e.g. path/checkpoint-12-output/)") - parser.add_argument( - "--max_shard_size", - type=str, - default="5GB", - help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" - "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" - "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" - "without CPU OOM issues.") - parser.add_argument( - "--safe_serialization", - default=False, - action='store_true', - help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") - parser.add_argument("-t", - "--tag", - type=str, - default=None, - help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") - parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") - parser.add_argument("-d", "--debug", action='store_true', help="enable debug") - args = parser.parse_args() - - debug = args.debug - - convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, - args.output_dir, - max_shard_size=args.max_shard_size, - safe_serialization=args.safe_serialization, - tag=args.tag, - exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed4-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/custom_checkpoint_0.pkl b/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed4-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/custom_checkpoint_0.pkl deleted file mode 100644 index 934d1829fe2c9fb79c035b298e9af9b7fd45867b..0000000000000000000000000000000000000000 --- a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed4-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/custom_checkpoint_0.pkl +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:4f2f99b7145922f6a1aa58ef156069bdb6ea34e0b3d24750bc0752f5b38c7fd1 -size 1525 diff --git a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed4-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/latest b/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed4-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/latest deleted file mode 100644 index 7b2c8602be034ae63f23b293f8d037fa7afa0c54..0000000000000000000000000000000000000000 --- a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed4-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/latest +++ /dev/null @@ -1 +0,0 @@ -pytorch_model \ No newline at end of file diff --git a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed4-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/pytorch_model/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed4-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/pytorch_model/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt deleted file mode 100644 index 324e49925e4410e851b8d972c1ecfefa15c068d9..0000000000000000000000000000000000000000 --- a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed4-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/pytorch_model/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:af0de2ead0c3e47d6a40efdb8d188089d682825bb4c65ba37a6b6a4b0abc3ded -size 27021673085 diff --git a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed4-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/pytorch_model/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed4-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/pytorch_model/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt deleted file mode 100644 index c814a32979e5f0084142e36898424eddf69cbf0f..0000000000000000000000000000000000000000 --- a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed4-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/pytorch_model/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:acf4693c6f19ff6102b4bdc313e8a1bdc0d179145857c0dbb082e92775b5c381 -size 27021651453 diff --git a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed4-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/pytorch_model/mp_rank_00_model_states.pt b/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed4-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/pytorch_model/mp_rank_00_model_states.pt deleted file mode 100644 index 73bbee3e0135857cf5a0649b81db80eab5b7ae6e..0000000000000000000000000000000000000000 --- a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed4-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/pytorch_model/mp_rank_00_model_states.pt +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:8f379910bf62c441a65a9b7e55d68640d2d3d9c0ec29759c01e52f896c76894b -size 9007481061 diff --git a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed4-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/random_states_0.pkl b/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed4-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/random_states_0.pkl deleted file mode 100644 index 643be72278b559711045cb44c18891b117b6cb94..0000000000000000000000000000000000000000 --- a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed4-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/random_states_0.pkl +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:d941a9657caba00045418ce4bd1c0a25f918c543af57e5825de2b1178cf047e4 -size 14953 diff --git a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed4-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/random_states_1.pkl b/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed4-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/random_states_1.pkl deleted file mode 100644 index 0e2d3e19d7258149472f70d502ea5b8ae5ed3990..0000000000000000000000000000000000000000 --- a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed4-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/random_states_1.pkl +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:e381c3041ee50113fb4c9114764f363c4699b73e6177124121e43238fec57bb7 -size 15017 diff --git a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed4-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/zero_to_fp32.py b/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed4-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/zero_to_fp32.py deleted file mode 100644 index 0e759146cadd92ddfefab3680146c2bd6a2b5c04..0000000000000000000000000000000000000000 --- a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenoft-h8/fixed4-prefix63-restart-20260925/training/bootstrap/step_05000/trainer_state/zero_to_fp32.py +++ /dev/null @@ -1,760 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) Microsoft Corporation. -# SPDX-License-Identifier: Apache-2.0 - -# DeepSpeed Team - -# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets -# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in -# the future. Once extracted, the weights don't require DeepSpeed and can be used in any -# application. -# -# example: -# python zero_to_fp32.py . output_dir/ -# or -# python zero_to_fp32.py . output_dir/ --safe_serialization - -import argparse -import torch -import glob -import math -import os -import re -import gc -import json -import numpy as np -from tqdm import tqdm -from collections import OrderedDict -from dataclasses import dataclass - -# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with -# DeepSpeed data structures it has to be available in the current python environment. -from deepspeed.utils import logger -from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, - FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, - FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) - - -@dataclass -class zero_model_state: - buffers: dict() - param_shapes: dict() - shared_params: list - ds_version: int - frozen_param_shapes: dict() - frozen_param_fragments: dict() - - -debug = 0 - -# load to cpu -device = torch.device('cpu') - - -def atoi(text): - return int(text) if text.isdigit() else text - - -def natural_keys(text): - ''' - alist.sort(key=natural_keys) sorts in human order - http://nedbatchelder.com/blog/200712/human_sorting.html - (See Toothy's implementation in the comments) - ''' - return [atoi(c) for c in re.split(r'(\d+)', text)] - - -def get_model_state_file(checkpoint_dir, zero_stage): - if not os.path.isdir(checkpoint_dir): - raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") - - # there should be only one file - if zero_stage <= 2: - file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") - elif zero_stage == 3: - file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") - - if not os.path.exists(file): - raise FileNotFoundError(f"can't find model states file at '{file}'") - - return file - - -def get_checkpoint_files(checkpoint_dir, glob_pattern): - # XXX: need to test that this simple glob rule works for multi-node setup too - ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) - - if len(ckpt_files) == 0: - raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") - - return ckpt_files - - -def get_optim_files(checkpoint_dir): - return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") - - -def get_model_state_files(checkpoint_dir): - return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") - - -def parse_model_states(files): - zero_model_states = [] - for file in files: - state_dict = torch.load(file, map_location=device, weights_only=False) - - if BUFFER_NAMES not in state_dict: - raise ValueError(f"{file} is not a model state checkpoint") - buffer_names = state_dict[BUFFER_NAMES] - if debug: - print("Found buffers:", buffer_names) - - # recover just the buffers while restoring them to fp32 if they were saved in fp16 - buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} - param_shapes = state_dict[PARAM_SHAPES] - - # collect parameters that are included in param_shapes - param_names = [] - for s in param_shapes: - for name in s.keys(): - param_names.append(name) - - # update with frozen parameters - frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) - if frozen_param_shapes is not None: - if debug: - print(f"Found frozen_param_shapes: {frozen_param_shapes}") - param_names += list(frozen_param_shapes.keys()) - - # handle shared params - shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] - - ds_version = state_dict.get(DS_VERSION, None) - - frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) - - z_model_state = zero_model_state(buffers=buffers, - param_shapes=param_shapes, - shared_params=shared_params, - ds_version=ds_version, - frozen_param_shapes=frozen_param_shapes, - frozen_param_fragments=frozen_param_fragments) - zero_model_states.append(z_model_state) - - return zero_model_states - - -def parse_optim_states(files, ds_checkpoint_dir): - total_files = len(files) - state_dicts = [] - for f in tqdm(files, desc='Loading checkpoint shards'): - state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False) - # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights - # and also handle the case where it was already removed by another helper script - state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) - state_dicts.append(state_dict) - - if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]: - raise ValueError(f"{files[0]} is not a zero checkpoint") - zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] - world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] - - # For ZeRO-2 each param group can have different partition_count as data parallelism for expert - # parameters can be different from data parallelism for non-expert parameters. So we can just - # use the max of the partition_count to get the dp world_size. - - if type(world_size) is list: - world_size = max(world_size) - - if world_size != total_files: - raise ValueError( - f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " - "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." - ) - - # the groups are named differently in each stage - if zero_stage <= 2: - fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS - elif zero_stage == 3: - fp32_groups_key = FP32_FLAT_GROUPS - else: - raise ValueError(f"unknown zero stage {zero_stage}") - - fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] - return zero_stage, world_size, fp32_flat_groups - - -def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): - """ - Returns fp32 state_dict reconstructed from ds checkpoint - - Args: - - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) - - """ - print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") - - optim_files = get_optim_files(ds_checkpoint_dir) - zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) - print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") - - model_files = get_model_state_files(ds_checkpoint_dir) - - zero_model_states = parse_model_states(model_files) - print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') - - if zero_stage <= 2: - return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, - exclude_frozen_parameters) - elif zero_stage == 3: - return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, - exclude_frozen_parameters) - - -def _zero2_merge_frozen_params(state_dict, zero_model_states): - if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: - return - - frozen_param_shapes = zero_model_states[0].frozen_param_shapes - frozen_param_fragments = zero_model_states[0].frozen_param_fragments - - if debug: - num_elem = sum(s.numel() for s in frozen_param_shapes.values()) - print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') - - wanted_params = len(frozen_param_shapes) - wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) - avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) - print(f'Frozen params: Have {avail_numel} numels to process.') - print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') - - total_params = 0 - total_numel = 0 - for name, shape in frozen_param_shapes.items(): - total_params += 1 - unpartitioned_numel = shape.numel() - total_numel += unpartitioned_numel - - state_dict[name] = frozen_param_fragments[name] - - if debug: - print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") - - print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") - - -def _has_callable(obj, fn): - attr = getattr(obj, fn, None) - return callable(attr) - - -def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): - param_shapes = zero_model_states[0].param_shapes - - # Reconstruction protocol: - # - # XXX: document this - - if debug: - for i in range(world_size): - for j in range(len(fp32_flat_groups[0])): - print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") - - # XXX: memory usage doubles here (zero2) - num_param_groups = len(fp32_flat_groups[0]) - merged_single_partition_of_fp32_groups = [] - for i in range(num_param_groups): - merged_partitions = [sd[i] for sd in fp32_flat_groups] - full_single_fp32_vector = torch.cat(merged_partitions, 0) - merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) - avail_numel = sum( - [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) - - if debug: - wanted_params = sum([len(shapes) for shapes in param_shapes]) - wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) - # not asserting if there is a mismatch due to possible padding - print(f"Have {avail_numel} numels to process.") - print(f"Need {wanted_numel} numels in {wanted_params} params.") - - # params - # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support - # out-of-core computing solution - total_numel = 0 - total_params = 0 - for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): - offset = 0 - avail_numel = full_single_fp32_vector.numel() - for name, shape in shapes.items(): - - unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) - total_numel += unpartitioned_numel - total_params += 1 - - if debug: - print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") - state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) - offset += unpartitioned_numel - - # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and - # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex - # paddings performed in the code it's almost impossible to predict the exact numbers w/o the - # live optimizer object, so we are checking that the numbers are within the right range - align_to = 2 * world_size - - def zero2_align(x): - return align_to * math.ceil(x / align_to) - - if debug: - print(f"original offset={offset}, avail_numel={avail_numel}") - - offset = zero2_align(offset) - avail_numel = zero2_align(avail_numel) - - if debug: - print(f"aligned offset={offset}, avail_numel={avail_numel}") - - # Sanity check - if offset != avail_numel: - raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") - - print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") - - -def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, - exclude_frozen_parameters): - state_dict = OrderedDict() - - # buffers - buffers = zero_model_states[0].buffers - state_dict.update(buffers) - if debug: - print(f"added {len(buffers)} buffers") - - if not exclude_frozen_parameters: - _zero2_merge_frozen_params(state_dict, zero_model_states) - - _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) - - # recover shared parameters - for pair in zero_model_states[0].shared_params: - if pair[1] in state_dict: - state_dict[pair[0]] = state_dict[pair[1]] - - return state_dict - - -def zero3_partitioned_param_info(unpartitioned_numel, world_size): - remainder = unpartitioned_numel % world_size - padding_numel = (world_size - remainder) if remainder else 0 - partitioned_numel = math.ceil(unpartitioned_numel / world_size) - return partitioned_numel, padding_numel - - -def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): - if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: - return - - if debug: - for i in range(world_size): - num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) - print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') - - frozen_param_shapes = zero_model_states[0].frozen_param_shapes - wanted_params = len(frozen_param_shapes) - wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) - avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size - print(f'Frozen params: Have {avail_numel} numels to process.') - print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') - - total_params = 0 - total_numel = 0 - for name, shape in zero_model_states[0].frozen_param_shapes.items(): - total_params += 1 - unpartitioned_numel = shape.numel() - total_numel += unpartitioned_numel - - param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) - state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) - - partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) - - if debug: - print( - f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" - ) - - print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") - - -class GatheredTensor: - """ - A pseudo tensor that collects partitioned weights. - It is more memory efficient when there are multiple groups. - """ - - def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape): - self.flat_groups = flat_groups - self.flat_groups_offset = flat_groups_offset - self.offset = offset - self.partitioned_numel = partitioned_numel - self.shape = shape - self.dtype = self.flat_groups[0][0].dtype - - def contiguous(self): - """ - Merge partitioned weights from flat_groups into a single tensor. - """ - end_idx = self.offset + self.partitioned_numel - world_size = len(self.flat_groups) - pad_flat_param_chunks = [] - - for rank_i in range(world_size): - # for each rank, we need to collect weights from related group/groups - flat_groups_at_rank_i = self.flat_groups[rank_i] - start_group_id = None - end_group_id = None - for group_id in range(len(self.flat_groups_offset)): - if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]: - start_group_id = group_id - if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]: - end_group_id = group_id - break - # collect weights from related group/groups - for group_id in range(start_group_id, end_group_id + 1): - flat_tensor = flat_groups_at_rank_i[group_id] - start_offset = self.offset - self.flat_groups_offset[group_id] - end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id] - pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset]) - - # collect weights from all ranks - pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0) - param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous() - return param - - -def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): - param_shapes = zero_model_states[0].param_shapes - avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size - - # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each - # param, re-consolidating each param, while dealing with padding if any - - # merge list of dicts, preserving order - param_shapes = {k: v for d in param_shapes for k, v in d.items()} - - if debug: - for i in range(world_size): - print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") - - wanted_params = len(param_shapes) - wanted_numel = sum(shape.numel() for shape in param_shapes.values()) - # not asserting if there is a mismatch due to possible padding - avail_numel = fp32_flat_groups[0].numel() * world_size - print(f"Trainable params: Have {avail_numel} numels to process.") - print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") - - # params - # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support - # out-of-core computing solution - offset = 0 - total_numel = 0 - total_params = 0 - flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]])) - for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'): - unpartitioned_numel = shape.numel() - total_numel += unpartitioned_numel - total_params += 1 - partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) - - if debug: - print( - f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" - ) - - # memory efficient tensor - tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape) - state_dict[name] = tensor - offset += partitioned_numel - - offset *= world_size - - # Sanity check - if offset != avail_numel: - raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") - - print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") - - -def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, - exclude_frozen_parameters): - state_dict = OrderedDict() - - # buffers - buffers = zero_model_states[0].buffers - state_dict.update(buffers) - if debug: - print(f"added {len(buffers)} buffers") - - if not exclude_frozen_parameters: - _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) - - _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) - - # recover shared parameters - for pair in zero_model_states[0].shared_params: - if pair[1] in state_dict: - state_dict[pair[0]] = state_dict[pair[1]] - - return state_dict - - -def to_torch_tensor(state_dict, return_empty_tensor=False): - """ - Convert state_dict of GatheredTensor to torch tensor - """ - torch_state_dict = {} - converted_tensors = {} - for name, tensor in state_dict.items(): - tensor_id = id(tensor) - if tensor_id in converted_tensors: # shared tensors - shared_tensor = torch_state_dict[converted_tensors[tensor_id]] - torch_state_dict[name] = shared_tensor - else: - converted_tensors[tensor_id] = name - if return_empty_tensor: - torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype) - else: - torch_state_dict[name] = tensor.contiguous() - return torch_state_dict - - -def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, - tag=None, - exclude_frozen_parameters=False, - lazy_mode=False): - """ - Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with - ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example - via a model hub. - - Args: - - ``checkpoint_dir``: path to the desired checkpoint folder - - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` - - ``exclude_frozen_parameters``: exclude frozen parameters - - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient. - Convert the pesduo tensor to torch tensor by ``.contiguous()`` - - Returns: - - pytorch ``state_dict`` - - A typical usage might be :: - - from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint - # do the training and checkpoint saving - state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu - model = model.cpu() # move to cpu - model.load_state_dict(state_dict) - # submit to model hub or save the model to share with others - - In this example the ``model`` will no longer be usable in the deepspeed context of the same - application. i.e. you will need to re-initialize the deepspeed engine, since - ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. - - If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. - - Note: the above usage may not work if your application doesn't have sufficient free CPU memory. - You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with - the checkpoint. Or you can load state_dict in lazy mode :: - - from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint - state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu - for name, lazy_tensor in state_dict.item(): - tensor = lazy_tensor.contiguous() # to cpu - print(name, tensor) - # del tensor to release memory if it no longer in use - """ - if tag is None: - latest_path = os.path.join(checkpoint_dir, 'latest') - if os.path.isfile(latest_path): - with open(latest_path, 'r') as fd: - tag = fd.read().strip() - else: - raise ValueError(f"Unable to find 'latest' file at {latest_path}") - - ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) - - if not os.path.isdir(ds_checkpoint_dir): - raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") - - state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) - if lazy_mode: - return state_dict - else: - return to_torch_tensor(state_dict) - - -def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, - output_dir, - max_shard_size="5GB", - safe_serialization=False, - tag=None, - exclude_frozen_parameters=False): - """ - Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be - loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. - - Args: - - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) - - ``output_dir``: directory to the pytorch fp32 state_dict output files - - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB - - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). - - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` - - ``exclude_frozen_parameters``: exclude frozen parameters - """ - - # Dependency pre-check - if safe_serialization: - try: - from safetensors.torch import save_file - except ImportError: - print('If you want to use `safe_serialization`, please `pip install safetensors`') - raise - if max_shard_size is not None: - try: - from huggingface_hub import split_torch_state_dict_into_shards - except ImportError: - print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') - raise - - # Convert zero checkpoint to state_dict - state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, - tag, - exclude_frozen_parameters, - lazy_mode=True) - - # Shard the model if it is too big. - weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" - if max_shard_size is not None: - filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") - # an memory-efficient approach for sharding - empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True) - state_dict_split = split_torch_state_dict_into_shards(empty_state_dict, - filename_pattern=filename_pattern, - max_shard_size=max_shard_size) - else: - from collections import namedtuple - StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) - state_dict_split = StateDictSplit(is_sharded=False, - filename_to_tensors={weights_name: list(state_dict.keys())}) - - # Save the model by shard - os.makedirs(output_dir, exist_ok=True) - filename_to_tensors = state_dict_split.filename_to_tensors.items() - for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): - shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors} - shard_state_dict = to_torch_tensor(shard_state_dict) - output_path = os.path.join(output_dir, shard_file) - if safe_serialization: - save_file(shard_state_dict, output_path, metadata={"format": "pt"}) - else: - torch.save(shard_state_dict, output_path) - # release the memory of current shard - for tensor_name in list(shard_state_dict.keys()): - del state_dict[tensor_name] - del shard_state_dict[tensor_name] - del shard_state_dict - gc.collect() - - # Save index if sharded - if state_dict_split.is_sharded: - index = { - "metadata": state_dict_split.metadata, - "weight_map": state_dict_split.tensor_to_filename, - } - save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" - save_index_file = os.path.join(output_dir, save_index_file) - with open(save_index_file, "w", encoding="utf-8") as f: - content = json.dumps(index, indent=2, sort_keys=True) + "\n" - f.write(content) - - -def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): - """ - 1. Put the provided model to cpu - 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` - 3. Load it into the provided model - - Args: - - ``model``: the model object to update - - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) - - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` - - Returns: - - ``model`: modified model - - Make sure you have plenty of CPU memory available before you call this function. If you don't - have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it - conveniently placed for you in the checkpoint folder. - - A typical usage might be :: - - from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint - model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) - # submit to model hub or save the model to share with others - - Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context - of the same application. i.e. you will need to re-initialize the deepspeed engine, since - ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. - - """ - logger.info(f"Extracting fp32 weights") - state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) - - logger.info(f"Overwriting model with fp32 weights") - model = model.cpu() - model.load_state_dict(state_dict, strict=False) - - return model - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("checkpoint_dir", - type=str, - help="path to the desired checkpoint folder, e.g., path/checkpoint-12") - parser.add_argument("output_dir", - type=str, - help="directory to the pytorch fp32 state_dict output files" - "(e.g. path/checkpoint-12-output/)") - parser.add_argument( - "--max_shard_size", - type=str, - default="5GB", - help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" - "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" - "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" - "without CPU OOM issues.") - parser.add_argument( - "--safe_serialization", - default=False, - action='store_true', - help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") - parser.add_argument("-t", - "--tag", - type=str, - default=None, - help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") - parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") - parser.add_argument("-d", "--debug", action='store_true', help="enable debug") - args = parser.parse_args() - - debug = args.debug - - convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, - args.output_dir, - max_shard_size=args.max_shard_size, - safe_serialization=args.safe_serialization, - tag=args.tag, - exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenpi_v3-h1/mikasa_qwenpi_v3_latency_3090_2xh200_20260922/README.md b/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenpi_v3-h1/mikasa_qwenpi_v3_latency_3090_2xh200_20260922/README.md index f19b3bdd6f0f2348a5ed78485ca2cbda97eb68f8..fe74ead8ebd039e4f98dd2c1c60cce3ba13b08ae 100644 --- a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenpi_v3-h1/mikasa_qwenpi_v3_latency_3090_2xh200_20260922/README.md +++ b/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenpi_v3-h1/mikasa_qwenpi_v3_latency_3090_2xh200_20260922/README.md @@ -1,3 +1,5 @@ # qwenpi_v3 — MIKASA InterceptGrabFast H1 Latency-aware SFT with the matched RTX 3090 temporal profile. See `eval_profile.json` and `provenance.json` for the 200-episode protocol and sources. + +The byte-identical weight is stored once in the corrected release: [checkpoint](https://huggingface.co/latency-sensitive-bench/benchmark-models/blob/6b07ea8e3f768a920e7d97ffbab4a5bedfa712ed/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenpi_v3-h1/mikasa_qwenpi_v3_latency_3090_2xh200_20260922_evalfix_20260923/checkpoints/model.pt). This directory retains the original run metadata. diff --git a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenpi_v3-h1/mikasa_qwenpi_v3_latency_3090_2xh200_20260922/checkpoints/model.pt b/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenpi_v3-h1/mikasa_qwenpi_v3_latency_3090_2xh200_20260922/checkpoints/model.pt deleted file mode 100644 index 419e5f36f0ec8b9321c7e8e2823504ea9c60246d..0000000000000000000000000000000000000000 --- a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenpi_v3-h1/mikasa_qwenpi_v3_latency_3090_2xh200_20260922/checkpoints/model.pt +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:6938375168dd24e186bbac9fa54268210ba33081116b9c8493a57cf49eecd93a -size 10922637469 diff --git a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenpi_v3-h1/mikasa_qwenpi_v3_latency_3090_2xh200_20260922/provenance.json b/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenpi_v3-h1/mikasa_qwenpi_v3_latency_3090_2xh200_20260922/provenance.json index 738fa4e5f6278edd5d3f32c92c64f28556c1239b..21c20c261b0b0bf3ecfe53ecf1dbbe1f11b53d87 100644 --- a/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenpi_v3-h1/mikasa_qwenpi_v3_latency_3090_2xh200_20260922/provenance.json +++ b/latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenpi_v3-h1/mikasa_qwenpi_v3_latency_3090_2xh200_20260922/provenance.json @@ -1126,5 +1126,11 @@ "worker_capacity": 1, "parallel_envs": 10, "max_transitions": 60 + }, + "checkpoint_reference": { + "repo_id": "latency-sensitive-bench/benchmark-models", + "repo_type": "model", + "revision": "6b07ea8e3f768a920e7d97ffbab4a5bedfa712ed", + "path": "latency-aware/mikasa-intercept-grab-fast/vla/starvla-qwenpi_v3-h1/mikasa_qwenpi_v3_latency_3090_2xh200_20260922_evalfix_20260923/checkpoints/model.pt" } } diff --git a/zero-latency/deadly-corridor/vla/starvla-qwenoft-h1/openvla_deadly_corridor_fix_latency_0/artifact.json b/zero-latency/deadly-corridor/vla/starvla-qwenoft-h1/openvla_deadly_corridor_fix_latency_0/artifact.json index 8b9a1055e517cb62ef3cbea1d3a83e5af4951e81..92169b39a6888b302a25b82f257b7f7c95f8a37c 100644 --- a/zero-latency/deadly-corridor/vla/starvla-qwenoft-h1/openvla_deadly_corridor_fix_latency_0/artifact.json +++ b/zero-latency/deadly-corridor/vla/starvla-qwenoft-h1/openvla_deadly_corridor_fix_latency_0/artifact.json @@ -5,11 +5,6 @@ "path": "historical/openvla_deadly_corridor_fix_latency_0" }, "checkpoint": [ - { - "repo": "latency-sensitive-bench/benchmark-models", - "revision": "46266ba5a184ac622f7fa3386ac689d8d755d64c", - "path": "zero-latency/deadly-corridor/vla/starvla-qwenoft-h1/openvla_deadly_corridor_fix_latency_0/checkpoints/best_model.safetensors" - }, { "repo": "latency-sensitive-bench/benchmark-models", "revision": "46266ba5a184ac622f7fa3386ac689d8d755d64c", diff --git a/zero-latency/deadly-corridor/vla/starvla-qwenoft-h1/openvla_deadly_corridor_fix_latency_0/checkpoints/best_model.safetensors b/zero-latency/deadly-corridor/vla/starvla-qwenoft-h1/openvla_deadly_corridor_fix_latency_0/checkpoints/best_model.safetensors deleted file mode 100644 index b52046573c0b8a22eac43023f7efb99f610f0d27..0000000000000000000000000000000000000000 --- a/zero-latency/deadly-corridor/vla/starvla-qwenoft-h1/openvla_deadly_corridor_fix_latency_0/checkpoints/best_model.safetensors +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:1f2b7941653b956dbb5c368b9da60e5538e04216c34371254ef6cfa5c34bce74 -size 9138230516 diff --git a/zero-latency/deadly-corridor/vla/starvla-qwenoft-h1/openvla_deadly_corridor_fix_latency_0_full/artifact.json b/zero-latency/deadly-corridor/vla/starvla-qwenoft-h1/openvla_deadly_corridor_fix_latency_0_full/artifact.json index abda22221ce22c1f0c26a0767fec72310fadbbdb..654c22b7e20acc36ac407d0c6484caaea0a1d7c8 100644 --- a/zero-latency/deadly-corridor/vla/starvla-qwenoft-h1/openvla_deadly_corridor_fix_latency_0_full/artifact.json +++ b/zero-latency/deadly-corridor/vla/starvla-qwenoft-h1/openvla_deadly_corridor_fix_latency_0_full/artifact.json @@ -5,11 +5,6 @@ "path": "historical/openvla_deadly_corridor_fix_latency_0_full" }, "checkpoint": [ - { - "repo": "latency-sensitive-bench/benchmark-models", - "revision": "46266ba5a184ac622f7fa3386ac689d8d755d64c", - "path": "zero-latency/deadly-corridor/vla/starvla-qwenoft-h1/openvla_deadly_corridor_fix_latency_0_full/checkpoints/best_model.safetensors" - }, { "repo": "latency-sensitive-bench/benchmark-models", "revision": "46266ba5a184ac622f7fa3386ac689d8d755d64c", diff --git a/zero-latency/deadly-corridor/vla/starvla-qwenoft-h1/openvla_deadly_corridor_fix_latency_0_full/checkpoints/best_model.safetensors b/zero-latency/deadly-corridor/vla/starvla-qwenoft-h1/openvla_deadly_corridor_fix_latency_0_full/checkpoints/best_model.safetensors deleted file mode 100644 index 4c3a81cd089150f7999853974ceeac0a237bede5..0000000000000000000000000000000000000000 --- a/zero-latency/deadly-corridor/vla/starvla-qwenoft-h1/openvla_deadly_corridor_fix_latency_0_full/checkpoints/best_model.safetensors +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:5de650dbc0c50201489cb391a18f95b76bc1b9fb5a3df03ee965050b90284e84 -size 9138230516 diff --git a/zero-latency/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_0/artifact.json b/zero-latency/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_0/artifact.json index c6bfbff3bb008f4908bb745400b04c8a8df2b601..03254b76430f6a67f5e7cde34fd9881e40995372 100644 --- a/zero-latency/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_0/artifact.json +++ b/zero-latency/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_0/artifact.json @@ -5,11 +5,6 @@ "path": "historical/openvla_flappy_fix_latency_0" }, "checkpoint": [ - { - "repo": "latency-sensitive-bench/benchmark-models", - "revision": "46266ba5a184ac622f7fa3386ac689d8d755d64c", - "path": "zero-latency/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_0/checkpoints/steps_5000_model.safetensors" - }, { "repo": "latency-sensitive-bench/benchmark-models", "revision": "46266ba5a184ac622f7fa3386ac689d8d755d64c", diff --git a/zero-latency/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_0/checkpoints/steps_5000_model.safetensors b/zero-latency/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_0/checkpoints/steps_5000_model.safetensors deleted file mode 100644 index 8e15010370cbbd700b69541c948bec52a176c56c..0000000000000000000000000000000000000000 --- a/zero-latency/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_0/checkpoints/steps_5000_model.safetensors +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:57b21a1d51a95495f08c60166b561aed0183fd4b680b665a440b9b0694aff4c8 -size 9138230516 diff --git a/zero-latency/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_0_small/artifact.json b/zero-latency/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_0_small/artifact.json index d1249be3af6915743bfffa51d50a17a5b410e02a..e65debb37dcbeb39c2964ebfab95ded95d298e18 100644 --- a/zero-latency/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_0_small/artifact.json +++ b/zero-latency/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_0_small/artifact.json @@ -5,11 +5,6 @@ "path": "historical/openvla_flappy_fix_latency_0_small" }, "checkpoint": [ - { - "repo": "latency-sensitive-bench/benchmark-models", - "revision": "46266ba5a184ac622f7fa3386ac689d8d755d64c", - "path": "zero-latency/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_0_small/checkpoints/best_model.safetensors" - }, { "repo": "latency-sensitive-bench/benchmark-models", "revision": "46266ba5a184ac622f7fa3386ac689d8d755d64c", diff --git a/zero-latency/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_0_small/checkpoints/best_model.safetensors b/zero-latency/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_0_small/checkpoints/best_model.safetensors deleted file mode 100644 index 61eaddbdc34aca54760c4037c7e4ef64ed5bcdc7..0000000000000000000000000000000000000000 --- a/zero-latency/flappy/vla/starvla-qwenoft-h1/openvla_flappy_fix_latency_0_small/checkpoints/best_model.safetensors +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:9117254092707a50375a50e48626afdfbc3e7ac1ea51320a09ec901cd53f0e37 -size 9138230516 diff --git a/zero-latency/mikasa-intercept-grab-fast/vla/starvla-qwengr00t-h8/h8-naive-dagger-20260923/checkpoints/latest_model.pt b/zero-latency/mikasa-intercept-grab-fast/vla/starvla-qwengr00t-h8/h8-naive-dagger-20260923/checkpoints/latest_model.pt deleted file mode 100644 index 1187d01e1f11a2de7890c3325972b505e5e80cf6..0000000000000000000000000000000000000000 --- a/zero-latency/mikasa-intercept-grab-fast/vla/starvla-qwengr00t-h8/h8-naive-dagger-20260923/checkpoints/latest_model.pt +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:bffc2153934ec34e414cc5b59c6243c6c8f792d087fa495e3e1b31e950a82beb -size 9976844067 diff --git a/zero-latency/mikasa-intercept-grab-fast/vla/starvla-qwengr00t-h8/h8-naive-dagger-20260923/provenance.json b/zero-latency/mikasa-intercept-grab-fast/vla/starvla-qwengr00t-h8/h8-naive-dagger-20260923/provenance.json index 6d11cde6a5f7f65511fcd553150b170c6b9d65b5..3c53bba64748549194ea2a4f55bf725df0980099 100644 --- a/zero-latency/mikasa-intercept-grab-fast/vla/starvla-qwengr00t-h8/h8-naive-dagger-20260923/provenance.json +++ b/zero-latency/mikasa-intercept-grab-fast/vla/starvla-qwengr00t-h8/h8-naive-dagger-20260923/provenance.json @@ -15,6 +15,5 @@ 4242424542, 4242424741 ], - "checkpoint": "checkpoints/model.pt", - "latest_checkpoint": "checkpoints/latest_model.pt" + "checkpoint": "checkpoints/model.pt" } diff --git a/zero-latency/mikasa-intercept-grab-fast/vla/starvla-qwenpi_v3-h8/h8-naive-dagger-20260923/checkpoints/latest_model.pt b/zero-latency/mikasa-intercept-grab-fast/vla/starvla-qwenpi_v3-h8/h8-naive-dagger-20260923/checkpoints/latest_model.pt deleted file mode 100644 index c0a3958da1e7393d3ebc29d0fe797cd2e38eb15a..0000000000000000000000000000000000000000 --- a/zero-latency/mikasa-intercept-grab-fast/vla/starvla-qwenpi_v3-h8/h8-naive-dagger-20260923/checkpoints/latest_model.pt +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:2eaea5f33ddae0dc517c66c177a65844093a436d5c4d766c6e8d117f13ad398d -size 10922637469 diff --git a/zero-latency/mikasa-intercept-grab-fast/vla/starvla-qwenpi_v3-h8/h8-naive-dagger-20260923/provenance.json b/zero-latency/mikasa-intercept-grab-fast/vla/starvla-qwenpi_v3-h8/h8-naive-dagger-20260923/provenance.json index a8045418da6b969b48e00eef347c0b4e9cd0c73e..a650bbf8fcfd677ea21e9fbece474b3201b61fe2 100644 --- a/zero-latency/mikasa-intercept-grab-fast/vla/starvla-qwenpi_v3-h8/h8-naive-dagger-20260923/provenance.json +++ b/zero-latency/mikasa-intercept-grab-fast/vla/starvla-qwenpi_v3-h8/h8-naive-dagger-20260923/provenance.json @@ -15,6 +15,5 @@ 4242424542, 4242424741 ], - "checkpoint": "checkpoints/model.pt", - "latest_checkpoint": "checkpoints/latest_model.pt" + "checkpoint": "checkpoints/model.pt" }