diff --git a/.gitattributes b/.gitattributes index d4d15b42d8bb4e6636ce275f150cfbdf7002e915..50c591095abe50c5ef6943d50b792bc286f4037e 100644 --- a/.gitattributes +++ b/.gitattributes @@ -45,3 +45,13 @@ settings/qwen_bridge/common_start/valid_anchor_manifest.json filter=lfs diff=lfs settings/qwen_bridge/frameskip/valid_anchor_manifest.json filter=lfs diff=lfs merge=lfs -text settings/qwen_bridge/keyframe/valid_anchor_manifest.json filter=lfs diff=lfs merge=lfs -text settings/qwen_bridge/loss_is/valid_anchor_manifest.json filter=lfs diff=lfs merge=lfs -text +settings/florence_robotwin/dependencies/content_fingerprints.json filter=lfs diff=lfs merge=lfs -text +settings/florence_robotwin/frameskip/cache/frameskip_full.json filter=lfs diff=lfs merge=lfs -text +settings/florence_robotwin/frameskip/valid_anchor_manifest.json filter=lfs diff=lfs merge=lfs -text +settings/florence_robotwin/keyframe/valid_anchor_manifest.json filter=lfs diff=lfs merge=lfs -text +settings/florence_robotwin/loss_is/valid_anchor_manifest.json filter=lfs diff=lfs merge=lfs -text +settings/qwen_robotwin/dependencies/content_fingerprints.json filter=lfs diff=lfs merge=lfs -text +settings/qwen_robotwin/frameskip/cache/frameskip_full.json filter=lfs diff=lfs merge=lfs -text +settings/qwen_robotwin/frameskip/valid_anchor_manifest.json filter=lfs diff=lfs merge=lfs -text +settings/qwen_robotwin/keyframe/valid_anchor_manifest.json filter=lfs diff=lfs merge=lfs -text +settings/qwen_robotwin/loss_is/valid_anchor_manifest.json filter=lfs diff=lfs merge=lfs -text diff --git a/settings/ASSET_FILES_sai.json b/settings/ASSET_FILES_sai.json new file mode 100644 index 0000000000000000000000000000000000000000..3be086785f00ec9dd14b75d14be473b05345714c --- /dev/null +++ b/settings/ASSET_FILES_sai.json @@ -0,0 +1,354 @@ +{ + "files": [ + { + "source": "/fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs/florence-robotwin-frameskip-10000-to-10500-seed484370/config.yaml", + "path_in_repo": "settings/florence_robotwin/frameskip/config.yaml", + "bytes": 2168, + "role": "run_metadata" + }, + { + "source": "/fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs/florence-robotwin-frameskip-10000-to-10500-seed484370/config.full.yaml", + "path_in_repo": "settings/florence_robotwin/frameskip/config.full.yaml", + "bytes": 3471, + "role": "run_metadata" + }, + { + "source": "/fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs/florence-robotwin-frameskip-10000-to-10500-seed484370/dataset_statistics.json", + "path_in_repo": "settings/florence_robotwin/frameskip/dataset_statistics.json", + "bytes": 5141, + "role": "run_metadata" + }, + { + "source": "/fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs/florence-robotwin-frameskip-10000-to-10500-seed484370/initialization_manifest.json", + "path_in_repo": "settings/florence_robotwin/frameskip/initialization_manifest.json", + "bytes": 200, + "role": "run_metadata" + }, + { + "source": "/fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs/florence-robotwin-frameskip-10000-to-10500-seed484370/external_resume_certificate.json", + "path_in_repo": "settings/florence_robotwin/frameskip/external_resume_certificate.json", + "bytes": 4675, + "role": "run_metadata" + }, + { + "source": "/fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs/florence-robotwin-frameskip-10000-to-10500-seed484370/valid_anchor_manifest.json", + "path_in_repo": "settings/florence_robotwin/frameskip/valid_anchor_manifest.json", + "bytes": 107646168, + "role": "run_metadata" + }, + { + "source": "/fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs/florence-robotwin-frameskip-10000-to-10500-seed484370/checkpoints/steps_10500_accelerate/checkpoint.json", + "path_in_repo": "settings/florence_robotwin/frameskip/checkpoint.json", + "bytes": 2090, + "role": "checkpoint_metadata" + }, + { + "source": "/fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs/florence-robotwin-keyframe-10000-to-10500-seed484370/config.yaml", + "path_in_repo": "settings/florence_robotwin/keyframe/config.yaml", + "bytes": 2166, + "role": "run_metadata" + }, + { + "source": "/fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs/florence-robotwin-keyframe-10000-to-10500-seed484370/config.full.yaml", + "path_in_repo": "settings/florence_robotwin/keyframe/config.full.yaml", + "bytes": 3510, + "role": "run_metadata" + }, + { + "source": "/fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs/florence-robotwin-keyframe-10000-to-10500-seed484370/dataset_statistics.json", + "path_in_repo": "settings/florence_robotwin/keyframe/dataset_statistics.json", + "bytes": 5141, + "role": "run_metadata" + }, + { + "source": "/fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs/florence-robotwin-keyframe-10000-to-10500-seed484370/initialization_manifest.json", + "path_in_repo": "settings/florence_robotwin/keyframe/initialization_manifest.json", + "bytes": 200, + "role": "run_metadata" + }, + { + "source": "/fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs/florence-robotwin-keyframe-10000-to-10500-seed484370/external_resume_certificate.json", + "path_in_repo": "settings/florence_robotwin/keyframe/external_resume_certificate.json", + "bytes": 4675, + "role": "run_metadata" + }, + { + "source": "/fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs/florence-robotwin-keyframe-10000-to-10500-seed484370/valid_anchor_manifest.json", + "path_in_repo": "settings/florence_robotwin/keyframe/valid_anchor_manifest.json", + "bytes": 107646168, + "role": "run_metadata" + }, + { + "source": "/fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs/florence-robotwin-keyframe-10000-to-10500-seed484370/checkpoints/steps_10500_accelerate/checkpoint.json", + "path_in_repo": "settings/florence_robotwin/keyframe/checkpoint.json", + "bytes": 2090, + "role": "checkpoint_metadata" + }, + { + "source": "/fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs/florence-robotwin-loss_is-10000-to-10500-seed484370/config.yaml", + "path_in_repo": "settings/florence_robotwin/loss_is/config.yaml", + "bytes": 2273, + "role": "run_metadata" + }, + { + "source": "/fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs/florence-robotwin-loss_is-10000-to-10500-seed484370/config.full.yaml", + "path_in_repo": "settings/florence_robotwin/loss_is/config.full.yaml", + "bytes": 3659, + "role": "run_metadata" + }, + { + "source": "/fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs/florence-robotwin-loss_is-10000-to-10500-seed484370/dataset_statistics.json", + "path_in_repo": "settings/florence_robotwin/loss_is/dataset_statistics.json", + "bytes": 5141, + "role": "run_metadata" + }, + { + "source": "/fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs/florence-robotwin-loss_is-10000-to-10500-seed484370/initialization_manifest.json", + "path_in_repo": "settings/florence_robotwin/loss_is/initialization_manifest.json", + "bytes": 200, + "role": "run_metadata" + }, + { + "source": "/fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs/florence-robotwin-loss_is-10000-to-10500-seed484370/external_resume_certificate.json", + "path_in_repo": "settings/florence_robotwin/loss_is/external_resume_certificate.json", + "bytes": 4675, + "role": "run_metadata" + }, + { + "source": "/fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs/florence-robotwin-loss_is-10000-to-10500-seed484370/valid_anchor_manifest.json", + "path_in_repo": "settings/florence_robotwin/loss_is/valid_anchor_manifest.json", + "bytes": 107646168, + "role": "run_metadata" + }, + { + "source": "/fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs/florence-robotwin-loss_is-10000-to-10500-seed484370/loss_is_state.json", + "path_in_repo": "settings/florence_robotwin/loss_is/loss_is_state.json", + "bytes": 595, + "role": "run_metadata" + }, + { + "source": "/fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs/florence-robotwin-loss_is-10000-to-10500-seed484370/checkpoints/steps_10500_accelerate/checkpoint.json", + "path_in_repo": "settings/florence_robotwin/loss_is/checkpoint.json", + "bytes": 2090, + "role": "checkpoint_metadata" + }, + { + "source": "/fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/cache/frameskip_full.json", + "path_in_repo": "settings/florence_robotwin/frameskip/cache/frameskip_full.json", + "bytes": 206357915, + "role": "frameskip_cache" + }, + { + "source": "/fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/cache/keyframe_full.npz", + "path_in_repo": "settings/florence_robotwin/keyframe/cache/keyframe_full.npz", + "bytes": 22442738, + "role": "keyframe_ape_cache" + }, + { + "source": "/fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/frameskip_repair_20260901/pipeline_frameskip.sh", + "path_in_repo": "settings/florence_robotwin/frameskip/cache/original_pipeline.sh", + "bytes": 5617, + "role": "historical_launcher" + }, + { + "source": "/fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/test_suite/pipeline_keyframe.sh", + "path_in_repo": "settings/florence_robotwin/keyframe/cache/original_pipeline.sh", + "bytes": 12855, + "role": "historical_launcher" + }, + { + "source": "/fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs/qwen-robotwin-frameskip-11000-to-11500-seed484370/config.yaml", + "path_in_repo": "settings/qwen_robotwin/frameskip/config.yaml", + "bytes": 2128, + "role": "run_metadata" + }, + { + "source": "/fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs/qwen-robotwin-frameskip-11000-to-11500-seed484370/config.full.yaml", + "path_in_repo": "settings/qwen_robotwin/frameskip/config.full.yaml", + "bytes": 3514, + "role": "run_metadata" + }, + { + "source": "/fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs/qwen-robotwin-frameskip-11000-to-11500-seed484370/dataset_statistics.json", + "path_in_repo": "settings/qwen_robotwin/frameskip/dataset_statistics.json", + "bytes": 5141, + "role": "run_metadata" + }, + { + "source": "/fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs/qwen-robotwin-frameskip-11000-to-11500-seed484370/initialization_manifest.json", + "path_in_repo": "settings/qwen_robotwin/frameskip/initialization_manifest.json", + "bytes": 200, + "role": "run_metadata" + }, + { + "source": "/fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs/qwen-robotwin-frameskip-11000-to-11500-seed484370/external_resume_certificate.json", + "path_in_repo": "settings/qwen_robotwin/frameskip/external_resume_certificate.json", + "bytes": 4792, + "role": "run_metadata" + }, + { + "source": "/fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs/qwen-robotwin-frameskip-11000-to-11500-seed484370/valid_anchor_manifest.json", + "path_in_repo": "settings/qwen_robotwin/frameskip/valid_anchor_manifest.json", + "bytes": 107646168, + "role": "run_metadata" + }, + { + "source": "/fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs/qwen-robotwin-frameskip-11000-to-11500-seed484370/checkpoints/steps_11500_accelerate/checkpoint.json", + "path_in_repo": "settings/qwen_robotwin/frameskip/checkpoint.json", + "bytes": 2138, + "role": "checkpoint_metadata" + }, + { + "source": "/fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs/qwen-robotwin-keyframe-11000-to-11500-seed484370/config.yaml", + "path_in_repo": "settings/qwen_robotwin/keyframe/config.yaml", + "bytes": 2126, + "role": "run_metadata" + }, + { + "source": "/fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs/qwen-robotwin-keyframe-11000-to-11500-seed484370/config.full.yaml", + "path_in_repo": "settings/qwen_robotwin/keyframe/config.full.yaml", + "bytes": 3553, + "role": "run_metadata" + }, + { + "source": "/fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs/qwen-robotwin-keyframe-11000-to-11500-seed484370/dataset_statistics.json", + "path_in_repo": "settings/qwen_robotwin/keyframe/dataset_statistics.json", + "bytes": 5141, + "role": "run_metadata" + }, + { + "source": "/fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs/qwen-robotwin-keyframe-11000-to-11500-seed484370/initialization_manifest.json", + "path_in_repo": "settings/qwen_robotwin/keyframe/initialization_manifest.json", + "bytes": 200, + "role": "run_metadata" + }, + { + "source": "/fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs/qwen-robotwin-keyframe-11000-to-11500-seed484370/external_resume_certificate.json", + "path_in_repo": "settings/qwen_robotwin/keyframe/external_resume_certificate.json", + "bytes": 4792, + "role": "run_metadata" + }, + { + "source": "/fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs/qwen-robotwin-keyframe-11000-to-11500-seed484370/valid_anchor_manifest.json", + "path_in_repo": "settings/qwen_robotwin/keyframe/valid_anchor_manifest.json", + "bytes": 107646168, + "role": "run_metadata" + }, + { + "source": "/fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs/qwen-robotwin-keyframe-11000-to-11500-seed484370/checkpoints/steps_11500_accelerate/checkpoint.json", + "path_in_repo": "settings/qwen_robotwin/keyframe/checkpoint.json", + "bytes": 2138, + "role": "checkpoint_metadata" + }, + { + "source": "/fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs/qwen-robotwin-loss_is-11000-to-11500-seed484370/config.yaml", + "path_in_repo": "settings/qwen_robotwin/loss_is/config.yaml", + "bytes": 2233, + "role": "run_metadata" + }, + { + "source": "/fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs/qwen-robotwin-loss_is-11000-to-11500-seed484370/config.full.yaml", + "path_in_repo": "settings/qwen_robotwin/loss_is/config.full.yaml", + "bytes": 3702, + "role": "run_metadata" + }, + { + "source": "/fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs/qwen-robotwin-loss_is-11000-to-11500-seed484370/dataset_statistics.json", + "path_in_repo": "settings/qwen_robotwin/loss_is/dataset_statistics.json", + "bytes": 5141, + "role": "run_metadata" + }, + { + "source": "/fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs/qwen-robotwin-loss_is-11000-to-11500-seed484370/initialization_manifest.json", + "path_in_repo": "settings/qwen_robotwin/loss_is/initialization_manifest.json", + "bytes": 200, + "role": "run_metadata" + }, + { + "source": "/fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs/qwen-robotwin-loss_is-11000-to-11500-seed484370/external_resume_certificate.json", + "path_in_repo": "settings/qwen_robotwin/loss_is/external_resume_certificate.json", + "bytes": 4792, + "role": "run_metadata" + }, + { + "source": "/fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs/qwen-robotwin-loss_is-11000-to-11500-seed484370/valid_anchor_manifest.json", + "path_in_repo": "settings/qwen_robotwin/loss_is/valid_anchor_manifest.json", + "bytes": 107646168, + "role": "run_metadata" + }, + { + "source": "/fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs/qwen-robotwin-loss_is-11000-to-11500-seed484370/loss_is_state.json", + "path_in_repo": "settings/qwen_robotwin/loss_is/loss_is_state.json", + "bytes": 595, + "role": "run_metadata" + }, + { + "source": "/fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs/qwen-robotwin-loss_is-11000-to-11500-seed484370/checkpoints/steps_11500_accelerate/checkpoint.json", + "path_in_repo": "settings/qwen_robotwin/loss_is/checkpoint.json", + "bytes": 2138, + "role": "checkpoint_metadata" + }, + { + "source": "/fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/cache/frameskip_full.json", + "path_in_repo": "settings/qwen_robotwin/frameskip/cache/frameskip_full.json", + "bytes": 206357915, + "role": "frameskip_cache" + }, + { + "source": "/fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/cache/keyframe_full.npz", + "path_in_repo": "settings/qwen_robotwin/keyframe/cache/keyframe_full.npz", + "bytes": 22442738, + "role": "keyframe_ape_cache" + }, + { + "source": "/fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/frameskip_repair_20260901/pipeline_frameskip.sh", + "path_in_repo": "settings/qwen_robotwin/frameskip/cache/original_pipeline.sh", + "bytes": 5617, + "role": "historical_launcher" + }, + { + "source": "/fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/test_suite/pipeline_keyframe.sh", + "path_in_repo": "settings/qwen_robotwin/keyframe/cache/original_pipeline.sh", + "bytes": 12855, + "role": "historical_launcher" + }, + { + "source": "/fs1/private/user/liuyu/student-workspace/outputs/keyframe_focused_adapted/cache/keyframe_ape_full_seed0_epoch300.npz", + "path_in_repo": "settings/florence_bridge/keyframe/cache/keyframe_ape_full_seed0_epoch300.npz", + "bytes": 20124275, + "role": "keyframe_ape_cache" + }, + { + "source": "/fs1/private/user/liuyu/student-workspace/outputs/keyframe_focused_adapted/logs/cache_support.json", + "path_in_repo": "settings/florence_bridge/keyframe/cache/cache_support.json", + "bytes": 358, + "role": "cache_support" + }, + { + "source": "/fs1/private/user/liuyu/student-workspace/data/robotwin/manifests/data_provenance.json", + "path_in_repo": "settings/florence_robotwin/dependencies/data_provenance.json", + "bytes": 641, + "role": "dataset_identity" + }, + { + "source": "/fs1/private/user/liuyu/student-workspace/data/robotwin/manifests/content_fingerprints.json", + "path_in_repo": "settings/florence_robotwin/dependencies/content_fingerprints.json", + "bytes": 14233373, + "role": "dataset_identity" + }, + { + "source": "/fs1/private/user/liuyu/student-workspace/data/robotwin/manifests/data_provenance.json", + "path_in_repo": "settings/qwen_robotwin/dependencies/data_provenance.json", + "bytes": 641, + "role": "dataset_identity" + }, + { + "source": "/fs1/private/user/liuyu/student-workspace/data/robotwin/manifests/content_fingerprints.json", + "path_in_repo": "settings/qwen_robotwin/dependencies/content_fingerprints.json", + "bytes": 14233373, + "role": "dataset_identity" + } + ], + "missing": [], + "checksum_gates": false +} \ No newline at end of file diff --git a/settings/florence_bridge/keyframe/cache/cache_support.json b/settings/florence_bridge/keyframe/cache/cache_support.json new file mode 100644 index 0000000000000000000000000000000000000000..3ccf127414c683a5dae8cbfb447652d4f009b6f6 --- /dev/null +++ b/settings/florence_bridge/keyframe/cache/cache_support.json @@ -0,0 +1 @@ +{"cache_sha256": "8b321259e0a2cb28778dccf934fc88d133aa9cb729bbeb3d9dbaa6069e2ce4bd", "dataset_counts": {"bridge_orig_lerobot": 4138, "fractal20220817_data_lerobot": 4054}, "draw_count": 8192, "draw_start": 0, "manifest_sha256": "a1627df8ff110c7298eaf025ffcd4f48d9ca17117cfcfb3926b962d0bb19c0ca", "status": "PASS", "weight_counts": {"1.0": 7364, "5.0": 828}} diff --git a/settings/florence_bridge/keyframe/cache/keyframe_ape_full_seed0_epoch300.npz b/settings/florence_bridge/keyframe/cache/keyframe_ape_full_seed0_epoch300.npz new file mode 100644 index 0000000000000000000000000000000000000000..15ca549b91ab6973ba66d83b7649656006d340f7 --- /dev/null +++ b/settings/florence_bridge/keyframe/cache/keyframe_ape_full_seed0_epoch300.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8b321259e0a2cb28778dccf934fc88d133aa9cb729bbeb3d9dbaa6069e2ce4bd +size 20124275 diff --git a/settings/florence_robotwin/dependencies/content_fingerprints.json b/settings/florence_robotwin/dependencies/content_fingerprints.json new file mode 100644 index 0000000000000000000000000000000000000000..f945c130e7871dcbe67f99368ba69978f57ba8ec --- /dev/null +++ b/settings/florence_robotwin/dependencies/content_fingerprints.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e6c3ef15cf746ad2a5f5ffa5f3c3c57fc2647507200873754718388643460d8 +size 14233373 diff --git a/settings/florence_robotwin/dependencies/data_provenance.json b/settings/florence_robotwin/dependencies/data_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..0c769a6507b50d7e64d9937924907ea3d4f633e6 --- /dev/null +++ b/settings/florence_robotwin/dependencies/data_provenance.json @@ -0,0 +1,21 @@ +{ + "checkpoint_repository": { + "repo_id": "StarVLA/Florence-GR00T-RoboTwin", + "revision": "unknown" + }, + "datasets": { + "RoboTwin/Clean": { + "repo_id": "local-mnt-miaochengyu-starvla-infoflow-runtime-playground-Datasets-RoboTwin-Clean", + "revision": "local-filesystem" + }, + "RoboTwin/Randomized": { + "repo_id": "local-mnt-miaochengyu-starvla-infoflow-runtime-playground-Datasets-RoboTwin-Randomized", + "revision": "local-filesystem" + } + }, + "florence_repository": { + "repo_id": "microsoft/Florence-2-large", + "revision": "21a599d414c4d928c9032694c424fb94458e3594" + }, + "schema_version": 1 +} \ No newline at end of file diff --git a/settings/florence_robotwin/frameskip/cache/frameskip_full.json b/settings/florence_robotwin/frameskip/cache/frameskip_full.json new file mode 100644 index 0000000000000000000000000000000000000000..55b4dbbe23819c222e1eea0ffa9c9696374cc642 --- /dev/null +++ b/settings/florence_robotwin/frameskip/cache/frameskip_full.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5fa058a703b62cf780621a9a495044923cd2289fd7b684c3dd8b4c217843b1f2 +size 206357915 diff --git a/settings/florence_robotwin/frameskip/cache/original_pipeline.sh b/settings/florence_robotwin/frameskip/cache/original_pipeline.sh new file mode 100644 index 0000000000000000000000000000000000000000..e3fa8491e858dc0d4396cdc575bb70c25cd1c220 --- /dev/null +++ b/settings/florence_robotwin/frameskip/cache/original_pipeline.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail +cd /home/liuyu/student-workspace/code/starVLA-loss-is-adapted +export OMP_NUM_THREADS=1 MKL_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 TORCHINDUCTOR_COMPILE_THREADS=1 MAX_JOBS=1 TOKENIZERS_PARALLELISM=false +trap 'rc=$?; printf "%s\n" "$rc" > /home/liuyu/student-workspace/outputs/robotwin_external_20260831/frameskip_repair_20260901/pipeline.exitcode' EXIT +ROOT=/fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831 +TEST=/fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/test_suite +mkdir -p "$ROOT/logs" "$ROOT/cache" "$ROOT/evidence" "$ROOT/formal_runs" "$ROOT/temporary_smoke_runs" +if [ ! -s /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/cache/frameskip_full.json ]; then +env PYTHONPATH=/fs1/private/user/liuyu/student-workspace/code/starVLA-infoflow-codex WANDB_MODE=offline PYTHONUNBUFFERED=1 PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True HF_HOME=/fs1/private/user/liuyu/student-workspace/cache/huggingface HF_HUB_OFFLINE=1 TMPDIR=/fs1/private/user/liuyu/student-workspace/tmp /fs1/private/user/liuyu/student-workspace/data/runtime/.venv/bin/python /fs1/private/user/liuyu/student-workspace/code/starVLA-infoflow-codex/external_baselines/frameskip_robotwin_adapted/build_robotwin_cache.py --manifest /fs1/private/user/liuyu/student-workspace/data/robotwin/manifests/valid_anchor_manifest.json --data-root /fs1/private/user/liuyu/student-workspace/data/robotwin/RoboTwin --upstream-root /fs1/private/user/liuyu/student-workspace/code/upstream/FrameSkip-965becf63e9cd5cc339fbf7fde40d1db47efa67a --visual-checkpoint /fs1/private/user/liuyu/student-workspace/cache/huggingface/hub/models--timm--vit_small_patch14_dinov2.lvd142m/snapshots/4610ca143709d58a633b6397a74412c2c3842454/model.safetensors --output /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/cache/frameskip_full.json --device cuda +fi +/fs1/private/user/liuyu/student-workspace/data/runtime/.venv/bin/python /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/test_suite/validate_frameskip_cache.py /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/cache/frameskip_full.json /fs1/private/user/liuyu/student-workspace/data/robotwin/manifests/valid_anchor_manifest.json +echo FORMAL florence frameskip +env PYTHONPATH=/fs1/private/user/liuyu/student-workspace/code/starVLA-loss-is-adapted WANDB_MODE=offline PYTHONUNBUFFERED=1 PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True HF_HOME=/fs1/private/user/liuyu/student-workspace/cache/huggingface HF_HUB_OFFLINE=1 TMPDIR=/fs1/private/user/liuyu/student-workspace/tmp FRAMESKIP_CACHE=/fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/cache/frameskip_full.json FRAMESKIP_START_GLOBAL_DRAW_ID=640000 FRAMESKIP_GLOBAL_MICROBATCH_SIZE=4 /fs1/private/user/liuyu/student-workspace/data/runtime/.venv/bin/python -m accelerate.commands.launch --config_file /fs1/private/user/liuyu/student-workspace/code/starVLA-loss-is-adapted/external_baselines/loss_is_adapted/configs/deepspeed_florence_2gpu_sai.yaml --num_processes 2 --mixed_precision bf16 --main_process_port 0 /fs1/private/user/liuyu/student-workspace/code/starVLA-infoflow-codex/external_baselines/frameskip_robotwin_adapted/train_frameskip_robotwin_continuation.py --config_yaml /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/configs/florence_frameskip_formal.yaml +/fs1/private/user/liuyu/student-workspace/data/runtime/.venv/bin/python /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/test_suite/validate_checkpoint.py /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs/florence-robotwin-frameskip-10000-to-10500-seed484370/checkpoints 10500 > /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/evidence/florence_frameskip_formal_checkpoint.json +echo FORMAL qwen frameskip +cd /home/liuyu/student-workspace/code/starVLA-qwen-bridge-adapted +env PYTHONPATH=/fs1/private/user/liuyu/student-workspace/code/starVLA-qwen-bridge-adapted WANDB_MODE=offline PYTHONUNBUFFERED=1 PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True HF_HOME=/fs1/private/user/liuyu/student-workspace/cache/huggingface HF_HUB_OFFLINE=1 TMPDIR=/fs1/private/user/liuyu/student-workspace/tmp FRAMESKIP_CACHE=/fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/cache/frameskip_full.json FRAMESKIP_START_GLOBAL_DRAW_ID=440000 FRAMESKIP_GLOBAL_MICROBATCH_SIZE=10 /fs1/private/user/liuyu/student-workspace/data/runtime/.venv/bin/python -m accelerate.commands.launch --config_file /fs1/private/user/liuyu/student-workspace/code/starVLA-qwen-bridge-adapted/starVLA/config/deepseeds/deepspeed_zero3.yaml --num_processes 2 --mixed_precision bf16 --main_process_port 0 /fs1/private/user/liuyu/student-workspace/code/starVLA-infoflow-codex/external_baselines/frameskip_robotwin_adapted/train_frameskip_robotwin_continuation.py --config_yaml /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/configs/qwen_frameskip_formal.yaml +/fs1/private/user/liuyu/student-workspace/data/runtime/.venv/bin/python /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/test_suite/validate_checkpoint.py /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs/qwen-robotwin-frameskip-11000-to-11500-seed484370/checkpoints 11500 > /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/evidence/qwen_frameskip_formal_checkpoint.json +echo PIPELINE_FRAMESKIP_PASS diff --git a/settings/florence_robotwin/frameskip/checkpoint.json b/settings/florence_robotwin/frameskip/checkpoint.json new file mode 100644 index 0000000000000000000000000000000000000000..8b1db5660201ccdcb2023a59177d5a928f078f1c --- /dev/null +++ b/settings/florence_robotwin/frameskip/checkpoint.json @@ -0,0 +1,65 @@ +{ + "completed_steps": 10500, + "files": [ + { + "bytes": 1525, + "name": "custom_checkpoint_0.pkl", + "sha256": "98276dada2a8c4a6309ae8c0dd0daa358ceac5a1dd932daea9ba4206e3dd5c78" + }, + { + "bytes": 13, + "name": "latest", + "sha256": "68d0ca762047fd48fdd6a06a5e6465209289c028bfd63f5557a110ff1ebc15ac" + }, + { + "bytes": 5509764721, + "name": "pytorch_model/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt", + "sha256": "af52ee4c1962395794922eba1b7ddea9da89714014342ddcc11c190a031f5f9a" + }, + { + "bytes": 5509778801, + "name": "pytorch_model/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt", + "sha256": "6ffef748dc7e1a35ca632d24ff4f77c8964a675cb2245c068e71af5679e1b2d5" + }, + { + "bytes": 1837527613, + "name": "pytorch_model/mp_rank_00_model_states.pt", + "sha256": "818a9dad0538df8dd89c721324cfe6e39896c130f250dc3cf883a309acdaeb99" + }, + { + "bytes": 2152621885, + "name": "pytorch_model.pt", + "sha256": "03745fecff8f7e98da348cb95d38ed27e840f4d6856d63b299a3468e30e73c55" + }, + { + "bytes": 15081, + "name": "random_states_0.pkl", + "sha256": "5e1eaba9845ffc86118d3545569c01055097bf21fd0069aeab3224571f4522e4" + }, + { + "bytes": 15081, + "name": "random_states_1.pkl", + "sha256": "4dc72dafa3023634bb820b6e722b6a060c2fd1622d432a9417caefe1c3c7d000" + }, + { + "bytes": 33270, + "name": "zero_to_fp32.py", + "sha256": "e8a2553b434ac043ec63b022db1e768277acf8d4e70f10a2842db6d238977135" + } + ], + "registered_checkpointables": [ + "lr_scheduler" + ], + "resume_identity": { + "config_sha256": "c1763edabdfd663076f244d685e52a432cb1e83c079948a9a7fbc92f09ab4641", + "identity_schema_version": 2, + "manifest_sha256": "dcb728a00da4b4ad7f989fa313d6d3355e13f57efefc7267e9ea092f2ffd8a50", + "model_structure_sha256": "2d1c8fd1dfa358c4975ad54934f42c24c8d10f6fcef9e8f7c2585b349b32d96b", + "proposal_alpha": null, + "proposal_sha256": null, + "seed": 484370, + "total_batch_size": 64 + }, + "schema_version": 1, + "world_size": 2 +} diff --git a/settings/florence_robotwin/frameskip/config.full.yaml b/settings/florence_robotwin/frameskip/config.full.yaml new file mode 100644 index 0000000000000000000000000000000000000000..84726cbf0fe8e1cb14b6e88505229db3822b9b10 --- /dev/null +++ b/settings/florence_robotwin/frameskip/config.full.yaml @@ -0,0 +1,111 @@ +run_id: florence-robotwin-frameskip-10000-to-10500-seed484370 +run_root_dir: /home/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs +seed: 484370 +wandb_entity: null +wandb_project: InfoFlow-StarVLA-RoboTwin +is_debug: false +version_id: '0.21' +framework: + name: QwenGR00T + qwenvl: + base_vlm: /home/liuyu/student-workspace/data/runtime/playground/Pretrained_models/Florence-2-large + attn_implementation: eager + vl_hidden_dim: 1024 + selected_view_indices: + - 0 + - 1 + - 2 + compile_vision_tower: true + action_model: + action_model_type: DiT-B + action_hidden_dim: 1024 + hidden_size: 1024 + add_pos_embed: true + max_seq_len: 1024 + action_dim: 14 + state_dim: 0 + action_horizon: 50 + repeated_diffusion_steps: 8 + noise_beta_alpha: 1.5 + noise_beta_beta: 1.0 + noise_s: 0.999 + num_timestep_buckets: 1000 + num_inference_timesteps: 4 + num_target_vision_tokens: 32 + diffusion_model_cfg: + cross_attention_dim: 1024 + dropout: 0.2 + final_dropout: true + interleave_self_attention: true + norm_type: ada_norm + num_layers: 16 + output_dim: 1024 + positional_embeddings: null + future_action_window_size: 49 + past_action_window_size: 0 + dino: + dino_backbone: dinov2_vits14 +datasets: + vla_data: + dataset_py: lerobot_datasets + data_root_dir: /home/liuyu/student-workspace/data/robotwin/RoboTwin + data_mix: robotwin_all_50 + action_type: abs_qpos + action_mode: abs + include_state: false + sequential_step_sampling: false + per_device_batch_size: 2 + load_all_data_for_training: true + image_size: + - 224 + - 224 + video_backend: pyav + num_workers: 4 + expected_world_size: 2 + traceable_sampling: true + dataset_provenance_path: /home/liuyu/student-workspace/data/robotwin/manifests/data_provenance.json + dataset_content_fingerprint_path: /home/liuyu/student-workspace/data/robotwin/manifests/content_fingerprints.json + valid_anchor_manifest_path: /home/liuyu/student-workspace/data/robotwin/manifests/valid_anchor_manifest.json + temporal_proposal_path: null + transform_rng_seed: 289726 +trainer: + use_deepspeed: true + max_train_steps: 150000 + num_warmup_steps: 5000 + save_interval: null + save_steps: + - 10500 + eval_interval: null + repeated_diffusion_steps: 4 + learning_rate: + base: 1.0e-05 + qwen_vl_interface: 1.0e-05 + action_model: 0.0001 + lr_scheduler_type: cosine_with_min_lr + scheduler_specific_kwargs: + min_lr: 5.0e-07 + freeze_modules: '' + trainable_modules: '' + frozen_modules_eval: false + logging_frequency: 50 + gradient_clipping: 1.0 + gradient_accumulation_steps: 16 + save_format: pt + save_final_checkpoint: false + optimizer: + name: AdamW + betas: + - 0.9 + - 0.95 + eps: 1.0e-08 + weight_decay: 1.0e-08 + stop_after_steps: 10500 + is_resume: true + allow_model_only_resume: false + resume_accelerate_checkpoint: /home/liuyu/student-workspace/data/robotwin/common_start/florence-groot-robotwin-xvla3view-official150k-seed42-20260804/checkpoints/steps_10000_accelerate + branch_parent_seed: 42 + branch_rng_seed: 484370 + model_rng_seed: 721927 + evaluation_snapshot_interval: null +config_yaml: /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/configs/florence_frameskip_formal.yaml +output_dir: /home/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs/florence-robotwin-frameskip-10000-to-10500-seed484370 diff --git a/settings/florence_robotwin/frameskip/config.yaml b/settings/florence_robotwin/frameskip/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b9fd9512f61d427d8b137a403dde679d7190f564 --- /dev/null +++ b/settings/florence_robotwin/frameskip/config.yaml @@ -0,0 +1,69 @@ +datasets: + vla_data: + action_mode: abs + data_mix: robotwin_all_50 + data_root_dir: /home/liuyu/student-workspace/data/robotwin/RoboTwin + dataset_content_fingerprint_path: /home/liuyu/student-workspace/data/robotwin/manifests/content_fingerprints.json + dataset_provenance_path: /home/liuyu/student-workspace/data/robotwin/manifests/data_provenance.json + dataset_py: lerobot_datasets + expected_world_size: 2 + num_workers: 4 + per_device_batch_size: 2 + temporal_proposal_path: null + traceable_sampling: true + valid_anchor_manifest_path: /home/liuyu/student-workspace/data/robotwin/manifests/valid_anchor_manifest.json + video_backend: pyav +framework: + action_model: + action_dim: 14 + action_horizon: 50 + action_model_type: DiT-B + add_pos_embed: true + diffusion_model_cfg: + cross_attention_dim: 1024 + dropout: 0.2 + final_dropout: true + interleave_self_attention: true + norm_type: ada_norm + num_layers: 16 + output_dim: 1024 + positional_embeddings: null + hidden_size: 1024 + max_seq_len: 1024 + noise_beta_alpha: 1.5 + noise_beta_beta: 1.0 + num_inference_timesteps: 4 + num_target_vision_tokens: 32 + num_timestep_buckets: 1000 + state_dim: 0 + name: QwenGR00T + qwenvl: + base_vlm: /home/liuyu/student-workspace/data/runtime/playground/Pretrained_models/Florence-2-large + compile_vision_tower: true + selected_view_indices: + - 0 + - 1 + - 2 +output_dir: /home/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs/florence-robotwin-frameskip-10000-to-10500-seed484370 +run_id: florence-robotwin-frameskip-10000-to-10500-seed484370 +run_root_dir: /home/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs +seed: 484370 +trainer: + freeze_modules: '' + gradient_accumulation_steps: 16 + learning_rate: + action_model: 0.0001 + base: 1.0e-05 + qwen_vl_interface: 1.0e-05 + lr_scheduler_type: cosine_with_min_lr + max_train_steps: 150000 + num_warmup_steps: 5000 + optimizer: + betas: + - 0.9 + - 0.95 + eps: 1.0e-08 + name: AdamW + weight_decay: 1.0e-08 + scheduler_specific_kwargs: + min_lr: 5.0e-07 diff --git a/settings/florence_robotwin/frameskip/dataset_statistics.json b/settings/florence_robotwin/frameskip/dataset_statistics.json new file mode 100644 index 0000000000000000000000000000000000000000..c920d76f3fc976e9e0b30e36eab0cff8a6bf1adb --- /dev/null +++ b/settings/florence_robotwin/frameskip/dataset_statistics.json @@ -0,0 +1,218 @@ +{ + "new_embodiment": { + "action": { + "mean": [ + -0.22423998869955536, + 1.0729362124204633, + 0.775079609751701, + -0.32701663097366684, + 0.06159586461764771, + -0.06420705380151046, + 0.22430071420967568, + 1.1274866509437556, + 0.831773659586906, + -0.3638282522559164, + -0.024959841337949897, + 0.05302203213563182, + 0.6774904182553295, + 0.6598676902055738 + ], + "std": [ + 0.39622931737429123, + 1.003590449339515, + 0.7811611351334437, + 0.6707485714474849, + 0.2686908204983921, + 0.6492019642472782, + 0.34733679400923234, + 1.0176226676795115, + 0.8039915805960646, + 0.7067249045233707, + 0.26259645177151686, + 0.6808759458519371, + 0.4482151151114418, + 0.4539921671160285 + ], + "max": [ + 6.45659065246582, + 4.179152488708496, + 5.346591472625732, + 1.7942548990249634, + 1.8604620695114136, + 5.43813943862915, + 7.142920017242432, + 4.157068729400635, + 5.672450065612793, + 1.95806884765625, + 1.5663840770721436, + 5.278968811035156, + 1.0, + 1.0 + ], + "min": [ + -7.552278995513916, + -0.36354875564575195, + -0.18577136099338531, + -1.956291913986206, + -1.6801013946533203, + -7.678869724273682, + -8.539926528930664, + -0.6294453740119934, + -0.07775841653347015, + -2.1328067779541016, + -2.1285502910614014, + -8.228925704956055, + 0.0, + 0.0 + ], + "q01": [ + -7.334665780067444, + -5.257390398583084e-07, + -2.8215323254698887e-05, + -1.841589093208313, + -1.2851550579071045, + -6.268352918624878, + -5.995517673492431, + -0.572821855545044, + -2.81171942333458e-05, + -1.8467556238174438, + -1.4545854091644288, + -5.945669202804565, + 0.0, + 0.0 + ], + "q99": [ + 5.016749763488781, + 3.4792869091033936, + 4.225283479690551, + 1.7064905214309687, + 1.4829185009002717, + 3.7177388882636553, + 1.258314609527588, + 2.967935800552368, + 3.2566452693939207, + 1.766038613319397, + 1.4066662788391113, + 3.5830023288726807, + 1.0, + 1.0 + ], + "mask": [ + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + false, + false + ] + }, + "state": { + "mean": [ + -0.22281882727518673, + 1.0668349397182466, + 0.770632639825344, + -0.3252330579422412, + 0.06118139047002841, + -0.06316296239456277, + 0.22253762468695631, + 1.1201750242710113, + 0.8263923591375351, + -0.36159489695448405, + -0.024702461528722772, + 0.05203246331657278, + 0.6793262696266176, + 0.6616973677277563 + ], + "std": [ + 0.39522870886731665, + 1.004018577033247, + 0.7809103811570285, + 0.6688088363177986, + 0.26775723818956776, + 0.6463212583712935, + 0.34648488321363663, + 1.018232630032839, + 0.8038798397682734, + 0.7045612787595747, + 0.26143414213134614, + 0.6774619870452644, + 0.44749220497027625, + 0.45333916022037934 + ], + "max": [ + 6.45659065246582, + 4.179152488708496, + 5.346591472625732, + 1.7942548990249634, + 1.8604620695114136, + 5.43813943862915, + 7.142920017242432, + 4.157068729400635, + 5.672450065612793, + 1.95806884765625, + 1.5663840770721436, + 5.278968811035156, + 1.0, + 1.0 + ], + "min": [ + -7.552278995513916, + -0.36354875564575195, + -0.18577136099338531, + -1.956291913986206, + -1.6801013946533203, + -7.678869724273682, + -8.539926528930664, + -0.6294453740119934, + -0.07775841653347015, + -2.1328067779541016, + -2.1285502910614014, + -8.228925704956055, + 0.0, + 0.0 + ], + "q01": [ + -7.334665780067444, + -5.257390398583084e-07, + -2.8215323254698887e-05, + -1.841589093208313, + -1.2851550579071045, + -6.268109321594238, + -5.912737417221069, + -0.572821855545044, + -2.81171942333458e-05, + -1.8467556238174438, + -1.440324923992157, + -5.8623305034637445, + 0.0, + 0.0 + ], + "q99": [ + 4.902149534225477, + 3.4775466918945312, + 4.224453401565551, + 1.7064905214309687, + 1.480757713317871, + 3.609884884357428, + 1.258314609527588, + 2.967935800552368, + 3.2566452693939207, + 1.7433889162540437, + 1.4066662788391113, + 3.581243851184845, + 1.0, + 1.0 + ] + }, + "num_transitions": 6075103, + "num_trajectories": 27500 + } +} \ No newline at end of file diff --git a/settings/florence_robotwin/frameskip/external_resume_certificate.json b/settings/florence_robotwin/frameskip/external_resume_certificate.json new file mode 100644 index 0000000000000000000000000000000000000000..fb328d39f46f24919ff5ca9a0e1c2c82e82057b0 --- /dev/null +++ b/settings/florence_robotwin/frameskip/external_resume_certificate.json @@ -0,0 +1,138 @@ +{ + "branch_resume_identity": { + "config_sha256": "c1763edabdfd663076f244d685e52a432cb1e83c079948a9a7fbc92f09ab4641", + "identity_schema_version": 2, + "manifest_sha256": "dcb728a00da4b4ad7f989fa313d6d3355e13f57efefc7267e9ea092f2ffd8a50", + "model_structure_sha256": "2d1c8fd1dfa358c4975ad54934f42c24c8d10f6fcef9e8f7c2585b349b32d96b", + "proposal_alpha": null, + "proposal_sha256": null, + "seed": 484370, + "total_batch_size": 64 + }, + "completed_steps": 10000, + "optimizer_param_groups": [ + { + "betas": [ + 0.9, + 0.95 + ], + "eps": 1e-08, + "initial_lr": 1e-05, + "lr": 9.97215529648321e-06, + "name": "qwen_vl_interface", + "weight_decay": 1e-08 + }, + { + "betas": [ + 0.9, + 0.95 + ], + "eps": 1e-08, + "initial_lr": 0.0001, + "lr": 9.972155296483209e-05, + "name": "action_model", + "weight_decay": 1e-08 + } + ], + "optimizer_state_entries": 4, + "parent_checkpoint": "/fs1/private/user/liuyu/student-workspace/data/robotwin/common_start/florence-groot-robotwin-xvla3view-official150k-seed42-20260804/checkpoints/steps_10000_accelerate", + "parent_checkpoint_commit_sha256": "f6f6408ead21ba9cb0d7bbfa26c74733950a504264a26e5b795129ffadff6958", + "parent_checkpoint_files": [ + { + "bytes": 1525, + "name": "custom_checkpoint_0.pkl", + "sha256": "031a5046639dfc967eebbf30b7184dbe2619e26675a74569827aa5edaf6cc85c" + }, + { + "bytes": 13, + "name": "latest", + "sha256": "68d0ca762047fd48fdd6a06a5e6465209289c028bfd63f5557a110ff1ebc15ac" + }, + { + "bytes": 5509764721, + "name": "pytorch_model/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt", + "sha256": "27d9b011b75f40c3d896c9681ee883eeb3d2277f201889faebf0b7fcf5492c52" + }, + { + "bytes": 5509778801, + "name": "pytorch_model/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt", + "sha256": "f657f6f6fb48956b2b406b79511e16b9c2712dc95216b78ba5179818b821d18f" + }, + { + "bytes": 1837527613, + "name": "pytorch_model/mp_rank_00_model_states.pt", + "sha256": "3d66f8b6c0e344c34f06d3b4708c3292cbb8a84ea5a25e650fb64b679a44eb3b" + }, + { + "bytes": 2152621885, + "name": "pytorch_model.pt", + "sha256": "4b6c4f47df7b0f05b8c9287e480d88f2b5e711b4fb6cccbdfffd677ed9bff778" + }, + { + "bytes": 15017, + "name": "random_states_0.pkl", + "sha256": "629d14d87e3cab680e32acec92c80c2578a7fbd1ebc0dee5475e774058f11d41" + }, + { + "bytes": 15081, + "name": "random_states_1.pkl", + "sha256": "3cb98283ee7eaaedcee320e8e3b17e74f2f01a22d9e3b958fa2422a86215c652" + }, + { + "bytes": 33270, + "name": "zero_to_fp32.py", + "sha256": "e8a2553b434ac043ec63b022db1e768277acf8d4e70f10a2842db6d238977135" + } + ], + "parent_checkpoint_files_sha256": "1f8985d0633f16ac3b0381b128cdb4e479c4fdfd67ff15d58736633e4b942bc0", + "parent_checkpoint_integrity_manifest_sha256": null, + "parent_config_sha256": "d35ab9e43634c7c4a74c5299657df3c17c7d78407dd0fe3d37296a927e353d28", + "parent_resume_identity": { + "config_sha256": "2440c6327cf5e8c14efc93b800209171c725dae0898181921c62f3490eb8378f", + "identity_schema_version": 2, + "manifest_sha256": null, + "model_structure_sha256": "2d1c8fd1dfa358c4975ad54934f42c24c8d10f6fcef9e8f7c2585b349b32d96b", + "proposal_alpha": null, + "proposal_sha256": null, + "seed": 42, + "total_batch_size": 64 + }, + "proposal_sha256": null, + "rank_state_summaries": [ + { + "optimizer_state_entries": 2, + "optimizer_state_sha256": "5a35add2c00249e41e72e1ab16261d87f205bfeaf23386c7263223e4ce168368", + "rank": 0, + "trainable_state_sha256": "667c6a6a7aa0dc49c32f7bef693f3272efee46064fda5d370d25788291da76a0" + }, + { + "optimizer_state_entries": 2, + "optimizer_state_sha256": "073f849cd1942522dbfde195bdc29e8ee015e4c5d4be3070381027a13b9784cf", + "rank": 1, + "trainable_state_sha256": "667c6a6a7aa0dc49c32f7bef693f3272efee46064fda5d370d25788291da76a0" + } + ], + "repeated_diffusion_steps": 4, + "rng_streams": { + "model_base": 721927, + "sampler": 484370, + "transform": 289726 + }, + "scheduler": { + "class": "LambdaLR", + "last_epoch": 10000, + "last_lr": [ + 9.97215529648321e-06, + 9.972155296483209e-05 + ], + "step_count": 10001 + }, + "scheduler_resume_mode": "accelerate_checkpoint", + "schema_version": 1, + "status": "exact_external_resume_loaded", + "total_batch_size": 64, + "trainable_parameter_elements": 918283022, + "trainable_parameter_sha256": "2e8f3de6aeee46bca56fab5993103ad30302b45330037cf98f96ffd472f0e9ce", + "trainable_parameter_tensors": 1161, + "world_size": 2 +} diff --git a/settings/florence_robotwin/frameskip/initialization_manifest.json b/settings/florence_robotwin/frameskip/initialization_manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..a67d8f6053e29aebeabfc485e6a85a2e79811fae --- /dev/null +++ b/settings/florence_robotwin/frameskip/initialization_manifest.json @@ -0,0 +1,6 @@ +{ + "schema_version": 1, + "initialization_seed": 484370, + "action_model_sha256": "0b5e42abcce1d0cc6aa164e9bb09ec77e0ea1c6719bb8dc55406f806c5d22b99", + "action_model_parameter_elements": 141815566 +} diff --git a/settings/florence_robotwin/frameskip/valid_anchor_manifest.json b/settings/florence_robotwin/frameskip/valid_anchor_manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..3f5c91478b5931286c9c84d021a7ffc10a0fb5c0 --- /dev/null +++ b/settings/florence_robotwin/frameskip/valid_anchor_manifest.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2090f6b9d7e1bc5ef41897436690f031c187d625fa6b71ae0c54f3793660c757 +size 107646168 diff --git a/settings/florence_robotwin/keyframe/cache/keyframe_full.npz b/settings/florence_robotwin/keyframe/cache/keyframe_full.npz new file mode 100644 index 0000000000000000000000000000000000000000..066942637c8ac5542d51f646f3f084381d3a3612 --- /dev/null +++ b/settings/florence_robotwin/keyframe/cache/keyframe_full.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:490da0439605b2b1acd7400add00c0e24a9c53c8bf68812ba47fcb34a0f18e40 +size 22442738 diff --git a/settings/florence_robotwin/keyframe/cache/original_pipeline.sh b/settings/florence_robotwin/keyframe/cache/original_pipeline.sh new file mode 100644 index 0000000000000000000000000000000000000000..8bafbc4ae32ad7525483d2fc03e6fd849f751576 --- /dev/null +++ b/settings/florence_robotwin/keyframe/cache/original_pipeline.sh @@ -0,0 +1,32 @@ +#!/bin/bash +set -euo pipefail +ROOT=/fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831 +TEST=/fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/test_suite +mkdir -p "$ROOT/logs" "$ROOT/cache" "$ROOT/evidence" "$ROOT/formal_runs" "$ROOT/temporary_smoke_runs" +if [ ! -s /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/cache/keyframe_full.npz ]; then +env PYTHONPATH=/fs1/private/user/liuyu/student-workspace/code/starVLA-loss-is-adapted WANDB_MODE=offline PYTHONUNBUFFERED=1 PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True HF_HOME=/fs1/private/user/liuyu/student-workspace/cache/huggingface HF_HUB_OFFLINE=1 TMPDIR=/fs1/private/user/liuyu/student-workspace/tmp /fs1/private/user/liuyu/student-workspace/data/runtime/.venv/bin/python /fs1/private/user/liuyu/student-workspace/code/starVLA-loss-is-adapted/external_baselines/robotwin_smoke/build_keyframe_ape_cache.py --manifest /fs1/private/user/liuyu/student-workspace/data/robotwin/manifests/valid_anchor_manifest.json --data-root /fs1/private/user/liuyu/student-workspace/data/robotwin/RoboTwin --statistics /fs1/private/user/liuyu/student-workspace/data/robotwin/common_start/florence-groot-robotwin-xvla3view-official150k-seed42-20260804/dataset_statistics.json --statistics-key new_embodiment --output /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/cache/keyframe_full.npz --scope full --epochs 300 --device cuda +fi +env PYTHONPATH=/fs1/private/user/liuyu/student-workspace/code/starVLA-loss-is-adapted /fs1/private/user/liuyu/student-workspace/data/runtime/.venv/bin/python /fs1/private/user/liuyu/student-workspace/code/starVLA-loss-is-adapted/external_baselines/keyframe_focused_adapted/validate_cache_support.py --manifest /fs1/private/user/liuyu/student-workspace/data/robotwin/manifests/valid_anchor_manifest.json --cache /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/cache/keyframe_full.npz --seed 484370 --draw-start 440000 --draw-count 4096 +echo SMOKE florence keyframe phase1 +env PYTHONPATH=/fs1/private/user/liuyu/student-workspace/code/starVLA-loss-is-adapted WANDB_MODE=offline PYTHONUNBUFFERED=1 PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True HF_HOME=/fs1/private/user/liuyu/student-workspace/cache/huggingface HF_HUB_OFFLINE=1 TMPDIR=/fs1/private/user/liuyu/student-workspace/tmp KEYFRAME_APE_CACHE=/fs1/private/user/liuyu/student-workspace/outputs/robotwin_smoke/data/keyframe_ape_full_smoke.npz /fs1/private/user/liuyu/student-workspace/data/runtime/.venv/bin/python -m accelerate.commands.launch --config_file /fs1/private/user/liuyu/student-workspace/code/starVLA-loss-is-adapted/external_baselines/loss_is_adapted/configs/deepspeed_florence_2gpu_sai.yaml --num_processes 2 --mixed_precision bf16 --main_process_port 0 /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/test_suite/smoke_runner.py --method keyframe --entry /fs1/private/user/liuyu/student-workspace/code/starVLA-loss-is-adapted/external_baselines/keyframe_focused_adapted/train_keyframe_continuation.py --config_yaml /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/test_suite/florence_keyframe_phase1.yaml >> /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/logs/keyframe_florence_smoke.log 2>&1 +/fs1/private/user/liuyu/student-workspace/data/runtime/.venv/bin/python /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/test_suite/validate_checkpoint.py /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/temporary_smoke_runs/florence-robotwin-keyframe-isolated-save-reload-smoke/checkpoints 1 >> /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/evidence/florence_keyframe_smoke.jsonl +echo SMOKE florence keyframe phase2 +env PYTHONPATH=/fs1/private/user/liuyu/student-workspace/code/starVLA-loss-is-adapted WANDB_MODE=offline PYTHONUNBUFFERED=1 PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True HF_HOME=/fs1/private/user/liuyu/student-workspace/cache/huggingface HF_HUB_OFFLINE=1 TMPDIR=/fs1/private/user/liuyu/student-workspace/tmp KEYFRAME_APE_CACHE=/fs1/private/user/liuyu/student-workspace/outputs/robotwin_smoke/data/keyframe_ape_full_smoke.npz /fs1/private/user/liuyu/student-workspace/data/runtime/.venv/bin/python -m accelerate.commands.launch --config_file /fs1/private/user/liuyu/student-workspace/code/starVLA-loss-is-adapted/external_baselines/loss_is_adapted/configs/deepspeed_florence_2gpu_sai.yaml --num_processes 2 --mixed_precision bf16 --main_process_port 0 /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/test_suite/smoke_runner.py --method keyframe --entry /fs1/private/user/liuyu/student-workspace/code/starVLA-loss-is-adapted/external_baselines/keyframe_focused_adapted/train_keyframe_continuation.py --config_yaml /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/test_suite/florence_keyframe_phase2.yaml >> /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/logs/keyframe_florence_smoke.log 2>&1 +/fs1/private/user/liuyu/student-workspace/data/runtime/.venv/bin/python /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/test_suite/validate_checkpoint.py /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/temporary_smoke_runs/florence-robotwin-keyframe-isolated-save-reload-smoke/checkpoints 2 >> /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/evidence/florence_keyframe_smoke.jsonl +/fs1/private/user/liuyu/student-workspace/data/runtime/.venv/bin/python /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/test_suite/cleanup_smoke.py florence-robotwin-keyframe-isolated-save-reload-smoke +rm -f -- /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/logs/keyframe_florence_smoke.log +echo SMOKE qwen keyframe phase1 +env PYTHONPATH=/fs1/private/user/liuyu/student-workspace/code/starVLA-qwen-bridge-adapted WANDB_MODE=offline PYTHONUNBUFFERED=1 PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True HF_HOME=/fs1/private/user/liuyu/student-workspace/cache/huggingface HF_HUB_OFFLINE=1 TMPDIR=/fs1/private/user/liuyu/student-workspace/tmp KEYFRAME_APE_CACHE=/fs1/private/user/liuyu/student-workspace/outputs/robotwin_smoke/data/keyframe_ape_full_smoke.npz /fs1/private/user/liuyu/student-workspace/data/runtime/.venv/bin/python -m accelerate.commands.launch --config_file /fs1/private/user/liuyu/student-workspace/code/starVLA-qwen-bridge-adapted/starVLA/config/deepseeds/deepspeed_zero3.yaml --num_processes 2 --mixed_precision bf16 --main_process_port 0 /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/test_suite/smoke_runner.py --method keyframe --entry /fs1/private/user/liuyu/student-workspace/code/starVLA-qwen-bridge-adapted/external_baselines/keyframe_focused_adapted/train_keyframe_continuation.py --config_yaml /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/test_suite/qwen_keyframe_phase1.yaml >> /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/logs/keyframe_qwen_smoke.log 2>&1 +/fs1/private/user/liuyu/student-workspace/data/runtime/.venv/bin/python /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/test_suite/validate_checkpoint.py /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/temporary_smoke_runs/qwen-robotwin-keyframe-isolated-save-reload-smoke/checkpoints 1 >> /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/evidence/qwen_keyframe_smoke.jsonl +echo SMOKE qwen keyframe phase2 +env PYTHONPATH=/fs1/private/user/liuyu/student-workspace/code/starVLA-qwen-bridge-adapted WANDB_MODE=offline PYTHONUNBUFFERED=1 PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True HF_HOME=/fs1/private/user/liuyu/student-workspace/cache/huggingface HF_HUB_OFFLINE=1 TMPDIR=/fs1/private/user/liuyu/student-workspace/tmp KEYFRAME_APE_CACHE=/fs1/private/user/liuyu/student-workspace/outputs/robotwin_smoke/data/keyframe_ape_full_smoke.npz /fs1/private/user/liuyu/student-workspace/data/runtime/.venv/bin/python -m accelerate.commands.launch --config_file /fs1/private/user/liuyu/student-workspace/code/starVLA-qwen-bridge-adapted/starVLA/config/deepseeds/deepspeed_zero3.yaml --num_processes 2 --mixed_precision bf16 --main_process_port 0 /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/test_suite/smoke_runner.py --method keyframe --entry /fs1/private/user/liuyu/student-workspace/code/starVLA-qwen-bridge-adapted/external_baselines/keyframe_focused_adapted/train_keyframe_continuation.py --config_yaml /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/test_suite/qwen_keyframe_phase2.yaml >> /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/logs/keyframe_qwen_smoke.log 2>&1 +/fs1/private/user/liuyu/student-workspace/data/runtime/.venv/bin/python /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/test_suite/validate_checkpoint.py /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/temporary_smoke_runs/qwen-robotwin-keyframe-isolated-save-reload-smoke/checkpoints 2 >> /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/evidence/qwen_keyframe_smoke.jsonl +/fs1/private/user/liuyu/student-workspace/data/runtime/.venv/bin/python /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/test_suite/cleanup_smoke.py qwen-robotwin-keyframe-isolated-save-reload-smoke +rm -f -- /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/logs/keyframe_qwen_smoke.log +echo FORMAL florence keyframe +env PYTHONPATH=/fs1/private/user/liuyu/student-workspace/code/starVLA-loss-is-adapted WANDB_MODE=offline PYTHONUNBUFFERED=1 PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True HF_HOME=/fs1/private/user/liuyu/student-workspace/cache/huggingface HF_HUB_OFFLINE=1 TMPDIR=/fs1/private/user/liuyu/student-workspace/tmp KEYFRAME_APE_CACHE=/fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/cache/keyframe_full.npz /fs1/private/user/liuyu/student-workspace/data/runtime/.venv/bin/python -m accelerate.commands.launch --config_file /fs1/private/user/liuyu/student-workspace/code/starVLA-loss-is-adapted/external_baselines/loss_is_adapted/configs/deepspeed_florence_2gpu_sai.yaml --num_processes 2 --mixed_precision bf16 --main_process_port 0 /fs1/private/user/liuyu/student-workspace/code/starVLA-loss-is-adapted/external_baselines/keyframe_focused_adapted/train_keyframe_continuation.py --config_yaml /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/configs/florence_keyframe_formal.yaml +/fs1/private/user/liuyu/student-workspace/data/runtime/.venv/bin/python /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/test_suite/validate_checkpoint.py /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs/florence-robotwin-keyframe-10000-to-10500-seed484370/checkpoints 10500 > /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/evidence/florence_keyframe_formal_checkpoint.json +echo FORMAL qwen keyframe +env PYTHONPATH=/fs1/private/user/liuyu/student-workspace/code/starVLA-qwen-bridge-adapted WANDB_MODE=offline PYTHONUNBUFFERED=1 PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True HF_HOME=/fs1/private/user/liuyu/student-workspace/cache/huggingface HF_HUB_OFFLINE=1 TMPDIR=/fs1/private/user/liuyu/student-workspace/tmp KEYFRAME_APE_CACHE=/fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/cache/keyframe_full.npz /fs1/private/user/liuyu/student-workspace/data/runtime/.venv/bin/python -m accelerate.commands.launch --config_file /fs1/private/user/liuyu/student-workspace/code/starVLA-qwen-bridge-adapted/starVLA/config/deepseeds/deepspeed_zero3.yaml --num_processes 2 --mixed_precision bf16 --main_process_port 0 /fs1/private/user/liuyu/student-workspace/code/starVLA-qwen-bridge-adapted/external_baselines/keyframe_focused_adapted/train_keyframe_continuation.py --config_yaml /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/configs/qwen_keyframe_formal.yaml +/fs1/private/user/liuyu/student-workspace/data/runtime/.venv/bin/python /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/test_suite/validate_checkpoint.py /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs/qwen-robotwin-keyframe-11000-to-11500-seed484370/checkpoints 11500 > /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/evidence/qwen_keyframe_formal_checkpoint.json +echo PIPELINE_KEYFRAME_PASS diff --git a/settings/florence_robotwin/keyframe/checkpoint.json b/settings/florence_robotwin/keyframe/checkpoint.json new file mode 100644 index 0000000000000000000000000000000000000000..d6d6ce339b666e4d9e4857c87f9b3def1023f108 --- /dev/null +++ b/settings/florence_robotwin/keyframe/checkpoint.json @@ -0,0 +1,65 @@ +{ + "completed_steps": 10500, + "files": [ + { + "bytes": 1525, + "name": "custom_checkpoint_0.pkl", + "sha256": "98276dada2a8c4a6309ae8c0dd0daa358ceac5a1dd932daea9ba4206e3dd5c78" + }, + { + "bytes": 13, + "name": "latest", + "sha256": "68d0ca762047fd48fdd6a06a5e6465209289c028bfd63f5557a110ff1ebc15ac" + }, + { + "bytes": 5509764721, + "name": "pytorch_model/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt", + "sha256": "107aa48c538fae8a89e7adccf5e986cb2b6c25f083360b1a80aac5620b72021f" + }, + { + "bytes": 5509778801, + "name": "pytorch_model/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt", + "sha256": "ae0ef3a9168316a1150abd22e26854e52f9ddaf5133284040ebd79f917fbd74a" + }, + { + "bytes": 1837527613, + "name": "pytorch_model/mp_rank_00_model_states.pt", + "sha256": "398f1ffb755be119c9e545549846654f44615f86e35efbb5507c9ffd852d457c" + }, + { + "bytes": 2152621885, + "name": "pytorch_model.pt", + "sha256": "1f2fcde1201f78dd169c9da1b2c07cc8e5e54720a5d613f9e34dffb2dd194010" + }, + { + "bytes": 15081, + "name": "random_states_0.pkl", + "sha256": "5e1eaba9845ffc86118d3545569c01055097bf21fd0069aeab3224571f4522e4" + }, + { + "bytes": 15081, + "name": "random_states_1.pkl", + "sha256": "4dc72dafa3023634bb820b6e722b6a060c2fd1622d432a9417caefe1c3c7d000" + }, + { + "bytes": 33270, + "name": "zero_to_fp32.py", + "sha256": "e8a2553b434ac043ec63b022db1e768277acf8d4e70f10a2842db6d238977135" + } + ], + "registered_checkpointables": [ + "lr_scheduler" + ], + "resume_identity": { + "config_sha256": "c1763edabdfd663076f244d685e52a432cb1e83c079948a9a7fbc92f09ab4641", + "identity_schema_version": 2, + "manifest_sha256": "dcb728a00da4b4ad7f989fa313d6d3355e13f57efefc7267e9ea092f2ffd8a50", + "model_structure_sha256": "2d1c8fd1dfa358c4975ad54934f42c24c8d10f6fcef9e8f7c2585b349b32d96b", + "proposal_alpha": null, + "proposal_sha256": null, + "seed": 484370, + "total_batch_size": 64 + }, + "schema_version": 1, + "world_size": 2 +} diff --git a/settings/florence_robotwin/keyframe/config.full.yaml b/settings/florence_robotwin/keyframe/config.full.yaml new file mode 100644 index 0000000000000000000000000000000000000000..53cc5d483d36ad039cce362e4117b8979180c99b --- /dev/null +++ b/settings/florence_robotwin/keyframe/config.full.yaml @@ -0,0 +1,112 @@ +run_id: florence-robotwin-keyframe-10000-to-10500-seed484370 +run_root_dir: /home/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs +seed: 484370 +wandb_entity: null +wandb_project: InfoFlow-StarVLA-RoboTwin +is_debug: false +version_id: '0.21' +framework: + name: QwenGR00T + qwenvl: + base_vlm: /home/liuyu/student-workspace/data/runtime/playground/Pretrained_models/Florence-2-large + attn_implementation: eager + vl_hidden_dim: 1024 + selected_view_indices: + - 0 + - 1 + - 2 + compile_vision_tower: true + action_model: + action_model_type: DiT-B + action_hidden_dim: 1024 + hidden_size: 1024 + add_pos_embed: true + max_seq_len: 1024 + action_dim: 14 + state_dim: 0 + action_horizon: 50 + repeated_diffusion_steps: 8 + noise_beta_alpha: 1.5 + noise_beta_beta: 1.0 + noise_s: 0.999 + num_timestep_buckets: 1000 + num_inference_timesteps: 4 + num_target_vision_tokens: 32 + diffusion_model_cfg: + cross_attention_dim: 1024 + dropout: 0.2 + final_dropout: true + interleave_self_attention: true + norm_type: ada_norm + num_layers: 16 + output_dim: 1024 + positional_embeddings: null + future_action_window_size: 49 + past_action_window_size: 0 + dino: + dino_backbone: dinov2_vits14 +datasets: + vla_data: + dataset_py: lerobot_datasets + data_root_dir: /home/liuyu/student-workspace/data/robotwin/RoboTwin + data_mix: robotwin_all_50 + action_type: abs_qpos + action_mode: abs + include_state: false + sequential_step_sampling: false + per_device_batch_size: 2 + load_all_data_for_training: true + image_size: + - 224 + - 224 + video_backend: pyav + num_workers: 4 + expected_world_size: 2 + traceable_sampling: true + dataset_provenance_path: /home/liuyu/student-workspace/data/robotwin/manifests/data_provenance.json + dataset_content_fingerprint_path: /home/liuyu/student-workspace/data/robotwin/manifests/content_fingerprints.json + valid_anchor_manifest_path: /home/liuyu/student-workspace/data/robotwin/manifests/valid_anchor_manifest.json + temporal_proposal_path: null + transform_rng_seed: 289726 +trainer: + use_deepspeed: true + max_train_steps: 150000 + num_warmup_steps: 5000 + save_interval: null + save_steps: + - 10500 + eval_interval: null + repeated_diffusion_steps: 4 + learning_rate: + base: 1.0e-05 + qwen_vl_interface: 1.0e-05 + action_model: 0.0001 + lr_scheduler_type: cosine_with_min_lr + scheduler_specific_kwargs: + min_lr: 5.0e-07 + freeze_modules: '' + trainable_modules: '' + frozen_modules_eval: false + logging_frequency: 50 + gradient_clipping: 1.0 + gradient_accumulation_steps: 16 + save_format: pt + save_final_checkpoint: false + optimizer: + name: AdamW + betas: + - 0.9 + - 0.95 + eps: 1.0e-08 + weight_decay: 1.0e-08 + stop_after_steps: 10500 + is_resume: true + allow_model_only_resume: false + resume_accelerate_checkpoint: /home/liuyu/student-workspace/data/robotwin/common_start/florence-groot-robotwin-xvla3view-official150k-seed42-20260804/checkpoints/steps_10000_accelerate + branch_parent_seed: 42 + branch_rng_seed: 484370 + model_rng_seed: 721927 + evaluation_snapshot_interval: null + sample_loss_weight_key: keyframe_weight +config_yaml: /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/configs/florence_keyframe_formal.yaml +output_dir: /home/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs/florence-robotwin-keyframe-10000-to-10500-seed484370 diff --git a/settings/florence_robotwin/keyframe/config.yaml b/settings/florence_robotwin/keyframe/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b59e2c95d3952e6e9560968f5310805998c695a1 --- /dev/null +++ b/settings/florence_robotwin/keyframe/config.yaml @@ -0,0 +1,69 @@ +datasets: + vla_data: + action_mode: abs + data_mix: robotwin_all_50 + data_root_dir: /home/liuyu/student-workspace/data/robotwin/RoboTwin + dataset_content_fingerprint_path: /home/liuyu/student-workspace/data/robotwin/manifests/content_fingerprints.json + dataset_provenance_path: /home/liuyu/student-workspace/data/robotwin/manifests/data_provenance.json + dataset_py: lerobot_datasets + expected_world_size: 2 + num_workers: 4 + per_device_batch_size: 2 + temporal_proposal_path: null + traceable_sampling: true + valid_anchor_manifest_path: /home/liuyu/student-workspace/data/robotwin/manifests/valid_anchor_manifest.json + video_backend: pyav +framework: + action_model: + action_dim: 14 + action_horizon: 50 + action_model_type: DiT-B + add_pos_embed: true + diffusion_model_cfg: + cross_attention_dim: 1024 + dropout: 0.2 + final_dropout: true + interleave_self_attention: true + norm_type: ada_norm + num_layers: 16 + output_dim: 1024 + positional_embeddings: null + hidden_size: 1024 + max_seq_len: 1024 + noise_beta_alpha: 1.5 + noise_beta_beta: 1.0 + num_inference_timesteps: 4 + num_target_vision_tokens: 32 + num_timestep_buckets: 1000 + state_dim: 0 + name: QwenGR00T + qwenvl: + base_vlm: /home/liuyu/student-workspace/data/runtime/playground/Pretrained_models/Florence-2-large + compile_vision_tower: true + selected_view_indices: + - 0 + - 1 + - 2 +output_dir: /home/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs/florence-robotwin-keyframe-10000-to-10500-seed484370 +run_id: florence-robotwin-keyframe-10000-to-10500-seed484370 +run_root_dir: /home/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs +seed: 484370 +trainer: + freeze_modules: '' + gradient_accumulation_steps: 16 + learning_rate: + action_model: 0.0001 + base: 1.0e-05 + qwen_vl_interface: 1.0e-05 + lr_scheduler_type: cosine_with_min_lr + max_train_steps: 150000 + num_warmup_steps: 5000 + optimizer: + betas: + - 0.9 + - 0.95 + eps: 1.0e-08 + name: AdamW + weight_decay: 1.0e-08 + scheduler_specific_kwargs: + min_lr: 5.0e-07 diff --git a/settings/florence_robotwin/keyframe/dataset_statistics.json b/settings/florence_robotwin/keyframe/dataset_statistics.json new file mode 100644 index 0000000000000000000000000000000000000000..c920d76f3fc976e9e0b30e36eab0cff8a6bf1adb --- /dev/null +++ b/settings/florence_robotwin/keyframe/dataset_statistics.json @@ -0,0 +1,218 @@ +{ + "new_embodiment": { + "action": { + "mean": [ + -0.22423998869955536, + 1.0729362124204633, + 0.775079609751701, + -0.32701663097366684, + 0.06159586461764771, + -0.06420705380151046, + 0.22430071420967568, + 1.1274866509437556, + 0.831773659586906, + -0.3638282522559164, + -0.024959841337949897, + 0.05302203213563182, + 0.6774904182553295, + 0.6598676902055738 + ], + "std": [ + 0.39622931737429123, + 1.003590449339515, + 0.7811611351334437, + 0.6707485714474849, + 0.2686908204983921, + 0.6492019642472782, + 0.34733679400923234, + 1.0176226676795115, + 0.8039915805960646, + 0.7067249045233707, + 0.26259645177151686, + 0.6808759458519371, + 0.4482151151114418, + 0.4539921671160285 + ], + "max": [ + 6.45659065246582, + 4.179152488708496, + 5.346591472625732, + 1.7942548990249634, + 1.8604620695114136, + 5.43813943862915, + 7.142920017242432, + 4.157068729400635, + 5.672450065612793, + 1.95806884765625, + 1.5663840770721436, + 5.278968811035156, + 1.0, + 1.0 + ], + "min": [ + -7.552278995513916, + -0.36354875564575195, + -0.18577136099338531, + -1.956291913986206, + -1.6801013946533203, + -7.678869724273682, + -8.539926528930664, + -0.6294453740119934, + -0.07775841653347015, + -2.1328067779541016, + -2.1285502910614014, + -8.228925704956055, + 0.0, + 0.0 + ], + "q01": [ + -7.334665780067444, + -5.257390398583084e-07, + -2.8215323254698887e-05, + -1.841589093208313, + -1.2851550579071045, + -6.268352918624878, + -5.995517673492431, + -0.572821855545044, + -2.81171942333458e-05, + -1.8467556238174438, + -1.4545854091644288, + -5.945669202804565, + 0.0, + 0.0 + ], + "q99": [ + 5.016749763488781, + 3.4792869091033936, + 4.225283479690551, + 1.7064905214309687, + 1.4829185009002717, + 3.7177388882636553, + 1.258314609527588, + 2.967935800552368, + 3.2566452693939207, + 1.766038613319397, + 1.4066662788391113, + 3.5830023288726807, + 1.0, + 1.0 + ], + "mask": [ + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + false, + false + ] + }, + "state": { + "mean": [ + -0.22281882727518673, + 1.0668349397182466, + 0.770632639825344, + -0.3252330579422412, + 0.06118139047002841, + -0.06316296239456277, + 0.22253762468695631, + 1.1201750242710113, + 0.8263923591375351, + -0.36159489695448405, + -0.024702461528722772, + 0.05203246331657278, + 0.6793262696266176, + 0.6616973677277563 + ], + "std": [ + 0.39522870886731665, + 1.004018577033247, + 0.7809103811570285, + 0.6688088363177986, + 0.26775723818956776, + 0.6463212583712935, + 0.34648488321363663, + 1.018232630032839, + 0.8038798397682734, + 0.7045612787595747, + 0.26143414213134614, + 0.6774619870452644, + 0.44749220497027625, + 0.45333916022037934 + ], + "max": [ + 6.45659065246582, + 4.179152488708496, + 5.346591472625732, + 1.7942548990249634, + 1.8604620695114136, + 5.43813943862915, + 7.142920017242432, + 4.157068729400635, + 5.672450065612793, + 1.95806884765625, + 1.5663840770721436, + 5.278968811035156, + 1.0, + 1.0 + ], + "min": [ + -7.552278995513916, + -0.36354875564575195, + -0.18577136099338531, + -1.956291913986206, + -1.6801013946533203, + -7.678869724273682, + -8.539926528930664, + -0.6294453740119934, + -0.07775841653347015, + -2.1328067779541016, + -2.1285502910614014, + -8.228925704956055, + 0.0, + 0.0 + ], + "q01": [ + -7.334665780067444, + -5.257390398583084e-07, + -2.8215323254698887e-05, + -1.841589093208313, + -1.2851550579071045, + -6.268109321594238, + -5.912737417221069, + -0.572821855545044, + -2.81171942333458e-05, + -1.8467556238174438, + -1.440324923992157, + -5.8623305034637445, + 0.0, + 0.0 + ], + "q99": [ + 4.902149534225477, + 3.4775466918945312, + 4.224453401565551, + 1.7064905214309687, + 1.480757713317871, + 3.609884884357428, + 1.258314609527588, + 2.967935800552368, + 3.2566452693939207, + 1.7433889162540437, + 1.4066662788391113, + 3.581243851184845, + 1.0, + 1.0 + ] + }, + "num_transitions": 6075103, + "num_trajectories": 27500 + } +} \ No newline at end of file diff --git a/settings/florence_robotwin/keyframe/external_resume_certificate.json b/settings/florence_robotwin/keyframe/external_resume_certificate.json new file mode 100644 index 0000000000000000000000000000000000000000..fb328d39f46f24919ff5ca9a0e1c2c82e82057b0 --- /dev/null +++ b/settings/florence_robotwin/keyframe/external_resume_certificate.json @@ -0,0 +1,138 @@ +{ + "branch_resume_identity": { + "config_sha256": "c1763edabdfd663076f244d685e52a432cb1e83c079948a9a7fbc92f09ab4641", + "identity_schema_version": 2, + "manifest_sha256": "dcb728a00da4b4ad7f989fa313d6d3355e13f57efefc7267e9ea092f2ffd8a50", + "model_structure_sha256": "2d1c8fd1dfa358c4975ad54934f42c24c8d10f6fcef9e8f7c2585b349b32d96b", + "proposal_alpha": null, + "proposal_sha256": null, + "seed": 484370, + "total_batch_size": 64 + }, + "completed_steps": 10000, + "optimizer_param_groups": [ + { + "betas": [ + 0.9, + 0.95 + ], + "eps": 1e-08, + "initial_lr": 1e-05, + "lr": 9.97215529648321e-06, + "name": "qwen_vl_interface", + "weight_decay": 1e-08 + }, + { + "betas": [ + 0.9, + 0.95 + ], + "eps": 1e-08, + "initial_lr": 0.0001, + "lr": 9.972155296483209e-05, + "name": "action_model", + "weight_decay": 1e-08 + } + ], + "optimizer_state_entries": 4, + "parent_checkpoint": "/fs1/private/user/liuyu/student-workspace/data/robotwin/common_start/florence-groot-robotwin-xvla3view-official150k-seed42-20260804/checkpoints/steps_10000_accelerate", + "parent_checkpoint_commit_sha256": "f6f6408ead21ba9cb0d7bbfa26c74733950a504264a26e5b795129ffadff6958", + "parent_checkpoint_files": [ + { + "bytes": 1525, + "name": "custom_checkpoint_0.pkl", + "sha256": "031a5046639dfc967eebbf30b7184dbe2619e26675a74569827aa5edaf6cc85c" + }, + { + "bytes": 13, + "name": "latest", + "sha256": "68d0ca762047fd48fdd6a06a5e6465209289c028bfd63f5557a110ff1ebc15ac" + }, + { + "bytes": 5509764721, + "name": "pytorch_model/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt", + "sha256": "27d9b011b75f40c3d896c9681ee883eeb3d2277f201889faebf0b7fcf5492c52" + }, + { + "bytes": 5509778801, + "name": "pytorch_model/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt", + "sha256": "f657f6f6fb48956b2b406b79511e16b9c2712dc95216b78ba5179818b821d18f" + }, + { + "bytes": 1837527613, + "name": "pytorch_model/mp_rank_00_model_states.pt", + "sha256": "3d66f8b6c0e344c34f06d3b4708c3292cbb8a84ea5a25e650fb64b679a44eb3b" + }, + { + "bytes": 2152621885, + "name": "pytorch_model.pt", + "sha256": "4b6c4f47df7b0f05b8c9287e480d88f2b5e711b4fb6cccbdfffd677ed9bff778" + }, + { + "bytes": 15017, + "name": "random_states_0.pkl", + "sha256": "629d14d87e3cab680e32acec92c80c2578a7fbd1ebc0dee5475e774058f11d41" + }, + { + "bytes": 15081, + "name": "random_states_1.pkl", + "sha256": "3cb98283ee7eaaedcee320e8e3b17e74f2f01a22d9e3b958fa2422a86215c652" + }, + { + "bytes": 33270, + "name": "zero_to_fp32.py", + "sha256": "e8a2553b434ac043ec63b022db1e768277acf8d4e70f10a2842db6d238977135" + } + ], + "parent_checkpoint_files_sha256": "1f8985d0633f16ac3b0381b128cdb4e479c4fdfd67ff15d58736633e4b942bc0", + "parent_checkpoint_integrity_manifest_sha256": null, + "parent_config_sha256": "d35ab9e43634c7c4a74c5299657df3c17c7d78407dd0fe3d37296a927e353d28", + "parent_resume_identity": { + "config_sha256": "2440c6327cf5e8c14efc93b800209171c725dae0898181921c62f3490eb8378f", + "identity_schema_version": 2, + "manifest_sha256": null, + "model_structure_sha256": "2d1c8fd1dfa358c4975ad54934f42c24c8d10f6fcef9e8f7c2585b349b32d96b", + "proposal_alpha": null, + "proposal_sha256": null, + "seed": 42, + "total_batch_size": 64 + }, + "proposal_sha256": null, + "rank_state_summaries": [ + { + "optimizer_state_entries": 2, + "optimizer_state_sha256": "5a35add2c00249e41e72e1ab16261d87f205bfeaf23386c7263223e4ce168368", + "rank": 0, + "trainable_state_sha256": "667c6a6a7aa0dc49c32f7bef693f3272efee46064fda5d370d25788291da76a0" + }, + { + "optimizer_state_entries": 2, + "optimizer_state_sha256": "073f849cd1942522dbfde195bdc29e8ee015e4c5d4be3070381027a13b9784cf", + "rank": 1, + "trainable_state_sha256": "667c6a6a7aa0dc49c32f7bef693f3272efee46064fda5d370d25788291da76a0" + } + ], + "repeated_diffusion_steps": 4, + "rng_streams": { + "model_base": 721927, + "sampler": 484370, + "transform": 289726 + }, + "scheduler": { + "class": "LambdaLR", + "last_epoch": 10000, + "last_lr": [ + 9.97215529648321e-06, + 9.972155296483209e-05 + ], + "step_count": 10001 + }, + "scheduler_resume_mode": "accelerate_checkpoint", + "schema_version": 1, + "status": "exact_external_resume_loaded", + "total_batch_size": 64, + "trainable_parameter_elements": 918283022, + "trainable_parameter_sha256": "2e8f3de6aeee46bca56fab5993103ad30302b45330037cf98f96ffd472f0e9ce", + "trainable_parameter_tensors": 1161, + "world_size": 2 +} diff --git a/settings/florence_robotwin/keyframe/initialization_manifest.json b/settings/florence_robotwin/keyframe/initialization_manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..a67d8f6053e29aebeabfc485e6a85a2e79811fae --- /dev/null +++ b/settings/florence_robotwin/keyframe/initialization_manifest.json @@ -0,0 +1,6 @@ +{ + "schema_version": 1, + "initialization_seed": 484370, + "action_model_sha256": "0b5e42abcce1d0cc6aa164e9bb09ec77e0ea1c6719bb8dc55406f806c5d22b99", + "action_model_parameter_elements": 141815566 +} diff --git a/settings/florence_robotwin/keyframe/valid_anchor_manifest.json b/settings/florence_robotwin/keyframe/valid_anchor_manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..3f5c91478b5931286c9c84d021a7ffc10a0fb5c0 --- /dev/null +++ b/settings/florence_robotwin/keyframe/valid_anchor_manifest.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2090f6b9d7e1bc5ef41897436690f031c187d625fa6b71ae0c54f3793660c757 +size 107646168 diff --git a/settings/florence_robotwin/loss_is/checkpoint.json b/settings/florence_robotwin/loss_is/checkpoint.json new file mode 100644 index 0000000000000000000000000000000000000000..0ebb3e29200eff1f3e99b26d5531a1ed9d55a259 --- /dev/null +++ b/settings/florence_robotwin/loss_is/checkpoint.json @@ -0,0 +1,65 @@ +{ + "completed_steps": 10500, + "files": [ + { + "bytes": 1525, + "name": "custom_checkpoint_0.pkl", + "sha256": "98276dada2a8c4a6309ae8c0dd0daa358ceac5a1dd932daea9ba4206e3dd5c78" + }, + { + "bytes": 13, + "name": "latest", + "sha256": "68d0ca762047fd48fdd6a06a5e6465209289c028bfd63f5557a110ff1ebc15ac" + }, + { + "bytes": 5509764721, + "name": "pytorch_model/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt", + "sha256": "a48c484b90407db991551c5aba01d783688e2d6ef79f07e2f184fc51cd914aa1" + }, + { + "bytes": 5509778801, + "name": "pytorch_model/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt", + "sha256": "4831d734987d54d691c5416b49f512d5a451d80108d1119c2c9dd560701acee8" + }, + { + "bytes": 1837527613, + "name": "pytorch_model/mp_rank_00_model_states.pt", + "sha256": "ee2458a823635528faf0677265bf06be7318482317f0f5d5a21089bfe6abdb6c" + }, + { + "bytes": 2152621885, + "name": "pytorch_model.pt", + "sha256": "e2b0d1c85f74ac1d72b2f6704e3d6acd1fd7595a5801629d6f96ba2246028ec7" + }, + { + "bytes": 15081, + "name": "random_states_0.pkl", + "sha256": "d4b7ae7e161085d27c793e70e0412ead82f4c8bcd29ec402aa119e8886bd7a8a" + }, + { + "bytes": 15081, + "name": "random_states_1.pkl", + "sha256": "4dc72dafa3023634bb820b6e722b6a060c2fd1622d432a9417caefe1c3c7d000" + }, + { + "bytes": 33270, + "name": "zero_to_fp32.py", + "sha256": "e8a2553b434ac043ec63b022db1e768277acf8d4e70f10a2842db6d238977135" + } + ], + "registered_checkpointables": [ + "lr_scheduler" + ], + "resume_identity": { + "config_sha256": "c1763edabdfd663076f244d685e52a432cb1e83c079948a9a7fbc92f09ab4641", + "identity_schema_version": 2, + "manifest_sha256": "dcb728a00da4b4ad7f989fa313d6d3355e13f57efefc7267e9ea092f2ffd8a50", + "model_structure_sha256": "2d1c8fd1dfa358c4975ad54934f42c24c8d10f6fcef9e8f7c2585b349b32d96b", + "proposal_alpha": null, + "proposal_sha256": null, + "seed": 484370, + "total_batch_size": 64 + }, + "schema_version": 1, + "world_size": 2 +} diff --git a/settings/florence_robotwin/loss_is/config.full.yaml b/settings/florence_robotwin/loss_is/config.full.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4a8b22148e8de22b9f87358a8cefa77cbaeb4bca --- /dev/null +++ b/settings/florence_robotwin/loss_is/config.full.yaml @@ -0,0 +1,118 @@ +run_id: florence-robotwin-loss_is-10000-to-10500-seed484370 +run_root_dir: /home/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs +seed: 484370 +wandb_entity: null +wandb_project: InfoFlow-StarVLA-RoboTwin +is_debug: false +version_id: '0.21' +framework: + name: QwenGR00T + qwenvl: + base_vlm: /home/liuyu/student-workspace/data/runtime/playground/Pretrained_models/Florence-2-large + attn_implementation: eager + vl_hidden_dim: 1024 + selected_view_indices: + - 0 + - 1 + - 2 + compile_vision_tower: true + action_model: + action_model_type: DiT-B + action_hidden_dim: 1024 + hidden_size: 1024 + add_pos_embed: true + max_seq_len: 1024 + action_dim: 14 + state_dim: 0 + action_horizon: 50 + repeated_diffusion_steps: 8 + noise_beta_alpha: 1.5 + noise_beta_beta: 1.0 + noise_s: 0.999 + num_timestep_buckets: 1000 + num_inference_timesteps: 4 + num_target_vision_tokens: 32 + diffusion_model_cfg: + cross_attention_dim: 1024 + dropout: 0.2 + final_dropout: true + interleave_self_attention: true + norm_type: ada_norm + num_layers: 16 + output_dim: 1024 + positional_embeddings: null + future_action_window_size: 49 + past_action_window_size: 0 + dino: + dino_backbone: dinov2_vits14 +datasets: + vla_data: + dataset_py: lerobot_datasets + data_root_dir: /home/liuyu/student-workspace/data/robotwin/RoboTwin + data_mix: robotwin_all_50 + action_type: abs_qpos + action_mode: abs + include_state: false + sequential_step_sampling: false + per_device_batch_size: 2 + load_all_data_for_training: true + image_size: + - 224 + - 224 + video_backend: pyav + num_workers: 4 + expected_world_size: 2 + traceable_sampling: true + dataset_provenance_path: /home/liuyu/student-workspace/data/robotwin/manifests/data_provenance.json + dataset_content_fingerprint_path: /home/liuyu/student-workspace/data/robotwin/manifests/content_fingerprints.json + valid_anchor_manifest_path: /home/liuyu/student-workspace/data/robotwin/manifests/valid_anchor_manifest.json + temporal_proposal_path: null + transform_rng_seed: 289726 +trainer: + use_deepspeed: true + max_train_steps: 150000 + num_warmup_steps: 5000 + save_interval: null + save_steps: + - 10500 + eval_interval: null + repeated_diffusion_steps: 4 + learning_rate: + base: 1.0e-05 + qwen_vl_interface: 1.0e-05 + action_model: 0.0001 + lr_scheduler_type: cosine_with_min_lr + scheduler_specific_kwargs: + min_lr: 5.0e-07 + freeze_modules: '' + trainable_modules: '' + frozen_modules_eval: false + logging_frequency: 50 + gradient_clipping: 1.0 + gradient_accumulation_steps: 16 + save_format: pt + save_final_checkpoint: false + optimizer: + name: AdamW + betas: + - 0.9 + - 0.95 + eps: 1.0e-08 + weight_decay: 1.0e-08 + stop_after_steps: 10500 + is_resume: true + allow_model_only_resume: false + resume_accelerate_checkpoint: /home/liuyu/student-workspace/data/robotwin/common_start/florence-groot-robotwin-xvla3view-official150k-seed42-20260804/checkpoints/steps_10000_accelerate + branch_parent_seed: 42 + branch_rng_seed: 484370 + model_rng_seed: 721927 + evaluation_snapshot_interval: null + sample_loss_weight_key: loss_is_importance_weight + model_rng_trace_key: loss_is_occurrence_id + loss_is: + candidate_factor: 3 + seed: 484370 + condition_threshold: 2.0 + condition_momentum: 0.9 +config_yaml: /home/liuyu/student-workspace/outputs/robotwin_external_20260831/configs/florence_loss_is_formal.yaml +output_dir: /home/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs/florence-robotwin-loss_is-10000-to-10500-seed484370 diff --git a/settings/florence_robotwin/loss_is/config.yaml b/settings/florence_robotwin/loss_is/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6a99e70de9b473b4c5ba899ae655d6ac31b09688 --- /dev/null +++ b/settings/florence_robotwin/loss_is/config.yaml @@ -0,0 +1,74 @@ +datasets: + vla_data: + action_mode: abs + data_mix: robotwin_all_50 + data_root_dir: /home/liuyu/student-workspace/data/robotwin/RoboTwin + dataset_content_fingerprint_path: /home/liuyu/student-workspace/data/robotwin/manifests/content_fingerprints.json + dataset_provenance_path: /home/liuyu/student-workspace/data/robotwin/manifests/data_provenance.json + dataset_py: lerobot_datasets + expected_world_size: 2 + num_workers: 4 + per_device_batch_size: 2 + temporal_proposal_path: null + traceable_sampling: true + valid_anchor_manifest_path: /home/liuyu/student-workspace/data/robotwin/manifests/valid_anchor_manifest.json + video_backend: pyav +framework: + action_model: + action_dim: 14 + action_horizon: 50 + action_model_type: DiT-B + add_pos_embed: true + diffusion_model_cfg: + cross_attention_dim: 1024 + dropout: 0.2 + final_dropout: true + interleave_self_attention: true + norm_type: ada_norm + num_layers: 16 + output_dim: 1024 + positional_embeddings: null + hidden_size: 1024 + max_seq_len: 1024 + noise_beta_alpha: 1.5 + noise_beta_beta: 1.0 + num_inference_timesteps: 4 + num_target_vision_tokens: 32 + num_timestep_buckets: 1000 + state_dim: 0 + name: QwenGR00T + qwenvl: + base_vlm: /home/liuyu/student-workspace/data/runtime/playground/Pretrained_models/Florence-2-large + compile_vision_tower: true + selected_view_indices: + - 0 + - 1 + - 2 +output_dir: /home/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs/florence-robotwin-loss_is-10000-to-10500-seed484370 +run_id: florence-robotwin-loss_is-10000-to-10500-seed484370 +run_root_dir: /home/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs +seed: 484370 +trainer: + freeze_modules: '' + gradient_accumulation_steps: 16 + learning_rate: + action_model: 0.0001 + base: 1.0e-05 + qwen_vl_interface: 1.0e-05 + loss_is: + candidate_factor: 3 + condition_momentum: 0.9 + condition_threshold: 2.0 + seed: 484370 + lr_scheduler_type: cosine_with_min_lr + max_train_steps: 150000 + num_warmup_steps: 5000 + optimizer: + betas: + - 0.9 + - 0.95 + eps: 1.0e-08 + name: AdamW + weight_decay: 1.0e-08 + scheduler_specific_kwargs: + min_lr: 5.0e-07 diff --git a/settings/florence_robotwin/loss_is/dataset_statistics.json b/settings/florence_robotwin/loss_is/dataset_statistics.json new file mode 100644 index 0000000000000000000000000000000000000000..c920d76f3fc976e9e0b30e36eab0cff8a6bf1adb --- /dev/null +++ b/settings/florence_robotwin/loss_is/dataset_statistics.json @@ -0,0 +1,218 @@ +{ + "new_embodiment": { + "action": { + "mean": [ + -0.22423998869955536, + 1.0729362124204633, + 0.775079609751701, + -0.32701663097366684, + 0.06159586461764771, + -0.06420705380151046, + 0.22430071420967568, + 1.1274866509437556, + 0.831773659586906, + -0.3638282522559164, + -0.024959841337949897, + 0.05302203213563182, + 0.6774904182553295, + 0.6598676902055738 + ], + "std": [ + 0.39622931737429123, + 1.003590449339515, + 0.7811611351334437, + 0.6707485714474849, + 0.2686908204983921, + 0.6492019642472782, + 0.34733679400923234, + 1.0176226676795115, + 0.8039915805960646, + 0.7067249045233707, + 0.26259645177151686, + 0.6808759458519371, + 0.4482151151114418, + 0.4539921671160285 + ], + "max": [ + 6.45659065246582, + 4.179152488708496, + 5.346591472625732, + 1.7942548990249634, + 1.8604620695114136, + 5.43813943862915, + 7.142920017242432, + 4.157068729400635, + 5.672450065612793, + 1.95806884765625, + 1.5663840770721436, + 5.278968811035156, + 1.0, + 1.0 + ], + "min": [ + -7.552278995513916, + -0.36354875564575195, + -0.18577136099338531, + -1.956291913986206, + -1.6801013946533203, + -7.678869724273682, + -8.539926528930664, + -0.6294453740119934, + -0.07775841653347015, + -2.1328067779541016, + -2.1285502910614014, + -8.228925704956055, + 0.0, + 0.0 + ], + "q01": [ + -7.334665780067444, + -5.257390398583084e-07, + -2.8215323254698887e-05, + -1.841589093208313, + -1.2851550579071045, + -6.268352918624878, + -5.995517673492431, + -0.572821855545044, + -2.81171942333458e-05, + -1.8467556238174438, + -1.4545854091644288, + -5.945669202804565, + 0.0, + 0.0 + ], + "q99": [ + 5.016749763488781, + 3.4792869091033936, + 4.225283479690551, + 1.7064905214309687, + 1.4829185009002717, + 3.7177388882636553, + 1.258314609527588, + 2.967935800552368, + 3.2566452693939207, + 1.766038613319397, + 1.4066662788391113, + 3.5830023288726807, + 1.0, + 1.0 + ], + "mask": [ + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + false, + false + ] + }, + "state": { + "mean": [ + -0.22281882727518673, + 1.0668349397182466, + 0.770632639825344, + -0.3252330579422412, + 0.06118139047002841, + -0.06316296239456277, + 0.22253762468695631, + 1.1201750242710113, + 0.8263923591375351, + -0.36159489695448405, + -0.024702461528722772, + 0.05203246331657278, + 0.6793262696266176, + 0.6616973677277563 + ], + "std": [ + 0.39522870886731665, + 1.004018577033247, + 0.7809103811570285, + 0.6688088363177986, + 0.26775723818956776, + 0.6463212583712935, + 0.34648488321363663, + 1.018232630032839, + 0.8038798397682734, + 0.7045612787595747, + 0.26143414213134614, + 0.6774619870452644, + 0.44749220497027625, + 0.45333916022037934 + ], + "max": [ + 6.45659065246582, + 4.179152488708496, + 5.346591472625732, + 1.7942548990249634, + 1.8604620695114136, + 5.43813943862915, + 7.142920017242432, + 4.157068729400635, + 5.672450065612793, + 1.95806884765625, + 1.5663840770721436, + 5.278968811035156, + 1.0, + 1.0 + ], + "min": [ + -7.552278995513916, + -0.36354875564575195, + -0.18577136099338531, + -1.956291913986206, + -1.6801013946533203, + -7.678869724273682, + -8.539926528930664, + -0.6294453740119934, + -0.07775841653347015, + -2.1328067779541016, + -2.1285502910614014, + -8.228925704956055, + 0.0, + 0.0 + ], + "q01": [ + -7.334665780067444, + -5.257390398583084e-07, + -2.8215323254698887e-05, + -1.841589093208313, + -1.2851550579071045, + -6.268109321594238, + -5.912737417221069, + -0.572821855545044, + -2.81171942333458e-05, + -1.8467556238174438, + -1.440324923992157, + -5.8623305034637445, + 0.0, + 0.0 + ], + "q99": [ + 4.902149534225477, + 3.4775466918945312, + 4.224453401565551, + 1.7064905214309687, + 1.480757713317871, + 3.609884884357428, + 1.258314609527588, + 2.967935800552368, + 3.2566452693939207, + 1.7433889162540437, + 1.4066662788391113, + 3.581243851184845, + 1.0, + 1.0 + ] + }, + "num_transitions": 6075103, + "num_trajectories": 27500 + } +} \ No newline at end of file diff --git a/settings/florence_robotwin/loss_is/external_resume_certificate.json b/settings/florence_robotwin/loss_is/external_resume_certificate.json new file mode 100644 index 0000000000000000000000000000000000000000..fb328d39f46f24919ff5ca9a0e1c2c82e82057b0 --- /dev/null +++ b/settings/florence_robotwin/loss_is/external_resume_certificate.json @@ -0,0 +1,138 @@ +{ + "branch_resume_identity": { + "config_sha256": "c1763edabdfd663076f244d685e52a432cb1e83c079948a9a7fbc92f09ab4641", + "identity_schema_version": 2, + "manifest_sha256": "dcb728a00da4b4ad7f989fa313d6d3355e13f57efefc7267e9ea092f2ffd8a50", + "model_structure_sha256": "2d1c8fd1dfa358c4975ad54934f42c24c8d10f6fcef9e8f7c2585b349b32d96b", + "proposal_alpha": null, + "proposal_sha256": null, + "seed": 484370, + "total_batch_size": 64 + }, + "completed_steps": 10000, + "optimizer_param_groups": [ + { + "betas": [ + 0.9, + 0.95 + ], + "eps": 1e-08, + "initial_lr": 1e-05, + "lr": 9.97215529648321e-06, + "name": "qwen_vl_interface", + "weight_decay": 1e-08 + }, + { + "betas": [ + 0.9, + 0.95 + ], + "eps": 1e-08, + "initial_lr": 0.0001, + "lr": 9.972155296483209e-05, + "name": "action_model", + "weight_decay": 1e-08 + } + ], + "optimizer_state_entries": 4, + "parent_checkpoint": "/fs1/private/user/liuyu/student-workspace/data/robotwin/common_start/florence-groot-robotwin-xvla3view-official150k-seed42-20260804/checkpoints/steps_10000_accelerate", + "parent_checkpoint_commit_sha256": "f6f6408ead21ba9cb0d7bbfa26c74733950a504264a26e5b795129ffadff6958", + "parent_checkpoint_files": [ + { + "bytes": 1525, + "name": "custom_checkpoint_0.pkl", + "sha256": "031a5046639dfc967eebbf30b7184dbe2619e26675a74569827aa5edaf6cc85c" + }, + { + "bytes": 13, + "name": "latest", + "sha256": "68d0ca762047fd48fdd6a06a5e6465209289c028bfd63f5557a110ff1ebc15ac" + }, + { + "bytes": 5509764721, + "name": "pytorch_model/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt", + "sha256": "27d9b011b75f40c3d896c9681ee883eeb3d2277f201889faebf0b7fcf5492c52" + }, + { + "bytes": 5509778801, + "name": "pytorch_model/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt", + "sha256": "f657f6f6fb48956b2b406b79511e16b9c2712dc95216b78ba5179818b821d18f" + }, + { + "bytes": 1837527613, + "name": "pytorch_model/mp_rank_00_model_states.pt", + "sha256": "3d66f8b6c0e344c34f06d3b4708c3292cbb8a84ea5a25e650fb64b679a44eb3b" + }, + { + "bytes": 2152621885, + "name": "pytorch_model.pt", + "sha256": "4b6c4f47df7b0f05b8c9287e480d88f2b5e711b4fb6cccbdfffd677ed9bff778" + }, + { + "bytes": 15017, + "name": "random_states_0.pkl", + "sha256": "629d14d87e3cab680e32acec92c80c2578a7fbd1ebc0dee5475e774058f11d41" + }, + { + "bytes": 15081, + "name": "random_states_1.pkl", + "sha256": "3cb98283ee7eaaedcee320e8e3b17e74f2f01a22d9e3b958fa2422a86215c652" + }, + { + "bytes": 33270, + "name": "zero_to_fp32.py", + "sha256": "e8a2553b434ac043ec63b022db1e768277acf8d4e70f10a2842db6d238977135" + } + ], + "parent_checkpoint_files_sha256": "1f8985d0633f16ac3b0381b128cdb4e479c4fdfd67ff15d58736633e4b942bc0", + "parent_checkpoint_integrity_manifest_sha256": null, + "parent_config_sha256": "d35ab9e43634c7c4a74c5299657df3c17c7d78407dd0fe3d37296a927e353d28", + "parent_resume_identity": { + "config_sha256": "2440c6327cf5e8c14efc93b800209171c725dae0898181921c62f3490eb8378f", + "identity_schema_version": 2, + "manifest_sha256": null, + "model_structure_sha256": "2d1c8fd1dfa358c4975ad54934f42c24c8d10f6fcef9e8f7c2585b349b32d96b", + "proposal_alpha": null, + "proposal_sha256": null, + "seed": 42, + "total_batch_size": 64 + }, + "proposal_sha256": null, + "rank_state_summaries": [ + { + "optimizer_state_entries": 2, + "optimizer_state_sha256": "5a35add2c00249e41e72e1ab16261d87f205bfeaf23386c7263223e4ce168368", + "rank": 0, + "trainable_state_sha256": "667c6a6a7aa0dc49c32f7bef693f3272efee46064fda5d370d25788291da76a0" + }, + { + "optimizer_state_entries": 2, + "optimizer_state_sha256": "073f849cd1942522dbfde195bdc29e8ee015e4c5d4be3070381027a13b9784cf", + "rank": 1, + "trainable_state_sha256": "667c6a6a7aa0dc49c32f7bef693f3272efee46064fda5d370d25788291da76a0" + } + ], + "repeated_diffusion_steps": 4, + "rng_streams": { + "model_base": 721927, + "sampler": 484370, + "transform": 289726 + }, + "scheduler": { + "class": "LambdaLR", + "last_epoch": 10000, + "last_lr": [ + 9.97215529648321e-06, + 9.972155296483209e-05 + ], + "step_count": 10001 + }, + "scheduler_resume_mode": "accelerate_checkpoint", + "schema_version": 1, + "status": "exact_external_resume_loaded", + "total_batch_size": 64, + "trainable_parameter_elements": 918283022, + "trainable_parameter_sha256": "2e8f3de6aeee46bca56fab5993103ad30302b45330037cf98f96ffd472f0e9ce", + "trainable_parameter_tensors": 1161, + "world_size": 2 +} diff --git a/settings/florence_robotwin/loss_is/initialization_manifest.json b/settings/florence_robotwin/loss_is/initialization_manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..a67d8f6053e29aebeabfc485e6a85a2e79811fae --- /dev/null +++ b/settings/florence_robotwin/loss_is/initialization_manifest.json @@ -0,0 +1,6 @@ +{ + "schema_version": 1, + "initialization_seed": 484370, + "action_model_sha256": "0b5e42abcce1d0cc6aa164e9bb09ec77e0ea1c6719bb8dc55406f806c5d22b99", + "action_model_parameter_elements": 141815566 +} diff --git a/settings/florence_robotwin/loss_is/loss_is_state.json b/settings/florence_robotwin/loss_is/loss_is_state.json new file mode 100644 index 0000000000000000000000000000000000000000..b4ecf487f5817c588335d0c8cf77cce8707a4eb5 --- /dev/null +++ b/settings/florence_robotwin/loss_is/loss_is_state.json @@ -0,0 +1,20 @@ +{ + "active_for_completed_step": false, + "candidate_factor": 3, + "completed_optimizer_step": 10500, + "condition": { + "momentum": 0.9, + "previous_value": 1.0878882060540187, + "threshold": 2.0, + "value": 1.0870531817606985 + }, + "metrics": { + "loss_is/active": 0.0, + "loss_is/candidate_factor": 3.0, + "loss_is/condition_threshold": 2.0, + "loss_is/variance_reduction_ema": 1.0870531817606985, + "loss_is/variance_reduction_observed": 1.0795379631208175 + }, + "schema_version": 1, + "selection_scope": "per-rank per-microbatch; globally averaged unbiased estimator" +} diff --git a/settings/florence_robotwin/loss_is/valid_anchor_manifest.json b/settings/florence_robotwin/loss_is/valid_anchor_manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..3f5c91478b5931286c9c84d021a7ffc10a0fb5c0 --- /dev/null +++ b/settings/florence_robotwin/loss_is/valid_anchor_manifest.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2090f6b9d7e1bc5ef41897436690f031c187d625fa6b71ae0c54f3793660c757 +size 107646168 diff --git a/settings/qwen_robotwin/dependencies/content_fingerprints.json b/settings/qwen_robotwin/dependencies/content_fingerprints.json new file mode 100644 index 0000000000000000000000000000000000000000..f945c130e7871dcbe67f99368ba69978f57ba8ec --- /dev/null +++ b/settings/qwen_robotwin/dependencies/content_fingerprints.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e6c3ef15cf746ad2a5f5ffa5f3c3c57fc2647507200873754718388643460d8 +size 14233373 diff --git a/settings/qwen_robotwin/dependencies/data_provenance.json b/settings/qwen_robotwin/dependencies/data_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..0c769a6507b50d7e64d9937924907ea3d4f633e6 --- /dev/null +++ b/settings/qwen_robotwin/dependencies/data_provenance.json @@ -0,0 +1,21 @@ +{ + "checkpoint_repository": { + "repo_id": "StarVLA/Florence-GR00T-RoboTwin", + "revision": "unknown" + }, + "datasets": { + "RoboTwin/Clean": { + "repo_id": "local-mnt-miaochengyu-starvla-infoflow-runtime-playground-Datasets-RoboTwin-Clean", + "revision": "local-filesystem" + }, + "RoboTwin/Randomized": { + "repo_id": "local-mnt-miaochengyu-starvla-infoflow-runtime-playground-Datasets-RoboTwin-Randomized", + "revision": "local-filesystem" + } + }, + "florence_repository": { + "repo_id": "microsoft/Florence-2-large", + "revision": "21a599d414c4d928c9032694c424fb94458e3594" + }, + "schema_version": 1 +} \ No newline at end of file diff --git a/settings/qwen_robotwin/frameskip/cache/frameskip_full.json b/settings/qwen_robotwin/frameskip/cache/frameskip_full.json new file mode 100644 index 0000000000000000000000000000000000000000..55b4dbbe23819c222e1eea0ffa9c9696374cc642 --- /dev/null +++ b/settings/qwen_robotwin/frameskip/cache/frameskip_full.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5fa058a703b62cf780621a9a495044923cd2289fd7b684c3dd8b4c217843b1f2 +size 206357915 diff --git a/settings/qwen_robotwin/frameskip/cache/original_pipeline.sh b/settings/qwen_robotwin/frameskip/cache/original_pipeline.sh new file mode 100644 index 0000000000000000000000000000000000000000..e3fa8491e858dc0d4396cdc575bb70c25cd1c220 --- /dev/null +++ b/settings/qwen_robotwin/frameskip/cache/original_pipeline.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail +cd /home/liuyu/student-workspace/code/starVLA-loss-is-adapted +export OMP_NUM_THREADS=1 MKL_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 TORCHINDUCTOR_COMPILE_THREADS=1 MAX_JOBS=1 TOKENIZERS_PARALLELISM=false +trap 'rc=$?; printf "%s\n" "$rc" > /home/liuyu/student-workspace/outputs/robotwin_external_20260831/frameskip_repair_20260901/pipeline.exitcode' EXIT +ROOT=/fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831 +TEST=/fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/test_suite +mkdir -p "$ROOT/logs" "$ROOT/cache" "$ROOT/evidence" "$ROOT/formal_runs" "$ROOT/temporary_smoke_runs" +if [ ! -s /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/cache/frameskip_full.json ]; then +env PYTHONPATH=/fs1/private/user/liuyu/student-workspace/code/starVLA-infoflow-codex WANDB_MODE=offline PYTHONUNBUFFERED=1 PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True HF_HOME=/fs1/private/user/liuyu/student-workspace/cache/huggingface HF_HUB_OFFLINE=1 TMPDIR=/fs1/private/user/liuyu/student-workspace/tmp /fs1/private/user/liuyu/student-workspace/data/runtime/.venv/bin/python /fs1/private/user/liuyu/student-workspace/code/starVLA-infoflow-codex/external_baselines/frameskip_robotwin_adapted/build_robotwin_cache.py --manifest /fs1/private/user/liuyu/student-workspace/data/robotwin/manifests/valid_anchor_manifest.json --data-root /fs1/private/user/liuyu/student-workspace/data/robotwin/RoboTwin --upstream-root /fs1/private/user/liuyu/student-workspace/code/upstream/FrameSkip-965becf63e9cd5cc339fbf7fde40d1db47efa67a --visual-checkpoint /fs1/private/user/liuyu/student-workspace/cache/huggingface/hub/models--timm--vit_small_patch14_dinov2.lvd142m/snapshots/4610ca143709d58a633b6397a74412c2c3842454/model.safetensors --output /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/cache/frameskip_full.json --device cuda +fi +/fs1/private/user/liuyu/student-workspace/data/runtime/.venv/bin/python /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/test_suite/validate_frameskip_cache.py /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/cache/frameskip_full.json /fs1/private/user/liuyu/student-workspace/data/robotwin/manifests/valid_anchor_manifest.json +echo FORMAL florence frameskip +env PYTHONPATH=/fs1/private/user/liuyu/student-workspace/code/starVLA-loss-is-adapted WANDB_MODE=offline PYTHONUNBUFFERED=1 PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True HF_HOME=/fs1/private/user/liuyu/student-workspace/cache/huggingface HF_HUB_OFFLINE=1 TMPDIR=/fs1/private/user/liuyu/student-workspace/tmp FRAMESKIP_CACHE=/fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/cache/frameskip_full.json FRAMESKIP_START_GLOBAL_DRAW_ID=640000 FRAMESKIP_GLOBAL_MICROBATCH_SIZE=4 /fs1/private/user/liuyu/student-workspace/data/runtime/.venv/bin/python -m accelerate.commands.launch --config_file /fs1/private/user/liuyu/student-workspace/code/starVLA-loss-is-adapted/external_baselines/loss_is_adapted/configs/deepspeed_florence_2gpu_sai.yaml --num_processes 2 --mixed_precision bf16 --main_process_port 0 /fs1/private/user/liuyu/student-workspace/code/starVLA-infoflow-codex/external_baselines/frameskip_robotwin_adapted/train_frameskip_robotwin_continuation.py --config_yaml /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/configs/florence_frameskip_formal.yaml +/fs1/private/user/liuyu/student-workspace/data/runtime/.venv/bin/python /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/test_suite/validate_checkpoint.py /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs/florence-robotwin-frameskip-10000-to-10500-seed484370/checkpoints 10500 > /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/evidence/florence_frameskip_formal_checkpoint.json +echo FORMAL qwen frameskip +cd /home/liuyu/student-workspace/code/starVLA-qwen-bridge-adapted +env PYTHONPATH=/fs1/private/user/liuyu/student-workspace/code/starVLA-qwen-bridge-adapted WANDB_MODE=offline PYTHONUNBUFFERED=1 PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True HF_HOME=/fs1/private/user/liuyu/student-workspace/cache/huggingface HF_HUB_OFFLINE=1 TMPDIR=/fs1/private/user/liuyu/student-workspace/tmp FRAMESKIP_CACHE=/fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/cache/frameskip_full.json FRAMESKIP_START_GLOBAL_DRAW_ID=440000 FRAMESKIP_GLOBAL_MICROBATCH_SIZE=10 /fs1/private/user/liuyu/student-workspace/data/runtime/.venv/bin/python -m accelerate.commands.launch --config_file /fs1/private/user/liuyu/student-workspace/code/starVLA-qwen-bridge-adapted/starVLA/config/deepseeds/deepspeed_zero3.yaml --num_processes 2 --mixed_precision bf16 --main_process_port 0 /fs1/private/user/liuyu/student-workspace/code/starVLA-infoflow-codex/external_baselines/frameskip_robotwin_adapted/train_frameskip_robotwin_continuation.py --config_yaml /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/configs/qwen_frameskip_formal.yaml +/fs1/private/user/liuyu/student-workspace/data/runtime/.venv/bin/python /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/test_suite/validate_checkpoint.py /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs/qwen-robotwin-frameskip-11000-to-11500-seed484370/checkpoints 11500 > /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/evidence/qwen_frameskip_formal_checkpoint.json +echo PIPELINE_FRAMESKIP_PASS diff --git a/settings/qwen_robotwin/frameskip/checkpoint.json b/settings/qwen_robotwin/frameskip/checkpoint.json new file mode 100644 index 0000000000000000000000000000000000000000..a10eace19327103220d803b54d51aca2fc5c036a --- /dev/null +++ b/settings/qwen_robotwin/frameskip/checkpoint.json @@ -0,0 +1,65 @@ +{ + "completed_steps": 11500, + "files": [ + { + "bytes": 1525, + "name": "custom_checkpoint_0.pkl", + "sha256": "b1002611dfae1393f28a634b9d947819b3df149c1e78485b44a31a200d2c1414" + }, + { + "bytes": 13, + "name": "latest", + "sha256": "68d0ca762047fd48fdd6a06a5e6465209289c028bfd63f5557a110ff1ebc15ac" + }, + { + "bytes": 13867389419, + "name": "pytorch_model/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt", + "sha256": "cc398a054987fb99ec2fbe75c1dc90f9f1b21f24c1e128aa1b806ea2eac8cda9" + }, + { + "bytes": 13867389419, + "name": "pytorch_model/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt", + "sha256": "3dcec647f0cf700ca5a9041f71763bc258bda5f0deea85579de1d07db67180b4" + }, + { + "bytes": 566626, + "name": "pytorch_model/zero_pp_rank_0_mp_rank_00_model_states.pt", + "sha256": "743ad760245b429d5387d7868a0b61bf310e72f887b02097c94265c46e0d6530" + }, + { + "bytes": 566498, + "name": "pytorch_model/zero_pp_rank_1_mp_rank_00_model_states.pt", + "sha256": "d2738a5c4991da9724574e3120ff7d06f32674a5d3074c07851d4fc5d41f811c" + }, + { + "bytes": 15081, + "name": "random_states_0.pkl", + "sha256": "7ab692cdaa60bfd0c655eca6c69dedad4650b57ed8c82d275be6435fe92e35fa" + }, + { + "bytes": 15081, + "name": "random_states_1.pkl", + "sha256": "ee39fa0a5ac8438faf6b549a41cc2021e0b22d0c7698aa67fcca9cafa74848a3" + }, + { + "bytes": 33270, + "name": "zero_to_fp32.py", + "sha256": "e8a2553b434ac043ec63b022db1e768277acf8d4e70f10a2842db6d238977135" + } + ], + "registered_checkpointables": [ + "lr_scheduler" + ], + "resume_identity": { + "config_sha256": "fe3d7fe8a58e48266616ea8d6084b359119c2f63d0f7a15661708b726f7cfddc", + "identity_schema_version": 2, + "manifest_sha256": "dcb728a00da4b4ad7f989fa313d6d3355e13f57efefc7267e9ea092f2ffd8a50", + "model_structure_sha256": "7809faa5245147f81b98f3dbd17c7a063a7cb07a21e18af588c748c5925e1407", + "proposal_alpha": null, + "proposal_sha256": null, + "seed": 484370, + "total_batch_size": 40 + }, + "schema_version": 1, + "world_size": 2 +} diff --git a/settings/qwen_robotwin/frameskip/config.full.yaml b/settings/qwen_robotwin/frameskip/config.full.yaml new file mode 100644 index 0000000000000000000000000000000000000000..beeba230cac3633f2d72d846dd404bae74c21ce1 --- /dev/null +++ b/settings/qwen_robotwin/frameskip/config.full.yaml @@ -0,0 +1,109 @@ +run_id: qwen-robotwin-frameskip-11000-to-11500-seed484370 +run_root_dir: /home/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs +seed: 484370 +wandb_entity: null +wandb_project: InfoFlow-StarVLA-RoboTwin +is_debug: false +version_id: '0.21' +framework: + name: QwenGR00T + qwenvl: + base_vlm: /home/liuyu/student-workspace/data/runtime/playground/Pretrained_models/Qwen3-VL-4B-Instruct + attn_implementation: flash_attention_2 + vl_hidden_dim: 2048 + action_model: + action_model_type: DiT-B + action_hidden_dim: 2048 + hidden_size: 1024 + add_pos_embed: true + max_seq_len: 1024 + action_dim: 14 + state_dim: 0 + action_horizon: 50 + repeated_diffusion_steps: 8 + noise_beta_alpha: 1.5 + noise_beta_beta: 1.0 + noise_s: 0.999 + num_timestep_buckets: 1000 + num_inference_timesteps: 4 + num_target_vision_tokens: 32 + diffusion_model_cfg: + cross_attention_dim: 2560 + dropout: 0.2 + final_dropout: true + interleave_self_attention: true + norm_type: ada_norm + num_layers: 16 + output_dim: 1024 + positional_embeddings: null + future_action_window_size: 49 + past_action_window_size: 0 + dino: + dino_backbone: dinov2_vits14 +datasets: + vla_data: + dataset_py: lerobot_datasets + data_root_dir: /home/liuyu/student-workspace/data/robotwin/RoboTwin + data_mix: robotwin_all_50 + action_type: abs_qpos + action_mode: abs + include_state: false + sequential_step_sampling: false + per_device_batch_size: 5 + load_all_data_for_training: true + image_size: + - 224 + - 224 + video_backend: pyav + num_workers: 4 + expected_world_size: 2 + traceable_sampling: true + dataset_provenance_path: /home/liuyu/student-workspace/data/robotwin/manifests/data_provenance.json + dataset_content_fingerprint_path: /home/liuyu/student-workspace/data/robotwin/manifests/content_fingerprints.json + valid_anchor_manifest_path: /home/liuyu/student-workspace/data/robotwin/manifests/valid_anchor_manifest.json + transform_rng_seed: 289726 + temporal_proposal_path: null +trainer: + use_deepspeed: true + max_train_steps: 150000 + num_warmup_steps: 5000 + save_interval: null + save_steps: + - 11500 + evaluation_snapshot_interval: null + evaluation_snapshot_start_step: 10000 + evaluation_snapshot_end_step: 20000 + eval_interval: null + repeated_diffusion_steps: 4 + learning_rate: + base: 1.0e-05 + qwen_vl_interface: 1.0e-05 + action_model: 0.0001 + lr_scheduler_type: cosine_with_min_lr + scheduler_specific_kwargs: + min_lr: 5.0e-07 + freeze_modules: '' + trainable_modules: '' + frozen_modules_eval: false + logging_frequency: 10 + gradient_clipping: 1.0 + gradient_accumulation_steps: 4 + save_format: pt + save_final_checkpoint: false + save_consolidated_model: false + optimizer: + name: AdamW8bit + betas: + - 0.9 + - 0.95 + eps: 1.0e-08 + weight_decay: 1.0e-08 + stop_after_steps: 11500 + is_resume: true + allow_model_only_resume: false + resume_accelerate_checkpoint: /home/liuyu/student-workspace/data/robotwin/common_start/qwen3vl-groot-robotwin-official150k-adamw8bit-mb5-ga4-seed42-20260810/checkpoints/steps_11000_accelerate + branch_parent_seed: 42 + branch_rng_seed: 484370 + model_rng_seed: 721927 +config_yaml: /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/configs/qwen_frameskip_formal.yaml +output_dir: /home/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs/qwen-robotwin-frameskip-11000-to-11500-seed484370 diff --git a/settings/qwen_robotwin/frameskip/config.yaml b/settings/qwen_robotwin/frameskip/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d55b81e29ffb8e9bf80104501bd2bfda5cf143dc --- /dev/null +++ b/settings/qwen_robotwin/frameskip/config.yaml @@ -0,0 +1,65 @@ +datasets: + vla_data: + action_mode: abs + data_mix: robotwin_all_50 + data_root_dir: /home/liuyu/student-workspace/data/robotwin/RoboTwin + dataset_content_fingerprint_path: /home/liuyu/student-workspace/data/robotwin/manifests/content_fingerprints.json + dataset_provenance_path: /home/liuyu/student-workspace/data/robotwin/manifests/data_provenance.json + dataset_py: lerobot_datasets + expected_world_size: 2 + num_workers: 4 + per_device_batch_size: 5 + temporal_proposal_path: null + traceable_sampling: true + valid_anchor_manifest_path: /home/liuyu/student-workspace/data/robotwin/manifests/valid_anchor_manifest.json + video_backend: pyav +framework: + action_model: + action_dim: 14 + action_horizon: 50 + action_model_type: DiT-B + add_pos_embed: true + diffusion_model_cfg: + cross_attention_dim: 2560 + dropout: 0.2 + final_dropout: true + interleave_self_attention: true + norm_type: ada_norm + num_layers: 16 + output_dim: 1024 + positional_embeddings: null + hidden_size: 1024 + max_seq_len: 1024 + noise_beta_alpha: 1.5 + noise_beta_beta: 1.0 + num_inference_timesteps: 4 + num_target_vision_tokens: 32 + num_timestep_buckets: 1000 + state_dim: 0 + name: QwenGR00T + qwenvl: + attn_implementation: flash_attention_2 + base_vlm: /home/liuyu/student-workspace/data/runtime/playground/Pretrained_models/Qwen3-VL-4B-Instruct +output_dir: /home/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs/qwen-robotwin-frameskip-11000-to-11500-seed484370 +run_id: qwen-robotwin-frameskip-11000-to-11500-seed484370 +run_root_dir: /home/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs +seed: 484370 +trainer: + freeze_modules: '' + gradient_accumulation_steps: 4 + learning_rate: + action_model: 0.0001 + base: 1.0e-05 + qwen_vl_interface: 1.0e-05 + lr_scheduler_type: cosine_with_min_lr + max_train_steps: 150000 + num_warmup_steps: 5000 + optimizer: + betas: + - 0.9 + - 0.95 + eps: 1.0e-08 + name: AdamW8bit + weight_decay: 1.0e-08 + scheduler_specific_kwargs: + min_lr: 5.0e-07 diff --git a/settings/qwen_robotwin/frameskip/dataset_statistics.json b/settings/qwen_robotwin/frameskip/dataset_statistics.json new file mode 100644 index 0000000000000000000000000000000000000000..c920d76f3fc976e9e0b30e36eab0cff8a6bf1adb --- /dev/null +++ b/settings/qwen_robotwin/frameskip/dataset_statistics.json @@ -0,0 +1,218 @@ +{ + "new_embodiment": { + "action": { + "mean": [ + -0.22423998869955536, + 1.0729362124204633, + 0.775079609751701, + -0.32701663097366684, + 0.06159586461764771, + -0.06420705380151046, + 0.22430071420967568, + 1.1274866509437556, + 0.831773659586906, + -0.3638282522559164, + -0.024959841337949897, + 0.05302203213563182, + 0.6774904182553295, + 0.6598676902055738 + ], + "std": [ + 0.39622931737429123, + 1.003590449339515, + 0.7811611351334437, + 0.6707485714474849, + 0.2686908204983921, + 0.6492019642472782, + 0.34733679400923234, + 1.0176226676795115, + 0.8039915805960646, + 0.7067249045233707, + 0.26259645177151686, + 0.6808759458519371, + 0.4482151151114418, + 0.4539921671160285 + ], + "max": [ + 6.45659065246582, + 4.179152488708496, + 5.346591472625732, + 1.7942548990249634, + 1.8604620695114136, + 5.43813943862915, + 7.142920017242432, + 4.157068729400635, + 5.672450065612793, + 1.95806884765625, + 1.5663840770721436, + 5.278968811035156, + 1.0, + 1.0 + ], + "min": [ + -7.552278995513916, + -0.36354875564575195, + -0.18577136099338531, + -1.956291913986206, + -1.6801013946533203, + -7.678869724273682, + -8.539926528930664, + -0.6294453740119934, + -0.07775841653347015, + -2.1328067779541016, + -2.1285502910614014, + -8.228925704956055, + 0.0, + 0.0 + ], + "q01": [ + -7.334665780067444, + -5.257390398583084e-07, + -2.8215323254698887e-05, + -1.841589093208313, + -1.2851550579071045, + -6.268352918624878, + -5.995517673492431, + -0.572821855545044, + -2.81171942333458e-05, + -1.8467556238174438, + -1.4545854091644288, + -5.945669202804565, + 0.0, + 0.0 + ], + "q99": [ + 5.016749763488781, + 3.4792869091033936, + 4.225283479690551, + 1.7064905214309687, + 1.4829185009002717, + 3.7177388882636553, + 1.258314609527588, + 2.967935800552368, + 3.2566452693939207, + 1.766038613319397, + 1.4066662788391113, + 3.5830023288726807, + 1.0, + 1.0 + ], + "mask": [ + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + false, + false + ] + }, + "state": { + "mean": [ + -0.22281882727518673, + 1.0668349397182466, + 0.770632639825344, + -0.3252330579422412, + 0.06118139047002841, + -0.06316296239456277, + 0.22253762468695631, + 1.1201750242710113, + 0.8263923591375351, + -0.36159489695448405, + -0.024702461528722772, + 0.05203246331657278, + 0.6793262696266176, + 0.6616973677277563 + ], + "std": [ + 0.39522870886731665, + 1.004018577033247, + 0.7809103811570285, + 0.6688088363177986, + 0.26775723818956776, + 0.6463212583712935, + 0.34648488321363663, + 1.018232630032839, + 0.8038798397682734, + 0.7045612787595747, + 0.26143414213134614, + 0.6774619870452644, + 0.44749220497027625, + 0.45333916022037934 + ], + "max": [ + 6.45659065246582, + 4.179152488708496, + 5.346591472625732, + 1.7942548990249634, + 1.8604620695114136, + 5.43813943862915, + 7.142920017242432, + 4.157068729400635, + 5.672450065612793, + 1.95806884765625, + 1.5663840770721436, + 5.278968811035156, + 1.0, + 1.0 + ], + "min": [ + -7.552278995513916, + -0.36354875564575195, + -0.18577136099338531, + -1.956291913986206, + -1.6801013946533203, + -7.678869724273682, + -8.539926528930664, + -0.6294453740119934, + -0.07775841653347015, + -2.1328067779541016, + -2.1285502910614014, + -8.228925704956055, + 0.0, + 0.0 + ], + "q01": [ + -7.334665780067444, + -5.257390398583084e-07, + -2.8215323254698887e-05, + -1.841589093208313, + -1.2851550579071045, + -6.268109321594238, + -5.912737417221069, + -0.572821855545044, + -2.81171942333458e-05, + -1.8467556238174438, + -1.440324923992157, + -5.8623305034637445, + 0.0, + 0.0 + ], + "q99": [ + 4.902149534225477, + 3.4775466918945312, + 4.224453401565551, + 1.7064905214309687, + 1.480757713317871, + 3.609884884357428, + 1.258314609527588, + 2.967935800552368, + 3.2566452693939207, + 1.7433889162540437, + 1.4066662788391113, + 3.581243851184845, + 1.0, + 1.0 + ] + }, + "num_transitions": 6075103, + "num_trajectories": 27500 + } +} \ No newline at end of file diff --git a/settings/qwen_robotwin/frameskip/external_resume_certificate.json b/settings/qwen_robotwin/frameskip/external_resume_certificate.json new file mode 100644 index 0000000000000000000000000000000000000000..1683e3ebced3d2ce2def520087db654c682e1449 --- /dev/null +++ b/settings/qwen_robotwin/frameskip/external_resume_certificate.json @@ -0,0 +1,138 @@ +{ + "branch_resume_identity": { + "config_sha256": "fe3d7fe8a58e48266616ea8d6084b359119c2f63d0f7a15661708b726f7cfddc", + "identity_schema_version": 2, + "manifest_sha256": "dcb728a00da4b4ad7f989fa313d6d3355e13f57efefc7267e9ea092f2ffd8a50", + "model_structure_sha256": "7809faa5245147f81b98f3dbd17c7a063a7cb07a21e18af588c748c5925e1407", + "proposal_alpha": null, + "proposal_sha256": null, + "seed": 484370, + "total_batch_size": 40 + }, + "completed_steps": 11000, + "optimizer_param_groups": [ + { + "betas": [ + 0.9, + 0.95 + ], + "eps": 1e-08, + "initial_lr": 1e-05, + "lr": 9.959920880962606e-06, + "name": "qwen_vl_interface", + "weight_decay": 1e-08 + }, + { + "betas": [ + 0.9, + 0.95 + ], + "eps": 1e-08, + "initial_lr": 0.0001, + "lr": 9.959920880962605e-05, + "name": "action_model", + "weight_decay": 1e-08 + } + ], + "optimizer_state_entries": 8, + "parent_checkpoint": "/fs1/private/user/liuyu/student-workspace/data/robotwin/common_start/qwen3vl-groot-robotwin-official150k-adamw8bit-mb5-ga4-seed42-20260810/checkpoints/steps_11000_accelerate", + "parent_checkpoint_commit_sha256": "180f1ed2d2b7a9e6c158974fc9ec951182dfb629f6a2806808ac5639d491ce1b", + "parent_checkpoint_files": [ + { + "bytes": 1525, + "name": "custom_checkpoint_0.pkl", + "sha256": "70788f8a2d7852a70fbb794ab9176d14ad37fb6d1b1d821a368572e1d8524390" + }, + { + "bytes": 13, + "name": "latest", + "sha256": "68d0ca762047fd48fdd6a06a5e6465209289c028bfd63f5557a110ff1ebc15ac" + }, + { + "bytes": 13867381301, + "name": "pytorch_model/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt", + "sha256": "42b6d10bb158b404056983ce88505c32738b29f3a062fe6835d24ce26c521a6a" + }, + { + "bytes": 13867381301, + "name": "pytorch_model/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt", + "sha256": "7b0a1bbc92efffbf056bc40f40e93a94b67ddf3f375dc6ae263bcb048ba6ec63" + }, + { + "bytes": 566626, + "name": "pytorch_model/zero_pp_rank_0_mp_rank_00_model_states.pt", + "sha256": "8a3ef88f9ab0df8197a836dca1d6e2416835f0b3d827cb10a78254b9a969ce7c" + }, + { + "bytes": 566498, + "name": "pytorch_model/zero_pp_rank_1_mp_rank_00_model_states.pt", + "sha256": "67cd03fceb5ee4c2be6affe504f71a48ee10127d6ea3a1f00d05721895ffba53" + }, + { + "bytes": 15017, + "name": "random_states_0.pkl", + "sha256": "636b1c5983ee2f8a5047025027453c9a478c0606389f2f6945b370f39cec9e65" + }, + { + "bytes": 15081, + "name": "random_states_1.pkl", + "sha256": "416b3be53ff049935984cc65385d7dae3b0e5a82df66c9516f61ee9100c41961" + }, + { + "bytes": 33270, + "name": "zero_to_fp32.py", + "sha256": "e8a2553b434ac043ec63b022db1e768277acf8d4e70f10a2842db6d238977135" + } + ], + "parent_checkpoint_files_sha256": "73692dae620e3d4368fe513e8adff4fe54bdb25341b4293e34ee2a6286554f8d", + "parent_checkpoint_integrity_manifest_sha256": null, + "parent_config_sha256": "4882855d94076fc0ec19a0d511f14b1cbe82860c8b5df3dcc64c5c5f6c0d2321", + "parent_resume_identity": { + "config_sha256": "1a48ea51cf89ce72a7956aaa29f710bba0326c9b726e52795561b381507d26fa", + "identity_schema_version": 2, + "manifest_sha256": "dcb728a00da4b4ad7f989fa313d6d3355e13f57efefc7267e9ea092f2ffd8a50", + "model_structure_sha256": "7809faa5245147f81b98f3dbd17c7a063a7cb07a21e18af588c748c5925e1407", + "proposal_alpha": null, + "proposal_sha256": null, + "seed": 42, + "total_batch_size": 40 + }, + "proposal_sha256": null, + "rank_state_summaries": [ + { + "optimizer_state_entries": 4, + "optimizer_state_sha256": "e1eea37514fa0478835597d1594b7a43b45cea9adc4ccb79a0de8fb9e10fe9f0", + "rank": 0, + "trainable_state_sha256": "ca2a19167d59aab693d0a0a1e17ea9adbf9cfd7367dfa7a5c90b04cf2183b717" + }, + { + "optimizer_state_entries": 4, + "optimizer_state_sha256": "62c4e2908a7e97fe9a1d60642a85968081b55345493a11b9e7f68e49fcdb22f3", + "rank": 1, + "trainable_state_sha256": "ca2a19167d59aab693d0a0a1e17ea9adbf9cfd7367dfa7a5c90b04cf2183b717" + } + ], + "repeated_diffusion_steps": 4, + "rng_streams": { + "model_base": 721927, + "sampler": 484370, + "transform": 289726 + }, + "scheduler": { + "class": "LambdaLR", + "last_epoch": 11000, + "last_lr": [ + 9.959920880962606e-06, + 9.959920880962605e-05 + ], + "step_count": 11001 + }, + "scheduler_resume_mode": "accelerate_checkpoint", + "schema_version": 1, + "status": "exact_external_resume_loaded", + "total_batch_size": 40, + "trainable_parameter_elements": 32022798, + "trainable_parameter_sha256": "1bc504b22e4375be40bbbd8250be5b5f380c14d2fb47ed85deba3604a84b5484", + "trainable_parameter_tensors": 957, + "world_size": 2 +} diff --git a/settings/qwen_robotwin/frameskip/initialization_manifest.json b/settings/qwen_robotwin/frameskip/initialization_manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..35160df6ed677c681d32574fe5119b91ef516bed --- /dev/null +++ b/settings/qwen_robotwin/frameskip/initialization_manifest.json @@ -0,0 +1,6 @@ +{ + "schema_version": 1, + "initialization_seed": 484370, + "action_model_sha256": "1cab43a1e801a94110a20384a9ad6cd76a239fedd9fe17890662da7e22d47e62", + "action_model_parameter_elements": 160689934 +} diff --git a/settings/qwen_robotwin/frameskip/valid_anchor_manifest.json b/settings/qwen_robotwin/frameskip/valid_anchor_manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..3f5c91478b5931286c9c84d021a7ffc10a0fb5c0 --- /dev/null +++ b/settings/qwen_robotwin/frameskip/valid_anchor_manifest.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2090f6b9d7e1bc5ef41897436690f031c187d625fa6b71ae0c54f3793660c757 +size 107646168 diff --git a/settings/qwen_robotwin/keyframe/cache/keyframe_full.npz b/settings/qwen_robotwin/keyframe/cache/keyframe_full.npz new file mode 100644 index 0000000000000000000000000000000000000000..066942637c8ac5542d51f646f3f084381d3a3612 --- /dev/null +++ b/settings/qwen_robotwin/keyframe/cache/keyframe_full.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:490da0439605b2b1acd7400add00c0e24a9c53c8bf68812ba47fcb34a0f18e40 +size 22442738 diff --git a/settings/qwen_robotwin/keyframe/cache/original_pipeline.sh b/settings/qwen_robotwin/keyframe/cache/original_pipeline.sh new file mode 100644 index 0000000000000000000000000000000000000000..8bafbc4ae32ad7525483d2fc03e6fd849f751576 --- /dev/null +++ b/settings/qwen_robotwin/keyframe/cache/original_pipeline.sh @@ -0,0 +1,32 @@ +#!/bin/bash +set -euo pipefail +ROOT=/fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831 +TEST=/fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/test_suite +mkdir -p "$ROOT/logs" "$ROOT/cache" "$ROOT/evidence" "$ROOT/formal_runs" "$ROOT/temporary_smoke_runs" +if [ ! -s /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/cache/keyframe_full.npz ]; then +env PYTHONPATH=/fs1/private/user/liuyu/student-workspace/code/starVLA-loss-is-adapted WANDB_MODE=offline PYTHONUNBUFFERED=1 PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True HF_HOME=/fs1/private/user/liuyu/student-workspace/cache/huggingface HF_HUB_OFFLINE=1 TMPDIR=/fs1/private/user/liuyu/student-workspace/tmp /fs1/private/user/liuyu/student-workspace/data/runtime/.venv/bin/python /fs1/private/user/liuyu/student-workspace/code/starVLA-loss-is-adapted/external_baselines/robotwin_smoke/build_keyframe_ape_cache.py --manifest /fs1/private/user/liuyu/student-workspace/data/robotwin/manifests/valid_anchor_manifest.json --data-root /fs1/private/user/liuyu/student-workspace/data/robotwin/RoboTwin --statistics /fs1/private/user/liuyu/student-workspace/data/robotwin/common_start/florence-groot-robotwin-xvla3view-official150k-seed42-20260804/dataset_statistics.json --statistics-key new_embodiment --output /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/cache/keyframe_full.npz --scope full --epochs 300 --device cuda +fi +env PYTHONPATH=/fs1/private/user/liuyu/student-workspace/code/starVLA-loss-is-adapted /fs1/private/user/liuyu/student-workspace/data/runtime/.venv/bin/python /fs1/private/user/liuyu/student-workspace/code/starVLA-loss-is-adapted/external_baselines/keyframe_focused_adapted/validate_cache_support.py --manifest /fs1/private/user/liuyu/student-workspace/data/robotwin/manifests/valid_anchor_manifest.json --cache /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/cache/keyframe_full.npz --seed 484370 --draw-start 440000 --draw-count 4096 +echo SMOKE florence keyframe phase1 +env PYTHONPATH=/fs1/private/user/liuyu/student-workspace/code/starVLA-loss-is-adapted WANDB_MODE=offline PYTHONUNBUFFERED=1 PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True HF_HOME=/fs1/private/user/liuyu/student-workspace/cache/huggingface HF_HUB_OFFLINE=1 TMPDIR=/fs1/private/user/liuyu/student-workspace/tmp KEYFRAME_APE_CACHE=/fs1/private/user/liuyu/student-workspace/outputs/robotwin_smoke/data/keyframe_ape_full_smoke.npz /fs1/private/user/liuyu/student-workspace/data/runtime/.venv/bin/python -m accelerate.commands.launch --config_file /fs1/private/user/liuyu/student-workspace/code/starVLA-loss-is-adapted/external_baselines/loss_is_adapted/configs/deepspeed_florence_2gpu_sai.yaml --num_processes 2 --mixed_precision bf16 --main_process_port 0 /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/test_suite/smoke_runner.py --method keyframe --entry /fs1/private/user/liuyu/student-workspace/code/starVLA-loss-is-adapted/external_baselines/keyframe_focused_adapted/train_keyframe_continuation.py --config_yaml /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/test_suite/florence_keyframe_phase1.yaml >> /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/logs/keyframe_florence_smoke.log 2>&1 +/fs1/private/user/liuyu/student-workspace/data/runtime/.venv/bin/python /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/test_suite/validate_checkpoint.py /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/temporary_smoke_runs/florence-robotwin-keyframe-isolated-save-reload-smoke/checkpoints 1 >> /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/evidence/florence_keyframe_smoke.jsonl +echo SMOKE florence keyframe phase2 +env PYTHONPATH=/fs1/private/user/liuyu/student-workspace/code/starVLA-loss-is-adapted WANDB_MODE=offline PYTHONUNBUFFERED=1 PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True HF_HOME=/fs1/private/user/liuyu/student-workspace/cache/huggingface HF_HUB_OFFLINE=1 TMPDIR=/fs1/private/user/liuyu/student-workspace/tmp KEYFRAME_APE_CACHE=/fs1/private/user/liuyu/student-workspace/outputs/robotwin_smoke/data/keyframe_ape_full_smoke.npz /fs1/private/user/liuyu/student-workspace/data/runtime/.venv/bin/python -m accelerate.commands.launch --config_file /fs1/private/user/liuyu/student-workspace/code/starVLA-loss-is-adapted/external_baselines/loss_is_adapted/configs/deepspeed_florence_2gpu_sai.yaml --num_processes 2 --mixed_precision bf16 --main_process_port 0 /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/test_suite/smoke_runner.py --method keyframe --entry /fs1/private/user/liuyu/student-workspace/code/starVLA-loss-is-adapted/external_baselines/keyframe_focused_adapted/train_keyframe_continuation.py --config_yaml /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/test_suite/florence_keyframe_phase2.yaml >> /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/logs/keyframe_florence_smoke.log 2>&1 +/fs1/private/user/liuyu/student-workspace/data/runtime/.venv/bin/python /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/test_suite/validate_checkpoint.py /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/temporary_smoke_runs/florence-robotwin-keyframe-isolated-save-reload-smoke/checkpoints 2 >> /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/evidence/florence_keyframe_smoke.jsonl +/fs1/private/user/liuyu/student-workspace/data/runtime/.venv/bin/python /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/test_suite/cleanup_smoke.py florence-robotwin-keyframe-isolated-save-reload-smoke +rm -f -- /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/logs/keyframe_florence_smoke.log +echo SMOKE qwen keyframe phase1 +env PYTHONPATH=/fs1/private/user/liuyu/student-workspace/code/starVLA-qwen-bridge-adapted WANDB_MODE=offline PYTHONUNBUFFERED=1 PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True HF_HOME=/fs1/private/user/liuyu/student-workspace/cache/huggingface HF_HUB_OFFLINE=1 TMPDIR=/fs1/private/user/liuyu/student-workspace/tmp KEYFRAME_APE_CACHE=/fs1/private/user/liuyu/student-workspace/outputs/robotwin_smoke/data/keyframe_ape_full_smoke.npz /fs1/private/user/liuyu/student-workspace/data/runtime/.venv/bin/python -m accelerate.commands.launch --config_file /fs1/private/user/liuyu/student-workspace/code/starVLA-qwen-bridge-adapted/starVLA/config/deepseeds/deepspeed_zero3.yaml --num_processes 2 --mixed_precision bf16 --main_process_port 0 /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/test_suite/smoke_runner.py --method keyframe --entry /fs1/private/user/liuyu/student-workspace/code/starVLA-qwen-bridge-adapted/external_baselines/keyframe_focused_adapted/train_keyframe_continuation.py --config_yaml /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/test_suite/qwen_keyframe_phase1.yaml >> /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/logs/keyframe_qwen_smoke.log 2>&1 +/fs1/private/user/liuyu/student-workspace/data/runtime/.venv/bin/python /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/test_suite/validate_checkpoint.py /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/temporary_smoke_runs/qwen-robotwin-keyframe-isolated-save-reload-smoke/checkpoints 1 >> /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/evidence/qwen_keyframe_smoke.jsonl +echo SMOKE qwen keyframe phase2 +env PYTHONPATH=/fs1/private/user/liuyu/student-workspace/code/starVLA-qwen-bridge-adapted WANDB_MODE=offline PYTHONUNBUFFERED=1 PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True HF_HOME=/fs1/private/user/liuyu/student-workspace/cache/huggingface HF_HUB_OFFLINE=1 TMPDIR=/fs1/private/user/liuyu/student-workspace/tmp KEYFRAME_APE_CACHE=/fs1/private/user/liuyu/student-workspace/outputs/robotwin_smoke/data/keyframe_ape_full_smoke.npz /fs1/private/user/liuyu/student-workspace/data/runtime/.venv/bin/python -m accelerate.commands.launch --config_file /fs1/private/user/liuyu/student-workspace/code/starVLA-qwen-bridge-adapted/starVLA/config/deepseeds/deepspeed_zero3.yaml --num_processes 2 --mixed_precision bf16 --main_process_port 0 /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/test_suite/smoke_runner.py --method keyframe --entry /fs1/private/user/liuyu/student-workspace/code/starVLA-qwen-bridge-adapted/external_baselines/keyframe_focused_adapted/train_keyframe_continuation.py --config_yaml /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/test_suite/qwen_keyframe_phase2.yaml >> /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/logs/keyframe_qwen_smoke.log 2>&1 +/fs1/private/user/liuyu/student-workspace/data/runtime/.venv/bin/python /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/test_suite/validate_checkpoint.py /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/temporary_smoke_runs/qwen-robotwin-keyframe-isolated-save-reload-smoke/checkpoints 2 >> /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/evidence/qwen_keyframe_smoke.jsonl +/fs1/private/user/liuyu/student-workspace/data/runtime/.venv/bin/python /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/test_suite/cleanup_smoke.py qwen-robotwin-keyframe-isolated-save-reload-smoke +rm -f -- /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/logs/keyframe_qwen_smoke.log +echo FORMAL florence keyframe +env PYTHONPATH=/fs1/private/user/liuyu/student-workspace/code/starVLA-loss-is-adapted WANDB_MODE=offline PYTHONUNBUFFERED=1 PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True HF_HOME=/fs1/private/user/liuyu/student-workspace/cache/huggingface HF_HUB_OFFLINE=1 TMPDIR=/fs1/private/user/liuyu/student-workspace/tmp KEYFRAME_APE_CACHE=/fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/cache/keyframe_full.npz /fs1/private/user/liuyu/student-workspace/data/runtime/.venv/bin/python -m accelerate.commands.launch --config_file /fs1/private/user/liuyu/student-workspace/code/starVLA-loss-is-adapted/external_baselines/loss_is_adapted/configs/deepspeed_florence_2gpu_sai.yaml --num_processes 2 --mixed_precision bf16 --main_process_port 0 /fs1/private/user/liuyu/student-workspace/code/starVLA-loss-is-adapted/external_baselines/keyframe_focused_adapted/train_keyframe_continuation.py --config_yaml /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/configs/florence_keyframe_formal.yaml +/fs1/private/user/liuyu/student-workspace/data/runtime/.venv/bin/python /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/test_suite/validate_checkpoint.py /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs/florence-robotwin-keyframe-10000-to-10500-seed484370/checkpoints 10500 > /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/evidence/florence_keyframe_formal_checkpoint.json +echo FORMAL qwen keyframe +env PYTHONPATH=/fs1/private/user/liuyu/student-workspace/code/starVLA-qwen-bridge-adapted WANDB_MODE=offline PYTHONUNBUFFERED=1 PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True HF_HOME=/fs1/private/user/liuyu/student-workspace/cache/huggingface HF_HUB_OFFLINE=1 TMPDIR=/fs1/private/user/liuyu/student-workspace/tmp KEYFRAME_APE_CACHE=/fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/cache/keyframe_full.npz /fs1/private/user/liuyu/student-workspace/data/runtime/.venv/bin/python -m accelerate.commands.launch --config_file /fs1/private/user/liuyu/student-workspace/code/starVLA-qwen-bridge-adapted/starVLA/config/deepseeds/deepspeed_zero3.yaml --num_processes 2 --mixed_precision bf16 --main_process_port 0 /fs1/private/user/liuyu/student-workspace/code/starVLA-qwen-bridge-adapted/external_baselines/keyframe_focused_adapted/train_keyframe_continuation.py --config_yaml /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/configs/qwen_keyframe_formal.yaml +/fs1/private/user/liuyu/student-workspace/data/runtime/.venv/bin/python /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/test_suite/validate_checkpoint.py /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs/qwen-robotwin-keyframe-11000-to-11500-seed484370/checkpoints 11500 > /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/evidence/qwen_keyframe_formal_checkpoint.json +echo PIPELINE_KEYFRAME_PASS diff --git a/settings/qwen_robotwin/keyframe/checkpoint.json b/settings/qwen_robotwin/keyframe/checkpoint.json new file mode 100644 index 0000000000000000000000000000000000000000..f1936785b247898b96cf12f449c991143c585012 --- /dev/null +++ b/settings/qwen_robotwin/keyframe/checkpoint.json @@ -0,0 +1,65 @@ +{ + "completed_steps": 11500, + "files": [ + { + "bytes": 1525, + "name": "custom_checkpoint_0.pkl", + "sha256": "b1002611dfae1393f28a634b9d947819b3df149c1e78485b44a31a200d2c1414" + }, + { + "bytes": 13, + "name": "latest", + "sha256": "68d0ca762047fd48fdd6a06a5e6465209289c028bfd63f5557a110ff1ebc15ac" + }, + { + "bytes": 13867389419, + "name": "pytorch_model/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt", + "sha256": "40dd5bdba0170520e88392d43ce82e4d4fb7baf89fc3ed63215e6f25e308e8d7" + }, + { + "bytes": 13867389419, + "name": "pytorch_model/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt", + "sha256": "152819d833dba8922ab4b6c11f19cbae7be92e5531cb154d13e29f486c2fab78" + }, + { + "bytes": 566626, + "name": "pytorch_model/zero_pp_rank_0_mp_rank_00_model_states.pt", + "sha256": "743ad760245b429d5387d7868a0b61bf310e72f887b02097c94265c46e0d6530" + }, + { + "bytes": 566498, + "name": "pytorch_model/zero_pp_rank_1_mp_rank_00_model_states.pt", + "sha256": "d2738a5c4991da9724574e3120ff7d06f32674a5d3074c07851d4fc5d41f811c" + }, + { + "bytes": 15081, + "name": "random_states_0.pkl", + "sha256": "7ab692cdaa60bfd0c655eca6c69dedad4650b57ed8c82d275be6435fe92e35fa" + }, + { + "bytes": 15081, + "name": "random_states_1.pkl", + "sha256": "ee39fa0a5ac8438faf6b549a41cc2021e0b22d0c7698aa67fcca9cafa74848a3" + }, + { + "bytes": 33270, + "name": "zero_to_fp32.py", + "sha256": "e8a2553b434ac043ec63b022db1e768277acf8d4e70f10a2842db6d238977135" + } + ], + "registered_checkpointables": [ + "lr_scheduler" + ], + "resume_identity": { + "config_sha256": "fe3d7fe8a58e48266616ea8d6084b359119c2f63d0f7a15661708b726f7cfddc", + "identity_schema_version": 2, + "manifest_sha256": "dcb728a00da4b4ad7f989fa313d6d3355e13f57efefc7267e9ea092f2ffd8a50", + "model_structure_sha256": "7809faa5245147f81b98f3dbd17c7a063a7cb07a21e18af588c748c5925e1407", + "proposal_alpha": null, + "proposal_sha256": null, + "seed": 484370, + "total_batch_size": 40 + }, + "schema_version": 1, + "world_size": 2 +} diff --git a/settings/qwen_robotwin/keyframe/config.full.yaml b/settings/qwen_robotwin/keyframe/config.full.yaml new file mode 100644 index 0000000000000000000000000000000000000000..662dff649570859f78546d28087515ae12872715 --- /dev/null +++ b/settings/qwen_robotwin/keyframe/config.full.yaml @@ -0,0 +1,110 @@ +run_id: qwen-robotwin-keyframe-11000-to-11500-seed484370 +run_root_dir: /home/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs +seed: 484370 +wandb_entity: null +wandb_project: InfoFlow-StarVLA-RoboTwin +is_debug: false +version_id: '0.21' +framework: + name: QwenGR00T + qwenvl: + base_vlm: /home/liuyu/student-workspace/data/runtime/playground/Pretrained_models/Qwen3-VL-4B-Instruct + attn_implementation: flash_attention_2 + vl_hidden_dim: 2048 + action_model: + action_model_type: DiT-B + action_hidden_dim: 2048 + hidden_size: 1024 + add_pos_embed: true + max_seq_len: 1024 + action_dim: 14 + state_dim: 0 + action_horizon: 50 + repeated_diffusion_steps: 8 + noise_beta_alpha: 1.5 + noise_beta_beta: 1.0 + noise_s: 0.999 + num_timestep_buckets: 1000 + num_inference_timesteps: 4 + num_target_vision_tokens: 32 + diffusion_model_cfg: + cross_attention_dim: 2560 + dropout: 0.2 + final_dropout: true + interleave_self_attention: true + norm_type: ada_norm + num_layers: 16 + output_dim: 1024 + positional_embeddings: null + future_action_window_size: 49 + past_action_window_size: 0 + dino: + dino_backbone: dinov2_vits14 +datasets: + vla_data: + dataset_py: lerobot_datasets + data_root_dir: /home/liuyu/student-workspace/data/robotwin/RoboTwin + data_mix: robotwin_all_50 + action_type: abs_qpos + action_mode: abs + include_state: false + sequential_step_sampling: false + per_device_batch_size: 5 + load_all_data_for_training: true + image_size: + - 224 + - 224 + video_backend: pyav + num_workers: 4 + expected_world_size: 2 + traceable_sampling: true + dataset_provenance_path: /home/liuyu/student-workspace/data/robotwin/manifests/data_provenance.json + dataset_content_fingerprint_path: /home/liuyu/student-workspace/data/robotwin/manifests/content_fingerprints.json + valid_anchor_manifest_path: /home/liuyu/student-workspace/data/robotwin/manifests/valid_anchor_manifest.json + transform_rng_seed: 289726 + temporal_proposal_path: null +trainer: + use_deepspeed: true + max_train_steps: 150000 + num_warmup_steps: 5000 + save_interval: null + save_steps: + - 11500 + evaluation_snapshot_interval: null + evaluation_snapshot_start_step: 10000 + evaluation_snapshot_end_step: 20000 + eval_interval: null + repeated_diffusion_steps: 4 + learning_rate: + base: 1.0e-05 + qwen_vl_interface: 1.0e-05 + action_model: 0.0001 + lr_scheduler_type: cosine_with_min_lr + scheduler_specific_kwargs: + min_lr: 5.0e-07 + freeze_modules: '' + trainable_modules: '' + frozen_modules_eval: false + logging_frequency: 10 + gradient_clipping: 1.0 + gradient_accumulation_steps: 4 + save_format: pt + save_final_checkpoint: false + save_consolidated_model: false + optimizer: + name: AdamW8bit + betas: + - 0.9 + - 0.95 + eps: 1.0e-08 + weight_decay: 1.0e-08 + stop_after_steps: 11500 + is_resume: true + allow_model_only_resume: false + resume_accelerate_checkpoint: /home/liuyu/student-workspace/data/robotwin/common_start/qwen3vl-groot-robotwin-official150k-adamw8bit-mb5-ga4-seed42-20260810/checkpoints/steps_11000_accelerate + branch_parent_seed: 42 + branch_rng_seed: 484370 + model_rng_seed: 721927 + sample_loss_weight_key: keyframe_weight +config_yaml: /fs1/private/user/liuyu/student-workspace/outputs/robotwin_external_20260831/configs/qwen_keyframe_formal.yaml +output_dir: /home/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs/qwen-robotwin-keyframe-11000-to-11500-seed484370 diff --git a/settings/qwen_robotwin/keyframe/config.yaml b/settings/qwen_robotwin/keyframe/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3b503d7b6eadbb37e32771a0cafbe3e3c208e69c --- /dev/null +++ b/settings/qwen_robotwin/keyframe/config.yaml @@ -0,0 +1,65 @@ +datasets: + vla_data: + action_mode: abs + data_mix: robotwin_all_50 + data_root_dir: /home/liuyu/student-workspace/data/robotwin/RoboTwin + dataset_content_fingerprint_path: /home/liuyu/student-workspace/data/robotwin/manifests/content_fingerprints.json + dataset_provenance_path: /home/liuyu/student-workspace/data/robotwin/manifests/data_provenance.json + dataset_py: lerobot_datasets + expected_world_size: 2 + num_workers: 4 + per_device_batch_size: 5 + temporal_proposal_path: null + traceable_sampling: true + valid_anchor_manifest_path: /home/liuyu/student-workspace/data/robotwin/manifests/valid_anchor_manifest.json + video_backend: pyav +framework: + action_model: + action_dim: 14 + action_horizon: 50 + action_model_type: DiT-B + add_pos_embed: true + diffusion_model_cfg: + cross_attention_dim: 2560 + dropout: 0.2 + final_dropout: true + interleave_self_attention: true + norm_type: ada_norm + num_layers: 16 + output_dim: 1024 + positional_embeddings: null + hidden_size: 1024 + max_seq_len: 1024 + noise_beta_alpha: 1.5 + noise_beta_beta: 1.0 + num_inference_timesteps: 4 + num_target_vision_tokens: 32 + num_timestep_buckets: 1000 + state_dim: 0 + name: QwenGR00T + qwenvl: + attn_implementation: flash_attention_2 + base_vlm: /home/liuyu/student-workspace/data/runtime/playground/Pretrained_models/Qwen3-VL-4B-Instruct +output_dir: /home/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs/qwen-robotwin-keyframe-11000-to-11500-seed484370 +run_id: qwen-robotwin-keyframe-11000-to-11500-seed484370 +run_root_dir: /home/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs +seed: 484370 +trainer: + freeze_modules: '' + gradient_accumulation_steps: 4 + learning_rate: + action_model: 0.0001 + base: 1.0e-05 + qwen_vl_interface: 1.0e-05 + lr_scheduler_type: cosine_with_min_lr + max_train_steps: 150000 + num_warmup_steps: 5000 + optimizer: + betas: + - 0.9 + - 0.95 + eps: 1.0e-08 + name: AdamW8bit + weight_decay: 1.0e-08 + scheduler_specific_kwargs: + min_lr: 5.0e-07 diff --git a/settings/qwen_robotwin/keyframe/dataset_statistics.json b/settings/qwen_robotwin/keyframe/dataset_statistics.json new file mode 100644 index 0000000000000000000000000000000000000000..c920d76f3fc976e9e0b30e36eab0cff8a6bf1adb --- /dev/null +++ b/settings/qwen_robotwin/keyframe/dataset_statistics.json @@ -0,0 +1,218 @@ +{ + "new_embodiment": { + "action": { + "mean": [ + -0.22423998869955536, + 1.0729362124204633, + 0.775079609751701, + -0.32701663097366684, + 0.06159586461764771, + -0.06420705380151046, + 0.22430071420967568, + 1.1274866509437556, + 0.831773659586906, + -0.3638282522559164, + -0.024959841337949897, + 0.05302203213563182, + 0.6774904182553295, + 0.6598676902055738 + ], + "std": [ + 0.39622931737429123, + 1.003590449339515, + 0.7811611351334437, + 0.6707485714474849, + 0.2686908204983921, + 0.6492019642472782, + 0.34733679400923234, + 1.0176226676795115, + 0.8039915805960646, + 0.7067249045233707, + 0.26259645177151686, + 0.6808759458519371, + 0.4482151151114418, + 0.4539921671160285 + ], + "max": [ + 6.45659065246582, + 4.179152488708496, + 5.346591472625732, + 1.7942548990249634, + 1.8604620695114136, + 5.43813943862915, + 7.142920017242432, + 4.157068729400635, + 5.672450065612793, + 1.95806884765625, + 1.5663840770721436, + 5.278968811035156, + 1.0, + 1.0 + ], + "min": [ + -7.552278995513916, + -0.36354875564575195, + -0.18577136099338531, + -1.956291913986206, + -1.6801013946533203, + -7.678869724273682, + -8.539926528930664, + -0.6294453740119934, + -0.07775841653347015, + -2.1328067779541016, + -2.1285502910614014, + -8.228925704956055, + 0.0, + 0.0 + ], + "q01": [ + -7.334665780067444, + -5.257390398583084e-07, + -2.8215323254698887e-05, + -1.841589093208313, + -1.2851550579071045, + -6.268352918624878, + -5.995517673492431, + -0.572821855545044, + -2.81171942333458e-05, + -1.8467556238174438, + -1.4545854091644288, + -5.945669202804565, + 0.0, + 0.0 + ], + "q99": [ + 5.016749763488781, + 3.4792869091033936, + 4.225283479690551, + 1.7064905214309687, + 1.4829185009002717, + 3.7177388882636553, + 1.258314609527588, + 2.967935800552368, + 3.2566452693939207, + 1.766038613319397, + 1.4066662788391113, + 3.5830023288726807, + 1.0, + 1.0 + ], + "mask": [ + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + false, + false + ] + }, + "state": { + "mean": [ + -0.22281882727518673, + 1.0668349397182466, + 0.770632639825344, + -0.3252330579422412, + 0.06118139047002841, + -0.06316296239456277, + 0.22253762468695631, + 1.1201750242710113, + 0.8263923591375351, + -0.36159489695448405, + -0.024702461528722772, + 0.05203246331657278, + 0.6793262696266176, + 0.6616973677277563 + ], + "std": [ + 0.39522870886731665, + 1.004018577033247, + 0.7809103811570285, + 0.6688088363177986, + 0.26775723818956776, + 0.6463212583712935, + 0.34648488321363663, + 1.018232630032839, + 0.8038798397682734, + 0.7045612787595747, + 0.26143414213134614, + 0.6774619870452644, + 0.44749220497027625, + 0.45333916022037934 + ], + "max": [ + 6.45659065246582, + 4.179152488708496, + 5.346591472625732, + 1.7942548990249634, + 1.8604620695114136, + 5.43813943862915, + 7.142920017242432, + 4.157068729400635, + 5.672450065612793, + 1.95806884765625, + 1.5663840770721436, + 5.278968811035156, + 1.0, + 1.0 + ], + "min": [ + -7.552278995513916, + -0.36354875564575195, + -0.18577136099338531, + -1.956291913986206, + -1.6801013946533203, + -7.678869724273682, + -8.539926528930664, + -0.6294453740119934, + -0.07775841653347015, + -2.1328067779541016, + -2.1285502910614014, + -8.228925704956055, + 0.0, + 0.0 + ], + "q01": [ + -7.334665780067444, + -5.257390398583084e-07, + -2.8215323254698887e-05, + -1.841589093208313, + -1.2851550579071045, + -6.268109321594238, + -5.912737417221069, + -0.572821855545044, + -2.81171942333458e-05, + -1.8467556238174438, + -1.440324923992157, + -5.8623305034637445, + 0.0, + 0.0 + ], + "q99": [ + 4.902149534225477, + 3.4775466918945312, + 4.224453401565551, + 1.7064905214309687, + 1.480757713317871, + 3.609884884357428, + 1.258314609527588, + 2.967935800552368, + 3.2566452693939207, + 1.7433889162540437, + 1.4066662788391113, + 3.581243851184845, + 1.0, + 1.0 + ] + }, + "num_transitions": 6075103, + "num_trajectories": 27500 + } +} \ No newline at end of file diff --git a/settings/qwen_robotwin/keyframe/external_resume_certificate.json b/settings/qwen_robotwin/keyframe/external_resume_certificate.json new file mode 100644 index 0000000000000000000000000000000000000000..1683e3ebced3d2ce2def520087db654c682e1449 --- /dev/null +++ b/settings/qwen_robotwin/keyframe/external_resume_certificate.json @@ -0,0 +1,138 @@ +{ + "branch_resume_identity": { + "config_sha256": "fe3d7fe8a58e48266616ea8d6084b359119c2f63d0f7a15661708b726f7cfddc", + "identity_schema_version": 2, + "manifest_sha256": "dcb728a00da4b4ad7f989fa313d6d3355e13f57efefc7267e9ea092f2ffd8a50", + "model_structure_sha256": "7809faa5245147f81b98f3dbd17c7a063a7cb07a21e18af588c748c5925e1407", + "proposal_alpha": null, + "proposal_sha256": null, + "seed": 484370, + "total_batch_size": 40 + }, + "completed_steps": 11000, + "optimizer_param_groups": [ + { + "betas": [ + 0.9, + 0.95 + ], + "eps": 1e-08, + "initial_lr": 1e-05, + "lr": 9.959920880962606e-06, + "name": "qwen_vl_interface", + "weight_decay": 1e-08 + }, + { + "betas": [ + 0.9, + 0.95 + ], + "eps": 1e-08, + "initial_lr": 0.0001, + "lr": 9.959920880962605e-05, + "name": "action_model", + "weight_decay": 1e-08 + } + ], + "optimizer_state_entries": 8, + "parent_checkpoint": "/fs1/private/user/liuyu/student-workspace/data/robotwin/common_start/qwen3vl-groot-robotwin-official150k-adamw8bit-mb5-ga4-seed42-20260810/checkpoints/steps_11000_accelerate", + "parent_checkpoint_commit_sha256": "180f1ed2d2b7a9e6c158974fc9ec951182dfb629f6a2806808ac5639d491ce1b", + "parent_checkpoint_files": [ + { + "bytes": 1525, + "name": "custom_checkpoint_0.pkl", + "sha256": "70788f8a2d7852a70fbb794ab9176d14ad37fb6d1b1d821a368572e1d8524390" + }, + { + "bytes": 13, + "name": "latest", + "sha256": "68d0ca762047fd48fdd6a06a5e6465209289c028bfd63f5557a110ff1ebc15ac" + }, + { + "bytes": 13867381301, + "name": "pytorch_model/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt", + "sha256": "42b6d10bb158b404056983ce88505c32738b29f3a062fe6835d24ce26c521a6a" + }, + { + "bytes": 13867381301, + "name": "pytorch_model/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt", + "sha256": "7b0a1bbc92efffbf056bc40f40e93a94b67ddf3f375dc6ae263bcb048ba6ec63" + }, + { + "bytes": 566626, + "name": "pytorch_model/zero_pp_rank_0_mp_rank_00_model_states.pt", + "sha256": "8a3ef88f9ab0df8197a836dca1d6e2416835f0b3d827cb10a78254b9a969ce7c" + }, + { + "bytes": 566498, + "name": "pytorch_model/zero_pp_rank_1_mp_rank_00_model_states.pt", + "sha256": "67cd03fceb5ee4c2be6affe504f71a48ee10127d6ea3a1f00d05721895ffba53" + }, + { + "bytes": 15017, + "name": "random_states_0.pkl", + "sha256": "636b1c5983ee2f8a5047025027453c9a478c0606389f2f6945b370f39cec9e65" + }, + { + "bytes": 15081, + "name": "random_states_1.pkl", + "sha256": "416b3be53ff049935984cc65385d7dae3b0e5a82df66c9516f61ee9100c41961" + }, + { + "bytes": 33270, + "name": "zero_to_fp32.py", + "sha256": "e8a2553b434ac043ec63b022db1e768277acf8d4e70f10a2842db6d238977135" + } + ], + "parent_checkpoint_files_sha256": "73692dae620e3d4368fe513e8adff4fe54bdb25341b4293e34ee2a6286554f8d", + "parent_checkpoint_integrity_manifest_sha256": null, + "parent_config_sha256": "4882855d94076fc0ec19a0d511f14b1cbe82860c8b5df3dcc64c5c5f6c0d2321", + "parent_resume_identity": { + "config_sha256": "1a48ea51cf89ce72a7956aaa29f710bba0326c9b726e52795561b381507d26fa", + "identity_schema_version": 2, + "manifest_sha256": "dcb728a00da4b4ad7f989fa313d6d3355e13f57efefc7267e9ea092f2ffd8a50", + "model_structure_sha256": "7809faa5245147f81b98f3dbd17c7a063a7cb07a21e18af588c748c5925e1407", + "proposal_alpha": null, + "proposal_sha256": null, + "seed": 42, + "total_batch_size": 40 + }, + "proposal_sha256": null, + "rank_state_summaries": [ + { + "optimizer_state_entries": 4, + "optimizer_state_sha256": "e1eea37514fa0478835597d1594b7a43b45cea9adc4ccb79a0de8fb9e10fe9f0", + "rank": 0, + "trainable_state_sha256": "ca2a19167d59aab693d0a0a1e17ea9adbf9cfd7367dfa7a5c90b04cf2183b717" + }, + { + "optimizer_state_entries": 4, + "optimizer_state_sha256": "62c4e2908a7e97fe9a1d60642a85968081b55345493a11b9e7f68e49fcdb22f3", + "rank": 1, + "trainable_state_sha256": "ca2a19167d59aab693d0a0a1e17ea9adbf9cfd7367dfa7a5c90b04cf2183b717" + } + ], + "repeated_diffusion_steps": 4, + "rng_streams": { + "model_base": 721927, + "sampler": 484370, + "transform": 289726 + }, + "scheduler": { + "class": "LambdaLR", + "last_epoch": 11000, + "last_lr": [ + 9.959920880962606e-06, + 9.959920880962605e-05 + ], + "step_count": 11001 + }, + "scheduler_resume_mode": "accelerate_checkpoint", + "schema_version": 1, + "status": "exact_external_resume_loaded", + "total_batch_size": 40, + "trainable_parameter_elements": 32022798, + "trainable_parameter_sha256": "1bc504b22e4375be40bbbd8250be5b5f380c14d2fb47ed85deba3604a84b5484", + "trainable_parameter_tensors": 957, + "world_size": 2 +} diff --git a/settings/qwen_robotwin/keyframe/initialization_manifest.json b/settings/qwen_robotwin/keyframe/initialization_manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..35160df6ed677c681d32574fe5119b91ef516bed --- /dev/null +++ b/settings/qwen_robotwin/keyframe/initialization_manifest.json @@ -0,0 +1,6 @@ +{ + "schema_version": 1, + "initialization_seed": 484370, + "action_model_sha256": "1cab43a1e801a94110a20384a9ad6cd76a239fedd9fe17890662da7e22d47e62", + "action_model_parameter_elements": 160689934 +} diff --git a/settings/qwen_robotwin/keyframe/valid_anchor_manifest.json b/settings/qwen_robotwin/keyframe/valid_anchor_manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..3f5c91478b5931286c9c84d021a7ffc10a0fb5c0 --- /dev/null +++ b/settings/qwen_robotwin/keyframe/valid_anchor_manifest.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2090f6b9d7e1bc5ef41897436690f031c187d625fa6b71ae0c54f3793660c757 +size 107646168 diff --git a/settings/qwen_robotwin/loss_is/checkpoint.json b/settings/qwen_robotwin/loss_is/checkpoint.json new file mode 100644 index 0000000000000000000000000000000000000000..104c27e8c2b227f4dff35031c5df4c2bc83d5e56 --- /dev/null +++ b/settings/qwen_robotwin/loss_is/checkpoint.json @@ -0,0 +1,65 @@ +{ + "completed_steps": 11500, + "files": [ + { + "bytes": 1525, + "name": "custom_checkpoint_0.pkl", + "sha256": "b1002611dfae1393f28a634b9d947819b3df149c1e78485b44a31a200d2c1414" + }, + { + "bytes": 13, + "name": "latest", + "sha256": "68d0ca762047fd48fdd6a06a5e6465209289c028bfd63f5557a110ff1ebc15ac" + }, + { + "bytes": 13867389419, + "name": "pytorch_model/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt", + "sha256": "3717d6ca254df847a066d5ffb6bcb5f359321a66b4f993884835b4f3169db253" + }, + { + "bytes": 13867389419, + "name": "pytorch_model/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt", + "sha256": "ee1b2ed950c9bb52c52d12952219a96572c76f3578de99996a7eb5fdb5446515" + }, + { + "bytes": 566626, + "name": "pytorch_model/zero_pp_rank_0_mp_rank_00_model_states.pt", + "sha256": "743ad760245b429d5387d7868a0b61bf310e72f887b02097c94265c46e0d6530" + }, + { + "bytes": 566498, + "name": "pytorch_model/zero_pp_rank_1_mp_rank_00_model_states.pt", + "sha256": "d2738a5c4991da9724574e3120ff7d06f32674a5d3074c07851d4fc5d41f811c" + }, + { + "bytes": 15081, + "name": "random_states_0.pkl", + "sha256": "8f73601e4e08b2e7ef10a33a59de5de8bf3247ab8a0e1ac578e9d2a4b70a6823" + }, + { + "bytes": 15081, + "name": "random_states_1.pkl", + "sha256": "ee39fa0a5ac8438faf6b549a41cc2021e0b22d0c7698aa67fcca9cafa74848a3" + }, + { + "bytes": 33270, + "name": "zero_to_fp32.py", + "sha256": "e8a2553b434ac043ec63b022db1e768277acf8d4e70f10a2842db6d238977135" + } + ], + "registered_checkpointables": [ + "lr_scheduler" + ], + "resume_identity": { + "config_sha256": "fe3d7fe8a58e48266616ea8d6084b359119c2f63d0f7a15661708b726f7cfddc", + "identity_schema_version": 2, + "manifest_sha256": "dcb728a00da4b4ad7f989fa313d6d3355e13f57efefc7267e9ea092f2ffd8a50", + "model_structure_sha256": "7809faa5245147f81b98f3dbd17c7a063a7cb07a21e18af588c748c5925e1407", + "proposal_alpha": null, + "proposal_sha256": null, + "seed": 484370, + "total_batch_size": 40 + }, + "schema_version": 1, + "world_size": 2 +} diff --git a/settings/qwen_robotwin/loss_is/config.full.yaml b/settings/qwen_robotwin/loss_is/config.full.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0ca715aefeae445b2c4b6f1980e251ff05746acf --- /dev/null +++ b/settings/qwen_robotwin/loss_is/config.full.yaml @@ -0,0 +1,116 @@ +run_id: qwen-robotwin-loss_is-11000-to-11500-seed484370 +run_root_dir: /home/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs +seed: 484370 +wandb_entity: null +wandb_project: InfoFlow-StarVLA-RoboTwin +is_debug: false +version_id: '0.21' +framework: + name: QwenGR00T + qwenvl: + base_vlm: /home/liuyu/student-workspace/data/runtime/playground/Pretrained_models/Qwen3-VL-4B-Instruct + attn_implementation: flash_attention_2 + vl_hidden_dim: 2048 + action_model: + action_model_type: DiT-B + action_hidden_dim: 2048 + hidden_size: 1024 + add_pos_embed: true + max_seq_len: 1024 + action_dim: 14 + state_dim: 0 + action_horizon: 50 + repeated_diffusion_steps: 8 + noise_beta_alpha: 1.5 + noise_beta_beta: 1.0 + noise_s: 0.999 + num_timestep_buckets: 1000 + num_inference_timesteps: 4 + num_target_vision_tokens: 32 + diffusion_model_cfg: + cross_attention_dim: 2560 + dropout: 0.2 + final_dropout: true + interleave_self_attention: true + norm_type: ada_norm + num_layers: 16 + output_dim: 1024 + positional_embeddings: null + future_action_window_size: 49 + past_action_window_size: 0 + dino: + dino_backbone: dinov2_vits14 +datasets: + vla_data: + dataset_py: lerobot_datasets + data_root_dir: /home/liuyu/student-workspace/data/robotwin/RoboTwin + data_mix: robotwin_all_50 + action_type: abs_qpos + action_mode: abs + include_state: false + sequential_step_sampling: false + per_device_batch_size: 5 + load_all_data_for_training: true + image_size: + - 224 + - 224 + video_backend: pyav + num_workers: 4 + expected_world_size: 2 + traceable_sampling: true + dataset_provenance_path: /home/liuyu/student-workspace/data/robotwin/manifests/data_provenance.json + dataset_content_fingerprint_path: /home/liuyu/student-workspace/data/robotwin/manifests/content_fingerprints.json + valid_anchor_manifest_path: /home/liuyu/student-workspace/data/robotwin/manifests/valid_anchor_manifest.json + transform_rng_seed: 289726 + temporal_proposal_path: null +trainer: + use_deepspeed: true + max_train_steps: 150000 + num_warmup_steps: 5000 + save_interval: null + save_steps: + - 11500 + evaluation_snapshot_interval: null + evaluation_snapshot_start_step: 10000 + evaluation_snapshot_end_step: 20000 + eval_interval: null + repeated_diffusion_steps: 4 + learning_rate: + base: 1.0e-05 + qwen_vl_interface: 1.0e-05 + action_model: 0.0001 + lr_scheduler_type: cosine_with_min_lr + scheduler_specific_kwargs: + min_lr: 5.0e-07 + freeze_modules: '' + trainable_modules: '' + frozen_modules_eval: false + logging_frequency: 10 + gradient_clipping: 1.0 + gradient_accumulation_steps: 4 + save_format: pt + save_final_checkpoint: false + save_consolidated_model: false + optimizer: + name: AdamW8bit + betas: + - 0.9 + - 0.95 + eps: 1.0e-08 + weight_decay: 1.0e-08 + stop_after_steps: 11500 + is_resume: true + allow_model_only_resume: false + resume_accelerate_checkpoint: /home/liuyu/student-workspace/data/robotwin/common_start/qwen3vl-groot-robotwin-official150k-adamw8bit-mb5-ga4-seed42-20260810/checkpoints/steps_11000_accelerate + branch_parent_seed: 42 + branch_rng_seed: 484370 + model_rng_seed: 721927 + sample_loss_weight_key: loss_is_importance_weight + model_rng_trace_key: loss_is_occurrence_id + loss_is: + candidate_factor: 3 + seed: 484370 + condition_threshold: 2.0 + condition_momentum: 0.9 +config_yaml: /home/liuyu/student-workspace/outputs/robotwin_external_20260831/configs/qwen_loss_is_formal.yaml +output_dir: /home/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs/qwen-robotwin-loss_is-11000-to-11500-seed484370 diff --git a/settings/qwen_robotwin/loss_is/config.yaml b/settings/qwen_robotwin/loss_is/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..910d057f2471e385f98f8498cc7fbaf2a18af60f --- /dev/null +++ b/settings/qwen_robotwin/loss_is/config.yaml @@ -0,0 +1,70 @@ +datasets: + vla_data: + action_mode: abs + data_mix: robotwin_all_50 + data_root_dir: /home/liuyu/student-workspace/data/robotwin/RoboTwin + dataset_content_fingerprint_path: /home/liuyu/student-workspace/data/robotwin/manifests/content_fingerprints.json + dataset_provenance_path: /home/liuyu/student-workspace/data/robotwin/manifests/data_provenance.json + dataset_py: lerobot_datasets + expected_world_size: 2 + num_workers: 4 + per_device_batch_size: 5 + temporal_proposal_path: null + traceable_sampling: true + valid_anchor_manifest_path: /home/liuyu/student-workspace/data/robotwin/manifests/valid_anchor_manifest.json + video_backend: pyav +framework: + action_model: + action_dim: 14 + action_horizon: 50 + action_model_type: DiT-B + add_pos_embed: true + diffusion_model_cfg: + cross_attention_dim: 2560 + dropout: 0.2 + final_dropout: true + interleave_self_attention: true + norm_type: ada_norm + num_layers: 16 + output_dim: 1024 + positional_embeddings: null + hidden_size: 1024 + max_seq_len: 1024 + noise_beta_alpha: 1.5 + noise_beta_beta: 1.0 + num_inference_timesteps: 4 + num_target_vision_tokens: 32 + num_timestep_buckets: 1000 + state_dim: 0 + name: QwenGR00T + qwenvl: + attn_implementation: flash_attention_2 + base_vlm: /home/liuyu/student-workspace/data/runtime/playground/Pretrained_models/Qwen3-VL-4B-Instruct +output_dir: /home/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs/qwen-robotwin-loss_is-11000-to-11500-seed484370 +run_id: qwen-robotwin-loss_is-11000-to-11500-seed484370 +run_root_dir: /home/liuyu/student-workspace/outputs/robotwin_external_20260831/formal_runs +seed: 484370 +trainer: + freeze_modules: '' + gradient_accumulation_steps: 4 + learning_rate: + action_model: 0.0001 + base: 1.0e-05 + qwen_vl_interface: 1.0e-05 + loss_is: + candidate_factor: 3 + condition_momentum: 0.9 + condition_threshold: 2.0 + seed: 484370 + lr_scheduler_type: cosine_with_min_lr + max_train_steps: 150000 + num_warmup_steps: 5000 + optimizer: + betas: + - 0.9 + - 0.95 + eps: 1.0e-08 + name: AdamW8bit + weight_decay: 1.0e-08 + scheduler_specific_kwargs: + min_lr: 5.0e-07 diff --git a/settings/qwen_robotwin/loss_is/dataset_statistics.json b/settings/qwen_robotwin/loss_is/dataset_statistics.json new file mode 100644 index 0000000000000000000000000000000000000000..c920d76f3fc976e9e0b30e36eab0cff8a6bf1adb --- /dev/null +++ b/settings/qwen_robotwin/loss_is/dataset_statistics.json @@ -0,0 +1,218 @@ +{ + "new_embodiment": { + "action": { + "mean": [ + -0.22423998869955536, + 1.0729362124204633, + 0.775079609751701, + -0.32701663097366684, + 0.06159586461764771, + -0.06420705380151046, + 0.22430071420967568, + 1.1274866509437556, + 0.831773659586906, + -0.3638282522559164, + -0.024959841337949897, + 0.05302203213563182, + 0.6774904182553295, + 0.6598676902055738 + ], + "std": [ + 0.39622931737429123, + 1.003590449339515, + 0.7811611351334437, + 0.6707485714474849, + 0.2686908204983921, + 0.6492019642472782, + 0.34733679400923234, + 1.0176226676795115, + 0.8039915805960646, + 0.7067249045233707, + 0.26259645177151686, + 0.6808759458519371, + 0.4482151151114418, + 0.4539921671160285 + ], + "max": [ + 6.45659065246582, + 4.179152488708496, + 5.346591472625732, + 1.7942548990249634, + 1.8604620695114136, + 5.43813943862915, + 7.142920017242432, + 4.157068729400635, + 5.672450065612793, + 1.95806884765625, + 1.5663840770721436, + 5.278968811035156, + 1.0, + 1.0 + ], + "min": [ + -7.552278995513916, + -0.36354875564575195, + -0.18577136099338531, + -1.956291913986206, + -1.6801013946533203, + -7.678869724273682, + -8.539926528930664, + -0.6294453740119934, + -0.07775841653347015, + -2.1328067779541016, + -2.1285502910614014, + -8.228925704956055, + 0.0, + 0.0 + ], + "q01": [ + -7.334665780067444, + -5.257390398583084e-07, + -2.8215323254698887e-05, + -1.841589093208313, + -1.2851550579071045, + -6.268352918624878, + -5.995517673492431, + -0.572821855545044, + -2.81171942333458e-05, + -1.8467556238174438, + -1.4545854091644288, + -5.945669202804565, + 0.0, + 0.0 + ], + "q99": [ + 5.016749763488781, + 3.4792869091033936, + 4.225283479690551, + 1.7064905214309687, + 1.4829185009002717, + 3.7177388882636553, + 1.258314609527588, + 2.967935800552368, + 3.2566452693939207, + 1.766038613319397, + 1.4066662788391113, + 3.5830023288726807, + 1.0, + 1.0 + ], + "mask": [ + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + false, + false + ] + }, + "state": { + "mean": [ + -0.22281882727518673, + 1.0668349397182466, + 0.770632639825344, + -0.3252330579422412, + 0.06118139047002841, + -0.06316296239456277, + 0.22253762468695631, + 1.1201750242710113, + 0.8263923591375351, + -0.36159489695448405, + -0.024702461528722772, + 0.05203246331657278, + 0.6793262696266176, + 0.6616973677277563 + ], + "std": [ + 0.39522870886731665, + 1.004018577033247, + 0.7809103811570285, + 0.6688088363177986, + 0.26775723818956776, + 0.6463212583712935, + 0.34648488321363663, + 1.018232630032839, + 0.8038798397682734, + 0.7045612787595747, + 0.26143414213134614, + 0.6774619870452644, + 0.44749220497027625, + 0.45333916022037934 + ], + "max": [ + 6.45659065246582, + 4.179152488708496, + 5.346591472625732, + 1.7942548990249634, + 1.8604620695114136, + 5.43813943862915, + 7.142920017242432, + 4.157068729400635, + 5.672450065612793, + 1.95806884765625, + 1.5663840770721436, + 5.278968811035156, + 1.0, + 1.0 + ], + "min": [ + -7.552278995513916, + -0.36354875564575195, + -0.18577136099338531, + -1.956291913986206, + -1.6801013946533203, + -7.678869724273682, + -8.539926528930664, + -0.6294453740119934, + -0.07775841653347015, + -2.1328067779541016, + -2.1285502910614014, + -8.228925704956055, + 0.0, + 0.0 + ], + "q01": [ + -7.334665780067444, + -5.257390398583084e-07, + -2.8215323254698887e-05, + -1.841589093208313, + -1.2851550579071045, + -6.268109321594238, + -5.912737417221069, + -0.572821855545044, + -2.81171942333458e-05, + -1.8467556238174438, + -1.440324923992157, + -5.8623305034637445, + 0.0, + 0.0 + ], + "q99": [ + 4.902149534225477, + 3.4775466918945312, + 4.224453401565551, + 1.7064905214309687, + 1.480757713317871, + 3.609884884357428, + 1.258314609527588, + 2.967935800552368, + 3.2566452693939207, + 1.7433889162540437, + 1.4066662788391113, + 3.581243851184845, + 1.0, + 1.0 + ] + }, + "num_transitions": 6075103, + "num_trajectories": 27500 + } +} \ No newline at end of file diff --git a/settings/qwen_robotwin/loss_is/external_resume_certificate.json b/settings/qwen_robotwin/loss_is/external_resume_certificate.json new file mode 100644 index 0000000000000000000000000000000000000000..1683e3ebced3d2ce2def520087db654c682e1449 --- /dev/null +++ b/settings/qwen_robotwin/loss_is/external_resume_certificate.json @@ -0,0 +1,138 @@ +{ + "branch_resume_identity": { + "config_sha256": "fe3d7fe8a58e48266616ea8d6084b359119c2f63d0f7a15661708b726f7cfddc", + "identity_schema_version": 2, + "manifest_sha256": "dcb728a00da4b4ad7f989fa313d6d3355e13f57efefc7267e9ea092f2ffd8a50", + "model_structure_sha256": "7809faa5245147f81b98f3dbd17c7a063a7cb07a21e18af588c748c5925e1407", + "proposal_alpha": null, + "proposal_sha256": null, + "seed": 484370, + "total_batch_size": 40 + }, + "completed_steps": 11000, + "optimizer_param_groups": [ + { + "betas": [ + 0.9, + 0.95 + ], + "eps": 1e-08, + "initial_lr": 1e-05, + "lr": 9.959920880962606e-06, + "name": "qwen_vl_interface", + "weight_decay": 1e-08 + }, + { + "betas": [ + 0.9, + 0.95 + ], + "eps": 1e-08, + "initial_lr": 0.0001, + "lr": 9.959920880962605e-05, + "name": "action_model", + "weight_decay": 1e-08 + } + ], + "optimizer_state_entries": 8, + "parent_checkpoint": "/fs1/private/user/liuyu/student-workspace/data/robotwin/common_start/qwen3vl-groot-robotwin-official150k-adamw8bit-mb5-ga4-seed42-20260810/checkpoints/steps_11000_accelerate", + "parent_checkpoint_commit_sha256": "180f1ed2d2b7a9e6c158974fc9ec951182dfb629f6a2806808ac5639d491ce1b", + "parent_checkpoint_files": [ + { + "bytes": 1525, + "name": "custom_checkpoint_0.pkl", + "sha256": "70788f8a2d7852a70fbb794ab9176d14ad37fb6d1b1d821a368572e1d8524390" + }, + { + "bytes": 13, + "name": "latest", + "sha256": "68d0ca762047fd48fdd6a06a5e6465209289c028bfd63f5557a110ff1ebc15ac" + }, + { + "bytes": 13867381301, + "name": "pytorch_model/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt", + "sha256": "42b6d10bb158b404056983ce88505c32738b29f3a062fe6835d24ce26c521a6a" + }, + { + "bytes": 13867381301, + "name": "pytorch_model/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt", + "sha256": "7b0a1bbc92efffbf056bc40f40e93a94b67ddf3f375dc6ae263bcb048ba6ec63" + }, + { + "bytes": 566626, + "name": "pytorch_model/zero_pp_rank_0_mp_rank_00_model_states.pt", + "sha256": "8a3ef88f9ab0df8197a836dca1d6e2416835f0b3d827cb10a78254b9a969ce7c" + }, + { + "bytes": 566498, + "name": "pytorch_model/zero_pp_rank_1_mp_rank_00_model_states.pt", + "sha256": "67cd03fceb5ee4c2be6affe504f71a48ee10127d6ea3a1f00d05721895ffba53" + }, + { + "bytes": 15017, + "name": "random_states_0.pkl", + "sha256": "636b1c5983ee2f8a5047025027453c9a478c0606389f2f6945b370f39cec9e65" + }, + { + "bytes": 15081, + "name": "random_states_1.pkl", + "sha256": "416b3be53ff049935984cc65385d7dae3b0e5a82df66c9516f61ee9100c41961" + }, + { + "bytes": 33270, + "name": "zero_to_fp32.py", + "sha256": "e8a2553b434ac043ec63b022db1e768277acf8d4e70f10a2842db6d238977135" + } + ], + "parent_checkpoint_files_sha256": "73692dae620e3d4368fe513e8adff4fe54bdb25341b4293e34ee2a6286554f8d", + "parent_checkpoint_integrity_manifest_sha256": null, + "parent_config_sha256": "4882855d94076fc0ec19a0d511f14b1cbe82860c8b5df3dcc64c5c5f6c0d2321", + "parent_resume_identity": { + "config_sha256": "1a48ea51cf89ce72a7956aaa29f710bba0326c9b726e52795561b381507d26fa", + "identity_schema_version": 2, + "manifest_sha256": "dcb728a00da4b4ad7f989fa313d6d3355e13f57efefc7267e9ea092f2ffd8a50", + "model_structure_sha256": "7809faa5245147f81b98f3dbd17c7a063a7cb07a21e18af588c748c5925e1407", + "proposal_alpha": null, + "proposal_sha256": null, + "seed": 42, + "total_batch_size": 40 + }, + "proposal_sha256": null, + "rank_state_summaries": [ + { + "optimizer_state_entries": 4, + "optimizer_state_sha256": "e1eea37514fa0478835597d1594b7a43b45cea9adc4ccb79a0de8fb9e10fe9f0", + "rank": 0, + "trainable_state_sha256": "ca2a19167d59aab693d0a0a1e17ea9adbf9cfd7367dfa7a5c90b04cf2183b717" + }, + { + "optimizer_state_entries": 4, + "optimizer_state_sha256": "62c4e2908a7e97fe9a1d60642a85968081b55345493a11b9e7f68e49fcdb22f3", + "rank": 1, + "trainable_state_sha256": "ca2a19167d59aab693d0a0a1e17ea9adbf9cfd7367dfa7a5c90b04cf2183b717" + } + ], + "repeated_diffusion_steps": 4, + "rng_streams": { + "model_base": 721927, + "sampler": 484370, + "transform": 289726 + }, + "scheduler": { + "class": "LambdaLR", + "last_epoch": 11000, + "last_lr": [ + 9.959920880962606e-06, + 9.959920880962605e-05 + ], + "step_count": 11001 + }, + "scheduler_resume_mode": "accelerate_checkpoint", + "schema_version": 1, + "status": "exact_external_resume_loaded", + "total_batch_size": 40, + "trainable_parameter_elements": 32022798, + "trainable_parameter_sha256": "1bc504b22e4375be40bbbd8250be5b5f380c14d2fb47ed85deba3604a84b5484", + "trainable_parameter_tensors": 957, + "world_size": 2 +} diff --git a/settings/qwen_robotwin/loss_is/initialization_manifest.json b/settings/qwen_robotwin/loss_is/initialization_manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..35160df6ed677c681d32574fe5119b91ef516bed --- /dev/null +++ b/settings/qwen_robotwin/loss_is/initialization_manifest.json @@ -0,0 +1,6 @@ +{ + "schema_version": 1, + "initialization_seed": 484370, + "action_model_sha256": "1cab43a1e801a94110a20384a9ad6cd76a239fedd9fe17890662da7e22d47e62", + "action_model_parameter_elements": 160689934 +} diff --git a/settings/qwen_robotwin/loss_is/loss_is_state.json b/settings/qwen_robotwin/loss_is/loss_is_state.json new file mode 100644 index 0000000000000000000000000000000000000000..da7336c065fd0de574c59f24c0bfc3e932c65eb6 --- /dev/null +++ b/settings/qwen_robotwin/loss_is/loss_is_state.json @@ -0,0 +1,20 @@ +{ + "active_for_completed_step": false, + "candidate_factor": 3, + "completed_optimizer_step": 11500, + "condition": { + "momentum": 0.9, + "previous_value": 1.1046745969347758, + "threshold": 2.0, + "value": 1.1086126240069607 + }, + "metrics": { + "loss_is/active": 0.0, + "loss_is/candidate_factor": 3.0, + "loss_is/condition_threshold": 2.0, + "loss_is/variance_reduction_ema": 1.1086126240069607, + "loss_is/variance_reduction_observed": 1.1440548676566253 + }, + "schema_version": 1, + "selection_scope": "per-rank per-microbatch; globally averaged unbiased estimator" +} diff --git a/settings/qwen_robotwin/loss_is/valid_anchor_manifest.json b/settings/qwen_robotwin/loss_is/valid_anchor_manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..3f5c91478b5931286c9c84d021a7ffc10a0fb5c0 --- /dev/null +++ b/settings/qwen_robotwin/loss_is/valid_anchor_manifest.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2090f6b9d7e1bc5ef41897436690f031c187d625fa6b71ae0c54f3793660c757 +size 107646168