Evanxw0715 commited on
Commit
b8a3b24
·
verified ·
1 Parent(s): 460fcba

Add BiCAT w/o CA VLA (latest, step 30000)

Browse files
BiCAT/CA_ablation/VLA/README.md ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ tags:
3
+ - model_hub_mixin
4
+ - pytorch_model_hub_mixin
5
+ ---
6
+
7
+ This model has been pushed to the Hub using the [PytorchModelHubMixin](https://huggingface.co/docs/huggingface_hub/package_reference/mixins#huggingface_hub.PyTorchModelHubMixin) integration:
8
+ - Code: [More Information Needed]
9
+ - Paper: [More Information Needed]
10
+ - Docs: [More Information Needed]
BiCAT/CA_ablation/VLA/config.json ADDED
@@ -0,0 +1,49 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "action_dim": 14,
3
+ "action_embed_init": "normal",
4
+ "action_horizon": 32,
5
+ "action_mask_token_template": "<|action_mask|>",
6
+ "action_token_template": "<|action_{i}|>",
7
+ "action_tokenizer": {
8
+ "kind": "tokens",
9
+ "name": "btt",
10
+ "pretrained_path": "/scratch/Projects/CFP-05/CFP05-CF-002/BiCAT/sim/BiCAT/CA_ablation/AT/checkpoints/latest/pretrained_model"
11
+ },
12
+ "block_ce_weighting": "block_mean",
13
+ "ce_vocab_scope": "full",
14
+ "decode_keep_k": null,
15
+ "decode_stop_mode": "fixed_budget",
16
+ "decode_temperature": null,
17
+ "decode_top_k": null,
18
+ "default_instruction": "Analyze the input image(s) and predict robot actions.",
19
+ "freeze_action_tokenizer": true,
20
+ "freeze_vision_encoder": false,
21
+ "gradient_checkpointing": true,
22
+ "lora_alpha": 16,
23
+ "lora_dropout": 0.05,
24
+ "lora_modules_to_save": [
25
+ "lm_head",
26
+ "embed_tokens"
27
+ ],
28
+ "lora_r": 64,
29
+ "lora_target_modules": [
30
+ "q_proj",
31
+ "v_proj",
32
+ "k_proj",
33
+ "o_proj",
34
+ "gate_proj",
35
+ "up_proj",
36
+ "down_proj"
37
+ ],
38
+ "max_new_tokens": null,
39
+ "n_action_steps": 16,
40
+ "n_obs_steps": 1,
41
+ "name": "bar_btt_pali",
42
+ "num_action_tokens": null,
43
+ "pretrained_path": null,
44
+ "state_prefix_encoding": "discrete_text",
45
+ "state_prefix_num_bins": 256,
46
+ "use_lora": false,
47
+ "vlm_dtype": "bfloat16",
48
+ "vlm_model_name": "google/paligemma2-3b-mix-224"
49
+ }
BiCAT/CA_ablation/VLA/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:45e5049b223e8da56c7dafeb00376508c8627672c177f4dac96b01815df81644
3
+ size 6073150736
BiCAT/CA_ablation/VLA/norm_stats.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8af2b215b59fbac4e1e7bbece5d95491528f31c95a6c06d017bcf7108b0cc6f5
3
+ size 3957
BiCAT/CA_ablation/VLA/policy_io.json ADDED
@@ -0,0 +1,54 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "obs_features": [
3
+ {
4
+ "key": "observation.state",
5
+ "type": "state",
6
+ "shape": [
7
+ 14
8
+ ],
9
+ "norm_mode": "min_max"
10
+ },
11
+ {
12
+ "key": "observation.images.cam_high",
13
+ "type": "image",
14
+ "shape": [
15
+ 3,
16
+ 240,
17
+ 320
18
+ ],
19
+ "norm_mode": "mean_std"
20
+ },
21
+ {
22
+ "key": "observation.images.cam_left_wrist",
23
+ "type": "image",
24
+ "shape": [
25
+ 3,
26
+ 240,
27
+ 320
28
+ ],
29
+ "norm_mode": "mean_std"
30
+ },
31
+ {
32
+ "key": "observation.images.cam_right_wrist",
33
+ "type": "image",
34
+ "shape": [
35
+ 3,
36
+ 240,
37
+ 320
38
+ ],
39
+ "norm_mode": "mean_std"
40
+ }
41
+ ],
42
+ "action_feature": {
43
+ "key": "action",
44
+ "type": "action",
45
+ "shape": [
46
+ 14
47
+ ],
48
+ "norm_mode": "min_max"
49
+ },
50
+ "n_obs_steps": 1,
51
+ "n_action_steps": 32,
52
+ "fps": 15,
53
+ "language_key": "task"
54
+ }
BiCAT/CA_ablation/VLA/train_config.yaml ADDED
@@ -0,0 +1,133 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ policy:
2
+ pretrained_path: null
3
+ vlm_model_name: google/paligemma2-3b-mix-224
4
+ vlm_dtype: bfloat16
5
+ freeze_vision_encoder: false
6
+ use_lora: false
7
+ lora_r: 64
8
+ lora_alpha: 16
9
+ lora_target_modules:
10
+ - q_proj
11
+ - v_proj
12
+ - k_proj
13
+ - o_proj
14
+ - gate_proj
15
+ - up_proj
16
+ - down_proj
17
+ lora_dropout: 0.05
18
+ lora_modules_to_save:
19
+ - lm_head
20
+ - embed_tokens
21
+ gradient_checkpointing: true
22
+ action_token_template: <|action_{i}|>
23
+ action_embed_init: normal
24
+ state_prefix_encoding: discrete_text
25
+ state_prefix_num_bins: 256
26
+ decode_temperature: null
27
+ decode_top_k: null
28
+ default_instruction: Analyze the input image(s) and predict robot actions.
29
+ action_dim: null
30
+ n_obs_steps: 1
31
+ action_horizon: 32
32
+ n_action_steps: 16
33
+ action_tokenizer:
34
+ name: btt
35
+ kind: tokens
36
+ pretrained_path: /scratch/Projects/CFP-05/CFP05-CF-002/BiCAT/sim/BiCAT/CA_ablation/AT/checkpoints/latest/pretrained_model
37
+ freeze_action_tokenizer: true
38
+ block_ce_weighting: block_mean
39
+ ce_vocab_scope: full
40
+ name: bar_btt_pali
41
+ dataset:
42
+ name: robotwin_new_demo_clean
43
+ repo_id: local/robotwin_new_demo_clean
44
+ root: /scratch/xuanweiliu/bimanual_tokenizer/data/lerobot/local/robotwin_new_demo_clean
45
+ hf_datasets_cache: ${oc.env:HF_DATASETS_CACHE,null}
46
+ episodes: null
47
+ exclude_episodes_file: null
48
+ action_stats_file: null
49
+ observation_keys:
50
+ - observation.state
51
+ - observation.images.cam_high
52
+ - observation.images.cam_left_wrist
53
+ - observation.images.cam_right_wrist
54
+ video_backend: null
55
+ bypass_force_cache_sync: true
56
+ trainer:
57
+ max_steps: 30000
58
+ batch_size: 64
59
+ global_batch_size: null
60
+ num_workers: 4
61
+ prefetch_factor: 4
62
+ use_episode_aware_sampler: true
63
+ drop_n_first_frames: 0
64
+ drop_n_last_frames: 0
65
+ weight_decay: 1.0e-06
66
+ betas:
67
+ - 0.95
68
+ - 0.999
69
+ eps: 1.0e-08
70
+ lr_scheduler: cosine
71
+ warmup_steps: 500
72
+ min_lr_ratio: 0.1
73
+ use_ema: true
74
+ ema_decay: 0.995
75
+ mixed_precision: bf16
76
+ gradient_accumulation_steps: 1
77
+ find_unused_parameters: false
78
+ max_grad_norm: 1.0
79
+ distributed_strategy: ddp
80
+ fsdp_min_num_params: 100000000
81
+ fsdp_reshard_after_forward: true
82
+ fsdp_cpu_offload: false
83
+ log_every: 200
84
+ val_every: 1000
85
+ checkpoint_every: 1000
86
+ checkpoint_dir: ${output_dir}/checkpoints
87
+ local_log_path: ${output_dir}/train_metrics.jsonl
88
+ val_ratio: 0.05
89
+ split_seed: ${seed}
90
+ val_batch_size: null
91
+ max_val_batches: 100
92
+ checkpoint_candidates:
93
+ best:
94
+ metric_key: val_reconst_loss
95
+ mode: min
96
+ k: 2
97
+ recent:
98
+ k: 0
99
+ use_wandb: true
100
+ wandb_project: praxis-vla
101
+ wandb_entity: evan_lxw-harbin-institute-of-technology
102
+ wandb_init_timeout: null
103
+ wandb_upload_eval_videos: true
104
+ wandb_max_eval_videos: 16
105
+ _trainer_type: policy
106
+ lrs:
107
+ vlm: 0.0001
108
+ vision: 0.0001
109
+ bridge: 0.0001
110
+ reconst_every: 1000
111
+ max_reconst_batches: 100
112
+ reconst_action_mode: policy
113
+ sim_eval_every: null
114
+ sim_eval_start_after_step: 0
115
+ sim_eval:
116
+ num_eval_per_task: 20
117
+ num_parallel_env: 1
118
+ record_episodes_per_task: 0
119
+ step_timeout_sec: null
120
+ rollout_failure_retries: 1
121
+ policy_kwargs: {}
122
+ name: train_policy
123
+ seed: 42
124
+ device: cuda
125
+ run_name: robotwin_new_bar_btt_wo_CA_pali_latest300k_30k
126
+ output_dir: /scratch/Projects/CFP-05/CFP05-CF-002/BiCAT/sim/BiCAT/CA_ablation/VLA
127
+ run_mode: overwrite
128
+ resume_checkpoint: null
129
+ env:
130
+ type: null
131
+ task: null
132
+ task_ids: null
133
+ python_bin: null