Robotics
LeRobot
act
so101
modalix
polima
Priyakarna commited on
Commit
307e6db
·
verified ·
1 Parent(s): 49848f4

Add compiled ACT SO-101 pick-and-place bundle

Browse files
Files changed (35) hide show
  1. LICENSE +233 -0
  2. README.md +112 -0
  3. SHA256SUMS +35 -0
  4. bundle/bundle.json +94 -0
  5. bundle/fixtures/act_fixture.npz +3 -0
  6. bundle/fixtures/expected/normalized_actions.f32 +3 -0
  7. bundle/fixtures/inputs/image0.f32 +3 -0
  8. bundle/fixtures/inputs/image0.u8 +3 -0
  9. bundle/fixtures/inputs/image1.f32 +3 -0
  10. bundle/fixtures/inputs/image1.u8 +3 -0
  11. bundle/fixtures/inputs/state.f32 +3 -0
  12. bundle/fixtures/normalization_stats.npz +3 -0
  13. bundle/fixtures/stages/decoder_action_tail_input.f32 +3 -0
  14. bundle/fixtures/stages/decoder_action_tail_output.f32 +3 -0
  15. bundle/fixtures/stages/encoder_layer_00_stem_input.f32 +3 -0
  16. bundle/fixtures/stages/encoder_layer_00_stem_output.f32 +3 -0
  17. bundle/fixtures/stages/encoder_layer_01_input.f32 +3 -0
  18. bundle/fixtures/stages/encoder_layer_01_output.f32 +3 -0
  19. bundle/fixtures/stages/encoder_layer_02_input.f32 +3 -0
  20. bundle/fixtures/stages/encoder_layer_02_output.f32 +3 -0
  21. bundle/fixtures/stages/encoder_layer_03_input.f32 +3 -0
  22. bundle/fixtures/stages/encoder_layer_03_output.f32 +3 -0
  23. bundle/fixtures/stages/vision_output_0.f32 +3 -0
  24. bundle/fixtures/stages/vision_output_1.f32 +3 -0
  25. bundle/models/decoder_action_tail/share/decoder_action_tail_stage1_mla.elf +3 -0
  26. bundle/models/encoder_layer_00_stem/share/encoder_layer_00_stem_stage1_mla.elf +3 -0
  27. bundle/models/encoder_layer_01/share/encoder_layer_01_stage1_mla.elf +3 -0
  28. bundle/models/encoder_layer_02/share/encoder_layer_02_stage1_mla.elf +3 -0
  29. bundle/models/encoder_layer_03/share/encoder_layer_03_stage1_mla.elf +3 -0
  30. bundle/models/vision_backbone/share/vision_backbone_stage1_mla.elf +3 -0
  31. bundle/plan.json +397 -0
  32. bundle/robot_client/preview_robot_cameras.py +5 -0
  33. bundle/robot_client/robot_rest_position.json +13 -0
  34. bundle/robot_client/run_robot_client_with_live_view.py +811 -0
  35. bundle/robot_client/start.sh +7 -0
LICENSE ADDED
@@ -0,0 +1,233 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Apache License
2
+ Version 2.0, January 2004
3
+ http://www.apache.org/licenses/
4
+
5
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
6
+
7
+ 1. Definitions.
8
+
9
+ "License" shall mean the terms and conditions for use, reproduction,
10
+ and distribution as defined by Sections 1 through 9 of this document.
11
+
12
+ "Licensor" shall mean the copyright owner or entity authorized by
13
+ the copyright owner that is granting the License.
14
+
15
+ "Legal Entity" shall mean the union of the acting entity and all
16
+ other entities that control, are controlled by, or are under common
17
+ control with that entity. For the purposes of this definition,
18
+ "control" means (i) the power, direct or indirect, to cause the
19
+ direction or management of such entity, whether by contract or
20
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
21
+ outstanding shares, or (iii) beneficial ownership of such entity.
22
+
23
+ "You" (or "Your") shall mean an individual or Legal Entity
24
+ exercising permissions granted by this License.
25
+
26
+ "Source" form shall mean the preferred form for making modifications,
27
+ including but not limited to software source code, documentation
28
+ source, and configuration files.
29
+
30
+ "Object" form shall mean any form resulting from mechanical
31
+ transformation or translation of a Source form, including but
32
+ not limited to compiled object code, generated documentation,
33
+ and conversions to other media types.
34
+
35
+ "Work" shall mean the work of authorship, whether in Source or
36
+ Object form, made available under the License, as indicated by a
37
+ copyright notice that is included in or attached to the work
38
+ (an example is provided in the Appendix below).
39
+
40
+ "Derivative Works" shall mean any work, whether in Source or Object
41
+ form, that is based on (or derived from) the Work and for which the
42
+ editorial revisions, annotations, elaborations, or other modifications
43
+ represent, as a whole, an original work of authorship. For the purposes
44
+ of this License, Derivative Works shall not include works that remain
45
+ separable from, or merely link (or bind by name) to the interfaces of,
46
+ the Work and Derivative Works thereof.
47
+
48
+ "Contribution" shall mean any work of authorship, including
49
+ the original version of the Work and any modifications or additions
50
+ to that Work or Derivative Works thereof, that is intentionally
51
+ submitted to Licensor for inclusion in the Work by the copyright owner
52
+ or by an individual or Legal Entity authorized to submit on behalf of
53
+ the copyright owner. For the purposes of this definition, "submitted"
54
+ means any form of electronic, verbal, or written communication sent
55
+ to the Licensor or its representatives, including but not limited to
56
+ communication on electronic mailing lists, source code control systems,
57
+ and issue tracking systems that are managed by, or on behalf of, the
58
+ Licensor for the purpose of discussing and improving the Work, but
59
+ excluding communication that is conspicuously marked or otherwise
60
+ designated in writing by the copyright owner as "Not a Contribution."
61
+
62
+ "Contributor" shall mean Licensor and any individual or Legal Entity
63
+ on behalf of whom a Contribution has been received by Licensor and
64
+ subsequently incorporated within the Work.
65
+
66
+ 2. Grant of Copyright License. Subject to the terms and conditions of
67
+ this License, each Contributor hereby grants to You a perpetual,
68
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
69
+ copyright license to reproduce, prepare Derivative Works of,
70
+ publicly display, publicly perform, sublicense, and distribute the
71
+ Work and such Derivative Works in Source or Object form.
72
+
73
+ 3. Grant of Patent License. Subject to the terms and conditions of
74
+ this License, each Contributor hereby grants to You a perpetual,
75
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
76
+ (except as stated in this section) patent license to make, have made,
77
+ use, offer to sell, sell, import, and otherwise transfer the Work,
78
+ where such license applies only to those patent claims licensable
79
+ by such Contributor that are necessarily infringed by their
80
+ Contribution(s) alone or by combination of their Contribution(s)
81
+ with the Work to which such Contribution(s) was submitted. If You
82
+ institute patent litigation against any entity (including a
83
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
84
+ or a Contribution incorporated within the Work constitutes direct
85
+ or contributory patent infringement, then any patent licenses
86
+ granted to You under this License for that Work shall terminate
87
+ as of the date such litigation is filed.
88
+
89
+ 4. Redistribution. You may reproduce and distribute copies of the
90
+ Work or Derivative Works thereof in any medium, with or without
91
+ modifications, and in Source or Object form, provided that You
92
+ meet the following conditions:
93
+
94
+ (a) You must give any other recipients of the Work or
95
+ Derivative Works a copy of this License; and
96
+
97
+ (b) You must cause any modified files to carry prominent notices
98
+ stating that You changed the files; and
99
+
100
+ (c) You must retain, in the Source form of any Derivative Works
101
+ that You distribute, all copyright, patent, trademark, and
102
+ attribution notices from the Source form of the Work,
103
+ excluding those notices that do not pertain to any part of
104
+ the Derivative Works; and
105
+
106
+ (d) If the Work includes a "NOTICE" text file as part of its
107
+ distribution, then any Derivative Works that You distribute must
108
+ include a readable copy of the attribution notices contained
109
+ within such NOTICE file, excluding those notices that do not
110
+ pertain to any part of the Derivative Works, in at least one
111
+ of the following places: within a NOTICE text file distributed
112
+ as part of the Derivative Works; within the Source form or
113
+ documentation, if provided along with the Derivative Works; or,
114
+ within a display generated by the Derivative Works, if and
115
+ wherever such third-party notices normally appear. The contents
116
+ of the NOTICE file are for informational purposes only and
117
+ do not modify the License. You may add Your own attribution
118
+ notices within Derivative Works that You distribute, alongside
119
+ or as an addendum to the NOTICE text from the Work, provided
120
+ that such additional attribution notices cannot be construed
121
+ as modifying the License.
122
+
123
+ You may add Your own copyright statement to Your modifications and
124
+ may provide additional or different license terms and conditions
125
+ for use, reproduction, or distribution of Your modifications, or
126
+ for any such Derivative Works as a whole, provided Your use,
127
+ reproduction, and distribution of the Work otherwise complies with
128
+ the conditions stated in this License.
129
+
130
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
131
+ any Contribution intentionally submitted for inclusion in the Work
132
+ by You to the Licensor shall be under the terms and conditions of
133
+ this License, without any additional terms or conditions.
134
+ Notwithstanding the above, nothing herein shall supersede or modify
135
+ the terms of any separate license agreement you may have executed
136
+ with Licensor regarding such Contributions.
137
+
138
+ 6. Trademarks. This License does not grant permission to use the trade
139
+ names, trademarks, service marks, or product names of the Licensor,
140
+ except as required for reasonable and customary use in describing the
141
+ origin of the Work and reproducing the content of the NOTICE file.
142
+
143
+ 7. Disclaimer of Warranty. Unless required by applicable law or
144
+ agreed to in writing, Licensor provides the Work (and each
145
+ Contributor provides its Contributions) on an "AS IS" BASIS,
146
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147
+ implied, including, without limitation, any warranties or conditions
148
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149
+ PARTICULAR PURPOSE. You are solely responsible for determining the
150
+ appropriateness of using or redistributing the Work and assume any
151
+ risks associated with Your exercise of permissions under this License.
152
+
153
+ 8. Limitation of Liability. In no event and under no legal theory,
154
+ whether in tort (including negligence), contract, or otherwise,
155
+ unless required by applicable law (such as deliberate and grossly
156
+ negligent acts) or agreed to in writing, shall any Contributor be
157
+ liable to You for damages, including any direct, indirect, special,
158
+ incidental, or consequential damages of any character arising as a
159
+ result of this License or out of the use or inability to use the
160
+ Work (including but not limited to damages for loss of goodwill,
161
+ work stoppage, computer failure or malfunction, or any and all
162
+ other commercial damages or losses), even if such Contributor
163
+ has been advised of the possibility of such damages.
164
+
165
+ 9. Accepting Warranty or Additional Liability. While redistributing
166
+ the Work or Derivative Works thereof, You may choose to offer,
167
+ and charge a fee for, acceptance of support, warranty, indemnity,
168
+ or other liability obligations and/or rights consistent with this
169
+ License. However, in accepting such obligations, You may act only
170
+ on Your own behalf and on Your sole responsibility, not on behalf
171
+ of any other Contributor, and only if You agree to indemnify,
172
+ defend, and hold each Contributor harmless for any liability
173
+ incurred by, or claims asserted against, such Contributor by reason
174
+ of your accepting any such warranty or additional liability.
175
+
176
+ END OF TERMS AND CONDITIONS
177
+
178
+ APPENDIX: How to apply the Apache License to your work.
179
+
180
+ To apply the Apache License to your work, attach the following
181
+ boilerplate notice, with the fields enclosed by brackets "[]"
182
+ replaced with your own identifying information. (Don't include
183
+ the brackets!) The text should be enclosed in the appropriate
184
+ comment syntax for the file format. We also recommend that a
185
+ file or class name and description of purpose be included on the
186
+ same "printed page" as the copyright notice for easier
187
+ identification within third-party archives.
188
+
189
+ Copyright [yyyy] [name of copyright owner]
190
+
191
+ Licensed under the Apache License, Version 2.0 (the "License");
192
+ you may not use this file except in compliance with the License.
193
+ You may obtain a copy of the License at
194
+
195
+ http://www.apache.org/licenses/LICENSE-2.0
196
+
197
+ Unless required by applicable law or agreed to in writing, software
198
+ distributed under the License is distributed on an "AS IS" BASIS,
199
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200
+ See the License for the specific language governing permissions and
201
+ limitations under the License.
202
+
203
+ ===============================================================================
204
+ PoLiMa copyright and bundled third-party software
205
+ ===============================================================================
206
+
207
+ PoLiMa: Copyright 2026 SiMa.ai. Licensed under Apache-2.0 above.
208
+
209
+ The bundled Studio UI includes Preact, @preact/signals and @preact/signals-core
210
+ under the following MIT license. This applies to those components only.
211
+
212
+ The MIT License (MIT)
213
+
214
+ Copyright (c) 2015-present Jason Miller
215
+ Copyright (c) 2022-present Preact Team
216
+
217
+ Permission is hereby granted, free of charge, to any person obtaining a copy
218
+ of this software and associated documentation files (the "Software"), to deal
219
+ in the Software without restriction, including without limitation the rights
220
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
221
+ copies of the Software, and to permit persons to whom the Software is
222
+ furnished to do so, subject to the following conditions:
223
+
224
+ The above copyright notice and this permission notice shall be included in all
225
+ copies or substantial portions of the Software.
226
+
227
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
228
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
229
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
230
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
231
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
232
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
233
+ SOFTWARE.
README.md ADDED
@@ -0,0 +1,112 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ license: apache-2.0
3
+ datasets:
4
+ - Priyakarna/SO101-GreyEraser-PickPlace
5
+ tags:
6
+ - robotics
7
+ - act
8
+ - lerobot
9
+ - so101
10
+ - modalix
11
+ - polima
12
+ ---
13
+
14
+ # ACT SO-101 Pick-and-Place
15
+
16
+ A task-specific ACT policy trained to place a grey eraser in a white basket,
17
+ compiled for the **SiMa.ai Modalix DevKit**.
18
+
19
+ | Property | Value |
20
+ |---|---|
21
+ | Training checkpoint | Step 100,000 |
22
+ | Compiled output | 100 actions × 6 joints |
23
+ | Precision | BF16 |
24
+ | Robot | SO-101 follower |
25
+ | Task | Place the grey eraser in the white basket |
26
+ | Bundle ID | `act-so101-grey-eraser-pickplace-100k-bf16-1879ca7c` |
27
+
28
+ This repository contains the deployable compiled bundle, reference fixtures and
29
+ SO-101 runtime assets. It does not contain the training checkpoint or dataset.
30
+ The policy is specific to the reference task and workspace.
31
+
32
+ ## Prerequisites
33
+
34
+ - Modalix DevKit with NVMe storage and a compatible SiMa.ai runtime.
35
+ - PoLiMa v0.3 installed from its source checkout on the deployment host and DevKit.
36
+ - A Linux deployment host, or WSL on Windows, with SSH, rsync and the Hugging Face CLI.
37
+ - SO-101 follower with six STS3215 motors (IDs 1–6), securely mounted and powered.
38
+ - Two USB cameras connected to the DevKit and framed like the training setup.
39
+
40
+ The release checks used ModelSDK 2.1.3, Neat 0.5.0 and `sima_lmm` 0.5.0.
41
+
42
+ ## Download and deploy
43
+
44
+ ```bash
45
+ hf download simaai/ACT-SO101-PickPlace --local-dir ACT-SO101-PickPlace
46
+ cd ACT-SO101-PickPlace
47
+ sha256sum --check SHA256SUMS
48
+ read -r -p "DevKit login (username@address): " devkit_login
49
+ polima deploy --bundle ./bundle --devkit "$devkit_login" --start
50
+ ```
51
+
52
+ Deployment installs and activates the bundle and starts the inference server.
53
+ It does not start robot motion.
54
+
55
+ ## Check inference
56
+
57
+ ```bash
58
+ polima run --bundle ./bundle --devkit "$devkit_login" --fixture --repeat 20
59
+ ```
60
+
61
+ This replays the packaged observations without cameras or robot motion and
62
+ checks numerical agreement. It does not measure physical task success.
63
+
64
+ ## Run the robot
65
+
66
+ Use Studio to inspect both camera views and confirm the selected arm, cameras
67
+ and task. Calibrate each physical follower arm before its first run.
68
+
69
+ ```bash
70
+ polima studio start
71
+ polima robot calibrate
72
+ polima robot run
73
+ ```
74
+
75
+ Reference cameras are a Logitech C920 overhead camera and a Sonix USB2.0 CAM1
76
+ wrist camera, capturing 640 × 480 MJPG at 30 frames/s. The overhead lens was
77
+ 49 cm above the table; the basket was approximately 32 cm from the arm end.
78
+ The basket measured 17.5 × 12 × 6 cm and the eraser 12.5 × 5 × 2 cm.
79
+
80
+ ## Training data
81
+
82
+ Trained on 769 demonstrations from
83
+ [SO-101 Grey Eraser Pick-and-Place](https://huggingface.co/datasets/Priyakarna/SO101-GreyEraser-PickPlace/tree/bfd7c1831092adfb0f02273e390d86bad193d419).
84
+ The full collection contains 961 episodes; 192 episodes were held out.
85
+
86
+ ## Validation
87
+
88
+ On the reference Modalix runtime, 30 measured requests after five warm-ups gave
89
+ 13.23 ms p50, 13.39 ms p95 and 13.45 ms p99 model-server latency. The packaged
90
+ fixture passed compiled-device fidelity checks. Sixteen held-out frames passed
91
+ cosine similarity ≥ 0.999 and normalized mean absolute error ≤ 0.01 against
92
+ PyTorch. Physical operation was verified on the reference SO-101 setup; a
93
+ controlled success rate was not recorded.
94
+
95
+ See the [PoLiMa benchmark documentation](https://github.com/sima-neat/polima/blob/develop/docs/benchmarks.md#act)
96
+ for methods, stage timings, memory, disk use and limitations.
97
+
98
+ ## Package contents
99
+
100
+ `bundle/` contains six compiled ELFs, the execution plan, normalization data,
101
+ reference observations and SO-101 launch assets. `SHA256SUMS` covers every
102
+ supplied file except the checksum file itself.
103
+
104
+ ## Limitations
105
+
106
+ The policy is specific to the reference robot, object, basket and camera views.
107
+ Different geometry, framing, lighting or calibration can change behavior. Text
108
+ changes do not teach ACT a new behavior at runtime.
109
+
110
+ ## License
111
+
112
+ The package is released under Apache-2.0. See [LICENSE](LICENSE).
SHA256SUMS ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 821ad037462159d808793900499dd75e625952130ed880618bf14bcbaf7e43b4 .gitattributes
2
+ e7e1a4d107119af198851d7a6c82f08d889f599caa09e882377215f255746871 LICENSE
3
+ b65f8081dfd0042d0653770b301163218e016ebac3764944a210a3c7aef63694 README.md
4
+ 70cc93f11f374867cbfd9e823b255601041565f4c0ea1f60a197d767312160f3 bundle/bundle.json
5
+ c072016dcda9f622fecccdb9be695565e3ad8099763582b4dd6adc639e6ec519 bundle/fixtures/act_fixture.npz
6
+ c002d33d8613992aacf26778abcd3c43621445040f8cc94537ccf73c9463c06d bundle/fixtures/expected/normalized_actions.f32
7
+ b852b48b8215793f8dbeff58b93094a73bfbdf8195719c48f03766ce44744fdf bundle/fixtures/inputs/image0.f32
8
+ a66f3a18960cafc1c9e2656d2f0b1c6da0b57d31d1e57e68663b56134704b1cd bundle/fixtures/inputs/image0.u8
9
+ 06d5425d8a03a01b4e7ed309d881cddb1ea859d8f0ff3cc7598b69fad8547603 bundle/fixtures/inputs/image1.f32
10
+ acdc303b4cab88d1ca0fd1bc807ea75b0d88a05fe95c469b09aef1afce18c9dd bundle/fixtures/inputs/image1.u8
11
+ 3558b6aa8f836fad4e547d36b8be4fbf12edc994582995106f7565bf8b116014 bundle/fixtures/inputs/state.f32
12
+ 8bee889b9e68d7a35140c6bea464265e3d6213f52ca2015a5d3ac549d600b508 bundle/fixtures/normalization_stats.npz
13
+ 98bab8ef627f4b5e0967a767e0f5805d2a9294345a8470df35b8ccfc2189c3a0 bundle/fixtures/stages/decoder_action_tail_input.f32
14
+ 3f8311a97617203e54da9719bbd7d2425f6f5bd17f96c1e2ccae1907bb4b0a79 bundle/fixtures/stages/decoder_action_tail_output.f32
15
+ 5274d06584f3a6db4056954a0564ca1c1a4c275c01a988b1bfe2c1c042ca7e18 bundle/fixtures/stages/encoder_layer_00_stem_input.f32
16
+ 67dcefea3a59727b7b962e1e07e7323c21f43708ba51329e75571617ad43577e bundle/fixtures/stages/encoder_layer_00_stem_output.f32
17
+ 67dcefea3a59727b7b962e1e07e7323c21f43708ba51329e75571617ad43577e bundle/fixtures/stages/encoder_layer_01_input.f32
18
+ 72631ff937cb2797129ca40f6797eaa54e63817a3bbdbb44b50e06ed5c108b2b bundle/fixtures/stages/encoder_layer_01_output.f32
19
+ 72631ff937cb2797129ca40f6797eaa54e63817a3bbdbb44b50e06ed5c108b2b bundle/fixtures/stages/encoder_layer_02_input.f32
20
+ 9cb7b795b5c9b8dd537293c501a1805c47a0ccb8d3e04f7f983d541f0f7837b3 bundle/fixtures/stages/encoder_layer_02_output.f32
21
+ 9cb7b795b5c9b8dd537293c501a1805c47a0ccb8d3e04f7f983d541f0f7837b3 bundle/fixtures/stages/encoder_layer_03_input.f32
22
+ 98bab8ef627f4b5e0967a767e0f5805d2a9294345a8470df35b8ccfc2189c3a0 bundle/fixtures/stages/encoder_layer_03_output.f32
23
+ ecf0780f392dfdbeecf8eb5ac2f316061b6aded2bdd443e470d986a7cc28297d bundle/fixtures/stages/vision_output_0.f32
24
+ 5c2ae7f2aa1c95cc2c756cbf830b5f9229fd3c3c034b61e1ab93b0cd720a5d2f bundle/fixtures/stages/vision_output_1.f32
25
+ 30c6027697dc2c60537be09058d6bdd0fd44f545c507f0166322ec1edc7daed5 bundle/models/decoder_action_tail/share/decoder_action_tail_stage1_mla.elf
26
+ 0a63ee05a584fa6d59dc4656423b0a0dba7fb667f7d5c61c0c26ccac92d1e7c1 bundle/models/encoder_layer_00_stem/share/encoder_layer_00_stem_stage1_mla.elf
27
+ 057ecaf847005b81a1927c3b49ea87e0e0392d86abea300d368c101d4aed84c1 bundle/models/encoder_layer_01/share/encoder_layer_01_stage1_mla.elf
28
+ 54425c45ed7bc80ad63f5f12abee17be0d440874dd742453e71997792b770af6 bundle/models/encoder_layer_02/share/encoder_layer_02_stage1_mla.elf
29
+ b3a6c48a91cca9ac07bad32ebeacdb6116762f848f0ddc2d5ab3a74e9bac4900 bundle/models/encoder_layer_03/share/encoder_layer_03_stage1_mla.elf
30
+ c1640f29bc4e679df5f8754e25dd817d9b8297565d0bdcf1de3816c3f3936f60 bundle/models/vision_backbone/share/vision_backbone_stage1_mla.elf
31
+ a335209b7fe72d73a770578ac316441ed99409efd6305cefbaf659265f6b1357 bundle/plan.json
32
+ 40bca20282a2ebacf70f995642df91f4f4ee3456a577afe209ddb8598ca1e470 bundle/robot_client/preview_robot_cameras.py
33
+ 649785ae3638a6b41cc3c83ba5c31fe975468eb2d386192fe3a6759f688f309d bundle/robot_client/robot_rest_position.json
34
+ cc07ef4e29be9f1dd02df5ce7e774d9c5667d6d65b734ca78b93778b47cd0a3b bundle/robot_client/run_robot_client_with_live_view.py
35
+ ae447bed934e5c8ae75a99a1758d624e6273ffbe8c8a41c2209e0adfb995871c bundle/robot_client/start.sh
bundle/bundle.json ADDED
@@ -0,0 +1,94 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bundle_id": "act-so101-grey-eraser-pickplace-100k-bf16-1879ca7c",
3
+ "checkpoint": "",
4
+ "default_task": 0,
5
+ "format": "polima-bundle-v1",
6
+ "graphs": [
7
+ {
8
+ "dram_layout": "plain",
9
+ "elf": "models/vision_backbone/share/vision_backbone_stage1_mla.elf",
10
+ "elf_bytes": 31027752,
11
+ "external_dram_layout": "compiler",
12
+ "input_elements": 921600,
13
+ "name": "vision_backbone",
14
+ "output_elements": 153600,
15
+ "precision": "bf16",
16
+ "sha256": "c1640f29bc4e679df5f8754e25dd817d9b8297565d0bdcf1de3816c3f3936f60"
17
+ },
18
+ {
19
+ "dram_layout": "plain",
20
+ "elf": "models/encoder_layer_00_stem/share/encoder_layer_00_stem_stage1_mla.elf",
21
+ "elf_bytes": 17343320,
22
+ "external_dram_layout": "compiler",
23
+ "input_elements": 307712,
24
+ "name": "encoder_layer_00_stem",
25
+ "output_elements": 308224,
26
+ "precision": "bf16",
27
+ "sha256": "0a63ee05a584fa6d59dc4656423b0a0dba7fb667f7d5c61c0c26ccac92d1e7c1"
28
+ },
29
+ {
30
+ "dram_layout": "plain",
31
+ "elf": "models/encoder_layer_01/share/encoder_layer_01_stage1_mla.elf",
32
+ "elf_bytes": 17013528,
33
+ "external_dram_layout": "compiler",
34
+ "input_elements": 308224,
35
+ "name": "encoder_layer_01",
36
+ "output_elements": 308224,
37
+ "precision": "bf16",
38
+ "sha256": "057ecaf847005b81a1927c3b49ea87e0e0392d86abea300d368c101d4aed84c1"
39
+ },
40
+ {
41
+ "dram_layout": "plain",
42
+ "elf": "models/encoder_layer_02/share/encoder_layer_02_stage1_mla.elf",
43
+ "elf_bytes": 17013528,
44
+ "external_dram_layout": "compiler",
45
+ "input_elements": 308224,
46
+ "name": "encoder_layer_02",
47
+ "output_elements": 308224,
48
+ "precision": "bf16",
49
+ "sha256": "54425c45ed7bc80ad63f5f12abee17be0d440874dd742453e71997792b770af6"
50
+ },
51
+ {
52
+ "dram_layout": "plain",
53
+ "elf": "models/encoder_layer_03/share/encoder_layer_03_stage1_mla.elf",
54
+ "elf_bytes": 17013528,
55
+ "external_dram_layout": "compiler",
56
+ "input_elements": 308224,
57
+ "name": "encoder_layer_03",
58
+ "output_elements": 308224,
59
+ "precision": "bf16",
60
+ "sha256": "b3a6c48a91cca9ac07bad32ebeacdb6116762f848f0ddc2d5ab3a74e9bac4900"
61
+ },
62
+ {
63
+ "dram_layout": "plain",
64
+ "elf": "models/decoder_action_tail/share/decoder_action_tail_stage1_mla.elf",
65
+ "elf_bytes": 13384824,
66
+ "external_dram_layout": "compiler",
67
+ "input_elements": 308224,
68
+ "name": "decoder_action_tail",
69
+ "output_elements": 1600,
70
+ "precision": "bf16",
71
+ "sha256": "30c6027697dc2c60537be09058d6bdd0fd44f545c507f0166322ec1edc7daed5"
72
+ }
73
+ ],
74
+ "policy": "act",
75
+ "sidecars": [
76
+ "robot_client/preview_robot_cameras.py",
77
+ "robot_client/robot_rest_position.json",
78
+ "robot_client/run_robot_client_with_live_view.py",
79
+ "robot_client/start.sh"
80
+ ],
81
+ "smoke": {
82
+ "cosine_min": 0.999,
83
+ "mean_abs_max": 0.01
84
+ },
85
+ "source": "polima-compile",
86
+ "tasks": [
87
+ "Place the grey eraser in the white basket"
88
+ ],
89
+ "tool_versions": {
90
+ "modelsdk": "2.1.3",
91
+ "source_format": "polima-export-v1"
92
+ },
93
+ "total_elf_bytes": 112796480
94
+ }
bundle/fixtures/act_fixture.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c072016dcda9f622fecccdb9be695565e3ad8099763582b4dd6adc639e6ec519
3
+ size 7378890
bundle/fixtures/expected/normalized_actions.f32 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c002d33d8613992aacf26778abcd3c43621445040f8cc94537ccf73c9463c06d
3
+ size 2400
bundle/fixtures/inputs/image0.f32 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b852b48b8215793f8dbeff58b93094a73bfbdf8195719c48f03766ce44744fdf
3
+ size 3686400
bundle/fixtures/inputs/image0.u8 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a66f3a18960cafc1c9e2656d2f0b1c6da0b57d31d1e57e68663b56134704b1cd
3
+ size 921600
bundle/fixtures/inputs/image1.f32 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:06d5425d8a03a01b4e7ed309d881cddb1ea859d8f0ff3cc7598b69fad8547603
3
+ size 3686400
bundle/fixtures/inputs/image1.u8 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:acdc303b4cab88d1ca0fd1bc807ea75b0d88a05fe95c469b09aef1afce18c9dd
3
+ size 921600
bundle/fixtures/inputs/state.f32 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3558b6aa8f836fad4e547d36b8be4fbf12edc994582995106f7565bf8b116014
3
+ size 24
bundle/fixtures/normalization_stats.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8bee889b9e68d7a35140c6bea464265e3d6213f52ca2015a5d3ac549d600b508
3
+ size 2440
bundle/fixtures/stages/decoder_action_tail_input.f32 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:98bab8ef627f4b5e0967a767e0f5805d2a9294345a8470df35b8ccfc2189c3a0
3
+ size 1232896
bundle/fixtures/stages/decoder_action_tail_output.f32 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3f8311a97617203e54da9719bbd7d2425f6f5bd17f96c1e2ccae1907bb4b0a79
3
+ size 6400
bundle/fixtures/stages/encoder_layer_00_stem_input.f32 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5274d06584f3a6db4056954a0564ca1c1a4c275c01a988b1bfe2c1c042ca7e18
3
+ size 1230848
bundle/fixtures/stages/encoder_layer_00_stem_output.f32 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:67dcefea3a59727b7b962e1e07e7323c21f43708ba51329e75571617ad43577e
3
+ size 1232896
bundle/fixtures/stages/encoder_layer_01_input.f32 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:67dcefea3a59727b7b962e1e07e7323c21f43708ba51329e75571617ad43577e
3
+ size 1232896
bundle/fixtures/stages/encoder_layer_01_output.f32 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:72631ff937cb2797129ca40f6797eaa54e63817a3bbdbb44b50e06ed5c108b2b
3
+ size 1232896
bundle/fixtures/stages/encoder_layer_02_input.f32 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:72631ff937cb2797129ca40f6797eaa54e63817a3bbdbb44b50e06ed5c108b2b
3
+ size 1232896
bundle/fixtures/stages/encoder_layer_02_output.f32 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9cb7b795b5c9b8dd537293c501a1805c47a0ccb8d3e04f7f983d541f0f7837b3
3
+ size 1232896
bundle/fixtures/stages/encoder_layer_03_input.f32 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9cb7b795b5c9b8dd537293c501a1805c47a0ccb8d3e04f7f983d541f0f7837b3
3
+ size 1232896
bundle/fixtures/stages/encoder_layer_03_output.f32 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:98bab8ef627f4b5e0967a767e0f5805d2a9294345a8470df35b8ccfc2189c3a0
3
+ size 1232896
bundle/fixtures/stages/vision_output_0.f32 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ecf0780f392dfdbeecf8eb5ac2f316061b6aded2bdd443e470d986a7cc28297d
3
+ size 614400
bundle/fixtures/stages/vision_output_1.f32 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5c2ae7f2aa1c95cc2c756cbf830b5f9229fd3c3c034b61e1ab93b0cd720a5d2f
3
+ size 614400
bundle/models/decoder_action_tail/share/decoder_action_tail_stage1_mla.elf ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:30c6027697dc2c60537be09058d6bdd0fd44f545c507f0166322ec1edc7daed5
3
+ size 13384824
bundle/models/encoder_layer_00_stem/share/encoder_layer_00_stem_stage1_mla.elf ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0a63ee05a584fa6d59dc4656423b0a0dba7fb667f7d5c61c0c26ccac92d1e7c1
3
+ size 17343320
bundle/models/encoder_layer_01/share/encoder_layer_01_stage1_mla.elf ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:057ecaf847005b81a1927c3b49ea87e0e0392d86abea300d368c101d4aed84c1
3
+ size 17013528
bundle/models/encoder_layer_02/share/encoder_layer_02_stage1_mla.elf ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:54425c45ed7bc80ad63f5f12abee17be0d440874dd742453e71997792b770af6
3
+ size 17013528
bundle/models/encoder_layer_03/share/encoder_layer_03_stage1_mla.elf ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b3a6c48a91cca9ac07bad32ebeacdb6116762f848f0ddc2d5ab3a74e9bac4900
3
+ size 17013528
bundle/models/vision_backbone/share/vision_backbone_stage1_mla.elf ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c1640f29bc4e679df5f8754e25dd817d9b8297565d0bdcf1de3816c3f3936f60
3
+ size 31027752
bundle/plan.json ADDED
@@ -0,0 +1,397 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "buffers": {
3
+ "actions": 600,
4
+ "image0": 921600,
5
+ "image1": 921600,
6
+ "packed_head": 512,
7
+ "padded": 1600,
8
+ "state": 6
9
+ },
10
+ "default_task": 0,
11
+ "device_buffers": {
12
+ "act_actions": 1600,
13
+ "act_hidden_a": 308224,
14
+ "act_hidden_b": 308224,
15
+ "act_image_0": 921600,
16
+ "act_image_1": 921600,
17
+ "act_stem": 307712
18
+ },
19
+ "graph_outputs": {
20
+ "decoder_action_tail": [
21
+ {
22
+ "dram_layout": "plain",
23
+ "dtype": "float32",
24
+ "logical_channels": null,
25
+ "logical_width": null,
26
+ "name": "normalized_actions",
27
+ "shape": [
28
+ 1,
29
+ 1,
30
+ 100,
31
+ 16
32
+ ]
33
+ }
34
+ ],
35
+ "encoder_layer_00_stem": [
36
+ {
37
+ "dram_layout": "plain",
38
+ "dtype": "float32",
39
+ "logical_channels": null,
40
+ "logical_width": null,
41
+ "name": "hidden_out",
42
+ "shape": [
43
+ 1,
44
+ 1,
45
+ 602,
46
+ 512
47
+ ]
48
+ }
49
+ ],
50
+ "encoder_layer_01": [
51
+ {
52
+ "dram_layout": "plain",
53
+ "dtype": "float32",
54
+ "logical_channels": null,
55
+ "logical_width": null,
56
+ "name": "hidden_out",
57
+ "shape": [
58
+ 1,
59
+ 1,
60
+ 602,
61
+ 512
62
+ ]
63
+ }
64
+ ],
65
+ "encoder_layer_02": [
66
+ {
67
+ "dram_layout": "plain",
68
+ "dtype": "float32",
69
+ "logical_channels": null,
70
+ "logical_width": null,
71
+ "name": "hidden_out",
72
+ "shape": [
73
+ 1,
74
+ 1,
75
+ 602,
76
+ 512
77
+ ]
78
+ }
79
+ ],
80
+ "encoder_layer_03": [
81
+ {
82
+ "dram_layout": "plain",
83
+ "dtype": "float32",
84
+ "logical_channels": null,
85
+ "logical_width": null,
86
+ "name": "hidden_out",
87
+ "shape": [
88
+ 1,
89
+ 1,
90
+ 602,
91
+ 512
92
+ ]
93
+ }
94
+ ],
95
+ "vision_backbone": [
96
+ {
97
+ "dram_layout": "plain",
98
+ "dtype": "float32",
99
+ "logical_channels": null,
100
+ "logical_width": null,
101
+ "name": "camera_tokens",
102
+ "shape": [
103
+ 1,
104
+ 300,
105
+ 512
106
+ ]
107
+ }
108
+ ]
109
+ },
110
+ "loops": {},
111
+ "profile": {
112
+ "action_dim": 6,
113
+ "actions_per_chunk": 100,
114
+ "calibration_id": "so-arm101",
115
+ "camera_keys": [
116
+ "observation.images.wrist",
117
+ "observation.images.overhead"
118
+ ],
119
+ "fps": 30,
120
+ "runtime_context": {},
121
+ "state_dim": 6,
122
+ "state_names": [
123
+ "shoulder_pan.pos",
124
+ "shoulder_lift.pos",
125
+ "elbow_flex.pos",
126
+ "wrist_flex.pos",
127
+ "wrist_roll.pos",
128
+ "gripper.pos"
129
+ ]
130
+ },
131
+ "result": "actions",
132
+ "robot": {
133
+ "actions_per_chunk": 100,
134
+ "aggregate_fn": "weighted_average",
135
+ "calibration_id": "so-arm101",
136
+ "camera_capture_fps": {},
137
+ "camera_fourcc": "MJPG",
138
+ "camera_hints": {
139
+ "overhead": "C920",
140
+ "wrist": "Sonix"
141
+ },
142
+ "camera_roles": [
143
+ [
144
+ "wrist",
145
+ "Wrist"
146
+ ],
147
+ [
148
+ "overhead",
149
+ "Overhead"
150
+ ]
151
+ ],
152
+ "fps": 30,
153
+ "joint_names": [
154
+ "shoulder_pan.pos",
155
+ "shoulder_lift.pos",
156
+ "elbow_flex.pos",
157
+ "wrist_flex.pos",
158
+ "wrist_roll.pos",
159
+ "gripper.pos"
160
+ ],
161
+ "max_relative_target": 12
162
+ },
163
+ "sidecars": [],
164
+ "steps": [
165
+ {
166
+ "args": {
167
+ "parts": [
168
+ {
169
+ "count": 6,
170
+ "dst_offset": 0,
171
+ "src": "state"
172
+ }
173
+ ],
174
+ "size": 512
175
+ },
176
+ "op": "pack",
177
+ "out": "packed_head"
178
+ },
179
+ {
180
+ "args": {
181
+ "calls": [
182
+ {
183
+ "graph": "vision_backbone",
184
+ "in": [
185
+ [
186
+ "act_image_0"
187
+ ]
188
+ ],
189
+ "out": [
190
+ [
191
+ "act_stem",
192
+ 512,
193
+ 153600
194
+ ]
195
+ ]
196
+ },
197
+ {
198
+ "graph": "vision_backbone",
199
+ "in": [
200
+ [
201
+ "act_image_1"
202
+ ]
203
+ ],
204
+ "out": [
205
+ [
206
+ "act_stem",
207
+ 154112,
208
+ 153600
209
+ ]
210
+ ]
211
+ },
212
+ {
213
+ "graph": "encoder_layer_00_stem",
214
+ "in": [
215
+ [
216
+ "act_stem"
217
+ ]
218
+ ],
219
+ "out": [
220
+ [
221
+ "act_hidden_a"
222
+ ]
223
+ ]
224
+ },
225
+ {
226
+ "graph": "encoder_layer_01",
227
+ "in": [
228
+ [
229
+ "act_hidden_a"
230
+ ]
231
+ ],
232
+ "out": [
233
+ [
234
+ "act_hidden_b"
235
+ ]
236
+ ]
237
+ },
238
+ {
239
+ "graph": "encoder_layer_02",
240
+ "in": [
241
+ [
242
+ "act_hidden_b"
243
+ ]
244
+ ],
245
+ "out": [
246
+ [
247
+ "act_hidden_a"
248
+ ]
249
+ ]
250
+ },
251
+ {
252
+ "graph": "encoder_layer_03",
253
+ "in": [
254
+ [
255
+ "act_hidden_a"
256
+ ]
257
+ ],
258
+ "out": [
259
+ [
260
+ "act_hidden_b"
261
+ ]
262
+ ]
263
+ },
264
+ {
265
+ "graph": "decoder_action_tail",
266
+ "in": [
267
+ [
268
+ "act_hidden_b"
269
+ ]
270
+ ],
271
+ "out": [
272
+ [
273
+ "act_actions"
274
+ ]
275
+ ]
276
+ }
277
+ ],
278
+ "downloads": [
279
+ {
280
+ "count": 1600,
281
+ "dst": "padded",
282
+ "dst_offset": 0,
283
+ "src": "act_actions",
284
+ "src_offset": 0
285
+ }
286
+ ],
287
+ "label": "act",
288
+ "uploads": [
289
+ {
290
+ "cache": false,
291
+ "count": 921600,
292
+ "dst": "act_image_0",
293
+ "dst_offset": 0,
294
+ "src": "image0",
295
+ "src_offset": 0
296
+ },
297
+ {
298
+ "cache": false,
299
+ "count": 921600,
300
+ "dst": "act_image_1",
301
+ "dst_offset": 0,
302
+ "src": "image1",
303
+ "src_offset": 0
304
+ },
305
+ {
306
+ "cache": false,
307
+ "count": 512,
308
+ "dst": "act_stem",
309
+ "dst_offset": 0,
310
+ "src": "packed_head",
311
+ "src_offset": 0
312
+ }
313
+ ]
314
+ },
315
+ "op": "run_device_chain",
316
+ "out": "padded"
317
+ },
318
+ {
319
+ "args": {
320
+ "count": 100,
321
+ "src": "padded",
322
+ "stride": 16,
323
+ "take": 6
324
+ },
325
+ "op": "gather_strided",
326
+ "out": "actions"
327
+ }
328
+ ],
329
+ "tasks": [
330
+ "Place the grey eraser in the white basket"
331
+ ],
332
+ "wire": {
333
+ "default_port": 8092,
334
+ "magic": 1297367873,
335
+ "request_tensors": [
336
+ {
337
+ "decode": {
338
+ "mean": [
339
+ 0.48500001430511475,
340
+ 0.4560000002384186,
341
+ 0.4059999883174896
342
+ ],
343
+ "scale": 255.0,
344
+ "std": [
345
+ 0.2290000170469284,
346
+ 0.2240000218153,
347
+ 0.22500000894069672
348
+ ]
349
+ },
350
+ "dtype": "uint8",
351
+ "elements": 921600,
352
+ "name": "image0",
353
+ "shape": [
354
+ 480,
355
+ 640,
356
+ 3
357
+ ]
358
+ },
359
+ {
360
+ "decode": {
361
+ "mean": [
362
+ 0.48500001430511475,
363
+ 0.4560000002384186,
364
+ 0.4059999883174896
365
+ ],
366
+ "scale": 255.0,
367
+ "std": [
368
+ 0.2290000170469284,
369
+ 0.2240000218153,
370
+ 0.22500000894069672
371
+ ]
372
+ },
373
+ "dtype": "uint8",
374
+ "elements": 921600,
375
+ "name": "image1",
376
+ "shape": [
377
+ 480,
378
+ 640,
379
+ 3
380
+ ]
381
+ },
382
+ {
383
+ "elements": 6,
384
+ "name": "state",
385
+ "shape": [
386
+ 6
387
+ ]
388
+ }
389
+ ],
390
+ "response_elements": 600,
391
+ "response_shape": [
392
+ 100,
393
+ 6
394
+ ],
395
+ "version": 2
396
+ }
397
+ }
bundle/robot_client/preview_robot_cameras.py ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ """Camera preview for `polima robot preview`; the code lives in polima.robot.preview."""
2
+
3
+ from polima.robot.preview import main
4
+
5
+ raise SystemExit(main())
bundle/robot_client/robot_rest_position.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "calibration_id": "so-arm101",
3
+ "method": "median observation.state at frame_index 0 across 961 episodes",
4
+ "positions": {
5
+ "elbow_flex.pos": 96.3956069946289,
6
+ "gripper.pos": 0.5996002554893494,
7
+ "shoulder_lift.pos": -103.29669952392578,
8
+ "shoulder_pan.pos": -6.549450397491455,
9
+ "wrist_flex.pos": 76.17582702636719,
10
+ "wrist_roll.pos": -92.0879135131836
11
+ },
12
+ "source_dataset": "SO101-GreyEraser-PickPlace"
13
+ }
bundle/robot_client/run_robot_client_with_live_view.py ADDED
@@ -0,0 +1,811 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+
3
+ """Run LeRobot's async client with a headless Flask live camera view."""
4
+
5
+ from __future__ import annotations
6
+
7
+ import json
8
+ import logging
9
+ import os
10
+ import secrets
11
+ import signal
12
+ import socket
13
+ import threading
14
+ import time
15
+ from collections.abc import Iterator
16
+ from queue import Empty, Full, Queue
17
+
18
+ import cv2
19
+ import numpy as np
20
+ import torch
21
+
22
+ _DIRECT_SOM_IMPORT = os.environ.get("ACT_SOM_DIRECT", "0") == "1"
23
+ if not _DIRECT_SOM_IMPORT:
24
+ from flask import Flask, Response, abort, redirect, render_template_string
25
+ from werkzeug.serving import make_server
26
+
27
+ import lerobot.robots.so_follower # noqa: F401 - register the SO-101 configuration
28
+ from lerobot.async_inference.helpers import TimedAction, TimedObservation
29
+ from lerobot.async_inference.robot_client import RobotClient, async_client
30
+ from lerobot.cameras.opencv.camera_opencv import OpenCVCamera
31
+ from lerobot.utils.import_utils import register_third_party_plugins
32
+
33
+ from polima.robot import camera_guard
34
+ from polima.robot.so101 import SO101PolicyClient as ACTSOMClient
35
+ from polima.robot.so101 import record
36
+
37
+ PAGE = """<!doctype html>
38
+ <html lang="en">
39
+ <head>
40
+ <meta charset="utf-8">
41
+ <meta name="viewport" content="width=device-width, initial-scale=1">
42
+ <title>ACT live camera feeds</title>
43
+ <style>
44
+ body { margin: 0; padding: 24px; color: #eee; background: #111; font: 16px system-ui, sans-serif; }
45
+ h1 { margin-top: 0; }
46
+ .grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(320px, 1fr)); gap: 16px; }
47
+ figure { margin: 0; padding: 12px; background: #222; border-radius: 8px; }
48
+ img { display: block; width: 100%; height: auto; background: #000; }
49
+ figcaption { margin-top: 8px; }
50
+ </style>
51
+ </head>
52
+ <body>
53
+ <h1>ACT live camera feeds</h1>
54
+ <p>The robot client is running. Stop it from its terminal with Ctrl-C.</p>
55
+ <div class="grid">
56
+ {% for camera_id, label in cameras %}
57
+ <figure><img src="{{ url_for("stream", token=token, camera_id=camera_id) }}" alt="{{ label }}"><figcaption>{{ label }}</figcaption></figure>
58
+ {% endfor %}
59
+ </div>
60
+ </body>
61
+ </html>"""
62
+
63
+ CAMERAS = (
64
+ ("overhead", "Overhead"),
65
+ ("wrist", "Wrist"),
66
+ )
67
+
68
+ _original_control_loop_observation = RobotClient.control_loop_observation
69
+ _original_control_loop_action = RobotClient.control_loop_action
70
+ _original_aggregate_action_queues = RobotClient._aggregate_action_queues
71
+ _original_init = RobotClient.__init__
72
+ _original_start = RobotClient.start
73
+ _original_stop = RobotClient.stop
74
+ _original_action_tensor_to_dict = RobotClient._action_tensor_to_action_dict
75
+ _original_camera_configure = OpenCVCamera._configure_capture_settings
76
+ _frames: dict[str, bytes] = {}
77
+ _frame_condition = threading.Condition()
78
+ _active_client: RobotClient | None = None
79
+ _last_web_encode_s = 0.0
80
+ _direct_som = _DIRECT_SOM_IMPORT
81
+ _JOINT_NAMES = (
82
+ "shoulder_pan.pos", "shoulder_lift.pos", "elbow_flex.pos",
83
+ "wrist_flex.pos", "wrist_roll.pos", "gripper.pos",
84
+ )
85
+
86
+
87
+ def _som_init(self: RobotClient, config) -> None:
88
+ _init_with_reset_position(self, config)
89
+ stats = os.environ.get("ACT_SOM_STATS")
90
+ if not stats:
91
+ raise ValueError("ACT_SOM_STATS must point to normalization_stats.npz")
92
+ self.som_client = ACTSOMClient(self.server_address, stats)
93
+ self.som_observations: Queue[TimedObservation] = Queue(maxsize=1)
94
+ self.logger.info("Using direct Modalix transport at %s", self.server_address)
95
+
96
+
97
+ def _som_start(self: RobotClient) -> bool:
98
+ self.logger.info("Startup safety reset: moving to the dataset resting position")
99
+ if not _return_to_reset_position(self, stop_inference=False, force=True):
100
+ record("error", message="The robot did not reach rest during startup")
101
+ self.logger.error("Client startup aborted because the robot did not reach rest")
102
+ _original_stop(self)
103
+ return False
104
+ self.reset_completed = False
105
+ self.shutdown_event.clear()
106
+ try:
107
+ with socket.create_connection((self.som_client.host, self.som_client.port), timeout=2.0):
108
+ pass
109
+ except OSError as error:
110
+ record("error", message=str(error))
111
+ self.logger.error("Cannot reach Modalix server: %s", error)
112
+ return False
113
+ self.logger.info("Connected to Modalix server")
114
+ return True
115
+
116
+
117
+ def _som_send_observation(self: RobotClient, observation: TimedObservation) -> bool:
118
+ if not self.running:
119
+ raise RuntimeError("Client is not running")
120
+ try:
121
+ self.som_observations.put_nowait(observation)
122
+ except Full:
123
+ try:
124
+ self.som_observations.get_nowait()
125
+ except Empty:
126
+ pass
127
+ self.som_observations.put_nowait(observation)
128
+ return True
129
+
130
+
131
+ def _som_receive_actions(self: RobotClient, verbose: bool = False) -> None:
132
+ self.start_barrier.wait()
133
+ self.logger.info("Modalix action receiver starting")
134
+ joint_names = tuple(self.robot.action_features)
135
+ if joint_names != _JOINT_NAMES:
136
+ record("error", message="Robot joint order does not match the policy")
137
+ self.logger.error("ACT joint order must be %s, robot exposes %s", _JOINT_NAMES, joint_names)
138
+ self.shutdown_event.set()
139
+ return
140
+ while self.running:
141
+ try:
142
+ observation = self.som_observations.get(timeout=0.2)
143
+ except Empty:
144
+ continue
145
+ raw = observation.get_observation()
146
+ try:
147
+ state = np.asarray([raw[name] for name in joint_names], dtype=np.float32)
148
+ actions, inference_ms = self.som_client.predict(raw["overhead"], raw["wrist"], state)
149
+ count = min(int(self.config.actions_per_chunk), actions.shape[0])
150
+ first_step = max(observation.get_timestep(), 0)
151
+ timestamp = observation.get_timestamp()
152
+ timed_actions = [
153
+ TimedAction(timestamp=timestamp, timestep=first_step + index,
154
+ action=torch.from_numpy(actions[index].copy()))
155
+ for index in range(count)
156
+ ]
157
+ self.action_chunk_size = max(self.action_chunk_size, count)
158
+ self._aggregate_action_queues(timed_actions, self.config.aggregate_fn)
159
+ self.must_go.set()
160
+ record("inference_completed", model_ms=float(inference_ms),
161
+ queue_ms=(time.time() - timestamp) * 1000.0, actions=count)
162
+ self.logger.info(
163
+ "Modalix produced %d actions: model %.1f ms, observation-to-queue %.1f ms",
164
+ count, inference_ms, (time.time() - timestamp) * 1000.0,
165
+ )
166
+ except Exception as error:
167
+ record("error", message=str(error))
168
+ self.logger.exception("Modalix inference request failed: %s", error) # noqa: TRY401 - preserve driver diagnostics
169
+ self.shutdown_event.set()
170
+ return
171
+
172
+
173
+ def _configure_camera_with_perspective_autofocus(self: OpenCVCamera) -> None:
174
+ """Restore the focus setting chosen on the preflight page."""
175
+ _original_camera_configure(self)
176
+ camera_ids = {
177
+ os.environ.get("ACT_PERSPECTIVE_CAMERA", "2"): "perspective",
178
+ os.environ.get("ACT_WRIST_CAMERA", "0"): "wrist",
179
+ }
180
+ camera_id = camera_ids.get(str(self.index_or_path))
181
+ if camera_id is None or self.videocapture is None:
182
+ return
183
+ autofocus = True
184
+ focus = 0
185
+ config_path = os.environ.get("ACT_CAMERA_FOCUS_CONFIG")
186
+ if config_path:
187
+ try:
188
+ with open(config_path, encoding="utf-8") as config_file:
189
+ setting = json.load(config_file).get(camera_id, {})
190
+ autofocus = bool(setting.get("autofocus", True))
191
+ focus = int(setting.get("focus", 0))
192
+ except (OSError, ValueError, TypeError, json.JSONDecodeError):
193
+ logging.getLogger(__name__).warning("Could not read camera focus config; using autofocus")
194
+ accepted = self.videocapture.set(cv2.CAP_PROP_AUTOFOCUS, 1 if autofocus else 0)
195
+ if accepted and (autofocus or self.videocapture.set(cv2.CAP_PROP_FOCUS, focus)):
196
+ mode = "autofocus" if autofocus else f"manual focus {focus}"
197
+ logging.getLogger(__name__).info("%s camera %s enabled", camera_id.title(), mode)
198
+ else:
199
+ logging.getLogger(__name__).warning(
200
+ "Perspective camera backend did not accept autofocus"
201
+ )
202
+
203
+
204
+ def _load_dataset_rest_position() -> dict[str, float]:
205
+ default_path = os.path.join(
206
+ os.path.dirname(os.path.abspath(__file__)),
207
+ "robot_rest_position.json",
208
+ )
209
+ path = os.environ.get("ACT_REST_POSITION", default_path)
210
+ with open(path, encoding="utf-8") as rest_file:
211
+ payload = json.load(rest_file)
212
+ positions = payload["positions"]
213
+ if not positions or any(not key.endswith(".pos") for key in positions):
214
+ raise ValueError(f"Invalid rest position file: {path}")
215
+ return {key: float(value) for key, value in positions.items()}
216
+
217
+
218
+ def _begin_task_cycle(self: RobotClient) -> None:
219
+ """Clear completion and rest-hold state before a new task cycle."""
220
+ self.reset_completed = False
221
+ self._auto_complete_started_s = None
222
+ self._auto_complete_close_count = 0
223
+ self._auto_complete_open_count = 0
224
+ self._auto_complete_initial_open_seen = False
225
+ self._auto_complete_grasp_seen = False
226
+ self._auto_complete_triggered = False
227
+ self._rest_snap_armed = False
228
+ self._rest_snap_latched = False
229
+ self._rest_snap_latched_s = None
230
+
231
+
232
+ def _action_tensor_to_dict_with_rest_snap(self: RobotClient, action_tensor):
233
+ action = _original_action_tensor_to_dict(self, action_tensor)
234
+ arm_keys = [
235
+ key for key in self.reset_position if key != "gripper.pos" and key in action
236
+ ]
237
+ if not arm_keys or self.rest_snap_tolerance <= 0:
238
+ return action
239
+
240
+ if self._rest_snap_latched:
241
+ if (
242
+ self.auto_repeat_enabled
243
+ and self._rest_snap_latched_s is not None
244
+ and time.monotonic() - self._rest_snap_latched_s
245
+ >= self.auto_repeat_pause_s
246
+ ):
247
+
248
+ with self.action_queue_lock:
249
+ self.action_queue.queue.clear()
250
+ self.must_go.set()
251
+ _begin_task_cycle(self)
252
+ self.logger.info(
253
+ "Rest hold complete after %.1fs; starting the next task cycle",
254
+ self.auto_repeat_pause_s,
255
+ )
256
+ action.update(self.reset_position)
257
+ return action
258
+ else:
259
+ action.update(self.reset_position)
260
+ return action
261
+
262
+ max_rest_error = max(
263
+ abs(float(action[key]) - float(self.reset_position[key])) for key in arm_keys
264
+ )
265
+ if not self._rest_snap_armed:
266
+ if max_rest_error >= self.rest_snap_arm_distance:
267
+ self._rest_snap_armed = True
268
+ self.logger.info(
269
+ "Rest snap armed after policy moved %.1f degrees away from rest",
270
+ max_rest_error,
271
+ )
272
+ return action
273
+
274
+ if not self._rest_snap_latched and max_rest_error <= self.rest_snap_tolerance:
275
+ self._rest_snap_latched = True
276
+ self._rest_snap_latched_s = time.monotonic()
277
+ self.logger.info(
278
+ "Policy returned within %.1f degrees of rest; snapping to exact rest pose",
279
+ max_rest_error,
280
+ )
281
+
282
+ if self._rest_snap_latched:
283
+
284
+ action.update(self.reset_position)
285
+ return action
286
+
287
+
288
+ def _init_with_reset_position(self: RobotClient, config) -> None:
289
+ global _active_client
290
+ cameras = getattr(config.robot, "cameras", {}) or {}
291
+ missing = camera_guard.missing_devices(
292
+ {role: getattr(camera, "index_or_path", None) for role, camera in cameras.items()})
293
+ if missing:
294
+ message = "Camera not enumerated before start: " + ", ".join(
295
+ f"{role} ({cameras[role].index_or_path})" for role in missing)
296
+ record("camera_fault", message=message, cameras=missing)
297
+ raise ConnectionError(message)
298
+ _original_init(self, config)
299
+ self.camera_watch = camera_guard.CameraWatch()
300
+ self.reset_position = _load_dataset_rest_position()
301
+ self.reset_completed = False
302
+ self.rest_snap_tolerance = float(
303
+ os.environ.get("ACT_REST_SNAP_TOLERANCE", "5")
304
+ )
305
+ self.rest_snap_arm_distance = max(15.0, self.rest_snap_tolerance * 3.0)
306
+ self._rest_snap_armed = False
307
+ self._rest_snap_latched = False
308
+ self._rest_snap_latched_s = None
309
+ self.auto_complete_enabled = os.environ.get("ACT_AUTO_COMPLETE", "1") != "0"
310
+ self.auto_complete_close_threshold = float(
311
+ os.environ.get("ACT_AUTO_CLOSE_THRESHOLD", "10")
312
+ )
313
+ self.auto_complete_open_threshold = float(
314
+ os.environ.get("ACT_AUTO_OPEN_THRESHOLD", "20")
315
+ )
316
+ self.auto_complete_min_runtime_s = float(
317
+ os.environ.get("ACT_AUTO_MIN_RUNTIME", "2")
318
+ )
319
+ self.auto_complete_close_frames = int(
320
+ os.environ.get("ACT_AUTO_CLOSE_FRAMES", "5")
321
+ )
322
+ self.auto_complete_open_frames = int(
323
+ os.environ.get("ACT_AUTO_OPEN_FRAMES", "8")
324
+ )
325
+ self.auto_repeat_enabled = os.environ.get("ACT_AUTO_REPEAT", "1") != "0"
326
+ self.auto_repeat_pause_s = float(
327
+ os.environ.get("ACT_AUTO_REPEAT_PAUSE", "2")
328
+ )
329
+ if self.auto_complete_open_threshold <= self.auto_complete_close_threshold:
330
+ raise ValueError(
331
+ "ACT_AUTO_OPEN_THRESHOLD must be higher than "
332
+ "ACT_AUTO_CLOSE_THRESHOLD"
333
+ )
334
+ if self.auto_complete_min_runtime_s < 0:
335
+ raise ValueError("ACT_AUTO_MIN_RUNTIME must be non-negative")
336
+ if self.rest_snap_tolerance < 0:
337
+ raise ValueError("ACT_REST_SNAP_TOLERANCE must be non-negative")
338
+ if self.auto_repeat_pause_s < 0:
339
+ raise ValueError("ACT_AUTO_REPEAT_PAUSE must be non-negative")
340
+ if self.auto_complete_close_frames < 1 or self.auto_complete_open_frames < 1:
341
+ raise ValueError("Auto-completion debounce frame counts must be positive")
342
+ self._auto_complete_started_s = None
343
+ self._auto_complete_close_count = 0
344
+ self._auto_complete_open_count = 0
345
+ self._auto_complete_initial_open_seen = False
346
+ self._auto_complete_grasp_seen = False
347
+ self._auto_complete_triggered = False
348
+ self.logger.info(
349
+ "Loaded dataset rest position (%d joints): %s",
350
+ len(self.reset_position),
351
+ self.reset_position,
352
+ )
353
+ if self.auto_complete_enabled:
354
+ self.logger.info(
355
+ "Automatic completion enabled: ignore initial closed rest pose, arm after "
356
+ "opening at >= %.1f for %d frames, detect grasp after closing at <= %.1f "
357
+ "for %d frames, then complete on the next opening (minimum runtime %.1fs)",
358
+ self.auto_complete_open_threshold,
359
+ self.auto_complete_open_frames,
360
+ self.auto_complete_close_threshold,
361
+ self.auto_complete_close_frames,
362
+ self.auto_complete_min_runtime_s,
363
+ )
364
+ if self.auto_repeat_enabled:
365
+ self.logger.info(
366
+ "Automatic repeat enabled: return to rest after release, wait %.1fs "
367
+ "for scene reset, then begin the next cycle",
368
+ self.auto_repeat_pause_s,
369
+ )
370
+ else:
371
+ self.logger.info("Automatic grasp-release completion detection disabled")
372
+ _active_client = self
373
+
374
+
375
+ def _return_to_reset_position(
376
+ self: RobotClient,
377
+ duration_s: float = 5.0,
378
+ lift_duration_s: float = 1.5,
379
+ lift_degrees: float = 20.0,
380
+ fps: int = 50,
381
+ tolerance_degrees: float = 2.0,
382
+ settle_timeout_s: float = 5.0,
383
+ *,
384
+ stop_inference: bool = True,
385
+ force: bool = False,
386
+ ) -> bool:
387
+ if (self.reset_completed and not force) or not self.reset_position:
388
+ return True
389
+
390
+ if stop_inference:
391
+
392
+ self.shutdown_event.set()
393
+ self.logger.info("Returning robot to reset position before shutdown...")
394
+ try:
395
+ current_observation = camera_guard.joint_positions(self.robot)
396
+ current_position = {
397
+ key: value
398
+ for key, value in current_observation.items()
399
+ if key in self.reset_position
400
+ }
401
+
402
+ lift_position = dict(current_position)
403
+ for key in ("shoulder_lift.pos", "wrist_flex.pos"):
404
+ if key not in lift_position:
405
+ continue
406
+ current = float(lift_position[key])
407
+ target = float(self.reset_position[key])
408
+ delta = max(-lift_degrees, min(lift_degrees, target - current))
409
+ lift_position[key] = current + delta
410
+
411
+ self.logger.info(
412
+ "Reset stage 1/2: lifting clear of the basket with shoulder pan held at %.2f",
413
+ float(current_position.get("shoulder_pan.pos", 0.0)),
414
+ )
415
+ lift_steps = max(int(lift_duration_s * fps), 1)
416
+ for step in range(1, lift_steps + 1):
417
+ ratio = step / lift_steps
418
+ action = {
419
+ key: current_position[key] * (1 - ratio)
420
+ + lift_position[key] * ratio
421
+ for key in current_position
422
+ }
423
+ self.robot.send_action(action)
424
+ time.sleep(1 / fps)
425
+
426
+ self.logger.info("Reset stage 2/2: returning to the resting position")
427
+ steps = max(int(duration_s * fps), 1)
428
+ for step in range(1, steps + 1):
429
+ ratio = step / steps
430
+ action = {
431
+ key: lift_position[key] * (1 - ratio)
432
+ + self.reset_position[key] * ratio
433
+ for key in lift_position
434
+ }
435
+ self.robot.send_action(action)
436
+ time.sleep(1 / fps)
437
+
438
+ settle_deadline = time.monotonic() + settle_timeout_s
439
+ while True:
440
+ current_observation = camera_guard.joint_positions(self.robot)
441
+ errors = {
442
+ key: abs(float(current_observation[key]) - target)
443
+ for key, target in self.reset_position.items()
444
+ }
445
+ max_error = max(errors.values())
446
+ if max_error <= tolerance_degrees:
447
+ break
448
+ if time.monotonic() >= settle_deadline:
449
+ raise TimeoutError(
450
+ f"Rest position did not settle within {tolerance_degrees} degrees; "
451
+ f"maximum remaining error is {max_error:.2f} degrees"
452
+ )
453
+ self.robot.send_action(dict(self.reset_position))
454
+ time.sleep(1 / fps)
455
+
456
+ self.robot.send_action(dict(self.reset_position))
457
+ self.reset_completed = True
458
+ self.logger.info("Robot reached reset position")
459
+ return True
460
+ except Exception as error: # noqa: BLE001 - driver failures must stop the reset
461
+ record("error", message=str(error))
462
+ self.logger.warning("Could not return robot to reset position: %s", error)
463
+ return False
464
+
465
+
466
+ def _stop_after_reset(self: RobotClient) -> None:
467
+ _return_to_reset_position(self)
468
+ _original_stop(self)
469
+
470
+
471
+ def _start_after_reset(self: RobotClient) -> bool:
472
+
473
+ self.logger.info(
474
+ "Startup safety reset: moving to the dataset resting position before inference"
475
+ )
476
+ if not _return_to_reset_position(
477
+ self,
478
+ stop_inference=False,
479
+ force=True,
480
+ ):
481
+ self.logger.error(
482
+ "Client startup aborted because the robot did not reach the resting position"
483
+ )
484
+ _original_stop(self)
485
+ return False
486
+
487
+ self.reset_completed = False
488
+ self.logger.info("Startup reset complete; connecting to the policy server")
489
+ return _original_start(self)
490
+
491
+
492
+ def _request_graceful_shutdown(signum: int, _frame) -> None:
493
+ record("operator_stop", signal=signum)
494
+ if _active_client is None:
495
+ raise KeyboardInterrupt
496
+ _active_client.logger.info(
497
+ "Received signal %s; requesting reset and shutdown", signum
498
+ )
499
+ _active_client.shutdown_event.set()
500
+ if signum == signal.SIGINT:
501
+
502
+ raise KeyboardInterrupt
503
+
504
+
505
+ def _aggregate_action_queues_with_latency(self, incoming_actions, aggregate_fn=None):
506
+ if incoming_actions:
507
+ latency_ms = max(0.0, (time.time() - incoming_actions[0].get_timestamp()) * 1000)
508
+ self.live_latency_ms = latency_ms
509
+ previous = getattr(self, "live_latency_ema_ms", latency_ms)
510
+ self.live_latency_ema_ms = 0.85 * previous + 0.15 * latency_ms
511
+ now = time.monotonic()
512
+ if now - getattr(self, "_last_latency_print_s", 0.0) >= 1.0:
513
+ with self.action_queue_lock:
514
+ queue_size = self.action_queue.qsize()
515
+ print(
516
+ f"[latency] observation-to-actions={latency_ms:.1f} ms "
517
+ f"(rolling={self.live_latency_ema_ms:.1f} ms), queue={queue_size}"
518
+ )
519
+ self._last_latency_print_s = now
520
+ return _original_aggregate_action_queues(self, incoming_actions, aggregate_fn)
521
+
522
+
523
+ def _control_loop_action_with_completion(
524
+ self: RobotClient,
525
+ verbose: bool = False,
526
+ ) -> dict[str, float]:
527
+ performed_action = _original_control_loop_action(self, verbose)
528
+ record("action_sent", positions=performed_action)
529
+ if not self.auto_complete_enabled or self._auto_complete_triggered:
530
+ return performed_action
531
+
532
+ gripper_position = performed_action.get("gripper.pos")
533
+ if gripper_position is None:
534
+ return performed_action
535
+
536
+ now = time.monotonic()
537
+ if self._auto_complete_started_s is None:
538
+ self._auto_complete_started_s = now
539
+
540
+ if not self._auto_complete_initial_open_seen:
541
+ if gripper_position >= self.auto_complete_open_threshold:
542
+ self._auto_complete_open_count += 1
543
+ else:
544
+ self._auto_complete_open_count = 0
545
+ if self._auto_complete_open_count >= self.auto_complete_open_frames:
546
+ self._auto_complete_initial_open_seen = True
547
+ self._auto_complete_open_count = 0
548
+ self.logger.info(
549
+ "Automatic completion armed after initial gripper opening at %.1f",
550
+ gripper_position,
551
+ )
552
+ return performed_action
553
+
554
+ if not self._auto_complete_grasp_seen:
555
+ if gripper_position <= self.auto_complete_close_threshold:
556
+ self._auto_complete_close_count += 1
557
+ else:
558
+ self._auto_complete_close_count = 0
559
+ if self._auto_complete_close_count >= self.auto_complete_close_frames:
560
+ self._auto_complete_grasp_seen = True
561
+ record("grasp_detected")
562
+ self.logger.info(
563
+ "Automatic completion: sustained gripper close detected at %.1f",
564
+ gripper_position,
565
+ )
566
+ return performed_action
567
+
568
+ runtime_s = now - self._auto_complete_started_s
569
+ if (
570
+ runtime_s >= self.auto_complete_min_runtime_s
571
+ and gripper_position >= self.auto_complete_open_threshold
572
+ ):
573
+ self._auto_complete_open_count += 1
574
+ else:
575
+ self._auto_complete_open_count = 0
576
+
577
+ if self._auto_complete_open_count >= self.auto_complete_open_frames:
578
+ self._auto_complete_triggered = True
579
+ record("cycle_completed")
580
+ if self.auto_repeat_enabled:
581
+ self.logger.info(
582
+ "Automatic completion: basket-drop release detected at gripper %.1f "
583
+ "after %.1fs; returning to rest for the next cycle",
584
+ gripper_position,
585
+ runtime_s,
586
+ )
587
+
588
+ with self.action_queue_lock:
589
+ self.action_queue.queue.clear()
590
+ reset_succeeded = _return_to_reset_position(
591
+ self,
592
+ stop_inference=False,
593
+ force=True,
594
+ )
595
+ if not reset_succeeded:
596
+ self.logger.error(
597
+ "Automatic repeat stopped because the robot did not reach rest"
598
+ )
599
+ self.shutdown_event.set()
600
+ return performed_action
601
+ self.logger.info(
602
+ "At rest; waiting %.1fs for the marker/basket scene to be reset",
603
+ self.auto_repeat_pause_s,
604
+ )
605
+ time.sleep(self.auto_repeat_pause_s)
606
+
607
+ with self.action_queue_lock:
608
+ self.action_queue.queue.clear()
609
+ self.must_go.set()
610
+ _begin_task_cycle(self)
611
+ self.logger.info("Starting the next automatic task cycle")
612
+ else:
613
+ self.logger.info(
614
+ "Automatic completion: basket-drop release detected at gripper %.1f "
615
+ "after %.1fs; stopping inference and returning to rest",
616
+ gripper_position,
617
+ runtime_s,
618
+ )
619
+ self.shutdown_event.set()
620
+
621
+ return performed_action
622
+
623
+
624
+ def _recover_from_camera_fault(self: RobotClient, fault: camera_guard.CameraFault) -> None:
625
+ """Stop inference, release any object, return to rest, close the gripper, then exit."""
626
+ record("camera_fault", message=str(fault), camera=fault.role)
627
+ self.logger.error("Camera fault: %s; stopping inference and returning to rest", fault)
628
+ self.shutdown_event.set()
629
+ with self.action_queue_lock:
630
+ self.action_queue.queue.clear()
631
+ rested = camera_guard.release_and_rest(
632
+ self.robot, self.reset_position,
633
+ lambda: _return_to_reset_position(self, stop_inference=False, force=True),
634
+ self.logger,
635
+ )
636
+ if rested:
637
+ self.logger.error("Camera recovery complete; the arm is at rest. Check the %s camera "
638
+ "and start the client again.", fault.role)
639
+ else:
640
+ record("error", message="Camera recovery did not reach rest")
641
+
642
+
643
+ def _guarded_control_loop_observation(self: RobotClient, task: str, verbose: bool = False):
644
+ """LeRobot logs and swallows camera errors; check the cameras first and recover instead."""
645
+ if not self.running:
646
+ return None
647
+ fault = self.camera_watch.check(getattr(self.robot, "cameras", {}) or {})
648
+ if fault is not None:
649
+ _recover_from_camera_fault(self, fault)
650
+ return None
651
+ if self.camera_watch.count:
652
+ return None
653
+ return _original_control_loop_observation(self, task, verbose)
654
+
655
+
656
+ def _control_loop_observation_with_web_view(
657
+ self: RobotClient,
658
+ task: str,
659
+ verbose: bool = False,
660
+ ):
661
+ global _last_web_encode_s
662
+
663
+ if not self.running:
664
+ return None
665
+ observation = _guarded_control_loop_observation(self, task, verbose)
666
+ if observation is None:
667
+ return None
668
+ now = time.monotonic()
669
+ if now - _last_web_encode_s < 0.1:
670
+ return observation
671
+ _last_web_encode_s = now
672
+
673
+ encoded_frames: dict[str, bytes] = {}
674
+ for camera_id, _ in CAMERAS:
675
+ frame = observation.get(camera_id)
676
+ if frame is None:
677
+ continue
678
+
679
+ frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR)
680
+ frame = cv2.resize(frame, (480, 360))
681
+ ok, encoded = cv2.imencode(".jpg", frame, [cv2.IMWRITE_JPEG_QUALITY, 60])
682
+ if ok:
683
+ encoded_frames[camera_id] = encoded.tobytes()
684
+
685
+ if encoded_frames:
686
+ with _frame_condition:
687
+ _frames.update(encoded_frames)
688
+ _frame_condition.notify_all()
689
+ return observation
690
+
691
+
692
+ def _stream(camera_id: str, stopped: threading.Event) -> Iterator[bytes]:
693
+ last_frame: bytes | None = None
694
+ while not stopped.is_set():
695
+ with _frame_condition:
696
+ _frame_condition.wait_for(
697
+ lambda: _frames.get(camera_id) is not last_frame or stopped.is_set(), # noqa: B023 - evaluated while holding the condition
698
+ timeout=2,
699
+ )
700
+ frame = _frames.get(camera_id)
701
+ if frame is not None and frame is not last_frame:
702
+ last_frame = frame
703
+ yield (
704
+ b"--frame\r\nContent-Type: image/jpeg\r\nContent-Length: "
705
+ + str(len(frame)).encode()
706
+ + b"\r\n\r\n"
707
+ + frame
708
+ + b"\r\n"
709
+ )
710
+
711
+
712
+ def _display_host(host: str) -> str:
713
+ if host not in {"0.0.0.0", "::"}:
714
+ return host
715
+ probe = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
716
+ try:
717
+ probe.connect(("8.8.8.8", 80))
718
+ return str(probe.getsockname()[0])
719
+ except OSError:
720
+ return "127.0.0.1"
721
+ finally:
722
+ probe.close()
723
+
724
+
725
+ def start_live_server() -> tuple[object, threading.Thread, threading.Event]:
726
+ host = os.environ.get("ACT_LIVE_HOST", "0.0.0.0")
727
+ port = int(os.environ.get("ACT_LIVE_PORT", "5001"))
728
+ token = secrets.token_urlsafe(18)
729
+ stopped = threading.Event()
730
+ app = Flask(__name__)
731
+
732
+ def require_token(candidate: str) -> None:
733
+ if not secrets.compare_digest(candidate, token):
734
+ abort(404)
735
+
736
+ @app.get("/")
737
+ def root() -> Response:
738
+ return redirect(f"/{token}/")
739
+
740
+ @app.get("/favicon.ico")
741
+ def favicon() -> Response:
742
+ return Response(status=204)
743
+
744
+ @app.get("/<token>/")
745
+ def index(token: str) -> str:
746
+ require_token(token)
747
+ return render_template_string(PAGE, token=token, cameras=CAMERAS)
748
+
749
+ @app.get("/<token>/stream/<camera_id>")
750
+ def stream(token: str, camera_id: str) -> Response:
751
+ require_token(token)
752
+ if camera_id not in dict(CAMERAS):
753
+ abort(404)
754
+ return Response(
755
+ _stream(camera_id, stopped),
756
+ mimetype="multipart/x-mixed-replace; boundary=frame",
757
+ headers={
758
+ "Cache-Control": "no-store, no-cache, must-revalidate, max-age=0",
759
+ "Pragma": "no-cache",
760
+ "X-Accel-Buffering": "no",
761
+ },
762
+ )
763
+
764
+ server = make_server(host, port, app, threaded=True)
765
+ thread = threading.Thread(target=server.serve_forever, daemon=True)
766
+ thread.start()
767
+ print(
768
+ f"Live camera server: http://{_display_host(host)}:{port}/{token}/",
769
+ flush=True,
770
+ )
771
+ return server, thread, stopped
772
+
773
+
774
+ RobotClient.__init__ = _som_init if _direct_som else _init_with_reset_position
775
+ RobotClient.start = _som_start if _direct_som else _start_after_reset
776
+ RobotClient.stop = _stop_after_reset
777
+ RobotClient.control_loop_action = _control_loop_action_with_completion
778
+ RobotClient.control_loop_observation = _guarded_control_loop_observation if _direct_som else _control_loop_observation_with_web_view
779
+ RobotClient._aggregate_action_queues = _aggregate_action_queues_with_latency
780
+ RobotClient._action_tensor_to_action_dict = _action_tensor_to_dict_with_rest_snap
781
+ if _direct_som:
782
+ RobotClient.send_observation = _som_send_observation
783
+ RobotClient.receive_actions = _som_receive_actions
784
+ OpenCVCamera._configure_capture_settings = _configure_camera_with_perspective_autofocus
785
+
786
+
787
+ if __name__ == "__main__":
788
+ register_third_party_plugins()
789
+ signal.signal(signal.SIGINT, _request_graceful_shutdown)
790
+ signal.signal(signal.SIGTERM, _request_graceful_shutdown)
791
+ if hasattr(signal, "SIGHUP"):
792
+ signal.signal(signal.SIGHUP, _request_graceful_shutdown)
793
+ live_server = live_thread = live_stopped = None
794
+ if not _direct_som:
795
+ live_server, live_thread, live_stopped = start_live_server()
796
+ else:
797
+ print("Live camera preview disabled for direct Modalix inference.", flush=True)
798
+ try:
799
+ async_client()
800
+ except KeyboardInterrupt:
801
+ print(
802
+ "Robot client interrupted; reset and shutdown complete.",
803
+ flush=True,
804
+ )
805
+ finally:
806
+ if live_stopped is not None:
807
+ live_stopped.set()
808
+ with _frame_condition:
809
+ _frame_condition.notify_all()
810
+ live_server.shutdown()
811
+ live_thread.join(timeout=2)
bundle/robot_client/start.sh ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env bash
2
+ # Run the SO-101 control loop for the bundle this script lives in.
3
+ set -euo pipefail
4
+ ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)"
5
+ VENV="${LEROBOT_VENV:-/media/nvme/lerobot}"
6
+ export PYTHONPATH="${POLIMA_PYTHON_ROOT:-/media/nvme/polima/python}:$ROOT/robot_client:${PYTHONPATH:-}"
7
+ exec "$VENV/bin/python" -m polima.robot.so101 --bundle "$ROOT" "$@"