Upload 31 files
Browse files- .gitattributes +9 -0
- LICENSE +202 -0
- README.md +164 -0
- code/cmu_to272.py +327 -0
- code/gen_ar.py +207 -0
- code/gen_prep.py +78 -0
- code/gen_sample.py +173 -0
- code/run_cmu.py +92 -0
- code/tok_cmu.py +136 -0
- code/tvq.py +383 -0
- code/tvq272.py +339 -0
- code/tvq272c.py +140 -0
- code/vq_mix.py +430 -0
- data/cmu_files.csv +0 -0
- generate.py +45 -0
- images/boxing.gif +3 -0
- images/jump.gif +3 -0
- images/kick.gif +3 -0
- images/live_ui.png +3 -0
- images/run_jog.gif +3 -0
- images/salsa.gif +3 -0
- images/switch.gif +3 -0
- images/walk.gif +3 -0
- images/wave.gif +3 -0
- live/live.html +276 -0
- live/live_server.py +312 -0
- live/start_windows.bat +7 -0
- requirements.txt +4 -0
- weights/Mean.npy +3 -0
- weights/Std.npy +3 -0
- weights/generator.safetensors +3 -0
- weights/tokenizer.safetensors +3 -0
.gitattributes
CHANGED
|
@@ -33,3 +33,12 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
images/boxing.gif filter=lfs diff=lfs merge=lfs -text
|
| 37 |
+
images/jump.gif filter=lfs diff=lfs merge=lfs -text
|
| 38 |
+
images/kick.gif filter=lfs diff=lfs merge=lfs -text
|
| 39 |
+
images/live_ui.png filter=lfs diff=lfs merge=lfs -text
|
| 40 |
+
images/run_jog.gif filter=lfs diff=lfs merge=lfs -text
|
| 41 |
+
images/salsa.gif filter=lfs diff=lfs merge=lfs -text
|
| 42 |
+
images/switch.gif filter=lfs diff=lfs merge=lfs -text
|
| 43 |
+
images/walk.gif filter=lfs diff=lfs merge=lfs -text
|
| 44 |
+
images/wave.gif filter=lfs diff=lfs merge=lfs -text
|
LICENSE
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
Apache License
|
| 3 |
+
Version 2.0, January 2004
|
| 4 |
+
http://www.apache.org/licenses/
|
| 5 |
+
|
| 6 |
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
| 7 |
+
|
| 8 |
+
1. Definitions.
|
| 9 |
+
|
| 10 |
+
"License" shall mean the terms and conditions for use, reproduction,
|
| 11 |
+
and distribution as defined by Sections 1 through 9 of this document.
|
| 12 |
+
|
| 13 |
+
"Licensor" shall mean the copyright owner or entity authorized by
|
| 14 |
+
the copyright owner that is granting the License.
|
| 15 |
+
|
| 16 |
+
"Legal Entity" shall mean the union of the acting entity and all
|
| 17 |
+
other entities that control, are controlled by, or are under common
|
| 18 |
+
control with that entity. For the purposes of this definition,
|
| 19 |
+
"control" means (i) the power, direct or indirect, to cause the
|
| 20 |
+
direction or management of such entity, whether by contract or
|
| 21 |
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
| 22 |
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
| 23 |
+
|
| 24 |
+
"You" (or "Your") shall mean an individual or Legal Entity
|
| 25 |
+
exercising permissions granted by this License.
|
| 26 |
+
|
| 27 |
+
"Source" form shall mean the preferred form for making modifications,
|
| 28 |
+
including but not limited to software source code, documentation
|
| 29 |
+
source, and configuration files.
|
| 30 |
+
|
| 31 |
+
"Object" form shall mean any form resulting from mechanical
|
| 32 |
+
transformation or translation of a Source form, including but
|
| 33 |
+
not limited to compiled object code, generated documentation,
|
| 34 |
+
and conversions to other media types.
|
| 35 |
+
|
| 36 |
+
"Work" shall mean the work of authorship, whether in Source or
|
| 37 |
+
Object form, made available under the License, as indicated by a
|
| 38 |
+
copyright notice that is included in or attached to the work
|
| 39 |
+
(an example is provided in the Appendix below).
|
| 40 |
+
|
| 41 |
+
"Derivative Works" shall mean any work, whether in Source or Object
|
| 42 |
+
form, that is based on (or derived from) the Work and for which the
|
| 43 |
+
editorial revisions, annotations, elaborations, or other modifications
|
| 44 |
+
represent, as a whole, an original work of authorship. For the purposes
|
| 45 |
+
of this License, Derivative Works shall not include works that remain
|
| 46 |
+
separable from, or merely link (or bind by name) to the interfaces of,
|
| 47 |
+
the Work and Derivative Works thereof.
|
| 48 |
+
|
| 49 |
+
"Contribution" shall mean any work of authorship, including
|
| 50 |
+
the original version of the Work and any modifications or additions
|
| 51 |
+
to that Work or Derivative Works thereof, that is intentionally
|
| 52 |
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
| 53 |
+
or by an individual or Legal Entity authorized to submit on behalf of
|
| 54 |
+
the copyright owner. For the purposes of this definition, "submitted"
|
| 55 |
+
means any form of electronic, verbal, or written communication sent
|
| 56 |
+
to the Licensor or its representatives, including but not limited to
|
| 57 |
+
communication on electronic mailing lists, source code control systems,
|
| 58 |
+
and issue tracking systems that are managed by, or on behalf of, the
|
| 59 |
+
Licensor for the purpose of discussing and improving the Work, but
|
| 60 |
+
excluding communication that is conspicuously marked or otherwise
|
| 61 |
+
designated in writing by the copyright owner as "Not a Contribution."
|
| 62 |
+
|
| 63 |
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
| 64 |
+
on behalf of whom a Contribution has been received by Licensor and
|
| 65 |
+
subsequently incorporated within the Work.
|
| 66 |
+
|
| 67 |
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
| 68 |
+
this License, each Contributor hereby grants to You a perpetual,
|
| 69 |
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
| 70 |
+
copyright license to reproduce, prepare Derivative Works of,
|
| 71 |
+
publicly display, publicly perform, sublicense, and distribute the
|
| 72 |
+
Work and such Derivative Works in Source or Object form.
|
| 73 |
+
|
| 74 |
+
3. Grant of Patent License. Subject to the terms and conditions of
|
| 75 |
+
this License, each Contributor hereby grants to You a perpetual,
|
| 76 |
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
| 77 |
+
(except as stated in this section) patent license to make, have made,
|
| 78 |
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
| 79 |
+
where such license applies only to those patent claims licensable
|
| 80 |
+
by such Contributor that are necessarily infringed by their
|
| 81 |
+
Contribution(s) alone or by combination of their Contribution(s)
|
| 82 |
+
with the Work to which such Contribution(s) was submitted. If You
|
| 83 |
+
institute patent litigation against any entity (including a
|
| 84 |
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
| 85 |
+
or a Contribution incorporated within the Work constitutes direct
|
| 86 |
+
or contributory patent infringement, then any patent licenses
|
| 87 |
+
granted to You under this License for that Work shall terminate
|
| 88 |
+
as of the date such litigation is filed.
|
| 89 |
+
|
| 90 |
+
4. Redistribution. You may reproduce and distribute copies of the
|
| 91 |
+
Work or Derivative Works thereof in any medium, with or without
|
| 92 |
+
modifications, and in Source or Object form, provided that You
|
| 93 |
+
meet the following conditions:
|
| 94 |
+
|
| 95 |
+
(a) You must give any other recipients of the Work or
|
| 96 |
+
Derivative Works a copy of this License; and
|
| 97 |
+
|
| 98 |
+
(b) You must cause any modified files to carry prominent notices
|
| 99 |
+
stating that You changed the files; and
|
| 100 |
+
|
| 101 |
+
(c) You must retain, in the Source form of any Derivative Works
|
| 102 |
+
that You distribute, all copyright, patent, trademark, and
|
| 103 |
+
attribution notices from the Source form of the Work,
|
| 104 |
+
excluding those notices that do not pertain to any part of
|
| 105 |
+
the Derivative Works; and
|
| 106 |
+
|
| 107 |
+
(d) If the Work includes a "NOTICE" text file as part of its
|
| 108 |
+
distribution, then any Derivative Works that You distribute must
|
| 109 |
+
include a readable copy of the attribution notices contained
|
| 110 |
+
within such NOTICE file, excluding those notices that do not
|
| 111 |
+
pertain to any part of the Derivative Works, in at least one
|
| 112 |
+
of the following places: within a NOTICE text file distributed
|
| 113 |
+
as part of the Derivative Works; within the Source form or
|
| 114 |
+
documentation, if provided along with the Derivative Works; or,
|
| 115 |
+
within a display generated by the Derivative Works, if and
|
| 116 |
+
wherever such third-party notices normally appear. The contents
|
| 117 |
+
of the NOTICE file are for informational purposes only and
|
| 118 |
+
do not modify the License. You may add Your own attribution
|
| 119 |
+
notices within Derivative Works that You distribute, alongside
|
| 120 |
+
or as an addendum to the NOTICE text from the Work, provided
|
| 121 |
+
that such additional attribution notices cannot be construed
|
| 122 |
+
as modifying the License.
|
| 123 |
+
|
| 124 |
+
You may add Your own copyright statement to Your modifications and
|
| 125 |
+
may provide additional or different license terms and conditions
|
| 126 |
+
for use, reproduction, or distribution of Your modifications, or
|
| 127 |
+
for any such Derivative Works as a whole, provided Your use,
|
| 128 |
+
reproduction, and distribution of the Work otherwise complies with
|
| 129 |
+
the conditions stated in this License.
|
| 130 |
+
|
| 131 |
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
| 132 |
+
any Contribution intentionally submitted for inclusion in the Work
|
| 133 |
+
by You to the Licensor shall be under the terms and conditions of
|
| 134 |
+
this License, without any additional terms or conditions.
|
| 135 |
+
Notwithstanding the above, nothing herein shall supersede or modify
|
| 136 |
+
the terms of any separate license agreement you may have executed
|
| 137 |
+
with Licensor regarding such Contributions.
|
| 138 |
+
|
| 139 |
+
6. Trademarks. This License does not grant permission to use the trade
|
| 140 |
+
names, trademarks, service marks, or product names of the Licensor,
|
| 141 |
+
except as required for reasonable and customary use in describing the
|
| 142 |
+
origin of the Work and reproducing the content of the NOTICE file.
|
| 143 |
+
|
| 144 |
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
| 145 |
+
agreed to in writing, Licensor provides the Work (and each
|
| 146 |
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
| 147 |
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
| 148 |
+
implied, including, without limitation, any warranties or conditions
|
| 149 |
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
| 150 |
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
| 151 |
+
appropriateness of using or redistributing the Work and assume any
|
| 152 |
+
risks associated with Your exercise of permissions under this License.
|
| 153 |
+
|
| 154 |
+
8. Limitation of Liability. In no event and under no legal theory,
|
| 155 |
+
whether in tort (including negligence), contract, or otherwise,
|
| 156 |
+
unless required by applicable law (such as deliberate and grossly
|
| 157 |
+
negligent acts) or agreed to in writing, shall any Contributor be
|
| 158 |
+
liable to You for damages, including any direct, indirect, special,
|
| 159 |
+
incidental, or consequential damages of any character arising as a
|
| 160 |
+
result of this License or out of the use or inability to use the
|
| 161 |
+
Work (including but not limited to damages for loss of goodwill,
|
| 162 |
+
work stoppage, computer failure or malfunction, or any and all
|
| 163 |
+
other commercial damages or losses), even if such Contributor
|
| 164 |
+
has been advised of the possibility of such damages.
|
| 165 |
+
|
| 166 |
+
9. Accepting Warranty or Additional Liability. While redistributing
|
| 167 |
+
the Work or Derivative Works thereof, You may choose to offer,
|
| 168 |
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
| 169 |
+
or other liability obligations and/or rights consistent with this
|
| 170 |
+
License. However, in accepting such obligations, You may act only
|
| 171 |
+
on Your own behalf and on Your sole responsibility, not on behalf
|
| 172 |
+
of any other Contributor, and only if You agree to indemnify,
|
| 173 |
+
defend, and hold each Contributor harmless for any liability
|
| 174 |
+
incurred by, or claims asserted against, such Contributor by reason
|
| 175 |
+
of your accepting any such warranty or additional liability.
|
| 176 |
+
|
| 177 |
+
END OF TERMS AND CONDITIONS
|
| 178 |
+
|
| 179 |
+
APPENDIX: How to apply the Apache License to your work.
|
| 180 |
+
|
| 181 |
+
To apply the Apache License to your work, attach the following
|
| 182 |
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
| 183 |
+
replaced with your own identifying information. (Don't include
|
| 184 |
+
the brackets!) The text should be enclosed in the appropriate
|
| 185 |
+
comment syntax for the file format. We also recommend that a
|
| 186 |
+
file or class name and description of purpose be included on the
|
| 187 |
+
same "printed page" as the copyright notice for easier
|
| 188 |
+
identification within third-party archives.
|
| 189 |
+
|
| 190 |
+
Copyright [yyyy] [name of copyright owner]
|
| 191 |
+
|
| 192 |
+
Licensed under the Apache License, Version 2.0 (the "License");
|
| 193 |
+
you may not use this file except in compliance with the License.
|
| 194 |
+
You may obtain a copy of the License at
|
| 195 |
+
|
| 196 |
+
http://www.apache.org/licenses/LICENSE-2.0
|
| 197 |
+
|
| 198 |
+
Unless required by applicable law or agreed to in writing, software
|
| 199 |
+
distributed under the License is distributed on an "AS IS" BASIS,
|
| 200 |
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 201 |
+
See the License for the specific language governing permissions and
|
| 202 |
+
limitations under the License.
|
README.md
ADDED
|
@@ -0,0 +1,164 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
license: apache-2.0
|
| 3 |
+
language:
|
| 4 |
+
- en
|
| 5 |
+
library_name: pytorch
|
| 6 |
+
pipeline_tag: text-to-3d
|
| 7 |
+
tags:
|
| 8 |
+
- text-to-motion
|
| 9 |
+
- motion-generation
|
| 10 |
+
- human-motion
|
| 11 |
+
- motion-capture
|
| 12 |
+
- cmu-mocap
|
| 13 |
+
- real-time
|
| 14 |
+
---
|
| 15 |
+
|
| 16 |
+
# PartLock-CMU
|
| 17 |
+
|
| 18 |
+
Real-time, endless text-to-motion trained **only on the CMU Motion Capture Database**, so the weights can be used in commercial products.
|
| 19 |
+
|
| 20 |
+
Type a short English caption. The model keeps generating full-body motion forever, and you can switch the caption at any moment. A new caption takes effect at the next step (4 frames = 0.13 s), and the motion so far is kept as context, so the transition is continuous.
|
| 21 |
+
|
| 22 |
+
Author: やきますく (Yakimasuku) · Code and weights: Apache License 2.0
|
| 23 |
+
|
| 24 |
+

|
| 25 |
+
|
| 26 |
+
*One continuous take (seed 1, not cherry-picked). The caption changes every 4 s: `walk` → `Wave` → `run/jog` → `jump` → `Standing still`.*
|
| 27 |
+
|
| 28 |
+
| `walk` | `run/jog` | `jump` | `Wave` |
|
| 29 |
+
|---|---|---|---|
|
| 30 |
+
|  |  |  |  |
|
| 31 |
+
| **`kick`** | **`salsa dance`** | **`boxing`** | |
|
| 32 |
+
|  |  |  | |
|
| 33 |
+
|
| 34 |
+
*Seed 1, first take of each. The figure is a simple preview body drawn from the generated joint positions and rotations; it is not an SMPL mesh.*
|
| 35 |
+
|
| 36 |
+
## Real-time demo
|
| 37 |
+
|
| 38 |
+

|
| 39 |
+
|
| 40 |
+
```bash
|
| 41 |
+
pip install -r requirements.txt
|
| 42 |
+
python live/live_server.py # then open http://localhost:8765
|
| 43 |
+
```
|
| 44 |
+
|
| 45 |
+
On Windows you can also double-click `live/start_windows.bat`.
|
| 46 |
+
|
| 47 |
+
- Type a caption and press the green button, or click one of the example captions. The page is in Japanese: 生成 = generate, 一時停止 = pause, 最初から = restart (clear the context), 文章の強さ = text strength (classifier-free guidance scale).
|
| 48 |
+
- The server generates only 0.8 s ahead of playback, so a new caption appears about 0.6 s after you press the button.
|
| 49 |
+
- Needs an NVIDIA GPU with CUDA. On an RTX 3060 (12 GB), one step (4 frames) takes 35–38 ms. Real time needs 133 ms or less, so there is about 3.5× headroom.
|
| 50 |
+
- At start-up the server checks itself. The CUDA-graph path is compared with the plain forward pass, and the largest logit difference was 0.06 (bf16 rounding; logits are up to about 12).
|
| 51 |
+
|
| 52 |
+
## Generate to a file
|
| 53 |
+
|
| 54 |
+
```bash
|
| 55 |
+
python generate.py --sec 4 --out motion.npz "walk" "Wave" "run/jog" "jump"
|
| 56 |
+
```
|
| 57 |
+
|
| 58 |
+
Each caption is used for `--sec` seconds, in order, in one continuous motion. The output `.npz` is at 30 fps:
|
| 59 |
+
|
| 60 |
+
| key | shape | content |
|
| 61 |
+
|---|---|---|
|
| 62 |
+
| `positions` | (T, 22, 3) | world joint positions in meters, y up, SMPL joint order |
|
| 63 |
+
| `rotations` | (T, 22, 3, 3) | world joint rotations |
|
| 64 |
+
| `root_xz` | (T, 2) | pelvis path on the floor |
|
| 65 |
+
| `prompt` | (T,) | caption used for each frame |
|
| 66 |
+
| `parents` | (22,) | skeleton parents |
|
| 67 |
+
|
| 68 |
+
Values are rounded to 0.001 (1 mm for positions).
|
| 69 |
+
|
| 70 |
+
## Captions that work
|
| 71 |
+
|
| 72 |
+
The model only knows the CMU motion descriptions. Short captions in the CMU style work best, for example:
|
| 73 |
+
|
| 74 |
+
`walk`, `run/jog`, `jump`, `Wave`, `kick`, `salsa dance`, `boxing`, `Standing still`, `sit on high stool, stand up`, `walk backwards and turn`, `cartwheel`, `Mickey sneaking walk`
|
| 75 |
+
|
| 76 |
+
The full list is in `data/cmu_files.csv` (column `desc`). Case doesn't matter: in our tests `Run` and `run` gave identical results, because the CLIP tokenizer lower-cases the text.
|
| 77 |
+
|
| 78 |
+
## Model
|
| 79 |
+
|
| 80 |
+
| | |
|
| 81 |
+
|---|---|
|
| 82 |
+
| Motion representation | 272 numbers per frame at 30 fps (root velocity and heading, joint positions, velocities and 6D rotations of 22 SMPL-order joints), following the 272-dim representation of [MotionStreamer](https://github.com/zju3dv/MotionStreamer). CMU ASF/AMC is converted with our own converter (`code/cmu_to272.py`). |
|
| 83 |
+
| Tokenizer (7.5 M parameters) | Causal hierarchical VQ. Every 4 frames become 7 tokens: 1 global code (codebook 4096) plus 6 part codes (codebook 128 each) for torso, head, right arm, left arm, right leg and left leg. Encoder and decoder only look at the past, which allows streaming. |
|
| 84 |
+
| Generator (30.5 M parameters) | Decoder-only Transformer: 8 layers, width 512, 8 heads, RoPE. Context is 64 steps (8.5 s) and slides forever, which makes it endless. The text embedding (CLIP ViT-B/32) is added at every position, so the caption can be swapped mid-stream. Classifier-free guidance s = 4 (null text for 10% of training samples). |
|
| 85 |
+
|
| 86 |
+
## Training data and results
|
| 87 |
+
|
| 88 |
+
- Data: CMU Motion Capture Database, 2,308 clips (9.9 h) with captions taken from the CMU motion descriptions. 179 clips were skipped (not in the CMU list or too short). Every clip is also mirrored left–right, which doubles the data.
|
| 89 |
+
- Split: by original clip, 90 / 5 / 5 (train 2,078, validation 115, test 115 clips). A mirrored copy always stays in the same split as its original.
|
| 90 |
+
- Hardware: one RTX 3060 (12 GB). Tokenizer 2 h, generator 75 min to its best validation loss. The whole pipeline, including conversion, took about 3 h 50 min.
|
| 91 |
+
|
| 92 |
+
Tokenizer reconstruction on the validation data (best step 18,000):
|
| 93 |
+
|
| 94 |
+
| part | position error (mm) | rotation error (°) |
|
| 95 |
+
|---|---|---|
|
| 96 |
+
| torso | 33.2 | 5.3 |
|
| 97 |
+
| head | 56.1 | 8.2 |
|
| 98 |
+
| right arm | 67.4 | 14.2 |
|
| 99 |
+
| left arm | 68.4 | 14.2 |
|
| 100 |
+
| right leg | 55.4 | 8.2 |
|
| 101 |
+
| left leg | 56.4 | 8.2 |
|
| 102 |
+
| **all** | **57.4** | 9.9 (global) |
|
| 103 |
+
|
| 104 |
+
Generator validation loss (cross-entropy per token) at the best step (9,500):
|
| 105 |
+
|
| 106 |
+
| global | torso | head | right arm | left arm | right leg | left leg | mean |
|
| 107 |
+
|---|---|---|---|---|---|---|---|
|
| 108 |
+
| 2.654 | 2.459 | 2.428 | 2.208 | 2.114 | 2.196 | 2.114 | 2.311 |
|
| 109 |
+
|
| 110 |
+
There is no standard text-to-motion benchmark for CMU-only models, so these numbers can't be compared directly with models trained on HumanML3D.
|
| 111 |
+
|
| 112 |
+
## Limitations
|
| 113 |
+
|
| 114 |
+
- **Only CMU motions.** Motions that are not in CMU didn't work in our tests. All 20 novel actions we tried came out nearly stationary.
|
| 115 |
+
- **Mixing two actions in one caption** (for example "wave while walking") usually loses most of the movement. Switching captions over time works much better; see the demo above.
|
| 116 |
+
- **`run` alone** runs for about 2 s and then slows to about 0.5 m/s. Use `run/jog`, which keeps running at about 2.2 m/s.
|
| 117 |
+
- English only, because of CLIP.
|
| 118 |
+
- CMU subjects have different body sizes and the data isn't normalized to one body, so bone lengths can vary a little within a motion.
|
| 119 |
+
- The output is a 22-joint skeleton with rotations. Retargeting to a character rig is up to you.
|
| 120 |
+
|
| 121 |
+
## Train from scratch
|
| 122 |
+
|
| 123 |
+
1. Download `allasfamc.zip` from [mocap.cs.cmu.edu](http://mocap.cs.cmu.edu/) and extract it. The folder contains `subjects/`.
|
| 124 |
+
2. Run:
|
| 125 |
+
|
| 126 |
+
```bash
|
| 127 |
+
python code/run_cmu.py --amc <extracted folder> --work <empty work folder>
|
| 128 |
+
```
|
| 129 |
+
|
| 130 |
+
This runs convert → tokenizer (2 h) → prep → generator (2 h). Checkpoints are written to `<work>/ckpt` as `.pt` files. To use them with the demo, convert them to `weights/*.safetensors`.
|
| 131 |
+
|
| 132 |
+
The released weights came from this pipeline. In the release copy, `run_cmu.py` was changed only to take the paths as arguments. That edited copy hasn't been run end to end again.
|
| 133 |
+
|
| 134 |
+
## Files
|
| 135 |
+
|
| 136 |
+
| path | content |
|
| 137 |
+
|---|---|
|
| 138 |
+
| `weights/generator.safetensors` | generator |
|
| 139 |
+
| `weights/tokenizer.safetensors` | causal tokenizer (encoder and decoder) |
|
| 140 |
+
| `weights/Mean.npy`, `weights/Std.npy` | normalization of the 272-dim features |
|
| 141 |
+
| `live/` | real-time server and page |
|
| 142 |
+
| `generate.py` | generate to `.npz` |
|
| 143 |
+
| `code/` | model code and the training pipeline |
|
| 144 |
+
| `data/cmu_files.csv` | CMU clip list with motion descriptions (used as captions) |
|
| 145 |
+
|
| 146 |
+
`code/tvq.py`, `code/tvq272.py` and `code/vq_mix.py` are shared modules from earlier research code, and their comments mention HumanML3D experiments. Only their model classes and helper functions are used here. No HumanML3D or AMASS data or weights were used to train this model.
|
| 147 |
+
|
| 148 |
+
## License and data
|
| 149 |
+
|
| 150 |
+
- Code and weights: Apache License 2.0 (`LICENSE`).
|
| 151 |
+
- Training data: the [CMU Motion Capture Database](http://mocap.cs.cmu.edu/). Its terms say: "This data is free for use in research projects. You may include this data in commercially-sold products, but you may not resell this data directly, even in converted form." This repository doesn't contain the motion data. `data/cmu_files.csv` is only the list of clip names and descriptions.
|
| 152 |
+
- The text encoder [openai/clip-vit-base-patch32](https://huggingface.co/openai/clip-vit-base-patch32) is downloaded at run time under its own license.
|
| 153 |
+
|
| 154 |
+
**Acknowledgment:** The data used in this project was obtained from mocap.cs.cmu.edu. The database was created with funding from NSF EIA-0196217.
|
| 155 |
+
|
| 156 |
+
---
|
| 157 |
+
|
| 158 |
+
## 日本語
|
| 159 |
+
|
| 160 |
+
CMU のモーションキャプチャだけで学習した、文章から全身の動きを作るモデルです。止まらずに動きを作り続け、途中で文章を切り替えられます。商用でも使えます。
|
| 161 |
+
|
| 162 |
+
- `python live/live_server.py` を実行して http://localhost:8765 を開くと、文章を入れてリアルタイムに動かせます(NVIDIA の GPU が必要です)。
|
| 163 |
+
- 学習データにある短い英語の文章(walk, run/jog, jump, Wave など)がよく効きます。学習データにない動きは、まだうまく作れません。
|
| 164 |
+
- 作者:やきますく
|
code/cmu_to272.py
ADDED
|
@@ -0,0 +1,327 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""CMU ASF/AMC -> 272-dim representation (same layout as the HumanML3D-272 data used before).
|
| 2 |
+
|
| 3 |
+
Layout (as read by tvq272.py / gen_sample.py recovery):
|
| 4 |
+
[0:2] root xz velocity in the heading frame of the previous frame (frame 0: 0)
|
| 5 |
+
[2:8] heading change, 6D (first two rows): Rd_0 = H_0, Rd_t = H_t H_{t-1}^T
|
| 6 |
+
[8:74] joint positions, heading removed, root xz at origin (22x3)
|
| 7 |
+
[74:140] joint velocities in the current heading frame (22x3)
|
| 8 |
+
[140:272] parent-relative rotations 6D (first two rows); root = H_t G_root
|
| 9 |
+
Body sizes are kept as captured (the HumanML3D-272 data was not size-normalised either).
|
| 10 |
+
Every clip is also mirrored (left/right swapped, caption words left/right swapped).
|
| 11 |
+
30 fps. Floor: lowest joint of the clip at y = 0.
|
| 12 |
+
|
| 13 |
+
usage: python cmu_to272.py <all_asfamc dir> <cmu_files.csv> <out h272 dir>
|
| 14 |
+
"""
|
| 15 |
+
import os, sys, csv, re, json, math
|
| 16 |
+
import numpy as np
|
| 17 |
+
|
| 18 |
+
SCALE = (1.0 / 0.45) * 2.54 / 100.0 # CMU FAQ
|
| 19 |
+
J22 = ["pelvis", "l_hip", "r_hip", "spine1", "l_knee", "r_knee", "spine2", "l_ankle", "r_ankle", "spine3",
|
| 20 |
+
"l_foot", "r_foot", "neck", "l_collar", "r_collar", "head", "l_shoulder", "r_shoulder", "l_elbow",
|
| 21 |
+
"r_elbow", "l_wrist", "r_wrist"]
|
| 22 |
+
PAR = [-1, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 9, 9, 12, 13, 14, 16, 17, 18, 19]
|
| 23 |
+
# position of each of the 22 joints = end point of this CMU bone (root = root position)
|
| 24 |
+
POS = {0: "root", 1: "lhipjoint", 2: "rhipjoint", 3: "lowerback", 4: "lfemur", 5: "rfemur", 6: "upperback",
|
| 25 |
+
7: "ltibia", 8: "rtibia", 9: "thorax", 10: "lfoot", 11: "rfoot", 12: "lowerneck", 15: "upperneck",
|
| 26 |
+
16: "lclavicle", 17: "rclavicle", 18: "lhumerus", 19: "rhumerus", 20: "lradius", 21: "rradius"}
|
| 27 |
+
# collars (13, 14): 35% of the way from thorax end to clavicle end
|
| 28 |
+
# rotation of each joint = global orientation of the CMU bone that leaves it towards its main child
|
| 29 |
+
ROT = {0: "root", 1: "lfemur", 2: "rfemur", 3: "upperback", 4: "ltibia", 5: "rtibia", 6: "thorax",
|
| 30 |
+
7: "lfoot", 8: "rfoot", 9: "lowerneck", 10: "ltoes", 11: "rtoes", 12: "upperneck", 13: "lclavicle",
|
| 31 |
+
14: "rclavicle", 15: "head", 16: "lhumerus", 17: "rhumerus", 18: "lradius", 19: "rradius",
|
| 32 |
+
20: "lhand", 21: "rhand"}
|
| 33 |
+
LR = [(1, 2), (4, 5), (7, 8), (10, 11), (13, 14), (16, 17), (18, 19), (20, 21)]
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
# ------------------------------------------------------------------ rotations
|
| 37 |
+
def rx(a):
|
| 38 |
+
c, s = np.cos(a), np.sin(a); o, z = np.ones_like(a), np.zeros_like(a)
|
| 39 |
+
return np.stack([np.stack([o, z, z], -1), np.stack([z, c, -s], -1), np.stack([z, s, c], -1)], -2)
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def ry(a):
|
| 43 |
+
c, s = np.cos(a), np.sin(a); o, z = np.ones_like(a), np.zeros_like(a)
|
| 44 |
+
return np.stack([np.stack([c, z, s], -1), np.stack([z, o, z], -1), np.stack([-s, z, c], -1)], -2)
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
def rz(a):
|
| 48 |
+
c, s = np.cos(a), np.sin(a); o, z = np.ones_like(a), np.zeros_like(a)
|
| 49 |
+
return np.stack([np.stack([c, -s, z], -1), np.stack([s, c, z], -1), np.stack([z, z, o], -1)], -2)
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def euler_xyz(x, y, z): # rotate about X, then Y, then Z -> Rz Ry Rx (degrees in)
|
| 53 |
+
return rz(np.radians(z)) @ ry(np.radians(y)) @ rx(np.radians(x))
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
# ------------------------------------------------------------------ ASF / AMC
|
| 57 |
+
def parse_asf(path):
|
| 58 |
+
lines = [l.strip() for l in open(path, encoding="latin-1")]
|
| 59 |
+
bones, hier, root_order, i = {}, {}, ["tx", "ty", "tz", "rx", "ry", "rz"], 0
|
| 60 |
+
root_axis = "XYZ"
|
| 61 |
+
while i < len(lines):
|
| 62 |
+
l = lines[i]
|
| 63 |
+
if l.startswith(":root"):
|
| 64 |
+
i += 1
|
| 65 |
+
while not lines[i].startswith(":"):
|
| 66 |
+
t = lines[i].split()
|
| 67 |
+
if t and t[0] == "order": root_order = [x.lower() for x in t[1:]]
|
| 68 |
+
if t and t[0] == "axis": root_axis = t[1]
|
| 69 |
+
i += 1
|
| 70 |
+
continue
|
| 71 |
+
if l.startswith(":bonedata"):
|
| 72 |
+
i += 1
|
| 73 |
+
while not lines[i].startswith(":"):
|
| 74 |
+
if lines[i] == "begin":
|
| 75 |
+
b = dict(dof=[])
|
| 76 |
+
i += 1
|
| 77 |
+
while lines[i] != "end":
|
| 78 |
+
t = lines[i].split()
|
| 79 |
+
if t[0] == "name": b["name"] = t[1]
|
| 80 |
+
elif t[0] == "direction": b["dir"] = np.array(list(map(float, t[1:4])))
|
| 81 |
+
elif t[0] == "length": b["length"] = float(t[1]) * SCALE
|
| 82 |
+
elif t[0] == "axis":
|
| 83 |
+
assert t[4] == "XYZ", t
|
| 84 |
+
b["axis"] = np.array(list(map(float, t[1:4])))
|
| 85 |
+
elif t[0] == "dof": b["dof"] = [x.lower() for x in t[1:]]
|
| 86 |
+
i += 1
|
| 87 |
+
bones[b["name"]] = b
|
| 88 |
+
i += 1
|
| 89 |
+
continue
|
| 90 |
+
if l.startswith(":hierarchy"):
|
| 91 |
+
i += 1
|
| 92 |
+
while i < len(lines) and lines[i] != "end":
|
| 93 |
+
t = lines[i].split()
|
| 94 |
+
if t and t[0] != "begin":
|
| 95 |
+
for ch in t[1:]: hier[ch] = t[0]
|
| 96 |
+
i += 1
|
| 97 |
+
continue
|
| 98 |
+
i += 1
|
| 99 |
+
assert root_axis == "XYZ"
|
| 100 |
+
order = ["root"]
|
| 101 |
+
k = 0
|
| 102 |
+
while k < len(order):
|
| 103 |
+
order += [c for c, p in hier.items() if p == order[k]]
|
| 104 |
+
k += 1
|
| 105 |
+
for b in bones.values():
|
| 106 |
+
b["C"] = euler_xyz(*b["axis"])
|
| 107 |
+
return bones, hier, order, root_order
|
| 108 |
+
|
| 109 |
+
|
| 110 |
+
def parse_amc(path):
|
| 111 |
+
frames, cur = [], None
|
| 112 |
+
for l in open(path, encoding="latin-1"):
|
| 113 |
+
l = l.strip()
|
| 114 |
+
if not l or l[0] in "#:": continue
|
| 115 |
+
if l.isdigit():
|
| 116 |
+
cur = {}; frames.append(cur); continue
|
| 117 |
+
t = l.split()
|
| 118 |
+
cur[t[0]] = [float(v) for v in t[1:]]
|
| 119 |
+
return frames
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
def fk(bones, hier, order, root_order, frames):
|
| 123 |
+
"""vectorised over frames -> global rotation M[name] (T,3,3) and end point E[name] (T,3), meters"""
|
| 124 |
+
T = len(frames)
|
| 125 |
+
rv = np.array([f["root"] for f in frames])
|
| 126 |
+
d = {k: rv[:, i] for i, k in enumerate(root_order)}
|
| 127 |
+
M, E = {}, {}
|
| 128 |
+
M["root"] = euler_xyz(d["rx"], d["ry"], d["rz"]) # root axis = 0 0 0 -> C = I
|
| 129 |
+
E["root"] = np.stack([d["tx"], d["ty"], d["tz"]], -1) * SCALE
|
| 130 |
+
for n in order[1:]:
|
| 131 |
+
b, p = bones[n], hier[n]
|
| 132 |
+
ang = {"rx": np.zeros(T), "ry": np.zeros(T), "rz": np.zeros(T)}
|
| 133 |
+
if b["dof"]:
|
| 134 |
+
v = np.array([f.get(n, [0.0] * len(b["dof"])) for f in frames])
|
| 135 |
+
for k, dof in enumerate(b["dof"]):
|
| 136 |
+
ang[dof] = v[:, k]
|
| 137 |
+
L = euler_xyz(ang["rx"], ang["ry"], ang["rz"])
|
| 138 |
+
M[n] = M[p] @ b["C"] @ L @ b["C"].T
|
| 139 |
+
E[n] = E[p] + b["length"] * (M[n] @ b["dir"])
|
| 140 |
+
return M, E
|
| 141 |
+
|
| 142 |
+
|
| 143 |
+
def to22(M, E):
|
| 144 |
+
P = np.zeros((len(E["root"]), 22, 3))
|
| 145 |
+
for j, n in POS.items():
|
| 146 |
+
P[:, j] = E[n]
|
| 147 |
+
P[:, 13] = E["thorax"] + 0.35 * (E["lclavicle"] - E["thorax"])
|
| 148 |
+
P[:, 14] = E["thorax"] + 0.35 * (E["rclavicle"] - E["thorax"])
|
| 149 |
+
G = np.stack([M[ROT[j]] for j in range(22)], 1) # (T,22,3,3)
|
| 150 |
+
return P, G
|
| 151 |
+
|
| 152 |
+
|
| 153 |
+
def mirror(P, G):
|
| 154 |
+
S = np.diag([-1.0, 1.0, 1.0])
|
| 155 |
+
P2, G2 = P * np.array([-1.0, 1.0, 1.0]), S @ G @ S
|
| 156 |
+
for a, b in LR:
|
| 157 |
+
P2[:, [a, b]] = P2[:, [b, a]]
|
| 158 |
+
G2[:, [a, b]] = G2[:, [b, a]]
|
| 159 |
+
return P2, G2
|
| 160 |
+
|
| 161 |
+
|
| 162 |
+
# ------------------------------------------------------------------ 272 features
|
| 163 |
+
def six(R): # first two ROWS
|
| 164 |
+
return R[..., :2, :].reshape(*R.shape[:-2], 6)
|
| 165 |
+
|
| 166 |
+
|
| 167 |
+
def features(P, G):
|
| 168 |
+
P = P.copy()
|
| 169 |
+
P[..., 1] -= P[..., 1].min() # floor
|
| 170 |
+
P[..., [0, 2]] -= P[0, 0, [0, 2]] # start at origin
|
| 171 |
+
T = P.shape[0]
|
| 172 |
+
f = G[:, 0] @ np.array([0.0, 0.0, 1.0]) # body forward
|
| 173 |
+
a = np.arctan2(f[:, 0], f[:, 2])
|
| 174 |
+
H = np.transpose(ry(a), (0, 2, 1)) # world -> heading frame
|
| 175 |
+
r = P[:, 0].copy(); r[:, 1] = 0
|
| 176 |
+
loc = np.einsum("tij,tkj->tki", H, P - r[:, None])
|
| 177 |
+
dP = np.zeros_like(P); dP[1:] = P[1:] - P[:-1]; dP[0] = dP[1] if T > 1 else 0
|
| 178 |
+
vel = np.einsum("tij,tkj->tki", H, dP)
|
| 179 |
+
rv = np.zeros((T, 3))
|
| 180 |
+
rv[1:] = np.einsum("tij,tj->ti", H[:-1], r[1:] - r[:-1])
|
| 181 |
+
Rd = np.empty_like(H); Rd[0] = H[0]; Rd[1:] = H[1:] @ np.transpose(H[:-1], (0, 2, 1))
|
| 182 |
+
R = np.empty_like(G)
|
| 183 |
+
R[:, 0] = H @ G[:, 0]
|
| 184 |
+
for j in range(1, 22):
|
| 185 |
+
R[:, j] = np.transpose(G[:, PAR[j]], (0, 2, 1)) @ G[:, j]
|
| 186 |
+
x = np.concatenate([rv[:, [0, 2]], six(Rd), loc.reshape(T, -1), vel.reshape(T, -1),
|
| 187 |
+
six(R).reshape(T, -1)], 1)
|
| 188 |
+
assert x.shape[1] == 272
|
| 189 |
+
return x.astype(np.float32), P, G
|
| 190 |
+
|
| 191 |
+
|
| 192 |
+
def recover(x):
|
| 193 |
+
"""numpy copy of gen_sample.py recovery -> world positions (T,22,3), global rotations"""
|
| 194 |
+
def mat(s):
|
| 195 |
+
a1, a2 = s[..., :3], s[..., 3:]
|
| 196 |
+
b1 = a1 / np.linalg.norm(a1, axis=-1, keepdims=True)
|
| 197 |
+
b2 = a2 - (b1 * a2).sum(-1, keepdims=True) * b1
|
| 198 |
+
b2 /= np.linalg.norm(b2, axis=-1, keepdims=True)
|
| 199 |
+
return np.stack([b1, b2, np.cross(b1, b2)], -2)
|
| 200 |
+
T = x.shape[0]
|
| 201 |
+
Rd = mat(x[:, 2:8])
|
| 202 |
+
tot = [Rd[0]]
|
| 203 |
+
for t in range(1, T):
|
| 204 |
+
tot.append(Rd[t] @ tot[-1])
|
| 205 |
+
inv = np.transpose(np.stack(tot), (0, 2, 1))
|
| 206 |
+
Pl = x[:, 8:74].reshape(T, 22, 3)
|
| 207 |
+
Pw = np.einsum("tij,tkj->tki", inv, Pl)
|
| 208 |
+
v = np.zeros((T, 3)); v[:, 0], v[:, 2] = x[:, 0], x[:, 1]
|
| 209 |
+
v[1:] = np.einsum("tij,tj->ti", inv[:-1], v[1:])
|
| 210 |
+
tr = np.cumsum(v, 0)
|
| 211 |
+
Pw[..., 0] += tr[:, None, 0]; Pw[..., 2] += tr[:, None, 2]
|
| 212 |
+
Rl = mat(x[:, 140:272].reshape(T, 22, 6))
|
| 213 |
+
Rl[:, 0] = inv @ Rl[:, 0]
|
| 214 |
+
Gr = [None] * 22
|
| 215 |
+
for j in range(22):
|
| 216 |
+
Gr[j] = Rl[:, j] if PAR[j] < 0 else Gr[PAR[j]] @ Rl[:, j]
|
| 217 |
+
return Pw, np.stack(Gr, 1)
|
| 218 |
+
|
| 219 |
+
|
| 220 |
+
# ------------------------------------------------------------------ captions
|
| 221 |
+
def clean_caption(s):
|
| 222 |
+
s = re.sub(r"(?<=[a-z])(?=[A-Z])", " ", s.strip()) # CarefulWalk -> Careful Walk
|
| 223 |
+
s = re.sub(r"\s+", " ", s).strip()
|
| 224 |
+
return s
|
| 225 |
+
|
| 226 |
+
|
| 227 |
+
def mirror_caption(s):
|
| 228 |
+
tmp = re.sub(r"\bleft\b", "\0L", s, flags=re.I)
|
| 229 |
+
tmp = re.sub(r"\bright\b", "left", tmp, flags=re.I)
|
| 230 |
+
tmp = tmp.replace("\0L", "right")
|
| 231 |
+
tmp = re.sub(r"\bcounter[- ]?clockwise\b", "\0CC", tmp, flags=re.I)
|
| 232 |
+
tmp = re.sub(r"\bclockwise\b", "counterclockwise", tmp, flags=re.I)
|
| 233 |
+
return tmp.replace("\0CC", "clockwise")
|
| 234 |
+
|
| 235 |
+
|
| 236 |
+
# ------------------------------------------------------------------ main
|
| 237 |
+
def main():
|
| 238 |
+
src, files_csv, out = sys.argv[1], sys.argv[2], sys.argv[3]
|
| 239 |
+
meta = list(csv.DictReader(open(files_csv, encoding="utf-8-sig")))
|
| 240 |
+
mdir = os.path.join(out, "motion_data", "cmu"); tdir = os.path.join(out, "texts", "texts")
|
| 241 |
+
for d in (mdir, tdir, os.path.join(out, "split"), os.path.join(out, "mean_std")):
|
| 242 |
+
os.makedirs(d, exist_ok=True)
|
| 243 |
+
report, kept, asf_cache = [], [], {}
|
| 244 |
+
for k, r in enumerate(meta):
|
| 245 |
+
key, desc = r["file"], r["desc"].strip()
|
| 246 |
+
rec = dict(file=key, desc=desc)
|
| 247 |
+
if desc == "(一覧になし)":
|
| 248 |
+
rec["skip"] = "not in CMU list (fps unknown)"; report.append(rec); continue
|
| 249 |
+
fps = int(r["fps"]); step = fps // 30
|
| 250 |
+
if fps % 30:
|
| 251 |
+
rec["skip"] = "fps %d" % fps; report.append(rec); continue
|
| 252 |
+
subj = key.split("_")[0]
|
| 253 |
+
try:
|
| 254 |
+
if subj not in asf_cache:
|
| 255 |
+
asf_cache[subj] = parse_asf(os.path.join(src, "subjects", subj, subj + ".asf"))
|
| 256 |
+
bones, hier, order, ro = asf_cache[subj]
|
| 257 |
+
frames = parse_amc(os.path.join(src, "subjects", subj, key + ".amc"))[::step]
|
| 258 |
+
if len(frames) < 16:
|
| 259 |
+
rec["skip"] = "too short (%d frames)" % len(frames); report.append(rec); continue
|
| 260 |
+
M, E = fk(bones, hier, order, ro, frames)
|
| 261 |
+
P, G = to22(M, E)
|
| 262 |
+
except Exception as e: # noqa
|
| 263 |
+
rec["skip"] = "error: %r" % e; report.append(rec); continue
|
| 264 |
+
T = P.shape[0]
|
| 265 |
+
sp = np.linalg.norm(np.diff(P, axis=0), axis=-1) * 30 # m/s
|
| 266 |
+
rec.update(frames=T, sec=round(T / 30, 2), max_joint_speed=round(float(sp.max()), 2),
|
| 267 |
+
pelvis_h_median=round(float(np.median(P[:, 0, 1] - P[..., 1].min())), 3))
|
| 268 |
+
flags = []
|
| 269 |
+
if not np.isfinite(P).all() or not np.isfinite(G).all():
|
| 270 |
+
flags.append("nan")
|
| 271 |
+
if sp.max() > 20:
|
| 272 |
+
flags.append("speed>20m/s")
|
| 273 |
+
caption = "" if desc.lower() in ("", "clean") else clean_caption(desc)
|
| 274 |
+
for tag, (PP, GG), cap in (("c", (P, G), caption), ("Mc", mirror(P, G), mirror_caption(caption))):
|
| 275 |
+
x, Pn, Gn = features(PP, GG)
|
| 276 |
+
Pr, Gr = recover(x)
|
| 277 |
+
err = float(np.abs(Pr - Pn).max()) * 1000
|
| 278 |
+
gerr = float(np.abs(Gr - Gn).max())
|
| 279 |
+
if tag == "c":
|
| 280 |
+
rec.update(recover_err_mm=round(err, 4), recover_rot_err=round(gerr, 6))
|
| 281 |
+
if err > 1.0 or gerr > 1e-4:
|
| 282 |
+
flags.append("recover %s %.3fmm %.1e" % (tag, err, gerr))
|
| 283 |
+
if flags:
|
| 284 |
+
continue
|
| 285 |
+
np.save(os.path.join(mdir, tag + key + ".npy"), x)
|
| 286 |
+
if cap:
|
| 287 |
+
with open(os.path.join(tdir, tag + key + ".txt"), "w", encoding="utf-8") as f:
|
| 288 |
+
f.write("%s#%s#0.0#0.0\n" % (cap, cap.lower()))
|
| 289 |
+
rec["flags"] = flags
|
| 290 |
+
if not flags:
|
| 291 |
+
kept.append(key)
|
| 292 |
+
report.append(rec)
|
| 293 |
+
if (k + 1) % 200 == 0:
|
| 294 |
+
print("%d / %d" % (k + 1, len(meta)), flush=True)
|
| 295 |
+
# split by original clip (mirror follows), 90/5/5
|
| 296 |
+
rng = np.random.default_rng(0)
|
| 297 |
+
ids = sorted(kept); rng.shuffle(ids)
|
| 298 |
+
n = len(ids); nv = nt = max(1, round(n * 0.05))
|
| 299 |
+
sp = {"val": ids[:nv], "test": ids[nv:nv + nt], "train": ids[nv + nt:]}
|
| 300 |
+
for name, lst in sp.items():
|
| 301 |
+
with open(os.path.join(out, "split", name + ".txt"), "w") as f:
|
| 302 |
+
f.write("\n".join(t + i for i in lst for t in ("c", "Mc")) + "\n")
|
| 303 |
+
# mean / std on train
|
| 304 |
+
s1 = np.zeros(272); s2 = np.zeros(272); cnt = 0
|
| 305 |
+
for i in sp["train"]:
|
| 306 |
+
for t in ("c", "Mc"):
|
| 307 |
+
x = np.load(os.path.join(mdir, t + i + ".npy")).astype(np.float64)
|
| 308 |
+
s1 += x.sum(0); s2 += (x ** 2).sum(0); cnt += x.shape[0]
|
| 309 |
+
mean = s1 / cnt; std = np.sqrt(np.maximum(s2 / cnt - mean ** 2, 0))
|
| 310 |
+
np.save(os.path.join(out, "mean_std", "Mean.npy"), mean.astype(np.float32))
|
| 311 |
+
np.save(os.path.join(out, "mean_std", "Std.npy"), np.maximum(std, 1e-3).astype(np.float32))
|
| 312 |
+
json.dump(report, open(os.path.join(out, "convert_report.json"), "w", encoding="utf-8"), ensure_ascii=False, indent=0)
|
| 313 |
+
flagged = [r for r in report if r.get("flags")]
|
| 314 |
+
skipped = [r for r in report if r.get("skip")]
|
| 315 |
+
sec = sum(r["sec"] for r in report if r.get("flags") == [])
|
| 316 |
+
print("kept %d clips (%.2f h, x2 with mirror) | flagged %d | skipped %d | split train %d val %d test %d"
|
| 317 |
+
% (len(kept), sec / 3600, len(flagged), len(skipped), len(sp["train"]), len(sp["val"]), len(sp["test"])))
|
| 318 |
+
print("recover err max mm:", max(r.get("recover_err_mm", 0) for r in report))
|
| 319 |
+
print("std floor hit dims:", int((std < 1e-3).sum()))
|
| 320 |
+
for r in flagged[:20]:
|
| 321 |
+
print(" FLAG", r["file"], r["flags"], r["desc"][:50])
|
| 322 |
+
from collections import Counter
|
| 323 |
+
print("skip reasons:", Counter(r["skip"].split(" (")[0].split(":")[0] for r in skipped))
|
| 324 |
+
|
| 325 |
+
|
| 326 |
+
if __name__ == "__main__":
|
| 327 |
+
main()
|
code/gen_ar.py
ADDED
|
@@ -0,0 +1,207 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Plan 2: text -> causal-H tokens, one token at a time.
|
| 2 |
+
Per 4 frames: 7 tokens in order [global(4096), torso, head, arm_R, arm_L, leg_R, leg_L (128 each)].
|
| 3 |
+
- decoder-only Transformer, 8 layers, d=512, RoPE over flattened position (slides forever)
|
| 4 |
+
- text (CLIP ViT-B/32, 512) added to EVERY position (per-slot text possible later)
|
| 5 |
+
- corruption: replace tau*100% input tokens with random ones, tau~U[0,1] per sample (T2M-GPT)
|
| 6 |
+
- CFG: 10% of samples get the null text (MoMask / MotionStreamer)
|
| 7 |
+
- training stops by wall-clock (--seconds), cosine schedule on elapsed/budget
|
| 8 |
+
"""
|
| 9 |
+
import os, sys, math, time, json, argparse
|
| 10 |
+
import torch
|
| 11 |
+
import torch.nn as nn
|
| 12 |
+
import torch.nn.functional as F
|
| 13 |
+
BASE = os.environ.get("PARTLOCK_DIR", ".") # folder that holds data/, ckpt/, runs/
|
| 14 |
+
|
| 15 |
+
GEN = os.path.join(BASE, "data", "gen")
|
| 16 |
+
CKPT = os.path.join(BASE, "ckpt")
|
| 17 |
+
RUNS = os.path.join(BASE, "runs")
|
| 18 |
+
VOC = [4096] + [128] * 6
|
| 19 |
+
OFF = [0]
|
| 20 |
+
for v in VOC[:-1]:
|
| 21 |
+
OFF.append(OFF[-1] + v)
|
| 22 |
+
NV = sum(VOC) # 4864
|
| 23 |
+
S = 64 # steps of context (= 8.5 s)
|
| 24 |
+
L = S * 7
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def rope(x, pos): # x (B,h,T,dh)
|
| 28 |
+
dh = x.shape[-1]
|
| 29 |
+
inv = 1.0 / (10000 ** (torch.arange(0, dh, 2, device=x.device).float() / dh))
|
| 30 |
+
a = pos[:, None].float() * inv[None]
|
| 31 |
+
c, s = a.cos()[None, None], a.sin()[None, None]
|
| 32 |
+
x1, x2 = x[..., ::2], x[..., 1::2]
|
| 33 |
+
return torch.stack([x1 * c - x2 * s, x1 * s + x2 * c], -1).flatten(-2)
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
class Block(nn.Module):
|
| 37 |
+
def __init__(self, d, h, drop):
|
| 38 |
+
super().__init__()
|
| 39 |
+
self.h = h
|
| 40 |
+
self.n1, self.n2 = nn.LayerNorm(d), nn.LayerNorm(d)
|
| 41 |
+
self.qkv, self.o = nn.Linear(d, 3 * d), nn.Linear(d, d)
|
| 42 |
+
self.ff = nn.Sequential(nn.Linear(d, 4 * d), nn.GELU(), nn.Linear(4 * d, d))
|
| 43 |
+
self.drop = drop
|
| 44 |
+
|
| 45 |
+
def forward(self, x, pos):
|
| 46 |
+
B, T, D = x.shape
|
| 47 |
+
q, k, v = self.qkv(self.n1(x)).view(B, T, 3, self.h, D // self.h).permute(2, 0, 3, 1, 4)
|
| 48 |
+
q, k = rope(q, pos), rope(k, pos)
|
| 49 |
+
a = F.scaled_dot_product_attention(q, k, v, is_causal=True, dropout_p=self.drop if self.training else 0)
|
| 50 |
+
x = x + F.dropout(self.o(a.transpose(1, 2).reshape(B, T, D)), self.drop, self.training)
|
| 51 |
+
return x + F.dropout(self.ff(self.n2(x)), self.drop, self.training)
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
class Gen(nn.Module):
|
| 55 |
+
def __init__(self, d=512, layers=8, heads=8, drop=0.1):
|
| 56 |
+
super().__init__()
|
| 57 |
+
self.tok = nn.Embedding(NV + 1, d) # last = BOS
|
| 58 |
+
self.slot = nn.Embedding(7, d) # slot of the token being PREDICTED
|
| 59 |
+
self.txt = nn.Linear(512, d)
|
| 60 |
+
self.null = nn.Parameter(torch.zeros(512))
|
| 61 |
+
self.blocks = nn.ModuleList([Block(d, heads, drop) for _ in range(layers)])
|
| 62 |
+
self.nf = nn.LayerNorm(d)
|
| 63 |
+
self.head = nn.Linear(d, NV)
|
| 64 |
+
mask = torch.full((7, NV), float("-inf"))
|
| 65 |
+
for s in range(7):
|
| 66 |
+
mask[s, OFF[s]:OFF[s] + VOC[s]] = 0
|
| 67 |
+
self.register_buffer("mask", mask)
|
| 68 |
+
|
| 69 |
+
def forward(self, inp, slots, text):
|
| 70 |
+
"""inp (B,T) flat ids (BOS=NV); slots (T,) slot predicted at each position; text (B,T,512)"""
|
| 71 |
+
x = self.tok(inp) + self.slot(slots)[None] + self.txt(text)
|
| 72 |
+
pos = torch.arange(inp.shape[1], device=inp.device)
|
| 73 |
+
for b in self.blocks:
|
| 74 |
+
x = b(x, pos)
|
| 75 |
+
return self.head(self.nf(x)) + self.mask[slots][None]
|
| 76 |
+
|
| 77 |
+
|
| 78 |
+
def flat(codes): # (n,7) local ids -> (n*7,) global ids
|
| 79 |
+
return (codes.long() + torch.tensor(OFF)).reshape(-1)
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
def batch(rec, bs, dev, g, corrupt=True, drop=0.1, fixed=False):
|
| 83 |
+
inps, tgts, txts = [], [], []
|
| 84 |
+
lens = torch.tensor([r["codes"].shape[0] for r in rec], dtype=torch.float)
|
| 85 |
+
idx = range(bs) if fixed else torch.multinomial(lens, bs, replacement=True, generator=g).tolist()
|
| 86 |
+
for i in idx:
|
| 87 |
+
r = rec[i]
|
| 88 |
+
n = r["codes"].shape[0]
|
| 89 |
+
st = 0 if fixed else int(torch.randint(0, max(1, n - S + 1), (1,), generator=g))
|
| 90 |
+
c = flat(r["codes"][st:st + S])
|
| 91 |
+
e = r["emb"][0 if fixed else int(torch.randint(0, r["emb"].shape[0], (1,), generator=g))].float()
|
| 92 |
+
if not fixed and torch.rand(1, generator=g).item() < drop:
|
| 93 |
+
e = None
|
| 94 |
+
tgt = torch.full((L,), -100, dtype=torch.long); tgt[:len(c)] = c
|
| 95 |
+
inp = torch.full((L,), NV, dtype=torch.long); inp[1:len(c)] = c[:-1]
|
| 96 |
+
if corrupt and not fixed:
|
| 97 |
+
tau = torch.rand(1, generator=g).item()
|
| 98 |
+
sel = torch.rand(L, generator=g) < tau
|
| 99 |
+
sel[0] = False
|
| 100 |
+
slot_of_inp = (torch.arange(L) - 1) % 7
|
| 101 |
+
rnd = torch.tensor(OFF)[slot_of_inp] + (torch.rand(L, generator=g) * torch.tensor(VOC)[slot_of_inp]).long()
|
| 102 |
+
inp = torch.where(sel, rnd, inp)
|
| 103 |
+
inps.append(inp); tgts.append(tgt); txts.append(e)
|
| 104 |
+
return torch.stack(inps).to(dev), torch.stack(tgts).to(dev), txts
|
| 105 |
+
|
| 106 |
+
|
| 107 |
+
def text_tensor(m, txts, dev):
|
| 108 |
+
rows = [m.null if t is None else t.to(dev) for t in txts]
|
| 109 |
+
return torch.stack(rows)[:, None, :].expand(-1, L, -1)
|
| 110 |
+
|
| 111 |
+
|
| 112 |
+
def main():
|
| 113 |
+
ap = argparse.ArgumentParser()
|
| 114 |
+
ap.add_argument("--seconds", type=float, default=7200)
|
| 115 |
+
ap.add_argument("--bs", type=int, default=64)
|
| 116 |
+
ap.add_argument("--lr", type=float, default=2e-4)
|
| 117 |
+
ap.add_argument("--overfit", type=int, default=0, help="N clips, no corruption/drop: memorization check")
|
| 118 |
+
ap.add_argument("--tag", default="gen_v1")
|
| 119 |
+
ap.add_argument("--val-every", type=int, default=500)
|
| 120 |
+
ap.add_argument("--patience", type=int, default=3)
|
| 121 |
+
ap.add_argument("--resume", action="store_true")
|
| 122 |
+
a = ap.parse_args()
|
| 123 |
+
dev = "cuda"
|
| 124 |
+
torch.manual_seed(0)
|
| 125 |
+
g = torch.Generator().manual_seed(0)
|
| 126 |
+
tr = torch.load(os.path.join(GEN, "train.pt"))
|
| 127 |
+
va = torch.load(os.path.join(GEN, "val.pt"))
|
| 128 |
+
if a.overfit:
|
| 129 |
+
tr = tr[: a.overfit]
|
| 130 |
+
m = Gen().to(dev)
|
| 131 |
+
print("params %.1fM train clips %d val clips %d" % (sum(p.numel() for p in m.parameters()) / 1e6, len(tr), len(va)), flush=True)
|
| 132 |
+
opt = torch.optim.AdamW(m.parameters(), lr=a.lr, weight_decay=0.05, betas=(0.9, 0.95))
|
| 133 |
+
slots = (torch.arange(L) % 7).to(dev)
|
| 134 |
+
vin, vtg, vtx = batch(va, min(256, len(va)), dev, g, fixed=True)
|
| 135 |
+
log = []
|
| 136 |
+
|
| 137 |
+
@torch.no_grad()
|
| 138 |
+
def val_loss():
|
| 139 |
+
m.eval()
|
| 140 |
+
tot, per = 0.0, torch.zeros(7, device=dev); cnt = torch.zeros(7, device=dev)
|
| 141 |
+
for i in range(0, vin.shape[0], 64):
|
| 142 |
+
with torch.autocast("cuda", dtype=torch.bfloat16):
|
| 143 |
+
lo = m(vin[i:i + 64], slots, text_tensor(m, vtx[i:i + 64], dev)).float()
|
| 144 |
+
ce = F.cross_entropy(lo.reshape(-1, NV), vtg[i:i + 64].reshape(-1), reduction="none").view(-1, L)
|
| 145 |
+
ok = (vtg[i:i + 64] != -100)
|
| 146 |
+
for s in range(7):
|
| 147 |
+
per[s] += (ce[:, s::7] * ok[:, s::7]).sum(); cnt[s] += ok[:, s::7].sum()
|
| 148 |
+
m.train()
|
| 149 |
+
p = (per / cnt)
|
| 150 |
+
return float(per.sum() / cnt.sum()), [round(float(v), 3) for v in p]
|
| 151 |
+
|
| 152 |
+
n_win = sum(r["codes"].shape[0] for r in tr) / S # data size in 64-step windows (1 epoch)
|
| 153 |
+
best, bad = [float("inf"), None, None], [0]
|
| 154 |
+
t0, step, warm = time.time(), 0, 300
|
| 155 |
+
if a.resume:
|
| 156 |
+
ck = torch.load(os.path.join(CKPT, a.tag + "_last.pt"), weights_only=False)
|
| 157 |
+
m.load_state_dict(ck["model"]); opt.load_state_dict(ck["opt"]); g.set_state(ck["rng"])
|
| 158 |
+
step, best, bad, log = ck["step"], ck["best"], ck["bad"], ck["log"]
|
| 159 |
+
t0 -= log[-1]["sec"] if log else 0
|
| 160 |
+
print("resumed at step %d (best val %.4f)" % (step, best[0]), flush=True)
|
| 161 |
+
while True:
|
| 162 |
+
el = time.time() - t0
|
| 163 |
+
if el > a.seconds:
|
| 164 |
+
break
|
| 165 |
+
lr = a.lr * min(1, (step + 1) / warm) * 0.5 * (1 + math.cos(math.pi * min(1.0, el / a.seconds)))
|
| 166 |
+
for pg in opt.param_groups:
|
| 167 |
+
pg["lr"] = lr
|
| 168 |
+
inp, tgt, txt = batch(tr, a.bs, dev, g, corrupt=not a.overfit, drop=0 if a.overfit else 0.1)
|
| 169 |
+
with torch.autocast("cuda", dtype=torch.bfloat16):
|
| 170 |
+
lo = m(inp, slots, text_tensor(m, txt, dev))
|
| 171 |
+
loss = F.cross_entropy(lo.float().reshape(-1, NV), tgt.reshape(-1))
|
| 172 |
+
opt.zero_grad(set_to_none=True)
|
| 173 |
+
loss.backward()
|
| 174 |
+
torch.nn.utils.clip_grad_norm_(m.parameters(), 1.0)
|
| 175 |
+
opt.step()
|
| 176 |
+
step += 1
|
| 177 |
+
if step == 20:
|
| 178 |
+
torch.cuda.synchronize()
|
| 179 |
+
print(" speed %.2f step/s -> ~%d steps in budget" % (20 / (time.time() - t0), a.seconds * 20 / (time.time() - t0)), flush=True)
|
| 180 |
+
if step % 100 == 0:
|
| 181 |
+
row = dict(step=step, sec=round(time.time() - t0), epochs=round(step * a.bs / n_win, 2),
|
| 182 |
+
train_loss=round(loss.item(), 4), lr=lr)
|
| 183 |
+
if step % a.val_every == 0 or a.overfit:
|
| 184 |
+
vl, per = val_loss(); row.update(val_loss=round(vl, 4), val_per_slot=per)
|
| 185 |
+
if vl < best[0]:
|
| 186 |
+
best[:] = [vl, step, per]; bad[0] = 0
|
| 187 |
+
torch.save(m.state_dict(), os.path.join(CKPT, a.tag + ".pt"))
|
| 188 |
+
row["saved"] = True
|
| 189 |
+
else:
|
| 190 |
+
bad[0] += 1
|
| 191 |
+
torch.save(dict(model=m.state_dict(), opt=opt.state_dict(), step=step, best=best, bad=bad,
|
| 192 |
+
log=log, rng=g.get_state()), os.path.join(CKPT, a.tag + "_last.pt"))
|
| 193 |
+
log.append(row)
|
| 194 |
+
print(" " + json.dumps(row), flush=True)
|
| 195 |
+
if not a.overfit and bad[0] >= a.patience:
|
| 196 |
+
print(" early stop: val not improved for %d checks" % a.patience, flush=True)
|
| 197 |
+
break
|
| 198 |
+
if a.overfit:
|
| 199 |
+
torch.save(m.state_dict(), os.path.join(CKPT, a.tag + ".pt"))
|
| 200 |
+
json.dump(dict(args=vars(a), steps=step, best_val_loss=best[0], best_step=best[1], best_val_per_slot=best[2],
|
| 201 |
+
log=log), open(os.path.join(RUNS, a.tag + ".json"), "w"), indent=1)
|
| 202 |
+
print("done steps %d (%.2f epochs) best val %.4f at step %s per-slot %s"
|
| 203 |
+
% (step, step * a.bs / n_win, best[0], best[1], best[2]), flush=True)
|
| 204 |
+
|
| 205 |
+
|
| 206 |
+
if __name__ == "__main__":
|
| 207 |
+
main()
|
code/gen_prep.py
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Plan 2 / prep: (a) whole clips -> causal-H tokens (n_steps, 7) ; (b) captions -> CLIP ViT-B/32 text embeddings.
|
| 2 |
+
Only captions with whole-clip timing (start=end=0) are used.
|
| 3 |
+
Output: $PARTLOCK_DIR/data/gen/{train,val}.pt
|
| 4 |
+
"""
|
| 5 |
+
import os, sys, time
|
| 6 |
+
import numpy as np
|
| 7 |
+
import torch
|
| 8 |
+
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
| 9 |
+
import tvq, tvq272 as B, tvq272c as C
|
| 10 |
+
|
| 11 |
+
dev = "cuda"
|
| 12 |
+
BASE = os.environ.get("PARTLOCK_DIR", ".") # folder that holds data/, ckpt/, runs/
|
| 13 |
+
OUT = os.path.join(BASE, "data", "gen")
|
| 14 |
+
os.makedirs(OUT, exist_ok=True)
|
| 15 |
+
t0 = time.time()
|
| 16 |
+
|
| 17 |
+
from transformers import CLIPTokenizer, CLIPTextModelWithProjection
|
| 18 |
+
name = "openai/clip-vit-base-patch32"
|
| 19 |
+
tok = CLIPTokenizer.from_pretrained(name)
|
| 20 |
+
clip = CLIPTextModelWithProjection.from_pretrained(name, use_safetensors=True).to(dev).eval()
|
| 21 |
+
print("clip loaded %.0fs" % (time.time() - t0), flush=True)
|
| 22 |
+
|
| 23 |
+
mean = torch.from_numpy(np.load(os.path.join(B.ROOT, "mean_std", "Mean.npy")).astype(np.float32)).to(dev)
|
| 24 |
+
std = torch.from_numpy(np.load(os.path.join(B.ROOT, "mean_std", "Std.npy")).astype(np.float32)).to(dev)
|
| 25 |
+
m = C.CHierTok().to(dev)
|
| 26 |
+
m.load_state_dict(torch.load(os.path.join(B.CKPT_DIR, os.environ.get("TOK_CKPT", "cmu_tok_v1_H.pt")), map_location=dev))
|
| 27 |
+
m.eval()
|
| 28 |
+
mdir = B.find_motion_dir()
|
| 29 |
+
tdir = os.path.join(B.ROOT, "texts", "texts")
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
@torch.no_grad()
|
| 33 |
+
def codes(z):
|
| 34 |
+
ig = tvq.vq_seq(m.gvq, m.genc(z.permute(0, 2, 1)))[2]
|
| 35 |
+
ip = [tvq.vq_seq(m.vq[p], m.enc[p](z[..., c].permute(0, 2, 1)))[2] for p, c in enumerate(B.PD)]
|
| 36 |
+
return torch.stack([ig] + ip, -1)[0] # (n, 7)
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
@torch.no_grad()
|
| 40 |
+
def embed(caps):
|
| 41 |
+
t = tok(caps, padding=True, truncation=True, max_length=77, return_tensors="pt").to(dev)
|
| 42 |
+
e = clip(**t).text_embeds
|
| 43 |
+
return torch.nn.functional.normalize(e, dim=-1).half().cpu()
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
for split in ("train", "val"):
|
| 47 |
+
ids = open(os.path.join(B.ROOT, "split", split + ".txt")).read().split()
|
| 48 |
+
rec = []
|
| 49 |
+
skipped = 0
|
| 50 |
+
for i in ids:
|
| 51 |
+
mp, tp = os.path.join(mdir, i + ".npy"), os.path.join(tdir, i + ".txt")
|
| 52 |
+
if not (os.path.exists(mp) and os.path.exists(tp)):
|
| 53 |
+
skipped += 1; continue
|
| 54 |
+
a = np.load(mp).astype(np.float32)
|
| 55 |
+
if a.ndim != 2 or a.shape[1] != 272 or a.shape[0] < 16:
|
| 56 |
+
skipped += 1; continue
|
| 57 |
+
caps = []
|
| 58 |
+
for line in open(tp, encoding="utf-8", errors="ignore"):
|
| 59 |
+
f = line.strip().split("#")
|
| 60 |
+
if len(f) >= 4 and f[0]:
|
| 61 |
+
try:
|
| 62 |
+
s, e = float(f[2]), float(f[3])
|
| 63 |
+
except ValueError:
|
| 64 |
+
continue
|
| 65 |
+
if s == 0.0 and e == 0.0:
|
| 66 |
+
caps.append(f[0])
|
| 67 |
+
if not caps:
|
| 68 |
+
skipped += 1; continue
|
| 69 |
+
a = a[: a.shape[0] // 4 * 4]
|
| 70 |
+
z = (torch.from_numpy(a).to(dev) - mean) / std
|
| 71 |
+
rec.append(dict(id=i, codes=codes(z[None]).short().cpu(), caps=caps, emb=embed(caps)))
|
| 72 |
+
if len(rec) % 2000 == 0:
|
| 73 |
+
print(" %s %d clips %.0fs" % (split, len(rec), time.time() - t0), flush=True)
|
| 74 |
+
torch.save(rec, os.path.join(OUT, split + ".pt"))
|
| 75 |
+
steps = sum(r["codes"].shape[0] for r in rec)
|
| 76 |
+
print("%s: %d clips (skipped %d), %d steps = %.1f h of motion, %d captions"
|
| 77 |
+
% (split, len(rec), skipped, steps, steps * 4 / 30 / 3600, sum(len(r["caps"]) for r in rec)), flush=True)
|
| 78 |
+
print("done %.0fs" % (time.time() - t0))
|
code/gen_sample.py
ADDED
|
@@ -0,0 +1,173 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Plan 2 / generate: 6 prompts x 10 s, plus 1 prompt x 60 s (endless check). CFG s=4 (MoMask form).
|
| 2 |
+
Tokens decoded by causal H. Global trajectory recovered as in representation_272 recover_visualize.py.
|
| 3 |
+
Writes runs/gen_v1_samples.json (metrics) and gen_viewer_data.json next to this script (viewer)."""
|
| 4 |
+
import os, sys, json, time, base64, argparse
|
| 5 |
+
import numpy as np
|
| 6 |
+
import torch
|
| 7 |
+
import torch.nn.functional as F
|
| 8 |
+
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
| 9 |
+
import tvq272 as B, tvq272c as C, gen_ar as G
|
| 10 |
+
|
| 11 |
+
dev = "cuda"
|
| 12 |
+
D = os.path.dirname(os.path.abspath(__file__))
|
| 13 |
+
# CMU-style captions (short phrases, as in the CMU list)
|
| 14 |
+
PROMPTS = [("walk", "walk"), ("run", "run"), ("stand", "Standing still"), ("jump", "jump"),
|
| 15 |
+
("dance", "salsa dance"), ("wave", "Wave"), ("kick", "kick"), ("sit", "sit on high stool, stand up")]
|
| 16 |
+
LONG = ("walk60", "walk")
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def load():
|
| 20 |
+
from transformers import CLIPTokenizer, CLIPTextModelWithProjection
|
| 21 |
+
name = "openai/clip-vit-base-patch32"
|
| 22 |
+
tok = CLIPTokenizer.from_pretrained(name)
|
| 23 |
+
clip = CLIPTextModelWithProjection.from_pretrained(name, use_safetensors=True).to(dev).eval()
|
| 24 |
+
return tok, clip
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
@torch.no_grad()
|
| 28 |
+
def embed(tok, clip, caps):
|
| 29 |
+
t = tok(caps, padding=True, truncation=True, max_length=77, return_tensors="pt").to(dev)
|
| 30 |
+
return F.normalize(clip(**t).text_embeds, dim=-1)
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
@torch.no_grad()
|
| 34 |
+
def generate(g, emb, steps, s=4.0, seed=0):
|
| 35 |
+
"""emb (N,512). returns codes (N, steps, 7). context = BOS + whole steps (<= 63 past + current)."""
|
| 36 |
+
torch.manual_seed(seed)
|
| 37 |
+
N = emb.shape[0]
|
| 38 |
+
toks = torch.zeros(N, 0, dtype=torch.long, device=dev)
|
| 39 |
+
cond = emb.float()
|
| 40 |
+
unc = g.null[None].expand(N, -1)
|
| 41 |
+
slots_all = (torch.arange(G.L, device=dev) % 7)
|
| 42 |
+
for k in range(steps * 7):
|
| 43 |
+
cur_step = k // 7
|
| 44 |
+
first = max(0, cur_step - (G.S - 1)) * 7
|
| 45 |
+
ctx = toks[:, first:]
|
| 46 |
+
inp = torch.cat([torch.full((N, 1), G.NV, device=dev, dtype=torch.long), ctx], 1)
|
| 47 |
+
T = inp.shape[1]
|
| 48 |
+
txt = torch.cat([cond, unc])[:, None, :].expand(-1, T, -1)
|
| 49 |
+
with torch.autocast("cuda", dtype=torch.bfloat16):
|
| 50 |
+
lo = g(torch.cat([inp, inp]), slots_all[:T], txt)[:, -1].float()
|
| 51 |
+
lc, lu = lo[:N], lo[N:]
|
| 52 |
+
valid = (g.mask[k % 7] == 0)[None]
|
| 53 |
+
lc = torch.where(valid, lc, torch.zeros_like(lc)); lu = torch.where(valid, lu, torch.zeros_like(lu))
|
| 54 |
+
lg = torch.where(valid, (1 + s) * lc - s * lu, torch.full_like(lc, float("-inf")))
|
| 55 |
+
nxt = torch.multinomial(F.softmax(lg, -1), 1)
|
| 56 |
+
toks = torch.cat([toks, nxt], 1)
|
| 57 |
+
loc = toks.view(N, steps, 7) - torch.tensor(G.OFF, device=dev)
|
| 58 |
+
return loc
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
def get_codebook(vq):
|
| 62 |
+
for n in ("embed", "codebook", "emb", "weight"):
|
| 63 |
+
if hasattr(vq, n):
|
| 64 |
+
t = getattr(vq, n)
|
| 65 |
+
t = t.weight if hasattr(t, "weight") else t
|
| 66 |
+
return t
|
| 67 |
+
raise AttributeError("codebook not found: " + str([k for k, _ in vq.named_buffers()] + [k for k, _ in vq.named_parameters()]))
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
@torch.no_grad()
|
| 71 |
+
def decode_codes(h, codes, mean, std):
|
| 72 |
+
cbg = get_codebook(h.gvq)
|
| 73 |
+
qg = cbg[codes[..., 0]].permute(0, 2, 1)
|
| 74 |
+
N, n = codes.shape[:2]
|
| 75 |
+
y = torch.zeros(N, n * 4, 272, device=dev)
|
| 76 |
+
for p, c in enumerate(B.PD):
|
| 77 |
+
q = get_codebook(h.vq[p])[codes[..., p + 1]].permute(0, 2, 1)
|
| 78 |
+
y[..., c] = h.dec[p](torch.cat([qg, q], 1)).permute(0, 2, 1)
|
| 79 |
+
return y * std + mean
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
def to_world(x):
|
| 83 |
+
"""x (T,272) raw -> world positions (T,22,3), world rotations (T,22,3,3)"""
|
| 84 |
+
T = x.shape[0]
|
| 85 |
+
Rd = B.rot6d_to_mat(x[:, 2:8])
|
| 86 |
+
tot = [Rd[0]]
|
| 87 |
+
for t in range(1, T):
|
| 88 |
+
tot.append(Rd[t] @ tot[-1])
|
| 89 |
+
inv = torch.stack(tot).transpose(-1, -2)
|
| 90 |
+
P = B.positions(x)
|
| 91 |
+
Pw = (inv[:, None] @ P[..., None]).squeeze(-1)
|
| 92 |
+
v = torch.zeros(T, 3, device=x.device)
|
| 93 |
+
v[:, 0], v[:, 2] = x[:, 0], x[:, 1]
|
| 94 |
+
v[1:] = (inv[:-1] @ v[1:, :, None]).squeeze(-1)
|
| 95 |
+
tr = torch.cumsum(v, 0)
|
| 96 |
+
Pw[..., 0] += tr[:, None, 0]
|
| 97 |
+
Pw[..., 2] += tr[:, None, 2]
|
| 98 |
+
Rl = B.local_rots(x).clone()
|
| 99 |
+
Rl[:, 0] = inv @ Rl[:, 0]
|
| 100 |
+
return Pw, B.global_rots(Rl), tr
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
def bone_dev_mm(P):
|
| 104 |
+
out = []
|
| 105 |
+
for j, p in enumerate(B.PARENTS):
|
| 106 |
+
if p < 0:
|
| 107 |
+
continue
|
| 108 |
+
l = (P[:, j] - P[:, p]).norm(dim=-1)
|
| 109 |
+
out.append((l - l.median()).abs().mean())
|
| 110 |
+
return float(torch.stack(out).mean()) * 1000
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
def pack(a, scale=1000):
|
| 114 |
+
v = np.round(np.asarray(a, np.float64) * scale).clip(-32767, 32767).astype(np.int16)
|
| 115 |
+
return base64.b64encode(v.tobytes()).decode("ascii")
|
| 116 |
+
|
| 117 |
+
|
| 118 |
+
def main():
|
| 119 |
+
ap = argparse.ArgumentParser()
|
| 120 |
+
ap.add_argument("--tag", default="gen_v1")
|
| 121 |
+
a = ap.parse_args()
|
| 122 |
+
t0 = time.time()
|
| 123 |
+
mean = torch.from_numpy(np.load(os.path.join(B.ROOT, "mean_std", "Mean.npy")).astype(np.float32)).to(dev)
|
| 124 |
+
std = torch.from_numpy(np.load(os.path.join(B.ROOT, "mean_std", "Std.npy")).astype(np.float32)).to(dev)
|
| 125 |
+
Wtr, _ = B.load_split("train", 30000, 16)
|
| 126 |
+
mt = B.Metrics(mean, std, torch.from_numpy(Wtr).to(dev))
|
| 127 |
+
h = C.CHierTok().to(dev)
|
| 128 |
+
h.load_state_dict(torch.load(os.path.join(B.CKPT_DIR, os.environ.get("TOK_CKPT", "cmu_tok_v1_H.pt")), map_location=dev)); h.eval()
|
| 129 |
+
g = G.Gen().to(dev)
|
| 130 |
+
g.load_state_dict(torch.load(os.path.join(G.CKPT, a.tag + ".pt"), map_location=dev)); g.eval()
|
| 131 |
+
tok, clip = load()
|
| 132 |
+
|
| 133 |
+
runs = []
|
| 134 |
+
e = embed(tok, clip, [p for _, p in PROMPTS])
|
| 135 |
+
codes = generate(g, e, 75) # 75 steps = 300 frames = 10 s
|
| 136 |
+
for i, (k, p) in enumerate(PROMPTS):
|
| 137 |
+
runs.append((k, p, codes[i]))
|
| 138 |
+
print("%d prompts done %.0fs" % (len(PROMPTS), time.time() - t0), flush=True)
|
| 139 |
+
e = embed(tok, clip, [LONG[1]])
|
| 140 |
+
codes = generate(g, e, 450) # 60 s
|
| 141 |
+
runs.append((LONG[0], LONG[1], codes[0]))
|
| 142 |
+
print("60 s done %.0fs" % (time.time() - t0), flush=True)
|
| 143 |
+
|
| 144 |
+
res, out = {}, dict(fps=30, parents=B.PARENTS, clips=[])
|
| 145 |
+
for k, p, c in runs:
|
| 146 |
+
x = decode_codes(h, c[None], mean, std)[0]
|
| 147 |
+
Pw, Gw, tr = to_world(x)
|
| 148 |
+
r = dict(prompt=p, frames=int(x.shape[0]), consistency_deg=mt.consistency(x[None]),
|
| 149 |
+
bone_dev_mm=bone_dev_mm(Pw), travel_m=float((tr[-1] - tr[0]).norm()),
|
| 150 |
+
distinct_global_codes=int(c[:, 0].unique().numel()))
|
| 151 |
+
if k == "walk60":
|
| 152 |
+
r["per_10s"] = [dict(consistency_deg=round(mt.consistency(x[None, i:i + 300]), 2),
|
| 153 |
+
bone_dev_mm=round(bone_dev_mm(Pw[i:i + 300]), 2),
|
| 154 |
+
pelvis_height_m=round(float(Pw[i:i + 300, 0, 1].mean()), 3))
|
| 155 |
+
for i in range(0, x.shape[0], 300)]
|
| 156 |
+
res[k] = r
|
| 157 |
+
print(k, json.dumps({kk: (round(v, 2) if isinstance(v, float) else v) for kk, v in r.items() if kk != "per_10s"}), flush=True)
|
| 158 |
+
if "per_10s" in r:
|
| 159 |
+
for i, row in enumerate(r["per_10s"]):
|
| 160 |
+
print(" %2d-%2ds %s" % (i * 10, i * 10 + 10, row), flush=True)
|
| 161 |
+
rel = Pw.clone(); rel[..., 0] -= tr[:, None, 0]; rel[..., 2] -= tr[:, None, 2]
|
| 162 |
+
out["clips"].append(dict(key=k, caption=p, frames=int(x.shape[0]),
|
| 163 |
+
p=pack(rel.cpu().numpy().ravel()), r=pack(Gw.reshape(-1, 22, 9).cpu().numpy().ravel()),
|
| 164 |
+
tr=[round(float(v), 3) for v in tr[:, [0, 2]].cpu().numpy().ravel()],
|
| 165 |
+
m=dict(cons=round(r["consistency_deg"], 2), bone=round(r["bone_dev_mm"], 1),
|
| 166 |
+
travel=round(r["travel_m"], 2))))
|
| 167 |
+
json.dump(res, open(os.path.join(G.RUNS, a.tag + "_samples.json"), "w", encoding="utf-8"), indent=1)
|
| 168 |
+
json.dump(out, open(os.path.join(D, "gen_viewer_data.json"), "w", encoding="utf-8"), separators=(",", ":"))
|
| 169 |
+
print("viewer data %.2f MB total %.0fs" % (os.path.getsize(os.path.join(D, "gen_viewer_data.json")) / 1024**2, time.time() - t0))
|
| 170 |
+
|
| 171 |
+
|
| 172 |
+
if __name__ == "__main__":
|
| 173 |
+
main()
|
code/run_cmu.py
ADDED
|
@@ -0,0 +1,92 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Full training pipeline: convert -> tokenizer (<=2h) -> prep -> generator (<=2h).
|
| 2 |
+
|
| 3 |
+
python code/run_cmu.py --amc <folder with the CMU .asf/.amc files> --work <empty work folder>
|
| 4 |
+
|
| 5 |
+
Each step logs to <work>/runs/<step>.log.
|
| 6 |
+
<work>/runs/STATUS.json = current step / state; <work>/runs/ALERT.txt = something needs attention.
|
| 7 |
+
Hang guard: a step whose log has not grown for --stall-min minutes is killed and reported.
|
| 8 |
+
Resumable: steps listed in runs/STATUS.json as done are skipped (use --from to force a start step).
|
| 9 |
+
Checkpoints are written to <work>/ckpt (cmu_tok_v1_H.pt, cmu_gen_v1.pt).
|
| 10 |
+
"""
|
| 11 |
+
import os, sys, json, time, subprocess, argparse
|
| 12 |
+
|
| 13 |
+
CODE = os.path.dirname(os.path.abspath(__file__))
|
| 14 |
+
REPO = os.path.dirname(CODE)
|
| 15 |
+
PY = sys.executable
|
| 16 |
+
W = RUNS = STEPS = None
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def setup(amc, work):
|
| 20 |
+
global W, RUNS, STEPS
|
| 21 |
+
W = os.path.abspath(work)
|
| 22 |
+
RUNS = os.path.join(W, "runs")
|
| 23 |
+
STEPS = [
|
| 24 |
+
("convert", [os.path.join(CODE, "cmu_to272.py"), amc, os.path.join(REPO, "data", "cmu_files.csv"),
|
| 25 |
+
os.path.join(W, "data", "motion_raw", "h272")]),
|
| 26 |
+
("tokenizer", [os.path.join(CODE, "tok_cmu.py"), "--seconds", "7200", "--tag", "cmu_tok_v1"]),
|
| 27 |
+
("prep", [os.path.join(CODE, "gen_prep.py")]),
|
| 28 |
+
("generator", [os.path.join(CODE, "gen_ar.py"), "--seconds", "7200", "--tag", "cmu_gen_v1"]),
|
| 29 |
+
]
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def now():
|
| 33 |
+
return time.strftime("%Y-%m-%d %H:%M:%S")
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def status(**kw):
|
| 37 |
+
p = os.path.join(RUNS, "STATUS.json")
|
| 38 |
+
st = json.load(open(p, encoding="utf-8")) if os.path.exists(p) else {"done": []}
|
| 39 |
+
st.update(kw, updated=now())
|
| 40 |
+
json.dump(st, open(p, "w", encoding="utf-8"), ensure_ascii=False, indent=1)
|
| 41 |
+
return st
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
def alert(msg):
|
| 45 |
+
with open(os.path.join(RUNS, "ALERT.txt"), "a", encoding="utf-8") as f:
|
| 46 |
+
f.write("[%s] %s\n" % (now(), msg))
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
def main():
|
| 50 |
+
ap = argparse.ArgumentParser()
|
| 51 |
+
ap.add_argument("--from", dest="start", default=None)
|
| 52 |
+
ap.add_argument("--stall-min", type=float, default=30)
|
| 53 |
+
ap.add_argument("--amc", required=True, help="folder with the CMU .asf/.amc files (allasfamc.zip, extracted)")
|
| 54 |
+
ap.add_argument("--work", required=True, help="work folder (data/, ckpt/, runs/ are created here)")
|
| 55 |
+
a = ap.parse_args()
|
| 56 |
+
setup(a.amc, a.work)
|
| 57 |
+
os.makedirs(RUNS, exist_ok=True)
|
| 58 |
+
os.makedirs(os.path.join(W, "ckpt"), exist_ok=True)
|
| 59 |
+
env = dict(os.environ, PARTLOCK_DIR=W, PYTHONIOENCODING="utf-8", PYTHONUNBUFFERED="1")
|
| 60 |
+
st = status(pid=os.getpid(), state="running")
|
| 61 |
+
names = [n for n, _ in STEPS]
|
| 62 |
+
first = names.index(a.start) if a.start else 0
|
| 63 |
+
for name, args in STEPS[first:]:
|
| 64 |
+
if not a.start and name in st.get("done", []):
|
| 65 |
+
continue
|
| 66 |
+
log = os.path.join(RUNS, name + ".log")
|
| 67 |
+
status(step=name, step_started=now(), state="running")
|
| 68 |
+
with open(log, "w", encoding="utf-8") as lf:
|
| 69 |
+
lf.write("[%s] start %s\n" % (now(), " ".join(args)))
|
| 70 |
+
lf.flush()
|
| 71 |
+
p = subprocess.Popen([PY] + args, stdout=lf, stderr=subprocess.STDOUT, env=env, cwd=CODE)
|
| 72 |
+
last_size, last_change = 0, time.time()
|
| 73 |
+
while p.poll() is None:
|
| 74 |
+
time.sleep(30)
|
| 75 |
+
sz = os.path.getsize(log)
|
| 76 |
+
if sz != last_size:
|
| 77 |
+
last_size, last_change = sz, time.time()
|
| 78 |
+
elif time.time() - last_change > a.stall_min * 60:
|
| 79 |
+
p.kill()
|
| 80 |
+
alert("%s: no log output for %d min -> killed (hang)" % (name, a.stall_min))
|
| 81 |
+
break
|
| 82 |
+
rc = p.wait()
|
| 83 |
+
if rc != 0:
|
| 84 |
+
alert("%s failed (exit %s). see runs/%s.log" % (name, rc, name))
|
| 85 |
+
status(state="failed", failed_step=name, exit_code=rc)
|
| 86 |
+
return
|
| 87 |
+
st = status(done=st.get("done", []) + [name], last_done=name)
|
| 88 |
+
status(state="all done", step=None)
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
if __name__ == "__main__":
|
| 92 |
+
main()
|
code/tok_cmu.py
ADDED
|
@@ -0,0 +1,136 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Causal hierarchical tokenizer (CHierTok, unchanged) trained on the CMU 272-dim data.
|
| 2 |
+
|
| 3 |
+
Differences from tvq272c.py (training loop only, model unchanged):
|
| 4 |
+
- all train frames are used; windows (64 frames) are cut at random from the clips on the fly
|
| 5 |
+
(the old code stacked every window on the GPU, which does not fit for this much data)
|
| 6 |
+
- stops by wall-clock (--seconds); lr: 5% warm-up, then cosine over elapsed / budget
|
| 7 |
+
- overfitting guard, same rule as gen_ar.py: every --val-every steps measure validation
|
| 8 |
+
reconstruction loss; keep the best checkpoint; stop when it has not improved for --patience checks
|
| 9 |
+
Validation = "val" split, fixed windows (stride 64).
|
| 10 |
+
"""
|
| 11 |
+
import os, sys, json, time, math, argparse
|
| 12 |
+
import numpy as np
|
| 13 |
+
import torch
|
| 14 |
+
import torch.nn.functional as F
|
| 15 |
+
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
| 16 |
+
import tvq272 as B
|
| 17 |
+
import tvq272c as C
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def load_clips(split):
|
| 21 |
+
mdir = B.find_motion_dir()
|
| 22 |
+
ids = open(os.path.join(B.ROOT, "split", split + ".txt")).read().split()
|
| 23 |
+
out = []
|
| 24 |
+
for i in ids:
|
| 25 |
+
p = os.path.join(mdir, i + ".npy")
|
| 26 |
+
if os.path.exists(p):
|
| 27 |
+
a = np.load(p).astype(np.float32)
|
| 28 |
+
if a.shape[0] >= B.T:
|
| 29 |
+
out.append(a)
|
| 30 |
+
return out
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def main():
|
| 34 |
+
ap = argparse.ArgumentParser()
|
| 35 |
+
ap.add_argument("--seconds", type=float, default=7200)
|
| 36 |
+
ap.add_argument("--bs", type=int, default=256)
|
| 37 |
+
ap.add_argument("--lr", type=float, default=4e-4)
|
| 38 |
+
ap.add_argument("--beta", type=float, default=0.25)
|
| 39 |
+
ap.add_argument("--val-every", type=int, default=1000)
|
| 40 |
+
ap.add_argument("--patience", type=int, default=3)
|
| 41 |
+
ap.add_argument("--tag", default="cmu_tok_v1")
|
| 42 |
+
a = ap.parse_args()
|
| 43 |
+
dev = "cuda"
|
| 44 |
+
t0 = time.time()
|
| 45 |
+
mean = torch.from_numpy(np.load(os.path.join(B.ROOT, "mean_std", "Mean.npy"))).to(dev)
|
| 46 |
+
std = torch.from_numpy(np.load(os.path.join(B.ROOT, "mean_std", "Std.npy"))).to(dev)
|
| 47 |
+
|
| 48 |
+
tr = load_clips("train")
|
| 49 |
+
lens = np.array([c.shape[0] for c in tr])
|
| 50 |
+
starts = np.concatenate([[0], np.cumsum(lens)[:-1]])
|
| 51 |
+
# normalise on the CPU so only ONE 2 GB copy ever lives on the GPU
|
| 52 |
+
big_np = np.concatenate(tr); del tr
|
| 53 |
+
big_np -= mean.cpu().numpy(); big_np /= std.cpu().numpy()
|
| 54 |
+
big = torch.from_numpy(big_np).to(dev); del big_np
|
| 55 |
+
valid = torch.from_numpy(np.concatenate([s + np.arange(n - B.T + 1) for s, n in zip(starts, lens)])).to(dev)
|
| 56 |
+
va = load_clips("val")
|
| 57 |
+
Wva = np.stack([c[s:s + B.T] for c in va for s in range(0, c.shape[0] - B.T + 1, B.T)])
|
| 58 |
+
Xva = torch.from_numpy(Wva).to(dev)
|
| 59 |
+
Zva = (Xva - mean) / std
|
| 60 |
+
print("train clips %d, %d frames (%.2f h), %d window starts | val %d windows | load %.0fs"
|
| 61 |
+
% (len(lens), big.shape[0], big.shape[0] / 30 / 3600, valid.numel(), Zva.shape[0], time.time() - t0), flush=True)
|
| 62 |
+
mt = B.Metrics(mean, std, torch.stack([big[s:s + B.T] for s in valid[:: max(1, valid.numel() // 2000)][:2000]]) * std + mean)
|
| 63 |
+
ar = torch.arange(B.T, device=dev)
|
| 64 |
+
torch.cuda.empty_cache()
|
| 65 |
+
print("gpu memory after load: allocated %.2f GB, reserved %.2f GB"
|
| 66 |
+
% (torch.cuda.memory_allocated() / 1e9, torch.cuda.memory_reserved() / 1e9), flush=True)
|
| 67 |
+
|
| 68 |
+
torch.manual_seed(0)
|
| 69 |
+
m = C.CHierTok().to(dev).train()
|
| 70 |
+
print("params %d" % B.nparams(m), flush=True)
|
| 71 |
+
opt = torch.optim.AdamW(m.parameters(), lr=a.lr)
|
| 72 |
+
|
| 73 |
+
@torch.no_grad()
|
| 74 |
+
def val_loss():
|
| 75 |
+
m.eval()
|
| 76 |
+
tot = 0.0
|
| 77 |
+
for i in range(0, Zva.shape[0], 256):
|
| 78 |
+
x = Zva[i:i + 256]
|
| 79 |
+
tot += float(F.mse_loss(m(x)[0], x, reduction="sum"))
|
| 80 |
+
m.train()
|
| 81 |
+
return tot / Zva.numel()
|
| 82 |
+
|
| 83 |
+
best, bad, log, step = [float("inf"), 0], 0, [], 0
|
| 84 |
+
tt = time.time()
|
| 85 |
+
while True:
|
| 86 |
+
el = time.time() - tt
|
| 87 |
+
if el > a.seconds:
|
| 88 |
+
print(" time budget reached", flush=True)
|
| 89 |
+
break
|
| 90 |
+
frac = el / a.seconds
|
| 91 |
+
lr = a.lr * min(1.0, frac / 0.05) if frac < 0.05 else a.lr * 0.5 * (1 + math.cos(math.pi * (frac - 0.05) / 0.95))
|
| 92 |
+
for g in opt.param_groups:
|
| 93 |
+
g["lr"] = max(lr, a.lr * 1e-3)
|
| 94 |
+
s = valid[torch.randint(0, valid.numel(), (a.bs,), device=dev)]
|
| 95 |
+
xb = big[s[:, None] + ar[None]]
|
| 96 |
+
y, commit = m(xb)
|
| 97 |
+
loss = F.mse_loss(y, xb) + a.beta * commit
|
| 98 |
+
opt.zero_grad(set_to_none=True)
|
| 99 |
+
loss.backward()
|
| 100 |
+
torch.nn.utils.clip_grad_norm_(m.parameters(), 1.0)
|
| 101 |
+
opt.step()
|
| 102 |
+
step += 1
|
| 103 |
+
if step == 20:
|
| 104 |
+
torch.cuda.synchronize()
|
| 105 |
+
print(" speed %.1f step/s -> ~%d steps in budget | gpu peak %.2f GB" % (20 / (time.time() - tt), a.seconds * 20 / (time.time() - tt), torch.cuda.max_memory_allocated() / 1e9), flush=True)
|
| 106 |
+
if step % a.val_every == 0:
|
| 107 |
+
vl = val_loss()
|
| 108 |
+
row = dict(step=step, sec=round(time.time() - tt), lr=round(lr, 7), train_loss=round(loss.item(), 5),
|
| 109 |
+
val_recon=round(vl, 5))
|
| 110 |
+
if vl < best[0]:
|
| 111 |
+
best, bad = [vl, step], 0
|
| 112 |
+
torch.save(m.state_dict(), os.path.join(B.CKPT_DIR, a.tag + "_H.pt"))
|
| 113 |
+
row["saved"] = True
|
| 114 |
+
else:
|
| 115 |
+
bad += 1
|
| 116 |
+
log.append(row)
|
| 117 |
+
print(" " + json.dumps(row), flush=True)
|
| 118 |
+
if bad >= a.patience:
|
| 119 |
+
print(" early stop: val not improved for %d checks" % a.patience, flush=True)
|
| 120 |
+
break
|
| 121 |
+
secs = time.time() - tt
|
| 122 |
+
m.load_state_dict(torch.load(os.path.join(B.CKPT_DIR, a.tag + "_H.pt"), map_location=dev))
|
| 123 |
+
r = B.evaluate(m, Zva, mt)
|
| 124 |
+
r.update(params=B.nparams(m), steps=step, best_step=best[1], best_val_recon=best[0], train_seconds=round(secs, 1),
|
| 125 |
+
causality_maxdiff=C.causality_check(m, Zva))
|
| 126 |
+
json.dump(dict(result=r, args=vars(a), log=log), open(os.path.join(B.OUTDIR, a.tag + ".json"), "w", encoding="utf-8"),
|
| 127 |
+
ensure_ascii=False, indent=1)
|
| 128 |
+
print("best step %d | pos %.2f mm | rot local %.2f / global %.2f deg | seam %.2f mm | consist %.2f (gt %.2f) | causal %.1e | %.0fs"
|
| 129 |
+
% (best[1], r["mpjpe_mm"], r["rot_local_deg"], r["rot_global_deg"], r["seam_bone_mm"], r["consistency_deg"],
|
| 130 |
+
r["consistency_gt_deg"], r["causality_maxdiff"], secs), flush=True)
|
| 131 |
+
for pn, v in r["per_part"].items():
|
| 132 |
+
print(" %-6s pos %7.2f mm rot %6.2f deg" % (pn, v["pos_mm"], v["rot_global_deg"]), flush=True)
|
| 133 |
+
|
| 134 |
+
|
| 135 |
+
if __name__ == "__main__":
|
| 136 |
+
main()
|
code/tvq.py
ADDED
|
@@ -0,0 +1,383 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Temporal tokenizer comparison at MoSCo's bit budget (54 bits / 4 frames).
|
| 3 |
+
|
| 4 |
+
M MoSCo/ParCo tokenizer: 6 independent part VQ-VAEs (conv1d, down x4),
|
| 5 |
+
codebook 512 x dim 128 each. No cross-part information anywhere.
|
| 6 |
+
H ours: global code (whole-body encoder, k=4096) + 6 part codes (k=128).
|
| 7 |
+
Part p decoded from [global code, part p code] only -> strictly local.
|
| 8 |
+
W reference: T2M-GPT style whole-body tokenizer, k=512 (9 bits / 4 frames,
|
| 9 |
+
i.e. LESS information; shown for scale only).
|
| 10 |
+
|
| 11 |
+
Bits per 4-frame token step: M = 6*9 = 54, H = 12 + 6*7 = 54, W = 9.
|
| 12 |
+
|
| 13 |
+
Data: HumanML3D new_joints, root-relative 21x3 positions, windows of 64 frames.
|
| 14 |
+
NOTE: MoSCo uses 263-dim features; we only have positions, so numbers are not
|
| 15 |
+
comparable to MoSCo's paper -- only M vs H under identical conditions.
|
| 16 |
+
"""
|
| 17 |
+
import os
|
| 18 |
+
import sys
|
| 19 |
+
import glob
|
| 20 |
+
import json
|
| 21 |
+
import time
|
| 22 |
+
import argparse
|
| 23 |
+
import numpy as np
|
| 24 |
+
import torch
|
| 25 |
+
import torch.nn as nn
|
| 26 |
+
import torch.nn.functional as F
|
| 27 |
+
|
| 28 |
+
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
| 29 |
+
import vq_mix as V # noqa: E402
|
| 30 |
+
BASE = os.environ.get("PARTLOCK_DIR", ".") # folder that holds data/, ckpt/, runs/
|
| 31 |
+
|
| 32 |
+
DEST = V.DEST
|
| 33 |
+
OUTDIR = os.path.join(BASE, "runs")
|
| 34 |
+
CKPT_DIR = os.path.join(BASE, "ckpt")
|
| 35 |
+
T = 64
|
| 36 |
+
|
| 37 |
+
PARENTS = [-1, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 9, 9, 12, 13, 14, 16, 17,
|
| 38 |
+
18, 19]
|
| 39 |
+
JPART = {0: "root"}
|
| 40 |
+
for pn, js in V.PART6.items():
|
| 41 |
+
for j in js:
|
| 42 |
+
JPART[j] = pn
|
| 43 |
+
BONES = [(j, p) for j, p in enumerate(PARENTS) if p >= 0]
|
| 44 |
+
CROSS = [(j, p) for j, p in BONES if JPART[j] != JPART[p]]
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
# ----------------------------------------------------------------- data
|
| 48 |
+
def load_windows(max_frames, val_frac=0.1, seed=0, stride_tr=16):
|
| 49 |
+
files = [p for p in glob.glob(os.path.join(DEST, "**", "*.npy"),
|
| 50 |
+
recursive=True)
|
| 51 |
+
if not os.path.basename(p).startswith(".")]
|
| 52 |
+
files.sort()
|
| 53 |
+
rng = np.random.default_rng(seed)
|
| 54 |
+
rng.shuffle(files)
|
| 55 |
+
nv = max(1, int(len(files) * val_frac))
|
| 56 |
+
vf, tf = files[:nv], files[nv:]
|
| 57 |
+
|
| 58 |
+
def grab(fs, budget, stride):
|
| 59 |
+
wins, total, used = [], 0, 0
|
| 60 |
+
for p in fs:
|
| 61 |
+
try:
|
| 62 |
+
a = np.load(p).astype(np.float32)
|
| 63 |
+
except Exception:
|
| 64 |
+
continue
|
| 65 |
+
if a.ndim != 3 or a.shape[1] != 22 or a.shape[0] < T:
|
| 66 |
+
continue
|
| 67 |
+
a = (a - a[:, 0:1, :])[:, 1:, :] # (n,21,3)
|
| 68 |
+
for s in range(0, a.shape[0] - T + 1, stride):
|
| 69 |
+
wins.append(a[s:s + T])
|
| 70 |
+
total += a.shape[0]
|
| 71 |
+
used += 1
|
| 72 |
+
if total >= budget:
|
| 73 |
+
break
|
| 74 |
+
return np.stack(wins), used
|
| 75 |
+
|
| 76 |
+
Wtr, ntr = grab(tf, max_frames, stride_tr)
|
| 77 |
+
Wva, nva = grab(vf, max(20000, max_frames // 10), T)
|
| 78 |
+
return Wtr, Wva, ntr, nva
|
| 79 |
+
|
| 80 |
+
|
| 81 |
+
# ----------------------------------------------------------------- conv blocks
|
| 82 |
+
class Res(nn.Module):
|
| 83 |
+
def __init__(self, ch, dil):
|
| 84 |
+
super().__init__()
|
| 85 |
+
self.c1 = nn.Conv1d(ch, ch, 3, 1, dil, dilation=dil)
|
| 86 |
+
self.c2 = nn.Conv1d(ch, ch, 1)
|
| 87 |
+
|
| 88 |
+
def forward(self, x):
|
| 89 |
+
return x + self.c2(F.relu(self.c1(F.relu(x))))
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
def enc1d(cin, ch, d):
|
| 93 |
+
L = [nn.Conv1d(cin, ch, 3, 1, 1), nn.ReLU()]
|
| 94 |
+
for _ in range(2):
|
| 95 |
+
L += [nn.Conv1d(ch, ch, 4, 2, 1), Res(ch, 1), Res(ch, 3)]
|
| 96 |
+
L += [nn.Conv1d(ch, d, 3, 1, 1)]
|
| 97 |
+
return nn.Sequential(*L)
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
def dec1d(d, ch, cout):
|
| 101 |
+
L = [nn.Conv1d(d, ch, 3, 1, 1), nn.ReLU()]
|
| 102 |
+
for _ in range(2):
|
| 103 |
+
L += [Res(ch, 3), Res(ch, 1), nn.Upsample(scale_factor=2, mode="nearest"),
|
| 104 |
+
nn.Conv1d(ch, ch, 3, 1, 1)]
|
| 105 |
+
L += [nn.ReLU(), nn.Conv1d(ch, cout, 3, 1, 1)]
|
| 106 |
+
return nn.Sequential(*L)
|
| 107 |
+
|
| 108 |
+
|
| 109 |
+
def vq_seq(vq, z):
|
| 110 |
+
"""z: (B,d,T') -> quantized (B,d,T'), commit, idx (B,T')"""
|
| 111 |
+
B, d, t = z.shape
|
| 112 |
+
zf = z.permute(0, 2, 1).reshape(B * t, d)
|
| 113 |
+
q, c, idx = vq(zf)
|
| 114 |
+
return q.reshape(B, t, d).permute(0, 2, 1), c, idx.reshape(B, t)
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
def part_in(x, p): # x (B,T,21,3) -> (B, 3n, T)
|
| 118 |
+
c = V.COLS[p]
|
| 119 |
+
B = x.shape[0]
|
| 120 |
+
return x[:, :, c, :].reshape(B, T, -1).permute(0, 2, 1)
|
| 121 |
+
|
| 122 |
+
|
| 123 |
+
def part_out(y, p, out): # y (B,3n,T) -> write into out (B,T,21,3)
|
| 124 |
+
c = V.COLS[p]
|
| 125 |
+
B = y.shape[0]
|
| 126 |
+
out[:, :, c, :] = y.permute(0, 2, 1).reshape(B, T, len(c), 3)
|
| 127 |
+
|
| 128 |
+
|
| 129 |
+
class PartTok(nn.Module): # M
|
| 130 |
+
def __init__(self, k=512, d=128, ch=128):
|
| 131 |
+
super().__init__()
|
| 132 |
+
self.enc = nn.ModuleList([enc1d(3 * n, ch, d) for n in V.NJ])
|
| 133 |
+
self.vq = nn.ModuleList([V.VQ(k, d) for _ in V.NJ])
|
| 134 |
+
self.dec = nn.ModuleList([dec1d(d, ch, 3 * n) for n in V.NJ])
|
| 135 |
+
self.kp = k
|
| 136 |
+
|
| 137 |
+
def encode(self, x):
|
| 138 |
+
idx = []
|
| 139 |
+
for p in range(6):
|
| 140 |
+
z = self.enc[p](part_in(x, p))
|
| 141 |
+
idx.append(self._nn(self.vq[p], z))
|
| 142 |
+
return None, torch.stack(idx, -1) # (B,T',6)
|
| 143 |
+
|
| 144 |
+
@staticmethod
|
| 145 |
+
def _nn(vq, z):
|
| 146 |
+
B, d, t = z.shape
|
| 147 |
+
zf = z.permute(0, 2, 1).reshape(B * t, d)
|
| 148 |
+
e = vq.emb
|
| 149 |
+
d2 = (zf * zf).sum(1, keepdim=True) - 2 * zf @ e.T + (e * e).sum(1)
|
| 150 |
+
return d2.argmin(1).reshape(B, t)
|
| 151 |
+
|
| 152 |
+
def decode(self, ig, ip):
|
| 153 |
+
B = ip.shape[0]
|
| 154 |
+
out = torch.empty(B, T, 21, 3, device=ip.device)
|
| 155 |
+
for p in range(6):
|
| 156 |
+
q = self.vq[p].emb[ip[..., p]].permute(0, 2, 1)
|
| 157 |
+
part_out(self.dec[p](q), p, out)
|
| 158 |
+
return out
|
| 159 |
+
|
| 160 |
+
def forward(self, x):
|
| 161 |
+
B = x.shape[0]
|
| 162 |
+
out = torch.empty_like(x)
|
| 163 |
+
commit = 0.0
|
| 164 |
+
for p in range(6):
|
| 165 |
+
q, c, _ = vq_seq(self.vq[p], self.enc[p](part_in(x, p)))
|
| 166 |
+
commit = commit + c
|
| 167 |
+
part_out(self.dec[p](q), p, out)
|
| 168 |
+
return out, commit / 6
|
| 169 |
+
|
| 170 |
+
|
| 171 |
+
class HierTok(nn.Module): # H
|
| 172 |
+
def __init__(self, kg=4096, kp=128, d=128, ch=128):
|
| 173 |
+
super().__init__()
|
| 174 |
+
self.genc = enc1d(63, 256, d)
|
| 175 |
+
self.gvq = V.VQ(kg, d)
|
| 176 |
+
self.enc = nn.ModuleList([enc1d(3 * n, ch, d) for n in V.NJ])
|
| 177 |
+
self.vq = nn.ModuleList([V.VQ(kp, d) for _ in V.NJ])
|
| 178 |
+
self.dec = nn.ModuleList([dec1d(2 * d, ch, 3 * n) for n in V.NJ])
|
| 179 |
+
self.kp = kp
|
| 180 |
+
|
| 181 |
+
def encode(self, x):
|
| 182 |
+
B = x.shape[0]
|
| 183 |
+
zg = self.genc(x.reshape(B, T, 63).permute(0, 2, 1))
|
| 184 |
+
ig = PartTok._nn(self.gvq, zg)
|
| 185 |
+
ip = torch.stack([PartTok._nn(self.vq[p], self.enc[p](part_in(x, p)))
|
| 186 |
+
for p in range(6)], -1)
|
| 187 |
+
return ig, ip
|
| 188 |
+
|
| 189 |
+
def decode(self, ig, ip):
|
| 190 |
+
B = ip.shape[0]
|
| 191 |
+
qg = self.gvq.emb[ig].permute(0, 2, 1)
|
| 192 |
+
out = torch.empty(B, T, 21, 3, device=ip.device)
|
| 193 |
+
for p in range(6):
|
| 194 |
+
q = self.vq[p].emb[ip[..., p]].permute(0, 2, 1)
|
| 195 |
+
part_out(self.dec[p](torch.cat([qg, q], 1)), p, out)
|
| 196 |
+
return out
|
| 197 |
+
|
| 198 |
+
def forward(self, x):
|
| 199 |
+
B = x.shape[0]
|
| 200 |
+
qg, cg, _ = vq_seq(self.gvq,
|
| 201 |
+
self.genc(x.reshape(B, T, 63).permute(0, 2, 1)))
|
| 202 |
+
out = torch.empty_like(x)
|
| 203 |
+
commit = cg
|
| 204 |
+
for p in range(6):
|
| 205 |
+
q, c, _ = vq_seq(self.vq[p], self.enc[p](part_in(x, p)))
|
| 206 |
+
commit = commit + c
|
| 207 |
+
part_out(self.dec[p](torch.cat([qg, q], 1)), p, out)
|
| 208 |
+
return out, commit / 7
|
| 209 |
+
|
| 210 |
+
|
| 211 |
+
class WholeTok(nn.Module): # W (reference)
|
| 212 |
+
def __init__(self, k=512, d=128, ch=256):
|
| 213 |
+
super().__init__()
|
| 214 |
+
self.enc = enc1d(63, ch, d)
|
| 215 |
+
self.vq = V.VQ(k, d)
|
| 216 |
+
self.dec = dec1d(d, ch, 63)
|
| 217 |
+
|
| 218 |
+
def forward(self, x):
|
| 219 |
+
B = x.shape[0]
|
| 220 |
+
q, c, _ = vq_seq(self.vq, self.enc(x.reshape(B, T, 63).permute(0, 2, 1)))
|
| 221 |
+
y = self.dec(q).permute(0, 2, 1).reshape(B, T, 21, 3)
|
| 222 |
+
return y, c
|
| 223 |
+
|
| 224 |
+
|
| 225 |
+
# ----------------------------------------------------------------- metrics
|
| 226 |
+
def full22(y):
|
| 227 |
+
B, t = y.shape[:2]
|
| 228 |
+
z = torch.zeros(B, t, 22, 3, device=y.device)
|
| 229 |
+
z[:, :, 1:] = y
|
| 230 |
+
return z
|
| 231 |
+
|
| 232 |
+
|
| 233 |
+
def bone_err(y, x, bones):
|
| 234 |
+
a, b = full22(y), full22(x)
|
| 235 |
+
e = []
|
| 236 |
+
for j, p in bones:
|
| 237 |
+
e.append(((a[:, :, j] - a[:, :, p]).norm(dim=-1)
|
| 238 |
+
- (b[:, :, j] - b[:, :, p]).norm(dim=-1)).abs())
|
| 239 |
+
return float(torch.stack(e, -1).mean()) * 1000
|
| 240 |
+
|
| 241 |
+
|
| 242 |
+
def mpjpe(y, x):
|
| 243 |
+
return float(((y - x) ** 2).sum(-1).mean().sqrt()) * 1000
|
| 244 |
+
|
| 245 |
+
|
| 246 |
+
@torch.no_grad()
|
| 247 |
+
def evaluate(m, Wva):
|
| 248 |
+
m.eval()
|
| 249 |
+
ys = []
|
| 250 |
+
for i in range(0, Wva.shape[0], 256):
|
| 251 |
+
ys.append(m(Wva[i:i + 256])[0])
|
| 252 |
+
y = torch.cat(ys)
|
| 253 |
+
m.train()
|
| 254 |
+
return y
|
| 255 |
+
|
| 256 |
+
|
| 257 |
+
def train(m, Wtr, Wva, steps, bs, lr, beta, log_every):
|
| 258 |
+
dev = Wtr.device
|
| 259 |
+
m.to(dev).train()
|
| 260 |
+
opt = torch.optim.AdamW(m.parameters(), lr=lr)
|
| 261 |
+
sch = torch.optim.lr_scheduler.OneCycleLR(opt, max_lr=lr, total_steps=steps,
|
| 262 |
+
pct_start=0.05)
|
| 263 |
+
n = Wtr.shape[0]
|
| 264 |
+
t0 = time.time()
|
| 265 |
+
for s in range(1, steps + 1):
|
| 266 |
+
xb = Wtr[torch.randint(0, n, (bs,), device=dev)]
|
| 267 |
+
y, commit = m(xb)
|
| 268 |
+
loss = F.mse_loss(y, xb) + beta * commit
|
| 269 |
+
opt.zero_grad(set_to_none=True)
|
| 270 |
+
loss.backward()
|
| 271 |
+
torch.nn.utils.clip_grad_norm_(m.parameters(), 1.0)
|
| 272 |
+
opt.step()
|
| 273 |
+
sch.step()
|
| 274 |
+
if s == 20:
|
| 275 |
+
torch.cuda.synchronize()
|
| 276 |
+
sps = 20 / (time.time() - t0)
|
| 277 |
+
print(" speed %.1f step/s -> est %.0f s for %d steps"
|
| 278 |
+
% (sps, steps / sps, steps), flush=True)
|
| 279 |
+
if log_every and s % log_every == 0:
|
| 280 |
+
y = evaluate(m, Wva)
|
| 281 |
+
print(" step %5d val MPJPE %7.2f mm (%.0fs)"
|
| 282 |
+
% (s, mpjpe(y, Wva), time.time() - t0), flush=True)
|
| 283 |
+
return time.time() - t0
|
| 284 |
+
|
| 285 |
+
|
| 286 |
+
@torch.no_grad()
|
| 287 |
+
def swap_test(m, Wva, g, rows, name):
|
| 288 |
+
m.eval()
|
| 289 |
+
ig, ip = m.encode(Wva)
|
| 290 |
+
y0 = m.decode(ig, ip)
|
| 291 |
+
base = dict(all=bone_err(y0, Wva, BONES), cross=bone_err(y0, Wva, CROSS))
|
| 292 |
+
print(" [%s] no swap: MPJPE %.2f bone %.2f cross-part bone %.2f mm"
|
| 293 |
+
% (name, mpjpe(y0, Wva), base["all"], base["cross"]), flush=True)
|
| 294 |
+
tpos = ip.shape[1] // 2
|
| 295 |
+
fr = slice(tpos * 4 - 4, tpos * 4 + 8) # frames influenced by the token
|
| 296 |
+
for p, pn in enumerate(V.PART_NAMES):
|
| 297 |
+
sw = ip.clone()
|
| 298 |
+
off = torch.randint(1, m.kp, (sw.shape[0],), generator=g, device=sw.device)
|
| 299 |
+
sw[:, tpos, p] = (sw[:, tpos, p] + off) % m.kp
|
| 300 |
+
y1 = m.decode(ig, sw)
|
| 301 |
+
d2 = ((y1 - y0) ** 2).sum(-1)[:, fr]
|
| 302 |
+
inside = torch.zeros(21, dtype=torch.bool, device=sw.device)
|
| 303 |
+
inside[V.COLS[p]] = True
|
| 304 |
+
r = dict(model=name, part=pn,
|
| 305 |
+
move_in_mm=round(float(d2[..., inside].mean().sqrt()) * 1000, 2),
|
| 306 |
+
move_out_mm=round(float(d2[..., ~inside].mean().sqrt()) * 1000, 2),
|
| 307 |
+
bone_mm=round(bone_err(y1[:, fr], Wva[:, fr], BONES), 2),
|
| 308 |
+
cross_bone_mm=round(bone_err(y1[:, fr], Wva[:, fr], CROSS), 2),
|
| 309 |
+
bone_noswap_mm=round(bone_err(y0[:, fr], Wva[:, fr], BONES), 2),
|
| 310 |
+
cross_noswap_mm=round(bone_err(y0[:, fr], Wva[:, fr], CROSS), 2))
|
| 311 |
+
rows.append(r)
|
| 312 |
+
print(" [%s] %-6s in %7.2f out %6.2f bone %6.2f (no swap %6.2f) "
|
| 313 |
+
"cross %6.2f (no swap %6.2f)" % (
|
| 314 |
+
name, pn, r["move_in_mm"], r["move_out_mm"], r["bone_mm"],
|
| 315 |
+
r["bone_noswap_mm"], r["cross_bone_mm"], r["cross_noswap_mm"]),
|
| 316 |
+
flush=True)
|
| 317 |
+
m.train()
|
| 318 |
+
|
| 319 |
+
|
| 320 |
+
def main():
|
| 321 |
+
ap = argparse.ArgumentParser()
|
| 322 |
+
ap.add_argument("--frames", type=int, default=400000)
|
| 323 |
+
ap.add_argument("--steps", type=int, default=6000)
|
| 324 |
+
ap.add_argument("--bs", type=int, default=256)
|
| 325 |
+
ap.add_argument("--lr", type=float, default=4e-4)
|
| 326 |
+
ap.add_argument("--beta", type=float, default=0.25)
|
| 327 |
+
ap.add_argument("--log-every", type=int, default=1000)
|
| 328 |
+
ap.add_argument("--models", nargs="+", default=["M", "H", "W"])
|
| 329 |
+
ap.add_argument("--m-ch", type=int, default=128)
|
| 330 |
+
ap.add_argument("--tag", default="tvq_v1")
|
| 331 |
+
args = ap.parse_args()
|
| 332 |
+
dev = "cuda"
|
| 333 |
+
t0 = time.time()
|
| 334 |
+
Wtr_np, Wva_np, ntr, nva = load_windows(args.frames)
|
| 335 |
+
Wtr = torch.from_numpy(Wtr_np).to(dev)
|
| 336 |
+
Wva = torch.from_numpy(Wva_np).to(dev)
|
| 337 |
+
print("train windows %d (%d files) val windows %d (%d files) T=%d "
|
| 338 |
+
"load_s %.1f" % (Wtr.shape[0], ntr, Wva.shape[0], nva, T,
|
| 339 |
+
time.time() - t0), flush=True)
|
| 340 |
+
print("cross-part bones:", [(V.JOINT_NAMES[j] if hasattr(V, "JOINT_NAMES")
|
| 341 |
+
else j, p) for j, p in CROSS], flush=True)
|
| 342 |
+
|
| 343 |
+
makers = {"M": lambda: PartTok(ch=args.m_ch), "H": lambda: HierTok(),
|
| 344 |
+
"W": lambda: WholeTok()}
|
| 345 |
+
bits = {"M": 54, "H": 54, "W": 9}
|
| 346 |
+
results, swaps = [], []
|
| 347 |
+
g = torch.Generator(device=dev).manual_seed(1)
|
| 348 |
+
for name in args.models:
|
| 349 |
+
torch.manual_seed(0)
|
| 350 |
+
m = makers[name]()
|
| 351 |
+
npar = sum(p.numel() for p in m.parameters())
|
| 352 |
+
print("\n=== %s bits/4frames=%d params=%.2fM ===" %
|
| 353 |
+
(name, bits[name], npar / 1e6), flush=True)
|
| 354 |
+
secs = train(m, Wtr, Wva, args.steps, args.bs, args.lr, args.beta,
|
| 355 |
+
args.log_every)
|
| 356 |
+
y = evaluate(m, Wva)
|
| 357 |
+
r = dict(model=name, bits_per_4frames=bits[name], params=npar,
|
| 358 |
+
val_mpjpe_mm=round(mpjpe(y, Wva), 3),
|
| 359 |
+
bone_mm=round(bone_err(y, Wva, BONES), 3),
|
| 360 |
+
cross_bone_mm=round(bone_err(y, Wva, CROSS), 3),
|
| 361 |
+
train_seconds=round(secs, 1))
|
| 362 |
+
results.append(r)
|
| 363 |
+
print("%s val MPJPE %.2f mm bone %.2f cross-part bone %.2f (%.0fs)"
|
| 364 |
+
% (name, r["val_mpjpe_mm"], r["bone_mm"], r["cross_bone_mm"], secs),
|
| 365 |
+
flush=True)
|
| 366 |
+
if name in ("M", "H"):
|
| 367 |
+
torch.save(m.state_dict(), os.path.join(CKPT_DIR, "%s_%s.pt"
|
| 368 |
+
% (args.tag, name)))
|
| 369 |
+
swap_test(m, Wva, g, swaps, name)
|
| 370 |
+
del m
|
| 371 |
+
torch.cuda.empty_cache()
|
| 372 |
+
|
| 373 |
+
out = os.path.join(OUTDIR, args.tag + ".json")
|
| 374 |
+
with open(out, "w", encoding="utf-8") as f:
|
| 375 |
+
json.dump(dict(args=vars(args), results=results, swaps=swaps,
|
| 376 |
+
cross_bones=CROSS,
|
| 377 |
+
total_seconds=round(time.time() - t0, 1)),
|
| 378 |
+
f, ensure_ascii=False, indent=2)
|
| 379 |
+
print("\nwrote", out, " total_s %.1f" % (time.time() - t0))
|
| 380 |
+
|
| 381 |
+
|
| 382 |
+
if __name__ == "__main__":
|
| 383 |
+
main()
|
code/tvq272.py
ADDED
|
@@ -0,0 +1,339 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Tokenizer comparison on the 272-dim HumanML3D representation (with rotations).
|
| 3 |
+
|
| 4 |
+
Layout (verified from representation_272.py):
|
| 5 |
+
[0:2] root xz velocity (heading removed)
|
| 6 |
+
[2:8] heading angular velocity, 6D
|
| 7 |
+
[8:74] local joint positions (22x3, heading removed, xz at origin)
|
| 8 |
+
[74:140] local joint velocities (22x3)
|
| 9 |
+
[140:272] SMPL parent-relative rotations (22x6D, first two ROWS of the matrix),
|
| 10 |
+
root (joint 0) with heading removed.
|
| 11 |
+
30 fps. Splits from split/{train,val}.txt.
|
| 12 |
+
|
| 13 |
+
Per-joint feature = pos3 + vel3 + rot6 = 12 dims. Root extras (8) go to the torso part.
|
| 14 |
+
torso: joints 0,3,6,9 (+8 root dims) = 56 head: 12,15 = 24
|
| 15 |
+
arm_R / arm_L / leg_R / leg_L: 4 joints = 48 each total 272
|
| 16 |
+
|
| 17 |
+
M 6 independent part VQ-VAEs, k=512 each -> 54 bits / 4 frames
|
| 18 |
+
H global (all 272 ch) k=4096 + 6 part codes k=128,
|
| 19 |
+
part p decoded from [global, part p] only -> 54 bits / 4 frames
|
| 20 |
+
Params matched by widening M's channels (--m-ch).
|
| 21 |
+
"""
|
| 22 |
+
import os
|
| 23 |
+
import sys
|
| 24 |
+
import json
|
| 25 |
+
import time
|
| 26 |
+
import argparse
|
| 27 |
+
import numpy as np
|
| 28 |
+
import torch
|
| 29 |
+
import torch.nn as nn
|
| 30 |
+
import torch.nn.functional as F
|
| 31 |
+
|
| 32 |
+
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
| 33 |
+
import vq_mix as V # noqa: E402 (VQ)
|
| 34 |
+
import tvq # noqa: E402 (enc1d, dec1d, vq_seq)
|
| 35 |
+
BASE = os.environ.get("PARTLOCK_DIR", ".") # folder that holds data/, ckpt/, runs/
|
| 36 |
+
|
| 37 |
+
ROOT = os.path.join(BASE, "data", "motion_raw", "h272")
|
| 38 |
+
OUTDIR = os.path.join(BASE, "runs")
|
| 39 |
+
CKPT_DIR = os.path.join(BASE, "ckpt")
|
| 40 |
+
T = 64
|
| 41 |
+
PARENTS = tvq.PARENTS
|
| 42 |
+
PART_J = {"torso": [0, 3, 6, 9], "head": [12, 15], "arm_R": [14, 17, 19, 21],
|
| 43 |
+
"arm_L": [13, 16, 18, 20], "leg_R": [2, 5, 8, 11], "leg_L": [1, 4, 7, 10]}
|
| 44 |
+
PNAMES = list(PART_J)
|
| 45 |
+
CROSS = tvq.CROSS
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
def joint_dims(j):
|
| 49 |
+
return (list(range(8 + 3 * j, 11 + 3 * j)) + list(range(74 + 3 * j, 77 + 3 * j))
|
| 50 |
+
+ list(range(140 + 6 * j, 146 + 6 * j)))
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
PART_DIMS = {}
|
| 54 |
+
for p, js in PART_J.items():
|
| 55 |
+
d = [] if p != "torso" else list(range(8))
|
| 56 |
+
for j in js:
|
| 57 |
+
d += joint_dims(j)
|
| 58 |
+
PART_DIMS[p] = d
|
| 59 |
+
assert sorted(sum(PART_DIMS.values(), [])) == list(range(272))
|
| 60 |
+
PD = [PART_DIMS[p] for p in PNAMES]
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
# ------------------------------------------------------------------ data
|
| 64 |
+
def find_motion_dir():
|
| 65 |
+
for dp, dn, fn in os.walk(os.path.join(ROOT, "motion_data")):
|
| 66 |
+
if sum(f.endswith(".npy") for f in fn) > 1000:
|
| 67 |
+
return dp
|
| 68 |
+
raise SystemExit("motion npy dir not found under " + ROOT)
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
def load_split(name, budget, stride, seed=0):
|
| 72 |
+
mdir = find_motion_dir()
|
| 73 |
+
ids = open(os.path.join(ROOT, "split", name + ".txt")).read().split()
|
| 74 |
+
rng = np.random.default_rng(seed)
|
| 75 |
+
rng.shuffle(ids)
|
| 76 |
+
wins, total, used = [], 0, 0
|
| 77 |
+
for i in ids:
|
| 78 |
+
p = os.path.join(mdir, i + ".npy")
|
| 79 |
+
if not os.path.exists(p):
|
| 80 |
+
continue
|
| 81 |
+
a = np.load(p).astype(np.float32)
|
| 82 |
+
if a.ndim != 2 or a.shape[1] != 272 or a.shape[0] < T:
|
| 83 |
+
continue
|
| 84 |
+
for s in range(0, a.shape[0] - T + 1, stride):
|
| 85 |
+
wins.append(a[s:s + T])
|
| 86 |
+
total += a.shape[0]
|
| 87 |
+
used += 1
|
| 88 |
+
if total >= budget:
|
| 89 |
+
break
|
| 90 |
+
return np.stack(wins), used
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
# ------------------------------------------------------------------ rotations
|
| 94 |
+
def rot6d_to_mat(x):
|
| 95 |
+
"""x (...,6) = first two ROWS -> (...,3,3) rows b1,b2,b3 (pytorch3d convention)."""
|
| 96 |
+
a1, a2 = x[..., :3], x[..., 3:]
|
| 97 |
+
b1 = F.normalize(a1, dim=-1)
|
| 98 |
+
b2 = F.normalize(a2 - (b1 * a2).sum(-1, keepdim=True) * b1, dim=-1)
|
| 99 |
+
b3 = torch.cross(b1, b2, dim=-1)
|
| 100 |
+
return torch.stack([b1, b2, b3], dim=-2)
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
def local_rots(x): # x (B,T,272) raw -> (B,T,22,3,3)
|
| 104 |
+
return rot6d_to_mat(x[..., 140:272].reshape(*x.shape[:-1], 22, 6))
|
| 105 |
+
|
| 106 |
+
|
| 107 |
+
def global_rots(R): # chain parent-relative rotations
|
| 108 |
+
G = [None] * 22
|
| 109 |
+
for j in range(22):
|
| 110 |
+
p = PARENTS[j]
|
| 111 |
+
G[j] = R[..., j, :, :] if p < 0 else G[p] @ R[..., j, :, :]
|
| 112 |
+
return torch.stack(G, dim=-3)
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
def geodesic_deg(A, B):
|
| 116 |
+
tr = (A.transpose(-1, -2) @ B).diagonal(dim1=-2, dim2=-1).sum(-1)
|
| 117 |
+
return torch.rad2deg(torch.acos(((tr - 1) / 2).clamp(-1 + 1e-7, 1 - 1e-7)))
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
def positions(x): # (B,T,22,3)
|
| 121 |
+
return x[..., 8:74].reshape(*x.shape[:-1], 22, 3)
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
# ------------------------------------------------------------------ models
|
| 125 |
+
class PartTok(nn.Module):
|
| 126 |
+
def __init__(self, k=512, d=128, ch=128):
|
| 127 |
+
super().__init__()
|
| 128 |
+
self.enc = nn.ModuleList([tvq.enc1d(len(c), ch, d) for c in PD])
|
| 129 |
+
self.vq = nn.ModuleList([V.VQ(k, d) for _ in PD])
|
| 130 |
+
self.dec = nn.ModuleList([tvq.dec1d(d, ch, len(c)) for c in PD])
|
| 131 |
+
self.kp = k
|
| 132 |
+
|
| 133 |
+
def codes(self, x):
|
| 134 |
+
out = []
|
| 135 |
+
for p, c in enumerate(PD):
|
| 136 |
+
out.append(tvq.vq_seq(self.vq[p], self.enc[p](x[..., c].permute(0, 2, 1))))
|
| 137 |
+
return out
|
| 138 |
+
|
| 139 |
+
def forward(self, x, swap=None):
|
| 140 |
+
y = torch.empty_like(x)
|
| 141 |
+
commit = 0.0
|
| 142 |
+
for p, (q, cm, _) in enumerate(self.codes(x)):
|
| 143 |
+
commit = commit + cm
|
| 144 |
+
y[..., PD[p]] = self.dec[p](q).permute(0, 2, 1)
|
| 145 |
+
return y, commit / 6
|
| 146 |
+
|
| 147 |
+
|
| 148 |
+
class HierTok(nn.Module):
|
| 149 |
+
def __init__(self, kg=4096, kp=128, d=128, ch=128, gch=256):
|
| 150 |
+
super().__init__()
|
| 151 |
+
self.genc = tvq.enc1d(272, gch, d)
|
| 152 |
+
self.gvq = V.VQ(kg, d)
|
| 153 |
+
self.enc = nn.ModuleList([tvq.enc1d(len(c), ch, d) for c in PD])
|
| 154 |
+
self.vq = nn.ModuleList([V.VQ(kp, d) for _ in PD])
|
| 155 |
+
self.dec = nn.ModuleList([tvq.dec1d(2 * d, ch, len(c)) for c in PD])
|
| 156 |
+
self.kp = kp
|
| 157 |
+
|
| 158 |
+
def forward(self, x):
|
| 159 |
+
qg, cg, _ = tvq.vq_seq(self.gvq, self.genc(x.permute(0, 2, 1)))
|
| 160 |
+
y = torch.empty_like(x)
|
| 161 |
+
commit = cg
|
| 162 |
+
for p, c in enumerate(PD):
|
| 163 |
+
q, cm, _ = tvq.vq_seq(self.vq[p], self.enc[p](x[..., c].permute(0, 2, 1)))
|
| 164 |
+
commit = commit + cm
|
| 165 |
+
y[..., c] = self.dec[p](torch.cat([qg, q], 1)).permute(0, 2, 1)
|
| 166 |
+
return y, commit / 7
|
| 167 |
+
|
| 168 |
+
|
| 169 |
+
def nparams(m):
|
| 170 |
+
return sum(p.numel() for p in m.parameters())
|
| 171 |
+
|
| 172 |
+
|
| 173 |
+
# ------------------------------------------------------------------ metrics
|
| 174 |
+
class Metrics:
|
| 175 |
+
def __init__(self, mean, std, Xtr_raw):
|
| 176 |
+
self.mean, self.std = mean, std
|
| 177 |
+
# rest bone direction in parent frame, estimated from GT train data
|
| 178 |
+
with torch.no_grad():
|
| 179 |
+
x = Xtr_raw[: min(2000, Xtr_raw.shape[0])]
|
| 180 |
+
G = global_rots(local_rots(x))
|
| 181 |
+
P = positions(x)
|
| 182 |
+
rest = torch.zeros(22, 3, device=x.device)
|
| 183 |
+
for j, p in enumerate(PARENTS):
|
| 184 |
+
if p < 0:
|
| 185 |
+
continue
|
| 186 |
+
v = P[..., j, :] - P[..., p, :]
|
| 187 |
+
loc = (G[..., p, :, :].transpose(-1, -2) @ v.unsqueeze(-1)).squeeze(-1)
|
| 188 |
+
rest[j] = F.normalize(F.normalize(loc, dim=-1).reshape(-1, 3).mean(0), dim=0)
|
| 189 |
+
self.rest = rest
|
| 190 |
+
|
| 191 |
+
def denorm(self, z):
|
| 192 |
+
return z * self.std + self.mean
|
| 193 |
+
|
| 194 |
+
@torch.no_grad()
|
| 195 |
+
def all(self, y_raw, x_raw):
|
| 196 |
+
Py, Px = positions(y_raw), positions(x_raw)
|
| 197 |
+
out = {}
|
| 198 |
+
out["mpjpe_mm"] = float(((Py - Px) ** 2).sum(-1).mean().sqrt()) * 1000
|
| 199 |
+
Ry, Rx = local_rots(y_raw), local_rots(x_raw)
|
| 200 |
+
Gy, Gx = global_rots(Ry), global_rots(Rx)
|
| 201 |
+
out["rot_local_deg"] = float(geodesic_deg(Ry, Rx).mean())
|
| 202 |
+
out["rot_global_deg"] = float(geodesic_deg(Gy, Gx).mean())
|
| 203 |
+
per = {}
|
| 204 |
+
for pn, js in PART_J.items():
|
| 205 |
+
per[pn] = dict(
|
| 206 |
+
pos_mm=round(float(((Py[..., js, :] - Px[..., js, :]) ** 2).sum(-1).mean().sqrt()) * 1000, 2),
|
| 207 |
+
rot_global_deg=round(float(geodesic_deg(Gy[..., js, :, :], Gx[..., js, :, :]).mean()), 2))
|
| 208 |
+
out["per_part"] = per
|
| 209 |
+
se = []
|
| 210 |
+
for j, p in CROSS:
|
| 211 |
+
se.append(((Py[..., j, :] - Py[..., p, :]).norm(dim=-1)
|
| 212 |
+
- (Px[..., j, :] - Px[..., p, :]).norm(dim=-1)).abs().mean())
|
| 213 |
+
out["seam_bone_mm"] = float(torch.stack(se).mean()) * 1000
|
| 214 |
+
out["consistency_deg"] = self.consistency(y_raw)
|
| 215 |
+
out["consistency_gt_deg"] = self.consistency(x_raw)
|
| 216 |
+
return out
|
| 217 |
+
|
| 218 |
+
def consistency(self, x_raw):
|
| 219 |
+
"""angle between bone dir from positions and bone dir predicted by rotations"""
|
| 220 |
+
G = global_rots(local_rots(x_raw))
|
| 221 |
+
P = positions(x_raw)
|
| 222 |
+
angs = []
|
| 223 |
+
for j, p in enumerate(PARENTS):
|
| 224 |
+
if p < 0:
|
| 225 |
+
continue
|
| 226 |
+
dpos = F.normalize(P[..., j, :] - P[..., p, :], dim=-1)
|
| 227 |
+
drot = (G[..., p, :, :] @ self.rest[j].view(3, 1)).squeeze(-1)
|
| 228 |
+
angs.append(torch.rad2deg(torch.acos((dpos * drot).sum(-1).clamp(-1, 1))).mean())
|
| 229 |
+
return float(torch.stack(angs).mean())
|
| 230 |
+
|
| 231 |
+
|
| 232 |
+
# ------------------------------------------------------------------ train
|
| 233 |
+
def train(m, Ztr, Zva, steps, bs, lr, beta, log_every, mt):
|
| 234 |
+
dev = Ztr.device
|
| 235 |
+
m.to(dev).train()
|
| 236 |
+
opt = torch.optim.AdamW(m.parameters(), lr=lr)
|
| 237 |
+
sch = torch.optim.lr_scheduler.OneCycleLR(opt, max_lr=lr, total_steps=steps, pct_start=0.05)
|
| 238 |
+
t0 = time.time()
|
| 239 |
+
for s in range(1, steps + 1):
|
| 240 |
+
xb = Ztr[torch.randint(0, Ztr.shape[0], (bs,), device=dev)]
|
| 241 |
+
y, commit = m(xb)
|
| 242 |
+
loss = F.mse_loss(y, xb) + beta * commit
|
| 243 |
+
opt.zero_grad(set_to_none=True)
|
| 244 |
+
loss.backward()
|
| 245 |
+
torch.nn.utils.clip_grad_norm_(m.parameters(), 1.0)
|
| 246 |
+
opt.step(); sch.step()
|
| 247 |
+
if s == 20:
|
| 248 |
+
torch.cuda.synchronize()
|
| 249 |
+
sps = 20 / (time.time() - t0)
|
| 250 |
+
print(" speed %.1f step/s -> est %.0f s" % (sps, steps / sps), flush=True)
|
| 251 |
+
if log_every and s % log_every == 0:
|
| 252 |
+
r = evaluate(m, Zva, mt)
|
| 253 |
+
print(" step %5d pos %6.2f mm rot(global) %5.2f deg (%.0fs)"
|
| 254 |
+
% (s, r["mpjpe_mm"], r["rot_global_deg"], time.time() - t0), flush=True)
|
| 255 |
+
return time.time() - t0
|
| 256 |
+
|
| 257 |
+
|
| 258 |
+
@torch.no_grad()
|
| 259 |
+
def evaluate(m, Zva, mt, return_y=False):
|
| 260 |
+
m.eval()
|
| 261 |
+
ys = [m(Zva[i:i + 256])[0] for i in range(0, Zva.shape[0], 256)]
|
| 262 |
+
m.train()
|
| 263 |
+
y = torch.cat(ys)
|
| 264 |
+
r = mt.all(mt.denorm(y), mt.denorm(Zva))
|
| 265 |
+
return (r, y) if return_y else r
|
| 266 |
+
|
| 267 |
+
|
| 268 |
+
def main():
|
| 269 |
+
ap = argparse.ArgumentParser()
|
| 270 |
+
ap.add_argument("--frames", type=int, default=500000)
|
| 271 |
+
ap.add_argument("--steps", type=int, default=6000)
|
| 272 |
+
ap.add_argument("--bs", type=int, default=256)
|
| 273 |
+
ap.add_argument("--lr", type=float, default=4e-4)
|
| 274 |
+
ap.add_argument("--beta", type=float, default=0.25)
|
| 275 |
+
ap.add_argument("--log-every", type=int, default=1000)
|
| 276 |
+
ap.add_argument("--models", nargs="+", default=["H", "M"])
|
| 277 |
+
ap.add_argument("--m-ch", type=int, default=0, help="0 = auto-match H params")
|
| 278 |
+
ap.add_argument("--tag", default="t272_v1")
|
| 279 |
+
ap.add_argument("--probe", action="store_true", help="only print data/param info")
|
| 280 |
+
args = ap.parse_args()
|
| 281 |
+
dev = "cuda"
|
| 282 |
+
t0 = time.time()
|
| 283 |
+
|
| 284 |
+
mean = torch.from_numpy(np.load(os.path.join(ROOT, "mean_std", "Mean.npy")).astype(np.float32)).to(dev)
|
| 285 |
+
std = torch.from_numpy(np.load(os.path.join(ROOT, "mean_std", "Std.npy")).astype(np.float32)).to(dev)
|
| 286 |
+
Wtr, ntr = load_split("train", args.frames, 16)
|
| 287 |
+
Wva, nva = load_split("val", max(40000, args.frames // 10), T)
|
| 288 |
+
Xtr = torch.from_numpy(Wtr).to(dev)
|
| 289 |
+
Xva = torch.from_numpy(Wva).to(dev)
|
| 290 |
+
print("train windows %d (%d files) val windows %d (%d files) load %.0fs"
|
| 291 |
+
% (Xtr.shape[0], ntr, Xva.shape[0], nva, time.time() - t0), flush=True)
|
| 292 |
+
Ztr, Zva = (Xtr - mean) / std, (Xva - mean) / std
|
| 293 |
+
mt = Metrics(mean, std, Xtr)
|
| 294 |
+
print("GT self-consistency (deg): %.3f" % mt.consistency(Xva[:256]), flush=True)
|
| 295 |
+
R = local_rots(Xva[:64])
|
| 296 |
+
orth = (R @ R.transpose(-1, -2) - torch.eye(3, device=dev)).abs().max()
|
| 297 |
+
print("6D->matrix orthonormality max err: %.2e" % float(orth), flush=True)
|
| 298 |
+
del Xtr
|
| 299 |
+
|
| 300 |
+
hp = nparams(HierTok())
|
| 301 |
+
mch = args.m_ch
|
| 302 |
+
if mch == 0:
|
| 303 |
+
best = None
|
| 304 |
+
for ch in range(128, 260, 2):
|
| 305 |
+
d = abs(nparams(PartTok(ch=ch)) - hp)
|
| 306 |
+
if best is None or d < best[0]:
|
| 307 |
+
best = (d, ch)
|
| 308 |
+
mch = best[1]
|
| 309 |
+
print("params H %d M(ch=%d) %d diff %+.2f%%" % (hp, mch, nparams(PartTok(ch=mch)),
|
| 310 |
+
(nparams(PartTok(ch=mch)) - hp) / hp * 100), flush=True)
|
| 311 |
+
if args.probe:
|
| 312 |
+
return
|
| 313 |
+
|
| 314 |
+
results = {}
|
| 315 |
+
for name in args.models:
|
| 316 |
+
torch.manual_seed(0)
|
| 317 |
+
m = HierTok() if name == "H" else PartTok(ch=mch)
|
| 318 |
+
print("\n=== %s params %d ===" % (name, nparams(m)), flush=True)
|
| 319 |
+
secs = train(m, Ztr, Zva, args.steps, args.bs, args.lr, args.beta, args.log_every, mt)
|
| 320 |
+
r = evaluate(m, Zva, mt)
|
| 321 |
+
r.update(params=nparams(m), train_seconds=round(secs, 1))
|
| 322 |
+
results[name] = r
|
| 323 |
+
torch.save(m.state_dict(), os.path.join(CKPT_DIR, "%s_%s.pt" % (args.tag, name)))
|
| 324 |
+
print("%s: pos %.2f mm | rot local %.2f / global %.2f deg | seam %.2f mm | "
|
| 325 |
+
"consistency %.2f deg (GT %.2f) | %.0fs" % (
|
| 326 |
+
name, r["mpjpe_mm"], r["rot_local_deg"], r["rot_global_deg"],
|
| 327 |
+
r["seam_bone_mm"], r["consistency_deg"], r["consistency_gt_deg"], secs), flush=True)
|
| 328 |
+
for pn, v in r["per_part"].items():
|
| 329 |
+
print(" %-6s pos %7.2f mm rot %6.2f deg" % (pn, v["pos_mm"], v["rot_global_deg"]), flush=True)
|
| 330 |
+
del m
|
| 331 |
+
torch.cuda.empty_cache()
|
| 332 |
+
with open(os.path.join(OUTDIR, args.tag + ".json"), "w", encoding="utf-8") as f:
|
| 333 |
+
json.dump(dict(args=vars(args), m_ch=mch, results=results,
|
| 334 |
+
total_seconds=round(time.time() - t0, 1)), f, ensure_ascii=False, indent=2)
|
| 335 |
+
print("done total %.0fs" % (time.time() - t0))
|
| 336 |
+
|
| 337 |
+
|
| 338 |
+
if __name__ == "__main__":
|
| 339 |
+
main()
|
code/tvq272c.py
ADDED
|
@@ -0,0 +1,140 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Stage 1: causal version of H (sees only current + past frames), for endless streaming.
|
| 3 |
+
Same layers / same parameter count as tvq272.HierTok; only padding moves to the left.
|
| 4 |
+
Compared against the already-measured H (runs/t272_v1.json) - H is NOT retrained.
|
| 5 |
+
|
| 6 |
+
Pass rule (fixed before running):
|
| 7 |
+
pos <= 61 mm, seam <= 12 mm, consistency <= 5.5 deg,
|
| 8 |
+
causality check: changing frames >= t never changes outputs before the token boundary.
|
| 9 |
+
"""
|
| 10 |
+
import os, sys, json, time, argparse
|
| 11 |
+
import numpy as np
|
| 12 |
+
import torch
|
| 13 |
+
import torch.nn as nn
|
| 14 |
+
import torch.nn.functional as F
|
| 15 |
+
|
| 16 |
+
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
| 17 |
+
import vq_mix as V
|
| 18 |
+
import tvq
|
| 19 |
+
import tvq272 as B
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
class CConv(nn.Module):
|
| 23 |
+
"""Conv1d with all padding on the left -> output i depends on inputs <= i*stride + (stride-1)."""
|
| 24 |
+
def __init__(self, cin, cout, k, s=1, dil=1):
|
| 25 |
+
super().__init__()
|
| 26 |
+
self.pad = (k - 1) * dil - (s - 1)
|
| 27 |
+
self.conv = nn.Conv1d(cin, cout, k, s, 0, dilation=dil)
|
| 28 |
+
|
| 29 |
+
def forward(self, x):
|
| 30 |
+
return self.conv(F.pad(x, (self.pad, 0)))
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
class CRes(nn.Module):
|
| 34 |
+
def __init__(self, ch, dil):
|
| 35 |
+
super().__init__()
|
| 36 |
+
self.c1 = CConv(ch, ch, 3, 1, dil)
|
| 37 |
+
self.c2 = nn.Conv1d(ch, ch, 1)
|
| 38 |
+
|
| 39 |
+
def forward(self, x):
|
| 40 |
+
return x + self.c2(F.relu(self.c1(F.relu(x))))
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def cenc(cin, ch, d):
|
| 44 |
+
L = [CConv(cin, ch, 3), nn.ReLU()]
|
| 45 |
+
for _ in range(2):
|
| 46 |
+
L += [CConv(ch, ch, 4, 2), CRes(ch, 1), CRes(ch, 3)]
|
| 47 |
+
L += [CConv(ch, d, 3)]
|
| 48 |
+
return nn.Sequential(*L)
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def cdec(d, ch, cout):
|
| 52 |
+
L = [CConv(d, ch, 3), nn.ReLU()]
|
| 53 |
+
for _ in range(2):
|
| 54 |
+
L += [CRes(ch, 3), CRes(ch, 1), nn.Upsample(scale_factor=2, mode="nearest"), CConv(ch, ch, 3)]
|
| 55 |
+
L += [nn.ReLU(), CConv(ch, cout, 3)]
|
| 56 |
+
return nn.Sequential(*L)
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
class CHierTok(nn.Module):
|
| 60 |
+
def __init__(self, kg=4096, kp=128, d=128, ch=128, gch=256):
|
| 61 |
+
super().__init__()
|
| 62 |
+
self.genc = cenc(272, gch, d)
|
| 63 |
+
self.gvq = V.VQ(kg, d)
|
| 64 |
+
self.enc = nn.ModuleList([cenc(len(c), ch, d) for c in B.PD])
|
| 65 |
+
self.vq = nn.ModuleList([V.VQ(kp, d) for _ in B.PD])
|
| 66 |
+
self.dec = nn.ModuleList([cdec(2 * d, ch, len(c)) for c in B.PD])
|
| 67 |
+
|
| 68 |
+
def forward(self, x):
|
| 69 |
+
qg, cg, _ = tvq.vq_seq(self.gvq, self.genc(x.permute(0, 2, 1)))
|
| 70 |
+
y = torch.empty_like(x)
|
| 71 |
+
commit = cg
|
| 72 |
+
for p, c in enumerate(B.PD):
|
| 73 |
+
q, cm, _ = tvq.vq_seq(self.vq[p], self.enc[p](x[..., c].permute(0, 2, 1)))
|
| 74 |
+
commit = commit + cm
|
| 75 |
+
y[..., c] = self.dec[p](torch.cat([qg, q], 1)).permute(0, 2, 1)
|
| 76 |
+
return y, commit / 7
|
| 77 |
+
|
| 78 |
+
|
| 79 |
+
@torch.no_grad()
|
| 80 |
+
def causality_check(m, Zva):
|
| 81 |
+
"""perturb frames from cut onward; outputs before the token block containing cut must not change."""
|
| 82 |
+
m.eval()
|
| 83 |
+
x = Zva[:32].clone()
|
| 84 |
+
y0 = m(x)[0]
|
| 85 |
+
worst = 0.0
|
| 86 |
+
for cut in (8, 20, 33, 47, 60):
|
| 87 |
+
x2 = x.clone()
|
| 88 |
+
x2[:, cut:] = torch.randn_like(x2[:, cut:]) * 3
|
| 89 |
+
y2 = m(x2)[0]
|
| 90 |
+
safe = (cut // 4) * 4 # frames of tokens fully before cut
|
| 91 |
+
worst = max(worst, float((y2[:, :safe] - y0[:, :safe]).abs().max()))
|
| 92 |
+
m.train()
|
| 93 |
+
return worst
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
def main():
|
| 97 |
+
ap = argparse.ArgumentParser()
|
| 98 |
+
ap.add_argument("--frames", type=int, default=500000)
|
| 99 |
+
ap.add_argument("--steps", type=int, default=6000)
|
| 100 |
+
ap.add_argument("--tag", default="t272_causal_v1")
|
| 101 |
+
ap.add_argument("--check-only", action="store_true")
|
| 102 |
+
a = ap.parse_args()
|
| 103 |
+
dev = "cuda"
|
| 104 |
+
t0 = time.time()
|
| 105 |
+
mean = torch.from_numpy(np.load(os.path.join(B.ROOT, "mean_std", "Mean.npy")).astype(np.float32)).to(dev)
|
| 106 |
+
std = torch.from_numpy(np.load(os.path.join(B.ROOT, "mean_std", "Std.npy")).astype(np.float32)).to(dev)
|
| 107 |
+
Wtr, ntr = B.load_split("train", a.frames, 16) # same data / seed as t272_v1
|
| 108 |
+
Wva, nva = B.load_split("val", max(40000, a.frames // 10), B.T)
|
| 109 |
+
Xtr = torch.from_numpy(Wtr).to(dev); Xva = torch.from_numpy(Wva).to(dev)
|
| 110 |
+
print("train %d win val %d win load %.0fs" % (Xtr.shape[0], Xva.shape[0], time.time() - t0), flush=True)
|
| 111 |
+
Ztr, Zva = (Xtr - mean) / std, (Xva - mean) / std
|
| 112 |
+
mt = B.Metrics(mean, std, Xtr)
|
| 113 |
+
del Xtr
|
| 114 |
+
|
| 115 |
+
torch.manual_seed(0)
|
| 116 |
+
m = CHierTok().to(dev)
|
| 117 |
+
print("params causal H %d (non-causal H %d)" % (B.nparams(m), B.nparams(B.HierTok())), flush=True)
|
| 118 |
+
print("causality (untrained) max diff %.2e" % causality_check(m, Zva), flush=True)
|
| 119 |
+
if a.check_only:
|
| 120 |
+
return
|
| 121 |
+
secs = B.train(m, Ztr, Zva, a.steps, 256, 4e-4, 0.25, 1000, mt)
|
| 122 |
+
r = B.evaluate(m, Zva, mt)
|
| 123 |
+
r.update(params=B.nparams(m), train_seconds=round(secs, 1), causality_maxdiff=causality_check(m, Zva))
|
| 124 |
+
torch.save(m.state_dict(), os.path.join(B.CKPT_DIR, a.tag + "_H.pt"))
|
| 125 |
+
ref = json.load(open(os.path.join(B.OUTDIR, "t272_v1.json"), encoding="utf-8"))["results"]["H"]
|
| 126 |
+
r["pass"] = dict(pos=r["mpjpe_mm"] <= 61, seam=r["seam_bone_mm"] <= 12,
|
| 127 |
+
consistency=r["consistency_deg"] <= 5.5, causal=r["causality_maxdiff"] == 0.0)
|
| 128 |
+
json.dump(dict(causal_H=r, ref_H=ref), open(os.path.join(B.OUTDIR, a.tag + ".json"), "w", encoding="utf-8"),
|
| 129 |
+
ensure_ascii=False, indent=2)
|
| 130 |
+
print("causal H: pos %.2f (ref %.2f) | rot g %.2f (ref %.2f) | seam %.2f (ref %.2f) | consist %.2f (ref %.2f) | causal diff %.1e | %.0fs"
|
| 131 |
+
% (r["mpjpe_mm"], ref["mpjpe_mm"], r["rot_global_deg"], ref["rot_global_deg"], r["seam_bone_mm"],
|
| 132 |
+
ref["seam_bone_mm"], r["consistency_deg"], ref["consistency_deg"], r["causality_maxdiff"], secs), flush=True)
|
| 133 |
+
for pn, v in r["per_part"].items():
|
| 134 |
+
print(" %-6s pos %7.2f rot %6.2f (ref %7.2f / %6.2f)" % (pn, v["pos_mm"], v["rot_global_deg"],
|
| 135 |
+
ref["per_part"][pn]["pos_mm"], ref["per_part"][pn]["rot_global_deg"]), flush=True)
|
| 136 |
+
print("PASS" if all(r["pass"].values()) else "FAIL", r["pass"])
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
if __name__ == "__main__":
|
| 140 |
+
main()
|
code/vq_mix.py
ADDED
|
@@ -0,0 +1,430 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Experiment A: does an encoder that mixes across parts close the gap between
|
| 3 |
+
per-part and whole-body quantization?
|
| 4 |
+
|
| 5 |
+
rd_joints_full.json (raw-coordinate k-means, in-sample) showed per-part VQ is
|
| 6 |
+
far worse than whole-body VQ at matched bitrate (12 bits: 61.17 vs 151.70 mm).
|
| 7 |
+
That test had no encoder. MoGenTS claims its joint-axis attention recovers the
|
| 8 |
+
cross-part information. This measures how much.
|
| 9 |
+
|
| 10 |
+
Conditions
|
| 11 |
+
C whole-body VQ-VAE (baseline)
|
| 12 |
+
A per-part VQ-VAE, no cross-part path at all
|
| 13 |
+
B per-part VQ-VAE, attention over part tokens in encoder AND decoder
|
| 14 |
+
B_enc per-part VQ-VAE, attention in encoder only (parts stay independently
|
| 15 |
+
decodable -> the version compositional control would need)
|
| 16 |
+
|
| 17 |
+
Plus KM_whole / KM_parts: raw-coordinate k-means, re-measured on the SAME
|
| 18 |
+
held-out split so every number in this file is comparable.
|
| 19 |
+
|
| 20 |
+
Data: HumanML3D new_joints, root-relative, 21 joints x 3, metres.
|
| 21 |
+
Metric: MPJPE in mm on held-out frames. Loss is MSE on positions, so the
|
| 22 |
+
training objective and the metric are the same quantity.
|
| 23 |
+
"""
|
| 24 |
+
import os
|
| 25 |
+
import glob
|
| 26 |
+
import json
|
| 27 |
+
import time
|
| 28 |
+
import math
|
| 29 |
+
import argparse
|
| 30 |
+
import numpy as np
|
| 31 |
+
import torch
|
| 32 |
+
import torch.nn as nn
|
| 33 |
+
import torch.nn.functional as F
|
| 34 |
+
BASE = os.environ.get("PARTLOCK_DIR", ".") # folder that holds data/, ckpt/, runs/
|
| 35 |
+
|
| 36 |
+
DEST = os.path.join(BASE, "data", "motion_raw", "humanml3d", "HumanML3D", "new_joints")
|
| 37 |
+
OUTDIR = os.path.join(BASE, "runs")
|
| 38 |
+
|
| 39 |
+
PART6 = {
|
| 40 |
+
"torso": [3, 6, 9],
|
| 41 |
+
"head": [12, 15],
|
| 42 |
+
"arm_R": [14, 17, 19, 21],
|
| 43 |
+
"arm_L": [13, 16, 18, 20],
|
| 44 |
+
"leg_R": [2, 5, 8, 11],
|
| 45 |
+
"leg_L": [1, 4, 7, 10],
|
| 46 |
+
}
|
| 47 |
+
PART_NAMES = list(PART6.keys())
|
| 48 |
+
COLS = [[j - 1 for j in PART6[p]] for p in PART_NAMES] # into 0..20
|
| 49 |
+
NJ = [len(c) for c in COLS] # [3,2,4,4,4,4]
|
| 50 |
+
assert sum(NJ) == 21
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
# ----------------------------------------------------------------- data
|
| 54 |
+
def load_split(max_frames, val_frac=0.1, seed=0):
|
| 55 |
+
files = [p for p in glob.glob(os.path.join(DEST, "**", "*.npy"),
|
| 56 |
+
recursive=True)
|
| 57 |
+
if not os.path.basename(p).startswith(".")]
|
| 58 |
+
files.sort()
|
| 59 |
+
rng = np.random.default_rng(seed)
|
| 60 |
+
rng.shuffle(files)
|
| 61 |
+
|
| 62 |
+
# split by FILE so val frames come from unseen motions
|
| 63 |
+
n_val_files = max(1, int(len(files) * val_frac))
|
| 64 |
+
val_files = files[:n_val_files]
|
| 65 |
+
train_files = files[n_val_files:]
|
| 66 |
+
|
| 67 |
+
def grab(fs, budget):
|
| 68 |
+
chunks, total, used = [], 0, 0
|
| 69 |
+
for p in fs:
|
| 70 |
+
try:
|
| 71 |
+
a = np.load(p)
|
| 72 |
+
except Exception:
|
| 73 |
+
continue
|
| 74 |
+
if a.ndim != 3 or a.shape[1] != 22 or a.shape[2] != 3:
|
| 75 |
+
continue
|
| 76 |
+
chunks.append(a.astype(np.float32))
|
| 77 |
+
total += a.shape[0]
|
| 78 |
+
used += 1
|
| 79 |
+
if total >= budget:
|
| 80 |
+
break
|
| 81 |
+
X = np.concatenate(chunks, axis=0)
|
| 82 |
+
if X.shape[0] > budget:
|
| 83 |
+
X = X[np.random.default_rng(seed).choice(X.shape[0], budget,
|
| 84 |
+
replace=False)]
|
| 85 |
+
return X, used
|
| 86 |
+
|
| 87 |
+
Xtr, ntr = grab(train_files, max_frames)
|
| 88 |
+
Xva, nva = grab(val_files, max(20000, int(max_frames * 0.1)))
|
| 89 |
+
rel = lambda X: (X - X[:, 0:1, :])[:, 1:, :]
|
| 90 |
+
return rel(Xtr), rel(Xva), ntr, nva, len(files)
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
# ----------------------------------------------------------------- k-means
|
| 94 |
+
def assign_chunked(X, C):
|
| 95 |
+
k = C.shape[0]
|
| 96 |
+
chunk = max(256, (1 << 25) // max(k, 1))
|
| 97 |
+
cn = (C * C).sum(1)
|
| 98 |
+
out = torch.empty(X.shape[0], dtype=torch.long, device=X.device)
|
| 99 |
+
for i in range(0, X.shape[0], chunk):
|
| 100 |
+
d = cn.unsqueeze(0) - 2.0 * (X[i:i + chunk] @ C.T)
|
| 101 |
+
out[i:i + chunk] = d.argmin(1)
|
| 102 |
+
return out
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
def kmeans_fit(X, k, iters=30, seed=0):
|
| 106 |
+
n = X.shape[0]
|
| 107 |
+
g = torch.Generator(device=X.device).manual_seed(seed)
|
| 108 |
+
C = X[torch.randperm(n, generator=g, device=X.device)[:k]].clone()
|
| 109 |
+
for _ in range(iters):
|
| 110 |
+
a = assign_chunked(X, C)
|
| 111 |
+
Cn = torch.zeros_like(C)
|
| 112 |
+
cnt = torch.zeros(k, device=X.device, dtype=X.dtype)
|
| 113 |
+
Cn.index_add_(0, a, X)
|
| 114 |
+
cnt.index_add_(0, a, torch.ones(n, device=X.device, dtype=X.dtype))
|
| 115 |
+
nz = cnt > 0
|
| 116 |
+
Cn[nz] /= cnt[nz].unsqueeze(1)
|
| 117 |
+
ne = int((~nz).sum())
|
| 118 |
+
if ne:
|
| 119 |
+
Cn[~nz] = X[torch.randint(0, n, (ne,), generator=g,
|
| 120 |
+
device=X.device)]
|
| 121 |
+
C = Cn
|
| 122 |
+
return C
|
| 123 |
+
|
| 124 |
+
|
| 125 |
+
# ----------------------------------------------------------------- VQ (EMA)
|
| 126 |
+
class VQ(nn.Module):
|
| 127 |
+
def __init__(self, k, d, decay=0.99, eps=1e-5):
|
| 128 |
+
super().__init__()
|
| 129 |
+
self.k, self.d, self.decay, self.eps = k, d, decay, eps
|
| 130 |
+
self.register_buffer("emb", torch.randn(k, d) * 0.1)
|
| 131 |
+
self.register_buffer("cluster_size", torch.zeros(k))
|
| 132 |
+
self.register_buffer("emb_avg", self.emb.clone())
|
| 133 |
+
self.register_buffer("inited", torch.zeros(1))
|
| 134 |
+
|
| 135 |
+
@torch.no_grad()
|
| 136 |
+
def _init_from(self, z):
|
| 137 |
+
n = z.shape[0]
|
| 138 |
+
idx = torch.randperm(n, device=z.device)[:self.k]
|
| 139 |
+
if idx.numel() < self.k:
|
| 140 |
+
idx = torch.randint(0, n, (self.k,), device=z.device)
|
| 141 |
+
self.emb.copy_(z[idx])
|
| 142 |
+
self.emb_avg.copy_(self.emb)
|
| 143 |
+
self.cluster_size.fill_(1.0)
|
| 144 |
+
self.inited.fill_(1)
|
| 145 |
+
|
| 146 |
+
def forward(self, z):
|
| 147 |
+
# z: (N, d)
|
| 148 |
+
if self.training and self.inited.item() == 0:
|
| 149 |
+
self._init_from(z.detach())
|
| 150 |
+
d2 = (z * z).sum(1, keepdim=True) - 2 * z @ self.emb.T \
|
| 151 |
+
+ (self.emb * self.emb).sum(1).unsqueeze(0)
|
| 152 |
+
idx = d2.argmin(1)
|
| 153 |
+
zq = self.emb[idx]
|
| 154 |
+
if self.training:
|
| 155 |
+
with torch.no_grad():
|
| 156 |
+
oh = F.one_hot(idx, self.k).to(z.dtype)
|
| 157 |
+
cs = oh.sum(0)
|
| 158 |
+
ea = oh.T @ z.detach()
|
| 159 |
+
self.cluster_size.mul_(self.decay).add_(cs, alpha=1 - self.decay)
|
| 160 |
+
self.emb_avg.mul_(self.decay).add_(ea, alpha=1 - self.decay)
|
| 161 |
+
n = self.cluster_size.sum()
|
| 162 |
+
cs_n = (self.cluster_size + self.eps) / (n + self.k * self.eps) * n
|
| 163 |
+
self.emb.copy_(self.emb_avg / cs_n.unsqueeze(1))
|
| 164 |
+
# revive dead codes
|
| 165 |
+
dead = self.cluster_size < 0.05
|
| 166 |
+
nd = int(dead.sum())
|
| 167 |
+
if nd:
|
| 168 |
+
r = torch.randint(0, z.shape[0], (nd,), device=z.device)
|
| 169 |
+
self.emb[dead] = z.detach()[r]
|
| 170 |
+
self.emb_avg[dead] = self.emb[dead]
|
| 171 |
+
self.cluster_size[dead] = 1.0
|
| 172 |
+
commit = F.mse_loss(z, zq.detach())
|
| 173 |
+
zq = z + (zq - z).detach() # straight-through
|
| 174 |
+
return zq, commit, idx
|
| 175 |
+
|
| 176 |
+
|
| 177 |
+
# ----------------------------------------------------------------- blocks
|
| 178 |
+
def mlp(i, h, o):
|
| 179 |
+
return nn.Sequential(nn.Linear(i, h), nn.GELU(), nn.Linear(h, h),
|
| 180 |
+
nn.GELU(), nn.Linear(h, o))
|
| 181 |
+
|
| 182 |
+
|
| 183 |
+
class MixBlock(nn.Module):
|
| 184 |
+
"""Self-attention over the 6 part tokens + FFN. Pre-norm."""
|
| 185 |
+
def __init__(self, d, heads=4):
|
| 186 |
+
super().__init__()
|
| 187 |
+
self.n1 = nn.LayerNorm(d)
|
| 188 |
+
self.at = nn.MultiheadAttention(d, heads, batch_first=True)
|
| 189 |
+
self.n2 = nn.LayerNorm(d)
|
| 190 |
+
self.ff = nn.Sequential(nn.Linear(d, d * 2), nn.GELU(),
|
| 191 |
+
nn.Linear(d * 2, d))
|
| 192 |
+
|
| 193 |
+
def forward(self, x):
|
| 194 |
+
h = self.n1(x)
|
| 195 |
+
x = x + self.at(h, h, h, need_weights=False)[0]
|
| 196 |
+
x = x + self.ff(self.n2(x))
|
| 197 |
+
return x
|
| 198 |
+
|
| 199 |
+
|
| 200 |
+
class WholeVQVAE(nn.Module):
|
| 201 |
+
"""Condition C."""
|
| 202 |
+
def __init__(self, k, d=64, h=256):
|
| 203 |
+
super().__init__()
|
| 204 |
+
self.enc = mlp(63, h, d)
|
| 205 |
+
self.vq = VQ(k, d)
|
| 206 |
+
self.dec = mlp(d, h, 63)
|
| 207 |
+
self.n_codes_per_frame = 1
|
| 208 |
+
|
| 209 |
+
def forward(self, x): # x: (B,21,3)
|
| 210 |
+
B = x.shape[0]
|
| 211 |
+
z = self.enc(x.reshape(B, -1))
|
| 212 |
+
zq, commit, _ = self.vq(z)
|
| 213 |
+
y = self.dec(zq).reshape(B, 21, 3)
|
| 214 |
+
return y, commit
|
| 215 |
+
|
| 216 |
+
|
| 217 |
+
class PartVQVAE(nn.Module):
|
| 218 |
+
"""Conditions A / B / B_enc. mix_enc / mix_dec toggle the cross-part path."""
|
| 219 |
+
def __init__(self, k, d=32, h=128, dt=128, mix_enc=False, mix_dec=False,
|
| 220 |
+
n_mix=2, heads=4):
|
| 221 |
+
super().__init__()
|
| 222 |
+
self.d, self.mix_enc, self.mix_dec = d, mix_enc, mix_dec
|
| 223 |
+
self.n_codes_per_frame = 6
|
| 224 |
+
self.e_in = nn.ModuleList([mlp(3 * n, h, dt) for n in NJ])
|
| 225 |
+
self.e_mix = nn.ModuleList([MixBlock(dt, heads) for _ in range(n_mix)]) \
|
| 226 |
+
if mix_enc else None
|
| 227 |
+
self.e_out = nn.ModuleList([nn.Linear(dt, d) for _ in NJ])
|
| 228 |
+
self.part_emb = nn.Parameter(torch.zeros(1, 6, dt))
|
| 229 |
+
self.vq = nn.ModuleList([VQ(k, d) for _ in NJ])
|
| 230 |
+
self.d_in = nn.ModuleList([nn.Linear(d, dt) for _ in NJ])
|
| 231 |
+
self.d_mix = nn.ModuleList([MixBlock(dt, heads) for _ in range(n_mix)]) \
|
| 232 |
+
if mix_dec else None
|
| 233 |
+
self.d_out = nn.ModuleList([mlp(dt, h, 3 * n) for n in NJ])
|
| 234 |
+
self.part_emb_d = nn.Parameter(torch.zeros(1, 6, dt))
|
| 235 |
+
|
| 236 |
+
def forward(self, x): # x: (B,21,3)
|
| 237 |
+
B = x.shape[0]
|
| 238 |
+
toks = []
|
| 239 |
+
for p, cols in enumerate(COLS):
|
| 240 |
+
toks.append(self.e_in[p](x[:, cols, :].reshape(B, -1)))
|
| 241 |
+
t = torch.stack(toks, 1) + self.part_emb # (B,6,dt)
|
| 242 |
+
if self.e_mix is not None:
|
| 243 |
+
for blk in self.e_mix:
|
| 244 |
+
t = blk(t)
|
| 245 |
+
commit = 0.0
|
| 246 |
+
zqs = []
|
| 247 |
+
for p in range(6):
|
| 248 |
+
z = self.e_out[p](t[:, p])
|
| 249 |
+
zq, c, _ = self.vq[p](z)
|
| 250 |
+
commit = commit + c
|
| 251 |
+
zqs.append(self.d_in[p](zq))
|
| 252 |
+
u = torch.stack(zqs, 1) + self.part_emb_d
|
| 253 |
+
if self.d_mix is not None:
|
| 254 |
+
for blk in self.d_mix:
|
| 255 |
+
u = blk(u)
|
| 256 |
+
y = x.new_empty(B, 21, 3)
|
| 257 |
+
for p, cols in enumerate(COLS):
|
| 258 |
+
y[:, cols, :] = self.d_out[p](u[:, p]).reshape(B, len(cols), 3)
|
| 259 |
+
return y, commit / 6.0
|
| 260 |
+
|
| 261 |
+
|
| 262 |
+
# ----------------------------------------------------------------- train/eval
|
| 263 |
+
def mpjpe_mm(y, x):
|
| 264 |
+
return float(((y - x) ** 2).sum(-1).mean().sqrt()) * 1000
|
| 265 |
+
|
| 266 |
+
|
| 267 |
+
def eval_mm(model, Xva):
|
| 268 |
+
model.eval()
|
| 269 |
+
se, cnt = 0.0, 0
|
| 270 |
+
with torch.no_grad():
|
| 271 |
+
for i in range(0, Xva.shape[0], 8192):
|
| 272 |
+
xb = Xva[i:i + 8192]
|
| 273 |
+
y, _ = model(xb)
|
| 274 |
+
se += float(((y - xb) ** 2).sum(-1).sum())
|
| 275 |
+
cnt += xb.shape[0] * 21
|
| 276 |
+
model.train()
|
| 277 |
+
return (se / cnt) ** 0.5 * 1000
|
| 278 |
+
|
| 279 |
+
|
| 280 |
+
def train_one(model, Xtr, Xva, epochs, bs, lr, beta, dev, log_every=0):
|
| 281 |
+
model.to(dev).train()
|
| 282 |
+
opt = torch.optim.AdamW(model.parameters(), lr=lr, weight_decay=0.0)
|
| 283 |
+
n = Xtr.shape[0]
|
| 284 |
+
steps = max(1, (n // bs)) * epochs
|
| 285 |
+
sch = torch.optim.lr_scheduler.OneCycleLR(
|
| 286 |
+
opt, max_lr=lr, total_steps=steps, pct_start=0.1)
|
| 287 |
+
step = 0
|
| 288 |
+
for ep in range(epochs):
|
| 289 |
+
perm = torch.randperm(n, device=dev)
|
| 290 |
+
for i in range(0, n - bs + 1, bs):
|
| 291 |
+
xb = Xtr[perm[i:i + bs]]
|
| 292 |
+
y, commit = model(xb)
|
| 293 |
+
rec = F.mse_loss(y, xb)
|
| 294 |
+
loss = rec + beta * commit
|
| 295 |
+
opt.zero_grad(set_to_none=True)
|
| 296 |
+
loss.backward()
|
| 297 |
+
torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
|
| 298 |
+
opt.step()
|
| 299 |
+
sch.step()
|
| 300 |
+
step += 1
|
| 301 |
+
if log_every and (ep + 1) % log_every == 0:
|
| 302 |
+
print(" ep %3d / %d step %5d train_rec %.6f "
|
| 303 |
+
"val %7.2f mm" % (ep + 1, epochs, step, float(rec),
|
| 304 |
+
eval_mm(model, Xva)), flush=True)
|
| 305 |
+
return eval_mm(model, Xva)
|
| 306 |
+
|
| 307 |
+
|
| 308 |
+
def main():
|
| 309 |
+
ap = argparse.ArgumentParser()
|
| 310 |
+
ap.add_argument("--frames", type=int, default=400000)
|
| 311 |
+
ap.add_argument("--epochs", type=int, default=15)
|
| 312 |
+
ap.add_argument("--bs", type=int, default=1024)
|
| 313 |
+
ap.add_argument("--lr", type=float, default=3e-4)
|
| 314 |
+
ap.add_argument("--beta", type=float, default=0.25)
|
| 315 |
+
ap.add_argument("--bits", type=int, nargs="+", default=[6, 12, 24, 36])
|
| 316 |
+
ap.add_argument("--km-iters", type=int, default=30)
|
| 317 |
+
ap.add_argument("--log-every", type=int, default=0)
|
| 318 |
+
ap.add_argument("--tag", default="vq_mix_v1")
|
| 319 |
+
args = ap.parse_args()
|
| 320 |
+
|
| 321 |
+
dev = "cuda" if torch.cuda.is_available() else "cpu"
|
| 322 |
+
torch.manual_seed(0)
|
| 323 |
+
t0 = time.time()
|
| 324 |
+
|
| 325 |
+
Xtr_np, Xva_np, ntr, nva, nfiles = load_split(args.frames)
|
| 326 |
+
Xtr = torch.from_numpy(np.ascontiguousarray(Xtr_np)).to(dev)
|
| 327 |
+
Xva = torch.from_numpy(np.ascontiguousarray(Xva_np)).to(dev)
|
| 328 |
+
print("device %s train %d frames / %d files val %d frames / %d files "
|
| 329 |
+
"(files total %d) load_s %.1f"
|
| 330 |
+
% (dev, Xtr.shape[0], ntr, Xva.shape[0], nva, nfiles,
|
| 331 |
+
time.time() - t0), flush=True)
|
| 332 |
+
print("NOTE: val motions are different FILES from train -> held out.\n",
|
| 333 |
+
flush=True)
|
| 334 |
+
|
| 335 |
+
results = []
|
| 336 |
+
|
| 337 |
+
def rec(cond, bits, k, mm, secs, extra=None):
|
| 338 |
+
row = dict(cond=cond, bits_per_frame=bits, k=k,
|
| 339 |
+
codes_per_frame=(1 if cond.endswith("whole") or cond == "C"
|
| 340 |
+
else 6),
|
| 341 |
+
mpjpe_mm=round(mm, 3), seconds=round(secs, 1))
|
| 342 |
+
if extra:
|
| 343 |
+
row.update(extra)
|
| 344 |
+
results.append(row)
|
| 345 |
+
print("%-10s bits=%-3d k=%-6d val MPJPE = %8.2f mm (%.1fs)"
|
| 346 |
+
% (cond, bits, k, mm, secs), flush=True)
|
| 347 |
+
|
| 348 |
+
# ---- k-means baselines on the SAME split (held-out eval) ----
|
| 349 |
+
print("=== k-means baselines (raw coordinates, held-out eval) ===",
|
| 350 |
+
flush=True)
|
| 351 |
+
tr_whole = Xtr.reshape(Xtr.shape[0], -1).contiguous()
|
| 352 |
+
va_whole = Xva.reshape(Xva.shape[0], -1).contiguous()
|
| 353 |
+
for bits in args.bits:
|
| 354 |
+
kw = 1 << bits
|
| 355 |
+
if kw > 65536:
|
| 356 |
+
print("KM_whole bits=%-3d k=%d -> skipped (codebook too large)"
|
| 357 |
+
% (bits, kw), flush=True)
|
| 358 |
+
continue
|
| 359 |
+
ts = time.time()
|
| 360 |
+
C = kmeans_fit(tr_whole, kw, iters=args.km_iters)
|
| 361 |
+
a = assign_chunked(va_whole, C)
|
| 362 |
+
y = C[a].reshape(Xva.shape)
|
| 363 |
+
rec("KM_whole", bits, kw, mpjpe_mm(y, Xva), time.time() - ts)
|
| 364 |
+
|
| 365 |
+
for bits in args.bits:
|
| 366 |
+
kp = 1 << (bits // 6)
|
| 367 |
+
if bits % 6:
|
| 368 |
+
continue
|
| 369 |
+
ts = time.time()
|
| 370 |
+
y = torch.empty_like(Xva)
|
| 371 |
+
for p, cols in enumerate(COLS):
|
| 372 |
+
Ctr = Xtr[:, cols, :].reshape(Xtr.shape[0], -1).contiguous()
|
| 373 |
+
Cva = Xva[:, cols, :].reshape(Xva.shape[0], -1).contiguous()
|
| 374 |
+
C = kmeans_fit(Ctr, kp, iters=args.km_iters)
|
| 375 |
+
a = assign_chunked(Cva, C)
|
| 376 |
+
y[:, cols, :] = C[a].reshape(Xva.shape[0], len(cols), 3)
|
| 377 |
+
rec("KM_parts", bits, kp, mpjpe_mm(y, Xva), time.time() - ts)
|
| 378 |
+
|
| 379 |
+
# ---- VQ-VAE conditions ----
|
| 380 |
+
print("\n=== VQ-VAE (%d epochs, bs %d, lr %g, beta %g) ==="
|
| 381 |
+
% (args.epochs, args.bs, args.lr, args.beta), flush=True)
|
| 382 |
+
for bits in args.bits:
|
| 383 |
+
# C: whole-body, k = 2^bits
|
| 384 |
+
kw = 1 << bits
|
| 385 |
+
if kw <= 65536:
|
| 386 |
+
ts = time.time()
|
| 387 |
+
torch.manual_seed(0)
|
| 388 |
+
m = WholeVQVAE(kw)
|
| 389 |
+
print(" -- C bits=%d k=%d --" % (bits, kw), flush=True)
|
| 390 |
+
mm = train_one(m, Xtr, Xva, args.epochs, args.bs, args.lr,
|
| 391 |
+
args.beta, dev, args.log_every)
|
| 392 |
+
rec("C", bits, kw, mm, time.time() - ts,
|
| 393 |
+
dict(params=sum(p.numel() for p in m.parameters())))
|
| 394 |
+
else:
|
| 395 |
+
print("C bits=%-3d k=%d -> skipped (codebook too large)"
|
| 396 |
+
% (bits, kw), flush=True)
|
| 397 |
+
|
| 398 |
+
if bits % 6:
|
| 399 |
+
continue
|
| 400 |
+
kp = 1 << (bits // 6)
|
| 401 |
+
for cond, me, md in (("A", False, False),
|
| 402 |
+
("B", True, True),
|
| 403 |
+
("B_enc", True, False)):
|
| 404 |
+
ts = time.time()
|
| 405 |
+
torch.manual_seed(0)
|
| 406 |
+
m = PartVQVAE(kp, mix_enc=me, mix_dec=md)
|
| 407 |
+
print(" -- %s bits=%d k=%d --" % (cond, bits, kp), flush=True)
|
| 408 |
+
mm = train_one(m, Xtr, Xva, args.epochs, args.bs, args.lr,
|
| 409 |
+
args.beta, dev, args.log_every)
|
| 410 |
+
rec(cond, bits, kp, mm, time.time() - ts,
|
| 411 |
+
dict(params=sum(p.numel() for p in m.parameters()),
|
| 412 |
+
mix_enc=me, mix_dec=md))
|
| 413 |
+
|
| 414 |
+
os.makedirs(OUTDIR, exist_ok=True)
|
| 415 |
+
out = os.path.join(OUTDIR, args.tag + ".json")
|
| 416 |
+
with open(out, "w", encoding="utf-8") as f:
|
| 417 |
+
json.dump(dict(train_frames=int(Xtr.shape[0]),
|
| 418 |
+
val_frames=int(Xva.shape[0]),
|
| 419 |
+
train_files=ntr, val_files=nva, files_total=nfiles,
|
| 420 |
+
epochs=args.epochs, bs=args.bs, lr=args.lr,
|
| 421 |
+
beta=args.beta, km_iters=args.km_iters, device=dev,
|
| 422 |
+
parts=PART6,
|
| 423 |
+
total_seconds=round(time.time() - t0, 1),
|
| 424 |
+
results=results), f, ensure_ascii=False, indent=2)
|
| 425 |
+
print("\nwrote", out)
|
| 426 |
+
print("total_seconds %.1f" % (time.time() - t0))
|
| 427 |
+
|
| 428 |
+
|
| 429 |
+
if __name__ == "__main__":
|
| 430 |
+
main()
|
data/cmu_files.csv
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
generate.py
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Offline generation to a file.
|
| 2 |
+
|
| 3 |
+
python generate.py --sec 4 --out motion.npz "walk" "Wave" "run/jog" "jump"
|
| 4 |
+
|
| 5 |
+
Each prompt is used for --sec seconds, in order, in ONE continuous motion (the context is never reset).
|
| 6 |
+
Output .npz (30 fps):
|
| 7 |
+
positions (T, 22, 3) world joint positions in meters (y up), SMPL joint order
|
| 8 |
+
rotations (T, 22, 3, 3) world joint rotations
|
| 9 |
+
root_xz (T, 2) pelvis path on the floor
|
| 10 |
+
prompt (T,) prompt used for each frame
|
| 11 |
+
parents (22,) skeleton parents
|
| 12 |
+
"""
|
| 13 |
+
import os, sys, argparse
|
| 14 |
+
import numpy as np, torch
|
| 15 |
+
sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "live"))
|
| 16 |
+
import live_server as L # loads the models (GPU)
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def main():
|
| 20 |
+
ap = argparse.ArgumentParser()
|
| 21 |
+
ap.add_argument("prompts", nargs="+")
|
| 22 |
+
ap.add_argument("--sec", type=float, default=6.0, help="seconds per prompt")
|
| 23 |
+
ap.add_argument("--s", type=float, default=4.0, help="text strength (classifier-free guidance)")
|
| 24 |
+
ap.add_argument("--seed", type=int, default=1)
|
| 25 |
+
ap.add_argument("--out", default="motion.npz")
|
| 26 |
+
a = ap.parse_args()
|
| 27 |
+
L.ENG = L.Engine()
|
| 28 |
+
torch.manual_seed(a.seed)
|
| 29 |
+
w = L.reset_world()
|
| 30 |
+
steps = max(1, int(round(a.sec * 30 / 4)))
|
| 31 |
+
P, R, T, txt = [], [], [], []
|
| 32 |
+
for p in a.prompts:
|
| 33 |
+
cond = L.embed(L.tok, L.clip, [p]).float()
|
| 34 |
+
for _ in range(steps):
|
| 35 |
+
for f in L.one_step(w, cond, a.s):
|
| 36 |
+
rel = np.array(f["p"], np.float32).reshape(22, 3)
|
| 37 |
+
rel[:, 0] += f["tr"][0]; rel[:, 2] += f["tr"][1]
|
| 38 |
+
P.append(rel); R.append(np.array(f["r"], np.float32).reshape(22, 3, 3)); T.append(f["tr"]); txt.append(p)
|
| 39 |
+
np.savez_compressed(a.out, positions=np.stack(P), rotations=np.stack(R), root_xz=np.array(T, np.float32),
|
| 40 |
+
prompt=np.array(txt), parents=np.array(L.B.PARENTS), fps=30)
|
| 41 |
+
print("saved %s: %d frames (%.1f s)" % (a.out, len(P), len(P) / 30))
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
if __name__ == "__main__":
|
| 45 |
+
main()
|
images/boxing.gif
ADDED
|
Git LFS Details
|
images/jump.gif
ADDED
|
Git LFS Details
|
images/kick.gif
ADDED
|
Git LFS Details
|
images/live_ui.png
ADDED
|
Git LFS Details
|
images/run_jog.gif
ADDED
|
Git LFS Details
|
images/salsa.gif
ADDED
|
Git LFS Details
|
images/switch.gif
ADDED
|
Git LFS Details
|
images/walk.gif
ADDED
|
Git LFS Details
|
images/wave.gif
ADDED
|
Git LFS Details
|
live/live.html
ADDED
|
@@ -0,0 +1,276 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
<meta charset="utf-8">
|
| 2 |
+
<meta name="viewport" content="width=device-width, initial-scale=1, viewport-fit=cover">
|
| 3 |
+
<title>リアルタイム生成</title>
|
| 4 |
+
<meta name="claude:viewport-fit-sides" content="cover">
|
| 5 |
+
<style>
|
| 6 |
+
:root {
|
| 7 |
+
--bg: #f6f7f5; --panel: #ffffff; --ink: #161a18; --ink-2: #4c5551; --muted: #84908a;
|
| 8 |
+
--line: #dde2df; --fig: #0f8a74; --trail: #b9c1bd; --floor: #e3e8e5; --front: #e8742c;
|
| 9 |
+
--f-ui: "Inter", system-ui, -apple-system, "Segoe UI", sans-serif;
|
| 10 |
+
--f-head: "Space Grotesk", "Inter", system-ui, sans-serif;
|
| 11 |
+
--f-num: "JetBrains Mono", ui-monospace, Menlo, monospace;
|
| 12 |
+
}
|
| 13 |
+
@media (prefers-color-scheme: dark) {
|
| 14 |
+
:root:not([data-theme="light"]) {
|
| 15 |
+
--bg: #121514; --panel: #1a1e1c; --ink: #edf1ee; --ink-2: #b9c2bd; --muted: #7f8a85;
|
| 16 |
+
--line: #2b312e; --fig: #2fc4a6; --trail: #4d5551; --floor: #262c29; --front: #ff9a4d; color-scheme: dark;
|
| 17 |
+
}
|
| 18 |
+
}
|
| 19 |
+
:root[data-theme="dark"] {
|
| 20 |
+
--bg: #121514; --panel: #1a1e1c; --ink: #edf1ee; --ink-2: #b9c2bd; --muted: #7f8a85;
|
| 21 |
+
--line: #2b312e; --fig: #2fc4a6; --trail: #4d5551; --floor: #262c29; --front: #ff9a4d; color-scheme: dark;
|
| 22 |
+
}
|
| 23 |
+
body { background: var(--bg); color: var(--ink); font: 14px/1.6 var(--f-ui); }
|
| 24 |
+
.wrap { max-width: 1080px; margin: 0 auto; display: grid; gap: 14px;
|
| 25 |
+
padding-left: max(16px, env(safe-area-inset-left, 0px)); padding-right: max(16px, env(safe-area-inset-right, 0px)); padding-block: 24px 56px; }
|
| 26 |
+
h1 { font: 600 22px/1.25 var(--f-head); margin: 0; text-wrap: balance; }
|
| 27 |
+
.sub { color: var(--ink-2); margin: 4px 0 0; max-width: 70ch; }
|
| 28 |
+
.row { display: flex; flex-wrap: wrap; gap: 6px; align-items: center; }
|
| 29 |
+
.row button, .ctl button { font: 500 13px var(--f-ui); color: var(--ink); background: var(--panel);
|
| 30 |
+
border: 1px solid var(--line); border-radius: 999px; padding: 5px 12px; cursor: pointer; }
|
| 31 |
+
.row button[aria-pressed="true"] { border-color: var(--ink); background: var(--ink); color: var(--bg); }
|
| 32 |
+
button:focus-visible, input:focus-visible, select:focus-visible { outline: 2px solid var(--fig); outline-offset: 2px; }
|
| 33 |
+
.caption { font-size: 15px; color: var(--ink); }
|
| 34 |
+
.stage { position: relative; background: var(--panel); border: 1px solid var(--line); border-radius: 12px;
|
| 35 |
+
overflow: hidden; height: min(60vh, 520px); min-height: 320px; }
|
| 36 |
+
.stage canvas { display: block; width: 100%; height: 100%; touch-action: none; cursor: grab; }
|
| 37 |
+
.hint { position: absolute; right: 12px; bottom: 10px; font-size: 12px; color: var(--muted); pointer-events: none; }
|
| 38 |
+
.ctl { display: flex; flex-wrap: wrap; gap: 10px 16px; align-items: center; }
|
| 39 |
+
.ctl label { display: inline-flex; align-items: center; gap: 6px; color: var(--ink-2); font-size: 13px; }
|
| 40 |
+
.ctl input[type=range] { width: min(320px, 60vw); accent-color: var(--fig); }
|
| 41 |
+
.fr { font-family: var(--f-num); font-size: 12.5px; color: var(--ink-2); font-variant-numeric: tabular-nums; }
|
| 42 |
+
.tbl { overflow-x: auto; background: var(--panel); border: 1px solid var(--line); border-radius: 12px; }
|
| 43 |
+
table { border-collapse: collapse; width: 100%; font-size: 13px; }
|
| 44 |
+
th, td { padding: 8px 12px; border-bottom: 1px solid var(--line); text-align: right; font-family: var(--f-num);
|
| 45 |
+
font-variant-numeric: tabular-nums; white-space: nowrap; }
|
| 46 |
+
th:first-child, td:first-child { text-align: left; font-family: var(--f-ui); }
|
| 47 |
+
tr:last-child td { border-bottom: none; }
|
| 48 |
+
thead th { color: var(--ink-2); font-weight: 600; font-family: var(--f-ui); }
|
| 49 |
+
.notes { color: var(--ink-2); font-size: 13px; max-width: 72ch; margin: 0; padding-left: 20px; }
|
| 50 |
+
.notes li { margin: 3px 0; }
|
| 51 |
+
.bar { display: flex; flex-wrap: wrap; gap: 8px; align-items: center; }
|
| 52 |
+
.bar input[type=text] { flex: 1 1 260px; min-width: 0; font: 15px var(--f-ui); color: var(--ink); background: var(--panel);
|
| 53 |
+
border: 1px solid var(--line); border-radius: 10px; padding: 9px 12px; }
|
| 54 |
+
.go { font: 600 15px var(--f-ui); color: var(--bg); background: var(--fig); border: none; border-radius: 10px; padding: 9px 20px; cursor: pointer; }
|
| 55 |
+
.ghost { font: 500 13px var(--f-ui); color: var(--ink); background: var(--panel); border: 1px solid var(--line); border-radius: 999px; padding: 5px 12px; cursor: pointer; }
|
| 56 |
+
.now { font-size: 16px; } .now b { font-weight: 600; }
|
| 57 |
+
.pend { color: var(--front); font-size: 13px; }
|
| 58 |
+
.stat { font-family: var(--f-num); font-size: 12.5px; color: var(--ink-2); font-variant-numeric: tabular-nums; }
|
| 59 |
+
.warn { color: var(--front); }
|
| 60 |
+
.log { font-size: 12.5px; color: var(--ink-2); margin: 0; padding-left: 18px; }
|
| 61 |
+
</style>
|
| 62 |
+
<link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600&family=Space+Grotesk:wght@600&family=JetBrains+Mono:wght@400&display=swap">
|
| 63 |
+
|
| 64 |
+
<div class="wrap">
|
| 65 |
+
<header>
|
| 66 |
+
<h1>リアルタイム生成(CMU モデル)</h1>
|
| 67 |
+
<p class="sub">止めずにずっと動きを作り続けて、その場で再生します。英語の文章を入れて「生成」を押すと、次のステップ(4コマ=0.13秒)から新しい文章に切り替わります。前の動きの流れは消さずにつなぎます。</p>
|
| 68 |
+
</header>
|
| 69 |
+
<form class="bar" id="f" autocomplete="off">
|
| 70 |
+
<input type="text" id="txt" placeholder="例: walk / run/jog / jump / Wave / kick" aria-label="文章(英語)">
|
| 71 |
+
<button class="go" type="submit">生成</button>
|
| 72 |
+
</form>
|
| 73 |
+
<div class="row" id="chips" role="group" aria-label="学習データにある文章"></div>
|
| 74 |
+
<div class="now" id="now"></div>
|
| 75 |
+
<div class="pend" id="pend"></div>
|
| 76 |
+
<div class="stage" id="stage">
|
| 77 |
+
<canvas id="cv" aria-label="生成中の動きの人形"></canvas>
|
| 78 |
+
<div class="hint">ドラッグで回転・ホイールで拡大</div>
|
| 79 |
+
</div>
|
| 80 |
+
<div class="ctl">
|
| 81 |
+
<button class="ghost" id="pause" type="button">一時停止</button>
|
| 82 |
+
<button class="ghost" id="reset" type="button">最初から(流れを消す)</button>
|
| 83 |
+
<label for="s">文章の強さ s <select id="s"><option>1</option><option>2</option><option>3</option><option selected>4</option><option>6</option><option>8</option></select></label>
|
| 84 |
+
</div>
|
| 85 |
+
<div class="stat" id="stat"></div>
|
| 86 |
+
<ol class="log" id="log" reversed></ol>
|
| 87 |
+
</div>
|
| 88 |
+
<script>
|
| 89 |
+
"use strict";
|
| 90 |
+
const RAD = { 1: .07, 2: .07, 3: .1, 6: .1, 9: .1, 4: .065, 5: .065, 7: .05, 8: .05, 10: .04, 11: .04, 12: .05,
|
| 91 |
+
13: .055, 14: .055, 15: .05, 16: .05, 17: .05, 18: .045, 19: .045, 20: .038, 21: .038 };
|
| 92 |
+
const css = n => getComputedStyle(document.documentElement).getPropertyValue(n).trim();
|
| 93 |
+
const stage = document.getElementById("stage"), cv = document.getElementById("cv"), ctx = cv.getContext("2d");
|
| 94 |
+
let W = 300, H = 300, DPR = 1;
|
| 95 |
+
const cam = { yaw: 0.5, pitch: 0.3, dist: 5.0, tx: 0, ty: 0.9, tz: 0 };
|
| 96 |
+
const st = { clip: 0, t: 0, playing: true };
|
| 97 |
+
function project(x, y, z) {
|
| 98 |
+
const dx = x - cam.tx, dy = y - cam.ty, dz = z - cam.tz;
|
| 99 |
+
const cy = Math.cos(cam.yaw), sy = Math.sin(cam.yaw);
|
| 100 |
+
const x1 = dx * cy - dz * sy, z1 = dx * sy + dz * cy;
|
| 101 |
+
const cp = Math.cos(cam.pitch), sp = Math.sin(cam.pitch);
|
| 102 |
+
const y2 = dy * cp - z1 * sp, z2 = dy * sp + z1 * cp;
|
| 103 |
+
const depth = cam.dist - z2;
|
| 104 |
+
if (!(depth > 0.1)) return null;
|
| 105 |
+
const f = Math.min(W, H) * 1.35 / depth;
|
| 106 |
+
return [W / 2 + x1 * f, H * 0.56 - y2 * f, depth];
|
| 107 |
+
}
|
| 108 |
+
function drawFloor() {
|
| 109 |
+
ctx.strokeStyle = css("--floor"); ctx.lineWidth = 1;
|
| 110 |
+
const cx = Math.round(cam.tx * 2) / 2, cz = Math.round(cam.tz * 2) / 2;
|
| 111 |
+
for (let i = -5; i <= 5.0001; i += 0.5)
|
| 112 |
+
for (const [a, b] of [[[cx + i, 0, cz - 5], [cx + i, 0, cz + 5]], [[cx - 5, 0, cz + i], [cx + 5, 0, cz + i]]]) {
|
| 113 |
+
const p = project(...a), q = project(...b); if (!p || !q) continue;
|
| 114 |
+
ctx.beginPath(); ctx.moveTo(p[0], p[1]); ctx.lineTo(q[0], q[1]); ctx.stroke();
|
| 115 |
+
}
|
| 116 |
+
}
|
| 117 |
+
const sub = (a, b) => [a[0] - b[0], a[1] - b[1], a[2] - b[2]];
|
| 118 |
+
const add = (a, b) => [a[0] + b[0], a[1] + b[1], a[2] + b[2]];
|
| 119 |
+
const mul = (a, s) => [a[0] * s, a[1] * s, a[2] * s];
|
| 120 |
+
const dot = (a, b) => a[0] * b[0] + a[1] * b[1] + a[2] * b[2];
|
| 121 |
+
const cross = (a, b) => [a[1] * b[2] - a[2] * b[1], a[2] * b[0] - a[0] * b[2], a[0] * b[1] - a[1] * b[0]];
|
| 122 |
+
const norm = a => { const n = Math.hypot(a[0], a[1], a[2]) || 1; return [a[0] / n, a[1] / n, a[2] / n]; };
|
| 123 |
+
function shade(hex, k) { const n = parseInt(hex.slice(1), 16), r = n >> 16 & 255, g = n >> 8 & 255, b = n & 255;
|
| 124 |
+
const f = v => Math.max(0, Math.min(255, Math.round(v * k))); return "rgb(" + f(r) + "," + f(g) + "," + f(b) + ")"; }
|
| 125 |
+
// SMPL joints: 0 pelvis 1 L_hip 2 R_hip 3 spine1 4 L_knee 5 R_knee 6 spine2 7 L_ankle 8 R_ankle 9 spine3
|
| 126 |
+
// 10 L_foot 11 R_foot 12 neck 13 L_collar 14 R_collar 15 head 16 L_shoulder 17 R_shoulder 18 L_elbow 19 R_elbow 20 L_wrist 21 R_wrist
|
| 127 |
+
// [a, b, radius at a, radius at b, side] side: 0 center, 1 left, 2 right (right side drawn darker)
|
| 128 |
+
// torso and head are boxes that follow the joint's rotation; front face (+z in SMPL) is orange, back face dark
|
| 129 |
+
// [joint for rotation, center = mix of joints a,b (t), offset in local axes, half sizes x,y,z]
|
| 130 |
+
const BOX = [[0, 0, 0, 0, [0, 0.01, 0], [.16, .09, .105]], // hips
|
| 131 |
+
[3, 3, 6, 0.5, [0, 0, 0], [.135, .085, .095]], // belly
|
| 132 |
+
[9, 9, 12, 0.35, [0, 0, 0], [.175, .13, .1]], // chest
|
| 133 |
+
[15, 15, 15, 0, [0, 0.085, 0.02], [.08, .11, .1]]]; // head
|
| 134 |
+
const SEG = [
|
| 135 |
+
[12, 15, .045, .045, 0],
|
| 136 |
+
[13, 16, .06, .055, 1], [14, 17, .06, .055, 2],
|
| 137 |
+
[16, 18, .052, .042, 1], [18, 20, .04, .031, 1], [17, 19, .052, .042, 2], [19, 21, .04, .031, 2],
|
| 138 |
+
[1, 4, .085, .057, 1], [4, 7, .055, .04, 1], [2, 5, .085, .057, 2], [5, 8, .055, .04, 2],
|
| 139 |
+
[7, 10, .042, .034, 1], [8, 11, .042, .034, 2]];
|
| 140 |
+
function figurePieces(c, k, color, out) {
|
| 141 |
+
const P = c.p, R = c.r, po = k * 66, ro = k * 198, ox = c.tr[2 * k], oz = c.tr[2 * k + 1];
|
| 142 |
+
const pos = j => [P[po + j * 3] + ox, P[po + j * 3 + 1], P[po + j * 3 + 2] + oz];
|
| 143 |
+
const col = (j, q) => [R[ro + j * 9 + q], R[ro + j * 9 + 3 + q], R[ro + j * 9 + 6 + q]];
|
| 144 |
+
const tone = s => s === 2 ? 0.78 : s === 1 ? 1.08 : 1;
|
| 145 |
+
const cap = (a, b, ra, rb, s, dz) => { const pa = project(...a), pb = project(...b); if (!pa || !pb) return;
|
| 146 |
+
out.push({ depth: (pa[2] + pb[2]) / 2 + (dz || 0), cap: [pa, pb, ra, rb], fill: color, k: tone(s) }); };
|
| 147 |
+
for (const [a, b, ra, rb, s] of SEG) cap(pos(a), pos(b), ra, rb, s, 0);
|
| 148 |
+
// hands follow the WRIST rotation (SMPL T-pose: hand along +x left / -x right, palm faces -y, thumb +z)
|
| 149 |
+
for (const [w, s, sx] of [[20, 1, 1], [21, 2, -1]]) {
|
| 150 |
+
const pw = pos(w), hx = mul(col(w, 0), sx), pn = mul(col(w, 1), -1), tz = col(w, 2);
|
| 151 |
+
const C = (l, t) => add(add(pw, mul(hx, l)), mul(tz, t));
|
| 152 |
+
const quad = [C(0.01, 0.035), C(0.17, 0.035), C(0.17, -0.035), C(0.01, -0.035)].map(p => project(...p));
|
| 153 |
+
if (quad.every(Boolean)) {
|
| 154 |
+
const dep = quad.reduce((a, q) => a + q[2], 0) / 4 - 0.003;
|
| 155 |
+
out.push({ depth: dep, palm: quad, fill: color, k: tone(s) * (0.75 + 0.45 * Math.abs(dot(pn, norm([-0.4, 0.8, 0.5])))) });
|
| 156 |
+
}
|
| 157 |
+
}
|
| 158 |
+
const front = css("--front");
|
| 159 |
+
for (const [jr, ja, jb, t, off, hs] of BOX) {
|
| 160 |
+
const ax = [col(jr, 0), col(jr, 1), col(jr, 2)];
|
| 161 |
+
let c0 = add(mul(pos(ja), 1 - t), mul(pos(jb), t));
|
| 162 |
+
for (let i = 0; i < 3; i++) c0 = add(c0, mul(ax[i], off[i]));
|
| 163 |
+
for (let a = 0; a < 3; a++) for (const sgn of [1, -1]) {
|
| 164 |
+
const b = (a + 1) % 3, cc = (a + 2) % 3;
|
| 165 |
+
const corner = (ub, uc) => add(add(add(c0, mul(ax[a], sgn * hs[a])), mul(ax[b], ub * hs[b])), mul(ax[cc], uc * hs[cc]));
|
| 166 |
+
let q = [corner(-1, -1), corner(1, -1), corner(1, 1), corner(-1, 1)];
|
| 167 |
+
if (sgn < 0) q.reverse();
|
| 168 |
+
q = q.map(p => project(...p)); if (!q.every(Boolean)) continue;
|
| 169 |
+
let area = 0; for (let i = 0; i < 4; i++) { const p = q[i], r = q[(i + 1) % 4]; area += p[0] * r[1] - r[0] * p[1]; }
|
| 170 |
+
if (area > 0) continue; // facing away from the camera
|
| 171 |
+
const nrm = mul(ax[a], sgn), lit = 0.7 + 0.45 * Math.max(0, dot(nrm, norm([-0.4, 0.8, 0.5])));
|
| 172 |
+
const isFront = a === 2 && sgn > 0, isBack = a === 2 && sgn < 0;
|
| 173 |
+
out.push({ depth: q.reduce((s, p) => s + p[2], 0) / 4, palm: q, fill: isFront ? front : color,
|
| 174 |
+
k: isFront ? 0.85 + 0.25 * lit / 1.15 : isBack ? 0.45 : lit * 0.85 });
|
| 175 |
+
}
|
| 176 |
+
}
|
| 177 |
+
const g = project(pos(0)[0], 0.002, pos(0)[2]); // floor shadow
|
| 178 |
+
if (g) out.push({ depth: 1e9, shadow: g });
|
| 179 |
+
}
|
| 180 |
+
function paintPiece(s, F) {
|
| 181 |
+
if (s.shadow) { const r = 0.28 * F / s.shadow[2]; ctx.fillStyle = "rgba(0,0,0,0.12)";
|
| 182 |
+
ctx.beginPath(); ctx.ellipse(s.shadow[0], s.shadow[1], r, r * 0.35, 0, 0, 7); ctx.fill(); return; }
|
| 183 |
+
if (s.palm) { const q = s.palm; ctx.fillStyle = shade(s.fill, s.k); ctx.strokeStyle = shade(s.fill, s.k * 0.8);
|
| 184 |
+
ctx.lineWidth = Math.max(2, 0.018 * F / q[0][2]); ctx.lineJoin = "round";
|
| 185 |
+
ctx.beginPath(); ctx.moveTo(q[0][0], q[0][1]); for (let i = 1; i < 4; i++) ctx.lineTo(q[i][0], q[i][1]);
|
| 186 |
+
ctx.closePath(); ctx.fill(); ctx.stroke(); return; }
|
| 187 |
+
if (s.ball) { const r = s.r * F / s.ball[2], [x, y] = s.ball;
|
| 188 |
+
const gr = ctx.createRadialGradient(x - r * 0.35, y - r * 0.4, r * 0.1, x, y, r);
|
| 189 |
+
const kb = s.k || 1; gr.addColorStop(0, shade(s.fill, 1.35 * kb)); gr.addColorStop(1, shade(s.fill, 0.7 * kb));
|
| 190 |
+
ctx.fillStyle = gr; ctx.beginPath(); ctx.arc(x, y, r, 0, 7); ctx.fill(); return; }
|
| 191 |
+
const [a, b, ra0, rb0] = s.cap, ra = ra0 * F / a[2], rb = rb0 * F / b[2];
|
| 192 |
+
let dx = b[0] - a[0], dy = b[1] - a[1]; const L = Math.hypot(dx, dy) || 1e-6; dx /= L; dy /= L;
|
| 193 |
+
const nx = -dy, ny = dx, mx = (a[0] + b[0]) / 2, my = (a[1] + b[1]) / 2, rm = Math.max(ra, rb);
|
| 194 |
+
const sg = nx * -0.6 + ny * -0.8 > 0 ? 1 : -1; // lit side faces the upper left
|
| 195 |
+
const gr = ctx.createLinearGradient(mx - nx * rm * sg, my - ny * rm * sg, mx + nx * rm * sg, my + ny * rm * sg);
|
| 196 |
+
const k = s.k || 1;
|
| 197 |
+
gr.addColorStop(0, shade(s.fill, 0.62 * k)); gr.addColorStop(0.62, shade(s.fill, 1.22 * k)); gr.addColorStop(1, shade(s.fill, 0.95 * k));
|
| 198 |
+
ctx.fillStyle = gr;
|
| 199 |
+
ctx.beginPath(); ctx.arc(a[0], a[1], ra, 0, 7); ctx.fill();
|
| 200 |
+
ctx.beginPath(); ctx.arc(b[0], b[1], rb, 0, 7); ctx.fill();
|
| 201 |
+
ctx.beginPath(); ctx.moveTo(a[0] + nx * ra, a[1] + ny * ra); ctx.lineTo(b[0] + nx * rb, b[1] + ny * rb);
|
| 202 |
+
ctx.lineTo(b[0] - nx * rb, b[1] - ny * rb); ctx.lineTo(a[0] - nx * ra, a[1] - ny * ra); ctx.closePath(); ctx.fill();
|
| 203 |
+
}
|
| 204 |
+
|
| 205 |
+
const CHIPS = ["Standing still", "walk", "run/jog", "jump", "Wave", "kick", "salsa dance", "boxing", "sit on high stool, stand up", "walk backwards and turn", "cartwheel", "Mickey sneaking walk"];
|
| 206 |
+
const PAR = [-1, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 9, 9, 12, 13, 14, 16, 17, 18, 19];
|
| 207 |
+
const FPS = 30;
|
| 208 |
+
let buf = [], bufStart = 0, have = -1, playF = 0, paused = false, trail = [], lastInfo = null, stalled = 0;
|
| 209 |
+
const $ = id => document.getElementById(id);
|
| 210 |
+
async function post(path, obj) { try { await fetch(path, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify(obj || {}) }); } catch (e) {} }
|
| 211 |
+
CHIPS.forEach(t => { const b = document.createElement("button"); b.type = "button"; b.textContent = t;
|
| 212 |
+
b.addEventListener("click", () => { $("txt").value = t; send(t); }); $("chips").appendChild(b); });
|
| 213 |
+
function send(t) { t = (t || "").trim(); if (!t) return; post("/prompt", { text: t }); $("pend").textContent = "切り替え待ち: “" + t + "”"; }
|
| 214 |
+
$("f").addEventListener("submit", e => { e.preventDefault(); send($("txt").value); });
|
| 215 |
+
$("pause").addEventListener("click", e => { paused = !paused; e.currentTarget.textContent = paused ? "再生" : "一時停止"; });
|
| 216 |
+
$("reset").addEventListener("click", () => { post("/reset"); buf = []; bufStart = 0; have = 0; playF = 0; trail = []; });
|
| 217 |
+
$("s").addEventListener("change", e => post("/s", { s: +e.target.value }));
|
| 218 |
+
async function poll() {
|
| 219 |
+
try {
|
| 220 |
+
const r = await fetch("/frames?from=" + have + "&play=" + Math.floor(playF), { cache: "no-store" });
|
| 221 |
+
const j = await r.json(); lastInfo = j;
|
| 222 |
+
if (have < 0) { buf = []; bufStart = j.start; have = j.start; playF = j.start; }
|
| 223 |
+
if (j.total < have) { buf = []; bufStart = j.start; have = j.start; playF = j.start; trail = []; }
|
| 224 |
+
if (j.start > have) { buf = []; bufStart = j.start; have = j.start; playF = Math.max(playF, j.start); }
|
| 225 |
+
for (const f of j.frames) { buf.push(f); have++; }
|
| 226 |
+
$("pend").textContent = j.pending ? "切り替え待ち: “" + j.pending + "”" : "";
|
| 227 |
+
const ok = j.step_ms <= j.realtime_ms;
|
| 228 |
+
$("stat").innerHTML = "1ステップの計算 " + j.step_ms.toFixed(0) + " ms(リアルタイムは " + j.realtime_ms.toFixed(0) + " ms 以下)" +
|
| 229 |
+
(ok ? "" : " <span class='warn'>← 間に合っていない</span>") + " 先読み " + ((have - playF) / FPS).toFixed(1) + " 秒" +
|
| 230 |
+
" 経過 " + (playF / FPS).toFixed(0) + " 秒" + (stalled > 0.2 ? " <span class='warn'>生成待ちで止まった合計 " + stalled.toFixed(1) + " 秒</span>" : "");
|
| 231 |
+
$("log").textContent = "";
|
| 232 |
+
j.log.slice().reverse().forEach(([f, t]) => { const li = document.createElement("li"); li.textContent = (f / FPS).toFixed(1) + " 秒から “" + t + "”"; $("log").appendChild(li); });
|
| 233 |
+
} catch (e) { $("stat").innerHTML = "<span class='warn'>サーバーにつながらない(live_server.py が動いているか確認)</span>"; }
|
| 234 |
+
setTimeout(poll, 120);
|
| 235 |
+
}
|
| 236 |
+
function frameAt(i) { const k = i - bufStart; return k >= 0 && k < buf.length ? buf[k] : null; }
|
| 237 |
+
function drawLive() {
|
| 238 |
+
const f = frameAt(Math.floor(playF));
|
| 239 |
+
ctx.setTransform(DPR, 0, 0, DPR, 0, 0); ctx.clearRect(0, 0, W, H);
|
| 240 |
+
if (!f) { drawFloor(); return; }
|
| 241 |
+
if (Math.hypot(f.tr[0] - cam.tx, f.tr[1] - cam.tz) > 3) { cam.tx = f.tr[0]; cam.tz = f.tr[1]; }
|
| 242 |
+
cam.tx += (f.tr[0] - cam.tx) * 0.15; cam.tz += (f.tr[1] - cam.tz) * 0.15;
|
| 243 |
+
drawFloor();
|
| 244 |
+
ctx.strokeStyle = css("--trail"); ctx.lineWidth = 2; ctx.beginPath(); let started = false;
|
| 245 |
+
for (const t of trail) { const p = project(t[0], 0.001, t[1]); if (!p) continue;
|
| 246 |
+
if (!started) { ctx.moveTo(p[0], p[1]); started = true; } else ctx.lineTo(p[0], p[1]); }
|
| 247 |
+
ctx.stroke();
|
| 248 |
+
const c = { p: f.p, r: f.r, tr: f.tr }, pieces = [];
|
| 249 |
+
figurePieces(c, 0, css("--fig"), pieces);
|
| 250 |
+
pieces.sort((a, b) => b.depth - a.depth);
|
| 251 |
+
const F = Math.min(W, H) * 1.35;
|
| 252 |
+
for (const s of pieces) paintPiece(s, F);
|
| 253 |
+
const txt = "いまの文章: “" + f.text + "”";
|
| 254 |
+
if ($("now").textContent !== txt) $("now").textContent = txt;
|
| 255 |
+
}
|
| 256 |
+
const ptrs = new Map();
|
| 257 |
+
cv.addEventListener("pointerdown", e => { cv.setPointerCapture(e.pointerId); ptrs.set(e.pointerId, [e.clientX, e.clientY]); });
|
| 258 |
+
cv.addEventListener("pointermove", e => { if (!ptrs.has(e.pointerId)) return; const prev = ptrs.get(e.pointerId); ptrs.set(e.pointerId, [e.clientX, e.clientY]);
|
| 259 |
+
cam.yaw += (e.clientX - prev[0]) * 0.01; cam.pitch = Math.max(-0.2, Math.min(1.3, cam.pitch + (e.clientY - prev[1]) * 0.01)); });
|
| 260 |
+
const up = e => ptrs.delete(e.pointerId); cv.addEventListener("pointerup", up); cv.addEventListener("pointercancel", up);
|
| 261 |
+
cv.addEventListener("wheel", e => { e.preventDefault(); cam.dist = Math.max(1.5, Math.min(16, cam.dist * Math.exp(e.deltaY * 0.001))); }, { passive: false });
|
| 262 |
+
function resize() { DPR = Math.min(2, window.devicePixelRatio || 1); W = stage.clientWidth || 300; H = stage.clientHeight || 300;
|
| 263 |
+
cv.width = Math.round(W * DPR); cv.height = Math.round(H * DPR); }
|
| 264 |
+
new ResizeObserver(resize).observe(stage); resize();
|
| 265 |
+
let last = performance.now(), lastTrail = -1;
|
| 266 |
+
function loop(now) { const dt = Math.min(0.1, (now - last) / 1000); last = now;
|
| 267 |
+
if (!paused) {
|
| 268 |
+
const next = playF + dt * FPS;
|
| 269 |
+
if (next < have - 1) playF = next; else if (have > 0) stalled += dt;
|
| 270 |
+
const fi = Math.floor(playF);
|
| 271 |
+
if (fi !== lastTrail && fi % 2 === 0) { const f = frameAt(fi); if (f) { trail.push(f.tr); if (trail.length > 900) trail.shift(); } lastTrail = fi; }
|
| 272 |
+
const drop = Math.floor(playF) - bufStart - 60; if (drop > 300) { buf.splice(0, drop); bufStart += drop; }
|
| 273 |
+
}
|
| 274 |
+
drawLive(); requestAnimationFrame(loop); }
|
| 275 |
+
poll(); requestAnimationFrame(loop);
|
| 276 |
+
</script>
|
live/live_server.py
ADDED
|
@@ -0,0 +1,312 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Real-time, endless text-to-motion with the PartLock CMU model.
|
| 2 |
+
The model keeps generating forever; the browser plays it in real time.
|
| 3 |
+
Typing a prompt and pressing the button swaps the text at the next step (4 frames), without resetting the context.
|
| 4 |
+
|
| 5 |
+
python live/live_server.py then open http://localhost:8765
|
| 6 |
+
|
| 7 |
+
Options (environment variables): LIVE_PORT (8765), LIVE_LEAD (6 steps generated ahead of playback = 0.8 s).
|
| 8 |
+
Generation is kept only LEAD steps ahead of playback, so a new prompt shows up after about LEAD*4/30 s."""
|
| 9 |
+
import os, sys, json, time, threading
|
| 10 |
+
from http.server import ThreadingHTTPServer, BaseHTTPRequestHandler
|
| 11 |
+
from urllib.parse import urlparse, parse_qs
|
| 12 |
+
import numpy as np, torch
|
| 13 |
+
import torch.nn.functional as F
|
| 14 |
+
from safetensors.torch import load_file
|
| 15 |
+
HERE = os.path.dirname(os.path.abspath(__file__))
|
| 16 |
+
REPO = os.path.dirname(HERE)
|
| 17 |
+
sys.path.insert(0, os.path.join(REPO, "code"))
|
| 18 |
+
import tvq272 as B, tvq272c as C, gen_ar as G
|
| 19 |
+
from gen_sample import load, embed, get_codebook
|
| 20 |
+
|
| 21 |
+
dev = "cuda"
|
| 22 |
+
WTS = os.path.join(REPO, "weights")
|
| 23 |
+
PORT = int(os.environ.get("LIVE_PORT", "8765"))
|
| 24 |
+
LEAD = int(os.environ.get("LIVE_LEAD", "6")) # steps generated ahead of playback (6 steps = 0.8 s)
|
| 25 |
+
DEC_CTX = 32 # decoder receptive field is ~16 steps; 32 is safe
|
| 26 |
+
KEEP = 30 * 120 # frames kept on the server (2 min)
|
| 27 |
+
|
| 28 |
+
mean = torch.from_numpy(np.load(os.path.join(WTS, "Mean.npy")).astype(np.float32)).to(dev)
|
| 29 |
+
std = torch.from_numpy(np.load(os.path.join(WTS, "Std.npy")).astype(np.float32)).to(dev)
|
| 30 |
+
h = C.CHierTok().to(dev); h.load_state_dict(load_file(os.path.join(WTS, "tokenizer.safetensors"), device=dev)); h.eval()
|
| 31 |
+
g = G.Gen().to(dev); g.load_state_dict(load_file(os.path.join(WTS, "generator.safetensors"), device=dev)); g.eval()
|
| 32 |
+
tok, clip = load()
|
| 33 |
+
CB = [get_codebook(h.gvq)] + [get_codebook(v) for v in h.vq]
|
| 34 |
+
OFFT = torch.tensor(G.OFF, device=dev)
|
| 35 |
+
SLOTS = torch.arange(G.L, device=dev) % 7
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
class State:
|
| 39 |
+
def __init__(self):
|
| 40 |
+
self.lock = threading.Lock()
|
| 41 |
+
self.frames = [] # list of dict per frame (json-ready)
|
| 42 |
+
self.base = 0 # global index of frames[0]
|
| 43 |
+
self.total = 0 # frames produced so far
|
| 44 |
+
self.play = 0 # browser playback frame (reported by polling)
|
| 45 |
+
self.prompt = "Standing still"
|
| 46 |
+
self.pending = None
|
| 47 |
+
self.reset = False
|
| 48 |
+
self.s = 4.0
|
| 49 |
+
self.step_ms = 0.0
|
| 50 |
+
self.log = [] # (frame, prompt)
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
S = State()
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def reset_world():
|
| 57 |
+
return dict(codes=torch.zeros(0, 7, dtype=torch.long, device=dev), toks=torch.zeros(1, 0, dtype=torch.long, device=dev),
|
| 58 |
+
tot=None, inv_prev=None, tr=torch.zeros(3, dtype=torch.float64))
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
def _attn_in(b, x, pos):
|
| 62 |
+
Bn, T, D = x.shape
|
| 63 |
+
q, k, v = b.qkv(b.n1(x)).view(Bn, T, 3, b.h, D // b.h).permute(2, 0, 3, 1, 4)
|
| 64 |
+
return G.rope(q, pos), G.rope(k, pos), v
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
def _attn_out(b, x, a):
|
| 68 |
+
Bn, T, D = x.shape
|
| 69 |
+
x = x + b.o(a.transpose(1, 2).reshape(Bn, T, D))
|
| 70 |
+
return x + b.ff(b.n2(x))
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
LMAX = G.L # 448 = BOS + 63 past steps + 7 tokens of the current step (most the original ever feeds)
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
class Engine:
|
| 77 |
+
"""CUDA-graph version of the same forward. Static K/V buffers of length LMAX.
|
| 78 |
+
full(): window padded to LMAX at the END (causal -> real positions are unaffected), logits taken at T-1.
|
| 79 |
+
one(): one new token at position p, attends to positions <= p."""
|
| 80 |
+
def __init__(self):
|
| 81 |
+
nh = g.blocks[0].h; dh = 512 // nh
|
| 82 |
+
self.K = [torch.zeros(2, nh, LMAX, dh, device=dev, dtype=torch.bfloat16) for _ in g.blocks]
|
| 83 |
+
self.V = [torch.zeros_like(k) for k in self.K]
|
| 84 |
+
self.ids = torch.full((2, LMAX), G.NV, device=dev, dtype=torch.long)
|
| 85 |
+
self.tc = torch.zeros(2, 1, 512, device=dev)
|
| 86 |
+
self.last = torch.zeros((), device=dev, dtype=torch.long) # index T-1
|
| 87 |
+
self.tid = torch.zeros(2, 1, device=dev, dtype=torch.long)
|
| 88 |
+
self.p = torch.zeros(1, device=dev, dtype=torch.long)
|
| 89 |
+
self.ar = torch.arange(LMAX, device=dev)
|
| 90 |
+
s = torch.cuda.Stream(); s.wait_stream(torch.cuda.current_stream())
|
| 91 |
+
with torch.cuda.stream(s):
|
| 92 |
+
for _ in range(3):
|
| 93 |
+
self._full(); self._one()
|
| 94 |
+
torch.cuda.current_stream().wait_stream(s)
|
| 95 |
+
self.gf = torch.cuda.CUDAGraph()
|
| 96 |
+
with torch.cuda.graph(self.gf):
|
| 97 |
+
self.out_f = self._full()
|
| 98 |
+
self.go = torch.cuda.CUDAGraph()
|
| 99 |
+
with torch.cuda.graph(self.go):
|
| 100 |
+
self.out_o = self._one()
|
| 101 |
+
|
| 102 |
+
@torch.no_grad()
|
| 103 |
+
def _full(self):
|
| 104 |
+
with torch.autocast("cuda", dtype=torch.bfloat16):
|
| 105 |
+
x = g.tok(self.ids) + g.slot(SLOTS)[None] + g.txt(self.tc)
|
| 106 |
+
pos = self.ar
|
| 107 |
+
for b, K, V in zip(g.blocks, self.K, self.V):
|
| 108 |
+
q, k, v = _attn_in(b, x, pos)
|
| 109 |
+
K.copy_(k); V.copy_(v)
|
| 110 |
+
x = _attn_out(b, x, F.scaled_dot_product_attention(q, k, v, is_causal=True))
|
| 111 |
+
xl = x.index_select(1, self.last.view(1))
|
| 112 |
+
return (g.head(g.nf(xl)) + g.mask[SLOTS.index_select(0, self.last.view(1))][None]).float()[:, 0]
|
| 113 |
+
|
| 114 |
+
@torch.no_grad()
|
| 115 |
+
def _one(self):
|
| 116 |
+
with torch.autocast("cuda", dtype=torch.bfloat16):
|
| 117 |
+
sl = SLOTS.index_select(0, self.p)
|
| 118 |
+
x = g.tok(self.tid) + g.slot(sl)[None] + g.txt(self.tc)
|
| 119 |
+
msk = (self.ar <= self.p)[None, None, None] # (1,1,1,LMAX)
|
| 120 |
+
for b, K, V in zip(g.blocks, self.K, self.V):
|
| 121 |
+
q, k, v = _attn_in(b, x, self.p)
|
| 122 |
+
K.index_copy_(2, self.p, k.to(K.dtype)); V.index_copy_(2, self.p, v.to(V.dtype))
|
| 123 |
+
x = _attn_out(b, x, F.scaled_dot_product_attention(q, K, V, attn_mask=msk))
|
| 124 |
+
return (g.head(g.nf(x)) + g.mask[sl][None]).float()[:, 0]
|
| 125 |
+
|
| 126 |
+
def full(self, inp, tc):
|
| 127 |
+
T = inp.shape[1]
|
| 128 |
+
self.ids.fill_(G.NV); self.ids[:, :T] = inp; self.tc.copy_(tc); self.last.fill_(T - 1)
|
| 129 |
+
self.gf.replay(); return self.out_f.clone()
|
| 130 |
+
|
| 131 |
+
def one(self, tid, p):
|
| 132 |
+
self.tid.copy_(tid); self.p.fill_(p)
|
| 133 |
+
self.go.replay(); return self.out_o.clone()
|
| 134 |
+
|
| 135 |
+
|
| 136 |
+
ENG = None
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
@torch.no_grad()
|
| 140 |
+
def one_step(w, cond, s, check=False):
|
| 141 |
+
"""generate the 7 tokens of one step (CFG), decode its 4 frames, integrate world pose."""
|
| 142 |
+
unc = g.null[None]
|
| 143 |
+
cur = w["toks"].shape[1] // 7
|
| 144 |
+
first = max(0, cur - (G.S - 1)) * 7
|
| 145 |
+
inp = torch.cat([torch.full((1, 1), G.NV, device=dev, dtype=torch.long), w["toks"][:, first:]], 1)
|
| 146 |
+
T = inp.shape[1]
|
| 147 |
+
tc = torch.cat([cond, unc])[:, None, :]
|
| 148 |
+
diffs = []
|
| 149 |
+
for k in range(7):
|
| 150 |
+
if ENG is not None:
|
| 151 |
+
lo = ENG.full(torch.cat([inp, inp]), tc) if k == 0 else ENG.one(torch.cat([nxt, nxt]), T + k - 1)
|
| 152 |
+
else:
|
| 153 |
+
with torch.autocast("cuda", dtype=torch.bfloat16):
|
| 154 |
+
lo = g(torch.cat([inp, inp]), SLOTS[:T], tc.expand(-1, T, -1))[:, -1].float() if k == 0 else None
|
| 155 |
+
if k:
|
| 156 |
+
full = torch.cat([inp, w["toks"][:, w["toks"].shape[1] - k:]], 1)
|
| 157 |
+
lo = g(torch.cat([full, full]), SLOTS[:full.shape[1]], tc.expand(-1, full.shape[1], -1))[:, -1].float()
|
| 158 |
+
if check: # compare with the original full forward (valid entries only)
|
| 159 |
+
with torch.autocast("cuda", dtype=torch.bfloat16):
|
| 160 |
+
full = torch.cat([inp, w["toks"][:, w["toks"].shape[1] - k:]], 1) if k else inp
|
| 161 |
+
ref = g(torch.cat([full, full]), SLOTS[:full.shape[1]], tc.expand(-1, full.shape[1], -1))[:, -1].float()
|
| 162 |
+
vv = g.mask[k] == 0
|
| 163 |
+
diffs.append(float((ref[:, vv] - lo[:, vv]).abs().max()))
|
| 164 |
+
lo = lo.float()
|
| 165 |
+
lc, lu = lo[:1], lo[1:]
|
| 166 |
+
valid = (g.mask[k] == 0)[None]
|
| 167 |
+
lc = torch.where(valid, lc, torch.zeros_like(lc)); lu = torch.where(valid, lu, torch.zeros_like(lu))
|
| 168 |
+
lg = torch.where(valid, (1 + s) * lc - s * lu, torch.full_like(lc, float("-inf")))
|
| 169 |
+
nxt = torch.multinomial(F.softmax(lg, -1), 1)
|
| 170 |
+
w["toks"] = torch.cat([w["toks"], nxt], 1)
|
| 171 |
+
if check:
|
| 172 |
+
w["diffs"] = diffs
|
| 173 |
+
w["toks"] = w["toks"][:, -G.S * 7:] # only the window is ever used
|
| 174 |
+
code = w["toks"][0, -7:] - OFFT
|
| 175 |
+
w["codes"] = torch.cat([w["codes"], code[None]])[-DEC_CTX:]
|
| 176 |
+
# causal decode of the last DEC_CTX steps, keep the newest 4 frames
|
| 177 |
+
c = w["codes"][None]
|
| 178 |
+
qg = CB[0][c[..., 0]].permute(0, 2, 1)
|
| 179 |
+
y = torch.zeros(1, c.shape[1] * 4, 272, device=dev)
|
| 180 |
+
for p, dims in enumerate(B.PD):
|
| 181 |
+
q = CB[p + 1][c[..., p + 1]].permute(0, 2, 1)
|
| 182 |
+
y[..., dims] = h.dec[p](torch.cat([qg, q], 1)).permute(0, 2, 1)
|
| 183 |
+
x = (y * std + mean)[0, -4:].double().cpu() # (4,272) -> small 3x3 math on the CPU
|
| 184 |
+
Rd = B.rot6d_to_mat(x[:, 2:8])
|
| 185 |
+
P = B.positions(x)
|
| 186 |
+
Rl = B.local_rots(x).clone()
|
| 187 |
+
out = []
|
| 188 |
+
for t in range(4):
|
| 189 |
+
w["tot"] = Rd[t] if w["tot"] is None else Rd[t] @ w["tot"]
|
| 190 |
+
U, _, Vh = torch.linalg.svd(w["tot"]); w["tot"] = U @ Vh # keep it a rotation forever
|
| 191 |
+
inv = w["tot"].T
|
| 192 |
+
v = torch.tensor([float(x[t, 0]), 0.0, float(x[t, 1])], dtype=torch.float64)
|
| 193 |
+
if w["inv_prev"] is not None:
|
| 194 |
+
v = w["inv_prev"] @ v
|
| 195 |
+
w["tr"] = w["tr"] + v
|
| 196 |
+
w["inv_prev"] = inv
|
| 197 |
+
Pw = (inv @ P[t].T).T # pelvis-relative in x/z, absolute y
|
| 198 |
+
Rl[t, 0] = inv @ Rl[t, 0]
|
| 199 |
+
out.append((Pw, w["tr"].clone()))
|
| 200 |
+
Gw = B.global_rots(Rl) # (4,22,3,3)
|
| 201 |
+
frames = []
|
| 202 |
+
for t in range(4):
|
| 203 |
+
Pw, tr = out[t]
|
| 204 |
+
frames.append(dict(p=np.round(Pw.numpy().ravel(), 3).tolist(),
|
| 205 |
+
r=np.round(Gw[t].numpy().ravel(), 3).tolist(),
|
| 206 |
+
tr=[round(float(tr[0]), 3), round(float(tr[2]), 3)]))
|
| 207 |
+
return frames
|
| 208 |
+
|
| 209 |
+
|
| 210 |
+
def gen_loop():
|
| 211 |
+
w = reset_world()
|
| 212 |
+
cond = embed(tok, clip, [S.prompt]).float()
|
| 213 |
+
torch.manual_seed(1)
|
| 214 |
+
while True:
|
| 215 |
+
with S.lock:
|
| 216 |
+
if S.reset:
|
| 217 |
+
S.reset = False; w = reset_world()
|
| 218 |
+
if S.pending is not None:
|
| 219 |
+
S.prompt, S.pending = S.pending, None
|
| 220 |
+
new = S.prompt
|
| 221 |
+
else:
|
| 222 |
+
new = None
|
| 223 |
+
ahead = S.total - S.play
|
| 224 |
+
s = S.s
|
| 225 |
+
if new is not None:
|
| 226 |
+
cond = embed(tok, clip, [new]).float()
|
| 227 |
+
with S.lock:
|
| 228 |
+
S.log.append([S.total, new]); S.log = S.log[-50:]
|
| 229 |
+
if ahead >= LEAD * 4:
|
| 230 |
+
time.sleep(0.01); continue
|
| 231 |
+
t0 = time.time()
|
| 232 |
+
fr = one_step(w, cond, s)
|
| 233 |
+
dt = (time.time() - t0) * 1000
|
| 234 |
+
with S.lock:
|
| 235 |
+
S.step_ms = dt if S.step_ms == 0 else 0.9 * S.step_ms + 0.1 * dt
|
| 236 |
+
for f in fr:
|
| 237 |
+
f["text"] = S.prompt
|
| 238 |
+
S.frames.extend(fr); S.total += 4
|
| 239 |
+
if len(S.frames) > KEEP:
|
| 240 |
+
cut = len(S.frames) - KEEP; S.frames = S.frames[cut:]; S.base += cut
|
| 241 |
+
|
| 242 |
+
|
| 243 |
+
class H(BaseHTTPRequestHandler):
|
| 244 |
+
def log_message(self, *a):
|
| 245 |
+
pass
|
| 246 |
+
|
| 247 |
+
def send(self, code, body, ctype="application/json; charset=utf-8"):
|
| 248 |
+
b = body.encode("utf-8") if isinstance(body, str) else body
|
| 249 |
+
self.send_response(code); self.send_header("Content-Type", ctype)
|
| 250 |
+
self.send_header("Cache-Control", "no-store"); self.send_header("Content-Length", str(len(b)))
|
| 251 |
+
self.end_headers(); self.wfile.write(b)
|
| 252 |
+
|
| 253 |
+
def do_GET(self):
|
| 254 |
+
u = urlparse(self.path); q = parse_qs(u.query)
|
| 255 |
+
if u.path in ("/", "/index.html"):
|
| 256 |
+
return self.send(200, open(os.path.join(HERE, "live.html"), encoding="utf-8").read(), "text/html; charset=utf-8")
|
| 257 |
+
if u.path == "/frames":
|
| 258 |
+
frm = int(q.get("from", ["0"])[0]); play = int(q.get("play", ["0"])[0])
|
| 259 |
+
with S.lock:
|
| 260 |
+
if frm < 0: # a page just opened: start at the newest frames
|
| 261 |
+
frm = max(S.base, S.total - 8)
|
| 262 |
+
S.play = frm
|
| 263 |
+
else:
|
| 264 |
+
S.play = max(S.play, min(play, S.total))
|
| 265 |
+
if frm < S.base:
|
| 266 |
+
frm = S.base
|
| 267 |
+
fr = S.frames[frm - S.base:]
|
| 268 |
+
body = json.dumps(dict(start=frm, frames=fr, total=S.total, prompt=S.prompt, pending=S.pending,
|
| 269 |
+
step_ms=round(S.step_ms, 1), realtime_ms=round(4000 / 30, 1), s=S.s,
|
| 270 |
+
lead=LEAD, log=S.log[-8:]), separators=(",", ":"))
|
| 271 |
+
return self.send(200, body)
|
| 272 |
+
self.send(404, "{}")
|
| 273 |
+
|
| 274 |
+
def do_POST(self):
|
| 275 |
+
u = urlparse(self.path)
|
| 276 |
+
n = int(self.headers.get("Content-Length", "0") or 0)
|
| 277 |
+
d = json.loads(self.rfile.read(n) or b"{}")
|
| 278 |
+
with S.lock:
|
| 279 |
+
if u.path == "/prompt":
|
| 280 |
+
t = str(d.get("text", "")).strip()[:300]
|
| 281 |
+
if t:
|
| 282 |
+
S.pending = t
|
| 283 |
+
elif u.path == "/reset":
|
| 284 |
+
S.reset = True
|
| 285 |
+
S.frames, S.base, S.total, S.play, S.log = [], 0, 0, 0, []
|
| 286 |
+
elif u.path == "/s":
|
| 287 |
+
S.s = float(max(0.0, min(10.0, float(d.get("s", 4.0)))))
|
| 288 |
+
else:
|
| 289 |
+
return self.send(404, "{}")
|
| 290 |
+
self.send(200, '{"ok":true}')
|
| 291 |
+
|
| 292 |
+
|
| 293 |
+
if __name__ == "__main__":
|
| 294 |
+
# build the CUDA graphs, then an exactness check (bf16 rounding only) and a speed check
|
| 295 |
+
ENG = Engine()
|
| 296 |
+
w = reset_world(); cond = embed(tok, clip, ["walk"]).float()
|
| 297 |
+
for _ in range(3):
|
| 298 |
+
one_step(w, cond, 4.0, check=True)
|
| 299 |
+
print("graph vs original forward, max |logit diff| per token:", ["%.3f" % v for v in w["diffs"]], flush=True)
|
| 300 |
+
torch.cuda.synchronize(); t0 = time.time()
|
| 301 |
+
for _ in range(70): # fills the full 64-step window
|
| 302 |
+
one_step(w, cond, 4.0)
|
| 303 |
+
torch.cuda.synchronize()
|
| 304 |
+
ms = (time.time() - t0) * 1000 / 70
|
| 305 |
+
print("1 step (4 frames) = %.1f ms ; real time needs <= %.1f ms" % (ms, 4000 / 30), flush=True)
|
| 306 |
+
for _ in range(2): # window already sliding here
|
| 307 |
+
one_step(w, cond, 4.0, check=True)
|
| 308 |
+
print("graph vs original forward (sliding window):", ["%.3f" % v for v in w["diffs"]], flush=True)
|
| 309 |
+
threading.Thread(target=gen_loop, daemon=True).start()
|
| 310 |
+
srv = ThreadingHTTPServer(("127.0.0.1", PORT), H)
|
| 311 |
+
print("open http://localhost:%d" % PORT, flush=True)
|
| 312 |
+
srv.serve_forever()
|
live/start_windows.bat
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
@echo off
|
| 2 |
+
rem Start the real-time server and open the browser after the models are loaded.
|
| 3 |
+
cd /d "%~dp0"
|
| 4 |
+
set PYTHONIOENCODING=utf-8
|
| 5 |
+
start "" cmd /c "timeout /t 40 >nul & start http://localhost:8765/"
|
| 6 |
+
python -u live_server.py
|
| 7 |
+
pause
|
requirements.txt
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
torch>=2.4
|
| 2 |
+
numpy>=1.26
|
| 3 |
+
transformers>=4.40
|
| 4 |
+
safetensors>=0.4
|
weights/Mean.npy
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:8585e2a583f5e50a37256fea483b55553cbd92943eae25ac4a22d31bbbbe5e4f
|
| 3 |
+
size 1216
|
weights/Std.npy
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:3baa9bbdfb6dfbdf031259deb4e433d56fc2d256685b61d4c3acfc931ebb4039
|
| 3 |
+
size 1216
|
weights/generator.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:537b4d769d4977ac890a3906a0658c87b67c28ddfce8eb1201231536d704680e
|
| 3 |
+
size 122037208
|
weights/tokenizer.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:a522ca7959239dbd263aa8fd36038a767105bc3c1e43d3fed8e090c13f7e4724
|
| 3 |
+
size 35096316
|