Initial mirror of localai-org/motion-bricks.cpp@6fdb75e
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- .gitignore +10 -0
- .gitmodules +3 -0
- CMakeLists.txt +141 -0
- CMakePresets.json +50 -0
- FORK_PROVENANCE.md +11 -0
- LICENSE +201 -0
- README.md +161 -0
- bindings/go/abi_test.go +104 -0
- bindings/go/go.mod +5 -0
- bindings/go/go.sum +2 -0
- bindings/go/motionbricks.go +410 -0
- demo/go.mod +21 -0
- demo/go.sum +23 -0
- demo/main.go +371 -0
- demo/main_test.go +236 -0
- demo/web/app.js +499 -0
- demo/web/index.html +50 -0
- demo/web/style.css +39 -0
- demo/web/vendor/THREE-LICENSE.txt +21 -0
- demo/web/vendor/three.core.min.js +0 -0
- demo/web/vendor/three.module.min.js +0 -0
- docs/DEMO.md +120 -0
- docs/FORMATS.md +50 -0
- docs/IMPLEMENTATION.md +806 -0
- docs/motions-bricks.md +42 -0
- flake.lock +27 -0
- flake.nix +39 -0
- include/motionbricks/motionbricks.h +178 -0
- include/motionbricks/motionbricks.hpp +55 -0
- patches/ggml/README.md +9 -0
- reference/Dockerfile +13 -0
- reference/README.md +117 -0
- reference/extract_safe.py +290 -0
- reference/generate_fixtures.py +338 -0
- scripts/convert_fixtures_to_gguf.py +44 -0
- scripts/convert_styles.py +85 -0
- scripts/convert_to_gguf.py +307 -0
- scripts/download_gguf_weights.py +117 -0
- scripts/hf/MotionBricks-G1-GGML/MANIFEST.json +114 -0
- scripts/hf/MotionBricks-G1-GGML/NOTICE +5 -0
- scripts/hf/MotionBricks-G1-GGML/README.md +36 -0
- scripts/hf/MotionBricks-G1-GGML/UPSTREAM_LICENSE +186 -0
- scripts/publish_gguf.py +151 -0
- src/agent.cpp +266 -0
- src/agent.hpp +22 -0
- src/capi.cpp +552 -0
- src/cli.cpp +58 -0
- src/decoder.cpp +223 -0
- src/decoder.hpp +29 -0
- src/error.cpp +25 -0
.gitignore
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
/build/
|
| 2 |
+
/dist/
|
| 3 |
+
/generated/
|
| 4 |
+
/models/
|
| 5 |
+
/fixtures/
|
| 6 |
+
/.cache/
|
| 7 |
+
/.direnv/
|
| 8 |
+
/result
|
| 9 |
+
compile_commands.json
|
| 10 |
+
__pycache__/
|
.gitmodules
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[submodule "ggml"]
|
| 2 |
+
path = ggml
|
| 3 |
+
url = https://github.com/ggml-org/ggml.git
|
CMakeLists.txt
ADDED
|
@@ -0,0 +1,141 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
cmake_minimum_required(VERSION 3.25)
|
| 2 |
+
project(motionbricks VERSION 0.1.0 LANGUAGES C CXX)
|
| 3 |
+
|
| 4 |
+
set(CMAKE_C_STANDARD 11)
|
| 5 |
+
set(CMAKE_C_STANDARD_REQUIRED ON)
|
| 6 |
+
set(CMAKE_C_EXTENSIONS OFF)
|
| 7 |
+
set(CMAKE_CXX_STANDARD 23)
|
| 8 |
+
set(CMAKE_CXX_STANDARD_REQUIRED ON)
|
| 9 |
+
set(CMAKE_CXX_EXTENSIONS OFF)
|
| 10 |
+
|
| 11 |
+
include(GNUInstallDirs)
|
| 12 |
+
|
| 13 |
+
option(MOTIONBRICKS_BUILD_TESTS "Build motion-bricks.cpp tests" ON)
|
| 14 |
+
option(MOTIONBRICKS_ENABLE_GGML "Use the pinned GGML submodule when present" ON)
|
| 15 |
+
option(MOTIONBRICKS_ENABLE_VULKAN "Enable the GGML Vulkan backend" ON)
|
| 16 |
+
option(MOTIONBRICKS_ENABLE_FUZZERS "Build libFuzzer targets" OFF)
|
| 17 |
+
option(MOTIONBRICKS_SANITIZE "Enable AddressSanitizer and UndefinedBehaviorSanitizer" OFF)
|
| 18 |
+
option(MOTIONBRICKS_DOWNLOAD_MODELS "Download and verify the published runtime model/style bundle" ON)
|
| 19 |
+
set(MOTIONBRICKS_GGML_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/ggml" CACHE PATH
|
| 20 |
+
"Pinned GGML submodule source directory")
|
| 21 |
+
set(MOTIONBRICKS_MODEL_REPOSITORY "LocalAI-io/MotionBricks-G1-GGML" CACHE STRING
|
| 22 |
+
"Hugging Face repository containing the runtime bundle")
|
| 23 |
+
set(MOTIONBRICKS_MODEL_REVISION "cc2a47603dbc203a4f18f35dd06ed3611833f506" CACHE STRING
|
| 24 |
+
"Pinned Hugging Face revision containing the runtime bundle")
|
| 25 |
+
set(MOTIONBRICKS_REFERENCE_BUNDLE "${CMAKE_CURRENT_SOURCE_DIR}/generated/g1-f32" CACHE PATH
|
| 26 |
+
"Downloaded or locally converted MotionBricks runtime bundle")
|
| 27 |
+
set(MOTIONBRICKS_REFERENCE_STYLES "${CMAKE_CURRENT_SOURCE_DIR}/generated/styles" CACHE PATH
|
| 28 |
+
"Downloaded or locally converted MotionBricks style bundle")
|
| 29 |
+
|
| 30 |
+
if(MOTIONBRICKS_ENABLE_GGML AND MOTIONBRICKS_DOWNLOAD_MODELS)
|
| 31 |
+
find_package(Python3 COMPONENTS Interpreter REQUIRED)
|
| 32 |
+
execute_process(
|
| 33 |
+
COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/scripts/download_gguf_weights.py"
|
| 34 |
+
--output "${CMAKE_CURRENT_SOURCE_DIR}/generated"
|
| 35 |
+
--repo "${MOTIONBRICKS_MODEL_REPOSITORY}"
|
| 36 |
+
--revision "${MOTIONBRICKS_MODEL_REVISION}"
|
| 37 |
+
RESULT_VARIABLE MOTIONBRICKS_MODEL_DOWNLOAD_RESULT
|
| 38 |
+
COMMAND_ECHO STDOUT)
|
| 39 |
+
if(NOT MOTIONBRICKS_MODEL_DOWNLOAD_RESULT EQUAL 0)
|
| 40 |
+
message(FATAL_ERROR
|
| 41 |
+
"MotionBricks model download/verification failed. Set "
|
| 42 |
+
"MOTIONBRICKS_DOWNLOAD_MODELS=OFF for a source-only build.")
|
| 43 |
+
endif()
|
| 44 |
+
endif()
|
| 45 |
+
|
| 46 |
+
set(MOTIONBRICKS_SOURCES
|
| 47 |
+
src/capi.cpp
|
| 48 |
+
src/decoder.cpp
|
| 49 |
+
src/error.cpp
|
| 50 |
+
src/model.cpp
|
| 51 |
+
src/neural_runtime.cpp
|
| 52 |
+
src/pose.cpp)
|
| 53 |
+
list(APPEND MOTIONBRICKS_SOURCES src/root.cpp)
|
| 54 |
+
list(APPEND MOTIONBRICKS_SOURCES src/motion_rep.cpp)
|
| 55 |
+
list(APPEND MOTIONBRICKS_SOURCES src/planner.cpp)
|
| 56 |
+
list(APPEND MOTIONBRICKS_SOURCES src/style.cpp)
|
| 57 |
+
list(APPEND MOTIONBRICKS_SOURCES src/agent.cpp)
|
| 58 |
+
|
| 59 |
+
add_library(motionbricks_shared SHARED ${MOTIONBRICKS_SOURCES})
|
| 60 |
+
add_library(motionbricks::motionbricks ALIAS motionbricks_shared)
|
| 61 |
+
set_target_properties(motionbricks_shared PROPERTIES
|
| 62 |
+
OUTPUT_NAME motionbricks
|
| 63 |
+
EXPORT_NAME motionbricks
|
| 64 |
+
CXX_VISIBILITY_PRESET hidden
|
| 65 |
+
VISIBILITY_INLINES_HIDDEN YES
|
| 66 |
+
VERSION ${PROJECT_VERSION}
|
| 67 |
+
SOVERSION 0)
|
| 68 |
+
target_compile_definitions(motionbricks_shared PRIVATE MOTIONBRICKS_BUILD MOTIONBRICKS_SHARED)
|
| 69 |
+
|
| 70 |
+
add_library(motionbricks_static STATIC ${MOTIONBRICKS_SOURCES})
|
| 71 |
+
add_library(motionbricks::motionbricks_static ALIAS motionbricks_static)
|
| 72 |
+
set_target_properties(motionbricks_static PROPERTIES
|
| 73 |
+
OUTPUT_NAME motionbricks
|
| 74 |
+
EXPORT_NAME motionbricks_static)
|
| 75 |
+
target_compile_definitions(motionbricks_static PRIVATE MOTIONBRICKS_BUILD)
|
| 76 |
+
|
| 77 |
+
foreach(target motionbricks_shared motionbricks_static)
|
| 78 |
+
target_include_directories(${target}
|
| 79 |
+
PUBLIC
|
| 80 |
+
"$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>"
|
| 81 |
+
"$<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>"
|
| 82 |
+
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/src")
|
| 83 |
+
if(MSVC)
|
| 84 |
+
target_compile_options(${target} PRIVATE /W4 /permissive-)
|
| 85 |
+
else()
|
| 86 |
+
target_compile_options(${target} PRIVATE
|
| 87 |
+
-Wall -Wextra -Wpedantic -Wconversion -Wshadow)
|
| 88 |
+
endif()
|
| 89 |
+
endforeach()
|
| 90 |
+
|
| 91 |
+
if(MOTIONBRICKS_ENABLE_GGML)
|
| 92 |
+
if(EXISTS "${MOTIONBRICKS_GGML_SOURCE_DIR}/CMakeLists.txt")
|
| 93 |
+
set(GGML_BUILD_TESTS OFF CACHE BOOL "" FORCE)
|
| 94 |
+
set(GGML_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE)
|
| 95 |
+
set(GGML_NATIVE OFF CACHE BOOL "" FORCE)
|
| 96 |
+
set(GGML_VULKAN ${MOTIONBRICKS_ENABLE_VULKAN} CACHE BOOL "" FORCE)
|
| 97 |
+
set(GGML_BACKEND_DL OFF CACHE BOOL "" FORCE)
|
| 98 |
+
add_subdirectory("${MOTIONBRICKS_GGML_SOURCE_DIR}"
|
| 99 |
+
"${CMAKE_BINARY_DIR}/ggml" EXCLUDE_FROM_ALL)
|
| 100 |
+
# GGML is currently an internal build-tree dependency. Its installed
|
| 101 |
+
# runtime/package contract will be added with the first neural graph.
|
| 102 |
+
target_link_libraries(motionbricks_shared PRIVATE "$<BUILD_INTERFACE:ggml>")
|
| 103 |
+
target_link_libraries(motionbricks_static PRIVATE "$<BUILD_INTERFACE:ggml>")
|
| 104 |
+
target_compile_definitions(motionbricks_shared PRIVATE MOTIONBRICKS_HAVE_GGML=1)
|
| 105 |
+
target_compile_definitions(motionbricks_static PRIVATE MOTIONBRICKS_HAVE_GGML=1)
|
| 106 |
+
if(MOTIONBRICKS_ENABLE_VULKAN AND TARGET ggml-vulkan)
|
| 107 |
+
target_link_libraries(motionbricks_shared PRIVATE "$<BUILD_INTERFACE:ggml-vulkan>")
|
| 108 |
+
target_link_libraries(motionbricks_static PRIVATE "$<BUILD_INTERFACE:ggml-vulkan>")
|
| 109 |
+
target_compile_definitions(motionbricks_shared PRIVATE MOTIONBRICKS_HAVE_VULKAN=1)
|
| 110 |
+
target_compile_definitions(motionbricks_static PRIVATE MOTIONBRICKS_HAVE_VULKAN=1)
|
| 111 |
+
endif()
|
| 112 |
+
else()
|
| 113 |
+
message(STATUS "GGML submodule not present; building the ABI scaffold only")
|
| 114 |
+
endif()
|
| 115 |
+
endif()
|
| 116 |
+
|
| 117 |
+
if(MOTIONBRICKS_SANITIZE)
|
| 118 |
+
if(MSVC)
|
| 119 |
+
message(FATAL_ERROR "MOTIONBRICKS_SANITIZE currently requires Clang or GCC")
|
| 120 |
+
endif()
|
| 121 |
+
foreach(target motionbricks_shared motionbricks_static)
|
| 122 |
+
target_compile_options(${target} PRIVATE -fsanitize=address,undefined -fno-omit-frame-pointer)
|
| 123 |
+
target_link_options(${target} PRIVATE -fsanitize=address,undefined)
|
| 124 |
+
endforeach()
|
| 125 |
+
endif()
|
| 126 |
+
|
| 127 |
+
add_executable(motionbricks-cli src/cli.cpp)
|
| 128 |
+
target_link_libraries(motionbricks-cli PRIVATE motionbricks::motionbricks)
|
| 129 |
+
set_target_properties(motionbricks-cli PROPERTIES
|
| 130 |
+
RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin")
|
| 131 |
+
|
| 132 |
+
include(CTest)
|
| 133 |
+
if(BUILD_TESTING AND MOTIONBRICKS_BUILD_TESTS)
|
| 134 |
+
add_subdirectory(tests)
|
| 135 |
+
endif()
|
| 136 |
+
|
| 137 |
+
install(TARGETS motionbricks_shared motionbricks_static motionbricks-cli
|
| 138 |
+
RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
|
| 139 |
+
LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
|
| 140 |
+
ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR})
|
| 141 |
+
install(DIRECTORY include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
|
CMakePresets.json
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"version": 6,
|
| 3 |
+
"configurePresets": [
|
| 4 |
+
{
|
| 5 |
+
"name": "debug",
|
| 6 |
+
"generator": "Ninja",
|
| 7 |
+
"binaryDir": "${sourceDir}/build/debug",
|
| 8 |
+
"cacheVariables": {
|
| 9 |
+
"CMAKE_BUILD_TYPE": "Debug",
|
| 10 |
+
"CMAKE_EXPORT_COMPILE_COMMANDS": "ON"
|
| 11 |
+
}
|
| 12 |
+
},
|
| 13 |
+
{
|
| 14 |
+
"name": "release",
|
| 15 |
+
"generator": "Ninja",
|
| 16 |
+
"binaryDir": "${sourceDir}/build/release",
|
| 17 |
+
"cacheVariables": {
|
| 18 |
+
"CMAKE_BUILD_TYPE": "Release"
|
| 19 |
+
}
|
| 20 |
+
},
|
| 21 |
+
{
|
| 22 |
+
"name": "asan-ubsan",
|
| 23 |
+
"inherits": "debug",
|
| 24 |
+
"binaryDir": "${sourceDir}/build/asan-ubsan",
|
| 25 |
+
"cacheVariables": {
|
| 26 |
+
"CMAKE_C_COMPILER": "clang",
|
| 27 |
+
"CMAKE_CXX_COMPILER": "clang++",
|
| 28 |
+
"MOTIONBRICKS_ENABLE_VULKAN": "OFF",
|
| 29 |
+
"MOTIONBRICKS_SANITIZE": "ON"
|
| 30 |
+
}
|
| 31 |
+
}
|
| 32 |
+
],
|
| 33 |
+
"buildPresets": [
|
| 34 |
+
{"name": "debug", "configurePreset": "debug"},
|
| 35 |
+
{"name": "release", "configurePreset": "release"},
|
| 36 |
+
{"name": "asan-ubsan", "configurePreset": "asan-ubsan"}
|
| 37 |
+
],
|
| 38 |
+
"testPresets": [
|
| 39 |
+
{
|
| 40 |
+
"name": "debug",
|
| 41 |
+
"configurePreset": "debug",
|
| 42 |
+
"output": {"outputOnFailure": true}
|
| 43 |
+
},
|
| 44 |
+
{
|
| 45 |
+
"name": "asan-ubsan",
|
| 46 |
+
"configurePreset": "asan-ubsan",
|
| 47 |
+
"output": {"outputOnFailure": true}
|
| 48 |
+
}
|
| 49 |
+
]
|
| 50 |
+
}
|
FORK_PROVENANCE.md
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Fork provenance
|
| 2 |
+
|
| 3 |
+
- **Upstream:** `localai-org/motion-bricks.cpp (github)`
|
| 4 |
+
- **Upstream commit / SHA:** `6fdb75e15ddb7f97dd1a4abb8017a57b936bc7a3`
|
| 5 |
+
- **License:** Apache-2.0
|
| 6 |
+
- **Kind:** Code mirror
|
| 7 |
+
- **Workspace consumer:** MotionBricks-G1-GGML (converter/inference)
|
| 8 |
+
- **Forked to:** `chibifire/motion-bricks.cpp`
|
| 9 |
+
- **Forked on:** 2026-09-05
|
| 10 |
+
- **Author:** Ernest Lee <ernest.lee@chibifire.com>
|
| 11 |
+
- **Reason:** shipping-surface completeness under our own HF org so a base/source we depend on cannot disappear or relicense out from under us
|
LICENSE
ADDED
|
@@ -0,0 +1,201 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Apache License
|
| 2 |
+
Version 2.0, January 2004
|
| 3 |
+
http://www.apache.org/licenses/
|
| 4 |
+
|
| 5 |
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
| 6 |
+
|
| 7 |
+
1. Definitions.
|
| 8 |
+
|
| 9 |
+
"License" shall mean the terms and conditions for use, reproduction,
|
| 10 |
+
and distribution as defined by Sections 1 through 9 of this document.
|
| 11 |
+
|
| 12 |
+
"Licensor" shall mean the copyright owner or entity authorized by
|
| 13 |
+
the copyright owner that is granting the License.
|
| 14 |
+
|
| 15 |
+
"Legal Entity" shall mean the union of the acting entity and all
|
| 16 |
+
other entities that control, are controlled by, or are under common
|
| 17 |
+
control with that entity. For the purposes of this definition,
|
| 18 |
+
"control" means (i) the power, direct or indirect, to cause the
|
| 19 |
+
direction or management of such entity, whether by contract or
|
| 20 |
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
| 21 |
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
| 22 |
+
|
| 23 |
+
"You" (or "Your") shall mean an individual or Legal Entity
|
| 24 |
+
exercising permissions granted by this License.
|
| 25 |
+
|
| 26 |
+
"Source" form shall mean the preferred form for making modifications,
|
| 27 |
+
including but not limited to software source code, documentation
|
| 28 |
+
source, and configuration files.
|
| 29 |
+
|
| 30 |
+
"Object" form shall mean any form resulting from mechanical
|
| 31 |
+
transformation or translation of a Source form, including but
|
| 32 |
+
not limited to compiled object code, generated documentation,
|
| 33 |
+
and conversions to other media types.
|
| 34 |
+
|
| 35 |
+
"Work" shall mean the work of authorship, whether in Source or
|
| 36 |
+
Object form, made available under the License, as indicated by a
|
| 37 |
+
copyright notice that is included in or attached to the work
|
| 38 |
+
(an example is provided in the Appendix below).
|
| 39 |
+
|
| 40 |
+
"Derivative Works" shall mean any work, whether in Source or Object
|
| 41 |
+
form, that is based on (or derived from) the Work and for which the
|
| 42 |
+
editorial revisions, annotations, elaborations, or other modifications
|
| 43 |
+
represent, as a whole, an original work of authorship. For the purposes
|
| 44 |
+
of this License, Derivative Works shall not include works that remain
|
| 45 |
+
separable from, or merely link (or bind by name) to the interfaces of,
|
| 46 |
+
the Work and Derivative Works thereof.
|
| 47 |
+
|
| 48 |
+
"Contribution" shall mean any work of authorship, including
|
| 49 |
+
the original version of the Work and any modifications or additions
|
| 50 |
+
to that Work or Derivative Works thereof, that is intentionally
|
| 51 |
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
| 52 |
+
or by an individual or Legal Entity authorized to submit on behalf of
|
| 53 |
+
the copyright owner. For the purposes of this definition, "submitted"
|
| 54 |
+
means any form of electronic, verbal, or written communication sent
|
| 55 |
+
to the Licensor or its representatives, including but not limited to
|
| 56 |
+
communication on electronic mailing lists, source code control systems,
|
| 57 |
+
and issue tracking systems that are managed by, or on behalf of, the
|
| 58 |
+
Licensor for the purpose of discussing and improving the Work, but
|
| 59 |
+
excluding communication that is conspicuously marked or otherwise
|
| 60 |
+
designated in writing by the copyright owner as "Not a Contribution."
|
| 61 |
+
|
| 62 |
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
| 63 |
+
on behalf of whom a Contribution has been received by Licensor and
|
| 64 |
+
subsequently incorporated within the Work.
|
| 65 |
+
|
| 66 |
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
| 67 |
+
this License, each Contributor hereby grants to You a perpetual,
|
| 68 |
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
| 69 |
+
copyright license to reproduce, prepare Derivative Works of,
|
| 70 |
+
publicly display, publicly perform, sublicense, and distribute the
|
| 71 |
+
Work and such Derivative Works in Source or Object form.
|
| 72 |
+
|
| 73 |
+
3. Grant of Patent License. Subject to the terms and conditions of
|
| 74 |
+
this License, each Contributor hereby grants to You a perpetual,
|
| 75 |
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
| 76 |
+
(except as stated in this section) patent license to make, have made,
|
| 77 |
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
| 78 |
+
where such license applies only to those patent claims licensable
|
| 79 |
+
by such Contributor that are necessarily infringed by their
|
| 80 |
+
Contribution(s) alone or by combination of their Contribution(s)
|
| 81 |
+
with the Work to which such Contribution(s) was submitted. If You
|
| 82 |
+
institute patent litigation against any entity (including a
|
| 83 |
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
| 84 |
+
or a Contribution incorporated within the Work constitutes direct
|
| 85 |
+
or contributory patent infringement, then any patent licenses
|
| 86 |
+
granted to You under this License for that Work shall terminate
|
| 87 |
+
as of the date such litigation is filed.
|
| 88 |
+
|
| 89 |
+
4. Redistribution. You may reproduce and distribute copies of the
|
| 90 |
+
Work or Derivative Works thereof in any medium, with or without
|
| 91 |
+
modifications, and in Source or Object form, provided that You
|
| 92 |
+
meet the following conditions:
|
| 93 |
+
|
| 94 |
+
(a) You must give any other recipients of the Work or
|
| 95 |
+
Derivative Works a copy of this License; and
|
| 96 |
+
|
| 97 |
+
(b) You must cause any modified files to carry prominent notices
|
| 98 |
+
stating that You changed the files; and
|
| 99 |
+
|
| 100 |
+
(c) You must retain, in the Source form of any Derivative Works
|
| 101 |
+
that You distribute, all copyright, patent, trademark, and
|
| 102 |
+
attribution notices from the Source form of the Work,
|
| 103 |
+
excluding those notices that do not pertain to any part of
|
| 104 |
+
the Derivative Works; and
|
| 105 |
+
|
| 106 |
+
(d) If the Work includes a "NOTICE" text file as part of its
|
| 107 |
+
distribution, then any Derivative Works that You distribute must
|
| 108 |
+
include a readable copy of the attribution notices contained
|
| 109 |
+
within such NOTICE file, excluding those notices that do not
|
| 110 |
+
pertain to any part of the Derivative Works, in at least one
|
| 111 |
+
of the following places: within a NOTICE text file distributed
|
| 112 |
+
as part of the Derivative Works; within the Source form or
|
| 113 |
+
documentation, if provided along with the Derivative Works; or,
|
| 114 |
+
within a display generated by the Derivative Works, if and
|
| 115 |
+
wherever such third-party notices normally appear. The contents
|
| 116 |
+
of the NOTICE file are for informational purposes only and
|
| 117 |
+
do not modify the License. You may add Your own attribution
|
| 118 |
+
notices within Derivative Works that You distribute, alongside
|
| 119 |
+
or as an addendum to the NOTICE text from the Work, provided
|
| 120 |
+
that such additional attribution notices cannot be construed
|
| 121 |
+
as modifying the License.
|
| 122 |
+
|
| 123 |
+
You may add Your own copyright statement to Your modifications and
|
| 124 |
+
may provide additional or different license terms and conditions
|
| 125 |
+
for use, reproduction, or distribution of Your modifications, or
|
| 126 |
+
for any such Derivative Works as a whole, provided Your use,
|
| 127 |
+
reproduction, and distribution of the Work otherwise complies with
|
| 128 |
+
the conditions stated in this License.
|
| 129 |
+
|
| 130 |
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
| 131 |
+
any Contribution intentionally submitted for inclusion in the Work
|
| 132 |
+
by You to the Licensor shall be under the terms and conditions of
|
| 133 |
+
this License, without any additional terms or conditions.
|
| 134 |
+
Notwithstanding the above, nothing herein shall supersede or modify
|
| 135 |
+
the terms of any separate license agreement you may have executed
|
| 136 |
+
with Licensor regarding such Contributions.
|
| 137 |
+
|
| 138 |
+
6. Trademarks. This License does not grant permission to use the trade
|
| 139 |
+
names, trademarks, service marks, or product names of the Licensor,
|
| 140 |
+
except as required for reasonable and customary use in describing the
|
| 141 |
+
origin of the Work and reproducing the content of the NOTICE file.
|
| 142 |
+
|
| 143 |
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
| 144 |
+
agreed to in writing, Licensor provides the Work (and each
|
| 145 |
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
| 146 |
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
| 147 |
+
implied, including, without limitation, any warranties or conditions
|
| 148 |
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
| 149 |
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
| 150 |
+
appropriateness of using or redistributing the Work and assume any
|
| 151 |
+
risks associated with Your exercise of permissions under this License.
|
| 152 |
+
|
| 153 |
+
8. Limitation of Liability. In no event and under no legal theory,
|
| 154 |
+
whether in tort (including negligence), contract, or otherwise,
|
| 155 |
+
unless required by applicable law (such as deliberate and grossly
|
| 156 |
+
negligent acts) or agreed to in writing, shall any Contributor be
|
| 157 |
+
liable to You for damages, including any direct, indirect, special,
|
| 158 |
+
incidental, or consequential damages of any character arising as a
|
| 159 |
+
result of this License or out of the use or inability to use the
|
| 160 |
+
Work (including but not limited to damages for loss of goodwill,
|
| 161 |
+
work stoppage, computer failure or malfunction, or any and all
|
| 162 |
+
other commercial damages or losses), even if such Contributor
|
| 163 |
+
has been advised of the possibility of such damages.
|
| 164 |
+
|
| 165 |
+
9. Accepting Warranty or Additional Liability. While redistributing
|
| 166 |
+
the Work or Derivative Works thereof, You may choose to offer,
|
| 167 |
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
| 168 |
+
or other liability obligations and/or rights consistent with this
|
| 169 |
+
License. However, in accepting such obligations, You may act only
|
| 170 |
+
on Your own behalf and on Your sole responsibility, not on behalf
|
| 171 |
+
of any other Contributor, and only if You agree to indemnify,
|
| 172 |
+
defend, and hold each Contributor harmless for any liability
|
| 173 |
+
incurred by, or claims asserted against, such Contributor by reason
|
| 174 |
+
of your accepting any such warranty or additional liability.
|
| 175 |
+
|
| 176 |
+
END OF TERMS AND CONDITIONS
|
| 177 |
+
|
| 178 |
+
APPENDIX: How to apply the Apache License to your work.
|
| 179 |
+
|
| 180 |
+
To apply the Apache License to your work, attach the following
|
| 181 |
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
| 182 |
+
replaced with your own identifying information. (Don't include
|
| 183 |
+
the brackets!) The text should be enclosed in the appropriate
|
| 184 |
+
comment syntax for the file format. We also recommend that a
|
| 185 |
+
file or class name and description of purpose be included on the
|
| 186 |
+
same "printed page" as the copyright notice for easier
|
| 187 |
+
identification within third-party archives.
|
| 188 |
+
|
| 189 |
+
Copyright [yyyy] [name of copyright owner]
|
| 190 |
+
|
| 191 |
+
Licensed under the Apache License, Version 2.0 (the "License");
|
| 192 |
+
you may not use this file except in compliance with the License.
|
| 193 |
+
You may obtain a copy of the License at
|
| 194 |
+
|
| 195 |
+
http://www.apache.org/licenses/LICENSE-2.0
|
| 196 |
+
|
| 197 |
+
Unless required by applicable law or agreed to in writing, software
|
| 198 |
+
distributed under the License is distributed on an "AS IS" BASIS,
|
| 199 |
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 200 |
+
See the License for the specific language governing permissions and
|
| 201 |
+
limitations under the License.
|
README.md
ADDED
|
@@ -0,0 +1,161 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# motion-bricks.cpp
|
| 2 |
+
|
| 3 |
+
A C++23/GGML port of NVIDIA MotionBricks for CPU and Vulkan inference, with a
|
| 4 |
+
stable C ABI suitable for PureGo.
|
| 5 |
+
|
| 6 |
+
The released batch-one G1 inference path is implemented end to end: strict
|
| 7 |
+
GGUF loading, root/duration planning, pose-token prediction, VQ decoding,
|
| 8 |
+
418/414/413 feature conversion, style alignment, and skeletal animation
|
| 9 |
+
output. CPU and Vulkan use the same public API and preserve the same duration
|
| 10 |
+
and pose-token decisions in the reference suite.
|
| 11 |
+
|
| 12 |
+
## Build
|
| 13 |
+
|
| 14 |
+
The normal build uses CMake and does not depend on Nix:
|
| 15 |
+
|
| 16 |
+
```sh
|
| 17 |
+
cmake --preset debug
|
| 18 |
+
cmake --build --preset debug
|
| 19 |
+
ctest --preset debug
|
| 20 |
+
```
|
| 21 |
+
|
| 22 |
+
Configuration downloads and SHA-256-verifies the published 0.73 GB G1 F32
|
| 23 |
+
GGUF and style bundles into `generated/` when they are not already present.
|
| 24 |
+
The same operation can be run explicitly:
|
| 25 |
+
|
| 26 |
+
```sh
|
| 27 |
+
python scripts/download_gguf_weights.py
|
| 28 |
+
```
|
| 29 |
+
|
| 30 |
+
For an offline or source-only build, preserve an existing local bundle or use
|
| 31 |
+
`cmake --preset debug -DMOTIONBRICKS_DOWNLOAD_MODELS=OFF`. The repository and
|
| 32 |
+
revision are configurable with `MOTIONBRICKS_MODEL_REPOSITORY` and
|
| 33 |
+
`MOTIONBRICKS_MODEL_REVISION`.
|
| 34 |
+
|
| 35 |
+
On NixOS, enter the reproducible development shell first:
|
| 36 |
+
|
| 37 |
+
```sh
|
| 38 |
+
nix develop
|
| 39 |
+
cmake --preset debug
|
| 40 |
+
cmake --build --preset debug
|
| 41 |
+
ctest --preset debug
|
| 42 |
+
```
|
| 43 |
+
|
| 44 |
+
When the pinned `ggml/` submodule is present, it is included automatically.
|
| 45 |
+
The non-neural ABI and validation subset can also be built without GGML:
|
| 46 |
+
|
| 47 |
+
```sh
|
| 48 |
+
cmake -S . -B build/debug -G Ninja -DMOTIONBRICKS_ENABLE_GGML=OFF
|
| 49 |
+
```
|
| 50 |
+
|
| 51 |
+
The Go binding and demo use PureGo to load `libmotionbricks` at runtime; they
|
| 52 |
+
do not use cgo or `import "C"`. Once the native shared library has been built,
|
| 53 |
+
the Go components therefore need no C compiler and can be built with cgo
|
| 54 |
+
explicitly disabled:
|
| 55 |
+
|
| 56 |
+
```sh
|
| 57 |
+
cd demo
|
| 58 |
+
CGO_ENABLED=0 go build -o ../build/debug/bin/motionbricks-demo .
|
| 59 |
+
```
|
| 60 |
+
|
| 61 |
+
`CGO_ENABLED=0` is optional but recommended for making this property explicit
|
| 62 |
+
in builds and CI. It affects only the Go build—the native C++ library is still
|
| 63 |
+
built separately with CMake.
|
| 64 |
+
|
| 65 |
+
The sanitizer lane is:
|
| 66 |
+
|
| 67 |
+
```sh
|
| 68 |
+
cmake --preset asan-ubsan
|
| 69 |
+
cmake --build --preset asan-ubsan
|
| 70 |
+
ctest --preset asan-ubsan
|
| 71 |
+
```
|
| 72 |
+
|
| 73 |
+
## Current ABI
|
| 74 |
+
|
| 75 |
+
The installed C API uses only fixed-width scalars, pointers, and opaque heap
|
| 76 |
+
handles. Callers never reproduce a C or C++ structure layout. All constructors
|
| 77 |
+
have matching free functions, and no C++ exception crosses the ABI boundary.
|
| 78 |
+
|
| 79 |
+
The current CLI can report ABI information:
|
| 80 |
+
|
| 81 |
+
```sh
|
| 82 |
+
./build/debug/bin/motionbricks-cli abi
|
| 83 |
+
```
|
| 84 |
+
|
| 85 |
+
After producing the trusted safetensors intermediates described in
|
| 86 |
+
`reference/README.md`, build and inspect an F32 runtime bundle with:
|
| 87 |
+
|
| 88 |
+
```sh
|
| 89 |
+
python scripts/convert_to_gguf.py \
|
| 90 |
+
--safe-directory generated/safe \
|
| 91 |
+
--output generated/g1-f32
|
| 92 |
+
./build/debug/bin/motionbricks-cli inspect generated/g1-f32
|
| 93 |
+
```
|
| 94 |
+
|
| 95 |
+
The released G1 inference path contains exactly **183,148,382 learned F32
|
| 96 |
+
parameters**. The bundle loader validates the upstream revision, source
|
| 97 |
+
checkpoint identities, component roles, tensor counts, parameter counts,
|
| 98 |
+
anchor shapes, and the 34-joint parent topology before accepting a model.
|
| 99 |
+
|
| 100 |
+
Convert the 15 original demo styles with:
|
| 101 |
+
|
| 102 |
+
```sh
|
| 103 |
+
python scripts/convert_styles.py \
|
| 104 |
+
--safe-directory generated/safe \
|
| 105 |
+
--output generated/styles
|
| 106 |
+
```
|
| 107 |
+
|
| 108 |
+
At runtime the high-level flow is: load one immutable model, load one or more
|
| 109 |
+
`.mbstyle` assets, create an agent, reset it from an initial style (or supply
|
| 110 |
+
at least four frames of G1 context), set movement/facing/style on a command,
|
| 111 |
+
then call `mb_agent_plan`. The returned motion owns row-major F32 root
|
| 112 |
+
translations `[frames,3]` and local XYZW rotations `[frames,34,4]`. Call
|
| 113 |
+
`mb_agent_advance` as playback progresses so replanning uses the generated
|
| 114 |
+
motion as its next context.
|
| 115 |
+
|
| 116 |
+
The current implementation covers original preprocessed G1 styles. Direct
|
| 117 |
+
Kimodo GLB-to-`.mbstyle` conversion remains subsequent integration work.
|
| 118 |
+
|
| 119 |
+
## Weights
|
| 120 |
+
|
| 121 |
+
Ready-to-run native weights and all 15 upstream style primitives are published
|
| 122 |
+
as [MotionBricks-G1-GGML](https://huggingface.co/LocalAI-io/MotionBricks-G1-GGML)
|
| 123 |
+
under the Hugging Face `LocalAI-io` organisation. NVIDIA currently distributes
|
| 124 |
+
MotionBricks checkpoints through Git LFS in
|
| 125 |
+
[`NVlabs/GR00T-WholeBodyControl`](https://github.com/NVlabs/GR00T-WholeBodyControl/tree/a0732b642c0333077e127a2f56ab0014c196bca4/motionbricks),
|
| 126 |
+
not a separate Hugging Face model repository, so the model card links to that
|
| 127 |
+
pinned upstream revision. The downloader verifies the version-controlled
|
| 128 |
+
distribution manifest before accepting any file.
|
| 129 |
+
|
| 130 |
+
The default build is pinned to Hugging Face commit
|
| 131 |
+
`cc2a47603dbc203a4f18f35dd06ed3611833f506` rather than the mutable `main`
|
| 132 |
+
branch.
|
| 133 |
+
|
| 134 |
+
## Interactive demo
|
| 135 |
+
|
| 136 |
+
The initial Go/Three.js demo renders the model's 34-joint skeleton alongside
|
| 137 |
+
the four actual placed target-keyframe ghosts. It lets you steer with W/A/S/D,
|
| 138 |
+
turn facing with the arrow keys, orbit/zoom the camera, and switch among the
|
| 139 |
+
converted upstream styles. It uses the reusable PureGo binding and the same
|
| 140 |
+
stateful native agent as other applications. See the
|
| 141 |
+
[demo guide](docs/DEMO.md) for build, run, architecture, and headless-Chromium
|
| 142 |
+
test instructions.
|
| 143 |
+
|
| 144 |
+
## Design
|
| 145 |
+
|
| 146 |
+
- [Human-led design](docs/motions-bricks.md)
|
| 147 |
+
- [Implementation sketch and plan](docs/IMPLEMENTATION.md)
|
| 148 |
+
- [Versioned formats](docs/FORMATS.md)
|
| 149 |
+
- [Go/Three.js demo](docs/DEMO.md)
|
| 150 |
+
- [Pinned upstream reference](reference/README.md)
|
| 151 |
+
|
| 152 |
+
## License
|
| 153 |
+
|
| 154 |
+
motion-bricks.cpp source code is licensed under the
|
| 155 |
+
[Apache License 2.0](LICENSE). NVIDIA's original model weights and converted
|
| 156 |
+
GGUF/style distributions remain under the NVIDIA Open Model License reproduced
|
| 157 |
+
with the published model. It permits derivative models and redistribution with
|
| 158 |
+
conditions including retention of the agreement and attribution, Trustworthy
|
| 159 |
+
AI terms, and trade compliance. Bundled third-party components retain their own
|
| 160 |
+
licenses; the vendored Three.js files are covered by
|
| 161 |
+
`demo/web/vendor/THREE-LICENSE.txt`.
|
bindings/go/abi_test.go
ADDED
|
@@ -0,0 +1,104 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
//go:build linux || darwin || freebsd
|
| 2 |
+
|
| 3 |
+
package motionbricks
|
| 4 |
+
|
| 5 |
+
import (
|
| 6 |
+
"math"
|
| 7 |
+
"os"
|
| 8 |
+
"testing"
|
| 9 |
+
"unsafe"
|
| 10 |
+
|
| 11 |
+
"github.com/ebitengine/purego"
|
| 12 |
+
)
|
| 13 |
+
|
| 14 |
+
const (
|
| 15 |
+
statusOK = uint32(0)
|
| 16 |
+
deviceAuto = uint32(0)
|
| 17 |
+
deviceVulkan = uint32(2)
|
| 18 |
+
abiVersion = uint32(1)
|
| 19 |
+
errorCapacity = uint64(256)
|
| 20 |
+
)
|
| 21 |
+
|
| 22 |
+
// TestOpaqueABIRoundTrip intentionally mirrors no C struct. Every object is an
|
| 23 |
+
// opaque uintptr and every record field is accessed through a function.
|
| 24 |
+
func TestOpaqueABIRoundTrip(t *testing.T) {
|
| 25 |
+
library := os.Getenv("MOTIONBRICKS_LIB")
|
| 26 |
+
if library == "" {
|
| 27 |
+
t.Skip("set MOTIONBRICKS_LIB to the motion-bricks.cpp shared library")
|
| 28 |
+
}
|
| 29 |
+
|
| 30 |
+
handle, err := purego.Dlopen(library, purego.RTLD_NOW|purego.RTLD_LOCAL)
|
| 31 |
+
if err != nil {
|
| 32 |
+
t.Fatalf("dlopen %s: %v", library, err)
|
| 33 |
+
}
|
| 34 |
+
defer purego.Dlclose(handle)
|
| 35 |
+
|
| 36 |
+
var version func() uint32
|
| 37 |
+
var optionsCreate func(unsafe.Pointer, unsafe.Pointer, uint64) uint32
|
| 38 |
+
var optionsFree func(uintptr)
|
| 39 |
+
var optionsSetDevice func(uintptr, uint32, unsafe.Pointer, uint64) uint32
|
| 40 |
+
var optionsGetDevice func(uintptr, unsafe.Pointer, unsafe.Pointer, uint64) uint32
|
| 41 |
+
var commandCreate func(unsafe.Pointer, unsafe.Pointer, uint64) uint32
|
| 42 |
+
var commandFree func(uintptr)
|
| 43 |
+
var commandSetMovement func(uintptr, float32, float32, float32, unsafe.Pointer, uint64) uint32
|
| 44 |
+
var commandGetMovement func(uintptr, unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, uint64) uint32
|
| 45 |
+
|
| 46 |
+
purego.RegisterLibFunc(&version, handle, "mb_abi_version")
|
| 47 |
+
purego.RegisterLibFunc(&optionsCreate, handle, "mb_runtime_options_create")
|
| 48 |
+
purego.RegisterLibFunc(&optionsFree, handle, "mb_runtime_options_free")
|
| 49 |
+
purego.RegisterLibFunc(&optionsSetDevice, handle, "mb_runtime_options_set_device")
|
| 50 |
+
purego.RegisterLibFunc(&optionsGetDevice, handle, "mb_runtime_options_get_device")
|
| 51 |
+
purego.RegisterLibFunc(&commandCreate, handle, "mb_command_create")
|
| 52 |
+
purego.RegisterLibFunc(&commandFree, handle, "mb_command_free")
|
| 53 |
+
purego.RegisterLibFunc(&commandSetMovement, handle, "mb_command_set_movement_direction")
|
| 54 |
+
purego.RegisterLibFunc(&commandGetMovement, handle, "mb_command_get_movement_direction")
|
| 55 |
+
|
| 56 |
+
if got := version(); got != abiVersion {
|
| 57 |
+
t.Fatalf("ABI version = %d, want %d", got, abiVersion)
|
| 58 |
+
}
|
| 59 |
+
|
| 60 |
+
errorBuffer := make([]byte, errorCapacity)
|
| 61 |
+
errorPointer := unsafe.Pointer(&errorBuffer[0])
|
| 62 |
+
var options uintptr
|
| 63 |
+
if status := optionsCreate(unsafe.Pointer(&options), errorPointer, errorCapacity); status != statusOK {
|
| 64 |
+
t.Fatalf("options create status %d: %q", status, errorBuffer)
|
| 65 |
+
}
|
| 66 |
+
if options == 0 {
|
| 67 |
+
t.Fatal("options create returned a null handle")
|
| 68 |
+
}
|
| 69 |
+
defer optionsFree(options)
|
| 70 |
+
|
| 71 |
+
var device uint32 = math.MaxUint32
|
| 72 |
+
if status := optionsGetDevice(options, unsafe.Pointer(&device), errorPointer, errorCapacity); status != statusOK {
|
| 73 |
+
t.Fatalf("get default device status %d", status)
|
| 74 |
+
}
|
| 75 |
+
if device != deviceAuto {
|
| 76 |
+
t.Fatalf("default device = %d, want auto", device)
|
| 77 |
+
}
|
| 78 |
+
if status := optionsSetDevice(options, deviceVulkan, errorPointer, errorCapacity); status != statusOK {
|
| 79 |
+
t.Fatalf("set Vulkan status %d", status)
|
| 80 |
+
}
|
| 81 |
+
if status := optionsGetDevice(options, unsafe.Pointer(&device), errorPointer, errorCapacity); status != statusOK || device != deviceVulkan {
|
| 82 |
+
t.Fatalf("Vulkan round trip status=%d device=%d", status, device)
|
| 83 |
+
}
|
| 84 |
+
|
| 85 |
+
var command uintptr
|
| 86 |
+
if status := commandCreate(unsafe.Pointer(&command), errorPointer, errorCapacity); status != statusOK {
|
| 87 |
+
t.Fatalf("command create status %d", status)
|
| 88 |
+
}
|
| 89 |
+
if command == 0 {
|
| 90 |
+
t.Fatal("command create returned a null handle")
|
| 91 |
+
}
|
| 92 |
+
defer commandFree(command)
|
| 93 |
+
|
| 94 |
+
if status := commandSetMovement(command, 1.0, 0.0, -0.25, errorPointer, errorCapacity); status != statusOK {
|
| 95 |
+
t.Fatalf("set movement status %d", status)
|
| 96 |
+
}
|
| 97 |
+
var x, y, z float32
|
| 98 |
+
if status := commandGetMovement(command, unsafe.Pointer(&x), unsafe.Pointer(&y), unsafe.Pointer(&z), errorPointer, errorCapacity); status != statusOK {
|
| 99 |
+
t.Fatalf("get movement status %d", status)
|
| 100 |
+
}
|
| 101 |
+
if x != 1.0 || y != 0.0 || z != -0.25 {
|
| 102 |
+
t.Fatalf("movement = (%g, %g, %g)", x, y, z)
|
| 103 |
+
}
|
| 104 |
+
}
|
bindings/go/go.mod
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
module github.com/localai/motion-bricks.cpp/bindings/go
|
| 2 |
+
|
| 3 |
+
go 1.23
|
| 4 |
+
|
| 5 |
+
require github.com/ebitengine/purego v0.10.0
|
bindings/go/go.sum
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
github.com/ebitengine/purego v0.10.0 h1:QIw4xfpWT6GWTzaW5XEKy3HXoqrJGx1ijYHzTF0/ISU=
|
| 2 |
+
github.com/ebitengine/purego v0.10.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
|
bindings/go/motionbricks.go
ADDED
|
@@ -0,0 +1,410 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
//go:build linux || darwin || freebsd
|
| 2 |
+
|
| 3 |
+
// Package motionbricks provides a PureGo wrapper around the stable opaque C ABI.
|
| 4 |
+
package motionbricks
|
| 5 |
+
|
| 6 |
+
import (
|
| 7 |
+
"errors"
|
| 8 |
+
"fmt"
|
| 9 |
+
"math"
|
| 10 |
+
"unsafe"
|
| 11 |
+
|
| 12 |
+
"github.com/ebitengine/purego"
|
| 13 |
+
)
|
| 14 |
+
|
| 15 |
+
type Device uint32
|
| 16 |
+
|
| 17 |
+
const (
|
| 18 |
+
DeviceAuto Device = 0
|
| 19 |
+
DeviceCPU Device = 1
|
| 20 |
+
DeviceVulkan Device = 2
|
| 21 |
+
)
|
| 22 |
+
|
| 23 |
+
const errorBufferSize = 1024
|
| 24 |
+
|
| 25 |
+
type Library struct {
|
| 26 |
+
handle uintptr
|
| 27 |
+
abiVersion func() uint32
|
| 28 |
+
statusString func(uint32) uintptr
|
| 29 |
+
optionsCreate func(unsafe.Pointer, unsafe.Pointer, uint64) uint32
|
| 30 |
+
optionsFree func(uintptr)
|
| 31 |
+
optionsSetDevice func(uintptr, uint32, unsafe.Pointer, uint64) uint32
|
| 32 |
+
modelLoad func(unsafe.Pointer, uintptr, unsafe.Pointer, unsafe.Pointer, uint64) uint32
|
| 33 |
+
modelFree func(uintptr)
|
| 34 |
+
modelJointCount func(uintptr, unsafe.Pointer, unsafe.Pointer, uint64) uint32
|
| 35 |
+
modelJointName func(uintptr, uint32, unsafe.Pointer, unsafe.Pointer, uint64) uint32
|
| 36 |
+
modelJointParent func(uintptr, uint32, unsafe.Pointer, unsafe.Pointer, uint64) uint32
|
| 37 |
+
modelNeutralPosition func(uintptr, uint32, unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, uint64) uint32
|
| 38 |
+
styleLoad func(uintptr, unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, uint64) uint32
|
| 39 |
+
styleFree func(uintptr)
|
| 40 |
+
styleName func(uintptr, unsafe.Pointer, unsafe.Pointer, uint64) uint32
|
| 41 |
+
styleSpeed func(uintptr, unsafe.Pointer, unsafe.Pointer, uint64) uint32
|
| 42 |
+
agentCreate func(uintptr, unsafe.Pointer, unsafe.Pointer, uint64) uint32
|
| 43 |
+
agentFree func(uintptr)
|
| 44 |
+
agentReset func(uintptr, uintptr, unsafe.Pointer, uint64) uint32
|
| 45 |
+
agentSetContext func(uintptr, unsafe.Pointer, unsafe.Pointer, uint64, uint64, unsafe.Pointer, uint64) uint32
|
| 46 |
+
agentPlan func(uintptr, uintptr, unsafe.Pointer, unsafe.Pointer, uint64) uint32
|
| 47 |
+
agentAdvance func(uintptr, uint32, unsafe.Pointer, uint64) uint32
|
| 48 |
+
commandCreate func(unsafe.Pointer, unsafe.Pointer, uint64) uint32
|
| 49 |
+
commandFree func(uintptr)
|
| 50 |
+
commandSetStyle func(uintptr, uintptr, unsafe.Pointer, uint64) uint32
|
| 51 |
+
commandSetMovement func(uintptr, float32, float32, float32, unsafe.Pointer, uint64) uint32
|
| 52 |
+
commandSetFacing func(uintptr, float32, float32, float32, unsafe.Pointer, uint64) uint32
|
| 53 |
+
commandSetSpeed func(uintptr, float32, unsafe.Pointer, uint64) uint32
|
| 54 |
+
commandSetWorldTarget func(uintptr, float32, float32, float32, float32, uint32, unsafe.Pointer, uint64) uint32
|
| 55 |
+
commandSetSeed func(uintptr, uint64, unsafe.Pointer, uint64) uint32
|
| 56 |
+
motionFree func(uintptr)
|
| 57 |
+
motionFrames func(uintptr, unsafe.Pointer, unsafe.Pointer, uint64) uint32
|
| 58 |
+
motionJoints func(uintptr, unsafe.Pointer, unsafe.Pointer, uint64) uint32
|
| 59 |
+
motionRoots func(uintptr, unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, uint64) uint32
|
| 60 |
+
motionRotations func(uintptr, unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, uint64) uint32
|
| 61 |
+
motionTargetFrames func(uintptr, unsafe.Pointer, unsafe.Pointer, uint64) uint32
|
| 62 |
+
motionTargetRoots func(uintptr, unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, uint64) uint32
|
| 63 |
+
motionTargetRotations func(uintptr, unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, uint64) uint32
|
| 64 |
+
}
|
| 65 |
+
|
| 66 |
+
func Open(path string) (*Library, error) {
|
| 67 |
+
handle, err := purego.Dlopen(path, purego.RTLD_NOW|purego.RTLD_LOCAL)
|
| 68 |
+
if err != nil {
|
| 69 |
+
return nil, err
|
| 70 |
+
}
|
| 71 |
+
library := &Library{handle: handle}
|
| 72 |
+
register := func(target any, name string) { purego.RegisterLibFunc(target, handle, name) }
|
| 73 |
+
register(&library.abiVersion, "mb_abi_version")
|
| 74 |
+
register(&library.statusString, "mb_status_string")
|
| 75 |
+
register(&library.optionsCreate, "mb_runtime_options_create")
|
| 76 |
+
register(&library.optionsFree, "mb_runtime_options_free")
|
| 77 |
+
register(&library.optionsSetDevice, "mb_runtime_options_set_device")
|
| 78 |
+
register(&library.modelLoad, "mb_model_load")
|
| 79 |
+
register(&library.modelFree, "mb_model_free")
|
| 80 |
+
register(&library.modelJointCount, "mb_model_get_joint_count")
|
| 81 |
+
register(&library.modelJointName, "mb_model_get_joint_name")
|
| 82 |
+
register(&library.modelJointParent, "mb_model_get_joint_parent")
|
| 83 |
+
register(&library.modelNeutralPosition, "mb_model_get_neutral_joint_position")
|
| 84 |
+
register(&library.styleLoad, "mb_style_load")
|
| 85 |
+
register(&library.styleFree, "mb_style_free")
|
| 86 |
+
register(&library.styleName, "mb_style_get_name")
|
| 87 |
+
register(&library.styleSpeed, "mb_style_get_speed")
|
| 88 |
+
register(&library.agentCreate, "mb_agent_create")
|
| 89 |
+
register(&library.agentFree, "mb_agent_free")
|
| 90 |
+
register(&library.agentReset, "mb_agent_reset")
|
| 91 |
+
register(&library.agentSetContext, "mb_agent_set_context")
|
| 92 |
+
register(&library.agentPlan, "mb_agent_plan")
|
| 93 |
+
register(&library.agentAdvance, "mb_agent_advance")
|
| 94 |
+
register(&library.commandCreate, "mb_command_create")
|
| 95 |
+
register(&library.commandFree, "mb_command_free")
|
| 96 |
+
register(&library.commandSetStyle, "mb_command_set_style")
|
| 97 |
+
register(&library.commandSetMovement, "mb_command_set_movement_direction")
|
| 98 |
+
register(&library.commandSetFacing, "mb_command_set_facing_direction")
|
| 99 |
+
register(&library.commandSetSpeed, "mb_command_set_target_speed")
|
| 100 |
+
register(&library.commandSetWorldTarget, "mb_command_set_world_target")
|
| 101 |
+
register(&library.commandSetSeed, "mb_command_set_seed")
|
| 102 |
+
register(&library.motionFree, "mb_motion_free")
|
| 103 |
+
register(&library.motionFrames, "mb_motion_get_frame_count")
|
| 104 |
+
register(&library.motionJoints, "mb_motion_get_joint_count")
|
| 105 |
+
register(&library.motionRoots, "mb_motion_get_root_translations")
|
| 106 |
+
register(&library.motionRotations, "mb_motion_get_local_rotations_xyzw")
|
| 107 |
+
register(&library.motionTargetFrames, "mb_motion_get_target_frame_count")
|
| 108 |
+
register(&library.motionTargetRoots, "mb_motion_get_target_root_translations")
|
| 109 |
+
register(&library.motionTargetRotations, "mb_motion_get_target_local_rotations_xyzw")
|
| 110 |
+
if version := library.abiVersion(); version != 1 {
|
| 111 |
+
library.Close()
|
| 112 |
+
return nil, fmt.Errorf("motionbricks ABI version %d is unsupported", version)
|
| 113 |
+
}
|
| 114 |
+
return library, nil
|
| 115 |
+
}
|
| 116 |
+
|
| 117 |
+
func (l *Library) Close() error {
|
| 118 |
+
if l == nil || l.handle == 0 {
|
| 119 |
+
return nil
|
| 120 |
+
}
|
| 121 |
+
err := purego.Dlclose(l.handle)
|
| 122 |
+
l.handle = 0
|
| 123 |
+
return err
|
| 124 |
+
}
|
| 125 |
+
|
| 126 |
+
func cString(value string) ([]byte, error) {
|
| 127 |
+
for _, char := range value {
|
| 128 |
+
if char == 0 {
|
| 129 |
+
return nil, errors.New("string contains NUL")
|
| 130 |
+
}
|
| 131 |
+
}
|
| 132 |
+
return append([]byte(value), 0), nil
|
| 133 |
+
}
|
| 134 |
+
|
| 135 |
+
func goString(pointer uintptr) string {
|
| 136 |
+
if pointer == 0 {
|
| 137 |
+
return ""
|
| 138 |
+
}
|
| 139 |
+
const maximum = 1 << 20
|
| 140 |
+
bytes := make([]byte, 0, 64)
|
| 141 |
+
for index := uintptr(0); index < maximum; index++ {
|
| 142 |
+
value := *(*byte)(unsafe.Pointer(pointer + index))
|
| 143 |
+
if value == 0 {
|
| 144 |
+
return string(bytes)
|
| 145 |
+
}
|
| 146 |
+
bytes = append(bytes, value)
|
| 147 |
+
}
|
| 148 |
+
return ""
|
| 149 |
+
}
|
| 150 |
+
|
| 151 |
+
func errorPointer(buffer []byte) unsafe.Pointer { return unsafe.Pointer(&buffer[0]) }
|
| 152 |
+
|
| 153 |
+
func (l *Library) check(operation string, status uint32, buffer []byte) error {
|
| 154 |
+
if status == 0 {
|
| 155 |
+
return nil
|
| 156 |
+
}
|
| 157 |
+
message := ""
|
| 158 |
+
for index, value := range buffer {
|
| 159 |
+
if value == 0 {
|
| 160 |
+
message = string(buffer[:index])
|
| 161 |
+
break
|
| 162 |
+
}
|
| 163 |
+
}
|
| 164 |
+
if message == "" {
|
| 165 |
+
message = goString(l.statusString(status))
|
| 166 |
+
}
|
| 167 |
+
return fmt.Errorf("%s: %s", operation, message)
|
| 168 |
+
}
|
| 169 |
+
|
| 170 |
+
type Model struct {
|
| 171 |
+
library *Library
|
| 172 |
+
handle uintptr
|
| 173 |
+
}
|
| 174 |
+
type Style struct {
|
| 175 |
+
library *Library
|
| 176 |
+
handle uintptr
|
| 177 |
+
Name string
|
| 178 |
+
Speed float32
|
| 179 |
+
}
|
| 180 |
+
type Agent struct {
|
| 181 |
+
model *Model
|
| 182 |
+
handle uintptr
|
| 183 |
+
}
|
| 184 |
+
type Command struct {
|
| 185 |
+
library *Library
|
| 186 |
+
handle uintptr
|
| 187 |
+
}
|
| 188 |
+
|
| 189 |
+
type Joint struct {
|
| 190 |
+
Name string `json:"name"`
|
| 191 |
+
Parent int32 `json:"parent"`
|
| 192 |
+
Position [3]float32 `json:"position"`
|
| 193 |
+
}
|
| 194 |
+
type Motion struct {
|
| 195 |
+
Frames uint64 `json:"frames"`
|
| 196 |
+
Joints uint64 `json:"joints"`
|
| 197 |
+
Roots []float32 `json:"roots"`
|
| 198 |
+
Rotations []float32 `json:"rotations"`
|
| 199 |
+
Targets *Keyframes `json:"-"`
|
| 200 |
+
}
|
| 201 |
+
type Keyframes struct {
|
| 202 |
+
Frames uint64 `json:"frames"`
|
| 203 |
+
Joints uint64 `json:"joints"`
|
| 204 |
+
Roots []float32 `json:"roots"`
|
| 205 |
+
Rotations []float32 `json:"rotations"`
|
| 206 |
+
}
|
| 207 |
+
|
| 208 |
+
func (l *Library) LoadModel(path string, device Device) (*Model, error) {
|
| 209 |
+
pathBytes, err := cString(path)
|
| 210 |
+
if err != nil {
|
| 211 |
+
return nil, err
|
| 212 |
+
}
|
| 213 |
+
buffer := make([]byte, errorBufferSize)
|
| 214 |
+
var options uintptr
|
| 215 |
+
if err = l.check("create runtime options", l.optionsCreate(unsafe.Pointer(&options), errorPointer(buffer), uint64(len(buffer))), buffer); err != nil {
|
| 216 |
+
return nil, err
|
| 217 |
+
}
|
| 218 |
+
defer l.optionsFree(options)
|
| 219 |
+
if err = l.check("select device", l.optionsSetDevice(options, uint32(device), errorPointer(buffer), uint64(len(buffer))), buffer); err != nil {
|
| 220 |
+
return nil, err
|
| 221 |
+
}
|
| 222 |
+
var handle uintptr
|
| 223 |
+
status := l.modelLoad(unsafe.Pointer(&pathBytes[0]), options, unsafe.Pointer(&handle), errorPointer(buffer), uint64(len(buffer)))
|
| 224 |
+
if err = l.check("load model", status, buffer); err != nil {
|
| 225 |
+
return nil, err
|
| 226 |
+
}
|
| 227 |
+
return &Model{library: l, handle: handle}, nil
|
| 228 |
+
}
|
| 229 |
+
|
| 230 |
+
func (m *Model) Close() {
|
| 231 |
+
if m != nil && m.handle != 0 {
|
| 232 |
+
m.library.modelFree(m.handle)
|
| 233 |
+
m.handle = 0
|
| 234 |
+
}
|
| 235 |
+
}
|
| 236 |
+
|
| 237 |
+
func (m *Model) Skeleton() ([]Joint, error) {
|
| 238 |
+
buffer := make([]byte, errorBufferSize)
|
| 239 |
+
var count uint32
|
| 240 |
+
if err := m.library.check("get joint count", m.library.modelJointCount(m.handle, unsafe.Pointer(&count), errorPointer(buffer), uint64(len(buffer))), buffer); err != nil {
|
| 241 |
+
return nil, err
|
| 242 |
+
}
|
| 243 |
+
joints := make([]Joint, count)
|
| 244 |
+
for index := uint32(0); index < count; index++ {
|
| 245 |
+
var name uintptr
|
| 246 |
+
if err := m.library.check("get joint name", m.library.modelJointName(m.handle, index, unsafe.Pointer(&name), errorPointer(buffer), uint64(len(buffer))), buffer); err != nil {
|
| 247 |
+
return nil, err
|
| 248 |
+
}
|
| 249 |
+
joints[index].Name = goString(name)
|
| 250 |
+
if err := m.library.check("get joint parent", m.library.modelJointParent(m.handle, index, unsafe.Pointer(&joints[index].Parent), errorPointer(buffer), uint64(len(buffer))), buffer); err != nil {
|
| 251 |
+
return nil, err
|
| 252 |
+
}
|
| 253 |
+
position := &joints[index].Position
|
| 254 |
+
if err := m.library.check("get neutral position", m.library.modelNeutralPosition(m.handle, index, unsafe.Pointer(&position[0]), unsafe.Pointer(&position[1]), unsafe.Pointer(&position[2]), errorPointer(buffer), uint64(len(buffer))), buffer); err != nil {
|
| 255 |
+
return nil, err
|
| 256 |
+
}
|
| 257 |
+
}
|
| 258 |
+
return joints, nil
|
| 259 |
+
}
|
| 260 |
+
|
| 261 |
+
func (m *Model) LoadStyle(path string) (*Style, error) {
|
| 262 |
+
pathBytes, err := cString(path)
|
| 263 |
+
if err != nil {
|
| 264 |
+
return nil, err
|
| 265 |
+
}
|
| 266 |
+
buffer := make([]byte, errorBufferSize)
|
| 267 |
+
var handle uintptr
|
| 268 |
+
if err = m.library.check("load style", m.library.styleLoad(m.handle, unsafe.Pointer(&pathBytes[0]), unsafe.Pointer(&handle), errorPointer(buffer), uint64(len(buffer))), buffer); err != nil {
|
| 269 |
+
return nil, err
|
| 270 |
+
}
|
| 271 |
+
style := &Style{library: m.library, handle: handle}
|
| 272 |
+
var name uintptr
|
| 273 |
+
if err = m.library.check("get style name", m.library.styleName(handle, unsafe.Pointer(&name), errorPointer(buffer), uint64(len(buffer))), buffer); err != nil {
|
| 274 |
+
style.Close()
|
| 275 |
+
return nil, err
|
| 276 |
+
}
|
| 277 |
+
style.Name = goString(name)
|
| 278 |
+
if err = m.library.check("get style speed", m.library.styleSpeed(handle, unsafe.Pointer(&style.Speed), errorPointer(buffer), uint64(len(buffer))), buffer); err != nil {
|
| 279 |
+
style.Close()
|
| 280 |
+
return nil, err
|
| 281 |
+
}
|
| 282 |
+
return style, nil
|
| 283 |
+
}
|
| 284 |
+
|
| 285 |
+
func (s *Style) Close() {
|
| 286 |
+
if s != nil && s.handle != 0 {
|
| 287 |
+
s.library.styleFree(s.handle)
|
| 288 |
+
s.handle = 0
|
| 289 |
+
}
|
| 290 |
+
}
|
| 291 |
+
|
| 292 |
+
func (m *Model) NewAgent() (*Agent, error) {
|
| 293 |
+
buffer := make([]byte, errorBufferSize)
|
| 294 |
+
var handle uintptr
|
| 295 |
+
if err := m.library.check("create agent", m.library.agentCreate(m.handle, unsafe.Pointer(&handle), errorPointer(buffer), uint64(len(buffer))), buffer); err != nil {
|
| 296 |
+
return nil, err
|
| 297 |
+
}
|
| 298 |
+
return &Agent{model: m, handle: handle}, nil
|
| 299 |
+
}
|
| 300 |
+
func (a *Agent) Close() {
|
| 301 |
+
if a != nil && a.handle != 0 {
|
| 302 |
+
a.model.library.agentFree(a.handle)
|
| 303 |
+
a.handle = 0
|
| 304 |
+
}
|
| 305 |
+
}
|
| 306 |
+
func (a *Agent) Reset(style *Style) error {
|
| 307 |
+
buffer := make([]byte, errorBufferSize)
|
| 308 |
+
return a.model.library.check("reset agent", a.model.library.agentReset(a.handle, style.handle, errorPointer(buffer), uint64(len(buffer))), buffer)
|
| 309 |
+
}
|
| 310 |
+
func (a *Agent) Advance(frames uint32) error {
|
| 311 |
+
buffer := make([]byte, errorBufferSize)
|
| 312 |
+
return a.model.library.check("advance agent", a.model.library.agentAdvance(a.handle, frames, errorPointer(buffer), uint64(len(buffer))), buffer)
|
| 313 |
+
}
|
| 314 |
+
func (a *Agent) SetContext(roots, rotations []float32, frames uint64) error {
|
| 315 |
+
if frames < 4 || len(roots) != int(frames*3) || len(rotations) != int(frames*34*4) {
|
| 316 |
+
return errors.New("invalid G1 context shape")
|
| 317 |
+
}
|
| 318 |
+
buffer := make([]byte, errorBufferSize)
|
| 319 |
+
return a.model.library.check("set agent context", a.model.library.agentSetContext(a.handle, unsafe.Pointer(&roots[0]), unsafe.Pointer(&rotations[0]), frames, 34, errorPointer(buffer), uint64(len(buffer))), buffer)
|
| 320 |
+
}
|
| 321 |
+
|
| 322 |
+
func (l *Library) NewCommand() (*Command, error) {
|
| 323 |
+
buffer := make([]byte, errorBufferSize)
|
| 324 |
+
var handle uintptr
|
| 325 |
+
if err := l.check("create command", l.commandCreate(unsafe.Pointer(&handle), errorPointer(buffer), uint64(len(buffer))), buffer); err != nil {
|
| 326 |
+
return nil, err
|
| 327 |
+
}
|
| 328 |
+
return &Command{library: l, handle: handle}, nil
|
| 329 |
+
}
|
| 330 |
+
func (c *Command) Close() {
|
| 331 |
+
if c != nil && c.handle != 0 {
|
| 332 |
+
c.library.commandFree(c.handle)
|
| 333 |
+
c.handle = 0
|
| 334 |
+
}
|
| 335 |
+
}
|
| 336 |
+
func (c *Command) SetStyle(style *Style) error {
|
| 337 |
+
buffer := make([]byte, errorBufferSize)
|
| 338 |
+
return c.library.check("set command style", c.library.commandSetStyle(c.handle, style.handle, errorPointer(buffer), uint64(len(buffer))), buffer)
|
| 339 |
+
}
|
| 340 |
+
func (c *Command) SetMovement(x, y, z float32) error {
|
| 341 |
+
buffer := make([]byte, errorBufferSize)
|
| 342 |
+
return c.library.check("set movement", c.library.commandSetMovement(c.handle, x, y, z, errorPointer(buffer), uint64(len(buffer))), buffer)
|
| 343 |
+
}
|
| 344 |
+
func (c *Command) SetFacing(x, y, z float32) error {
|
| 345 |
+
buffer := make([]byte, errorBufferSize)
|
| 346 |
+
return c.library.check("set facing", c.library.commandSetFacing(c.handle, x, y, z, errorPointer(buffer), uint64(len(buffer))), buffer)
|
| 347 |
+
}
|
| 348 |
+
func (c *Command) SetSpeed(speed float32) error {
|
| 349 |
+
if math.IsNaN(float64(speed)) {
|
| 350 |
+
return errors.New("speed is NaN")
|
| 351 |
+
}
|
| 352 |
+
buffer := make([]byte, errorBufferSize)
|
| 353 |
+
return c.library.check("set speed", c.library.commandSetSpeed(c.handle, speed, errorPointer(buffer), uint64(len(buffer))), buffer)
|
| 354 |
+
}
|
| 355 |
+
func (c *Command) SetSeed(seed uint64) error {
|
| 356 |
+
buffer := make([]byte, errorBufferSize)
|
| 357 |
+
return c.library.check("set seed", c.library.commandSetSeed(c.handle, seed, errorPointer(buffer), uint64(len(buffer))), buffer)
|
| 358 |
+
}
|
| 359 |
+
func (c *Command) SetWorldTarget(x, y, z, heading float32, enabled bool) error {
|
| 360 |
+
var flag uint32
|
| 361 |
+
if enabled {
|
| 362 |
+
flag = 1
|
| 363 |
+
}
|
| 364 |
+
buffer := make([]byte, errorBufferSize)
|
| 365 |
+
return c.library.check("set world target", c.library.commandSetWorldTarget(c.handle, x, y, z, heading, flag, errorPointer(buffer), uint64(len(buffer))), buffer)
|
| 366 |
+
}
|
| 367 |
+
|
| 368 |
+
func (a *Agent) Plan(command *Command) (*Motion, error) {
|
| 369 |
+
buffer := make([]byte, errorBufferSize)
|
| 370 |
+
var handle uintptr
|
| 371 |
+
if err := a.model.library.check("plan motion", a.model.library.agentPlan(a.handle, command.handle, unsafe.Pointer(&handle), errorPointer(buffer), uint64(len(buffer))), buffer); err != nil {
|
| 372 |
+
return nil, err
|
| 373 |
+
}
|
| 374 |
+
defer a.model.library.motionFree(handle)
|
| 375 |
+
motion := &Motion{}
|
| 376 |
+
var rootsPointer, rotationsPointer, targetRootsPointer, targetRotationsPointer uintptr
|
| 377 |
+
var rootsCount, rotationsCount, targetRootsCount, targetRotationsCount uint64
|
| 378 |
+
if err := a.model.library.check("get frame count", a.model.library.motionFrames(handle, unsafe.Pointer(&motion.Frames), errorPointer(buffer), uint64(len(buffer))), buffer); err != nil {
|
| 379 |
+
return nil, err
|
| 380 |
+
}
|
| 381 |
+
if err := a.model.library.check("get joint count", a.model.library.motionJoints(handle, unsafe.Pointer(&motion.Joints), errorPointer(buffer), uint64(len(buffer))), buffer); err != nil {
|
| 382 |
+
return nil, err
|
| 383 |
+
}
|
| 384 |
+
if err := a.model.library.check("get roots", a.model.library.motionRoots(handle, unsafe.Pointer(&rootsPointer), unsafe.Pointer(&rootsCount), errorPointer(buffer), uint64(len(buffer))), buffer); err != nil {
|
| 385 |
+
return nil, err
|
| 386 |
+
}
|
| 387 |
+
if err := a.model.library.check("get rotations", a.model.library.motionRotations(handle, unsafe.Pointer(&rotationsPointer), unsafe.Pointer(&rotationsCount), errorPointer(buffer), uint64(len(buffer))), buffer); err != nil {
|
| 388 |
+
return nil, err
|
| 389 |
+
}
|
| 390 |
+
targets := &Keyframes{Joints: motion.Joints}
|
| 391 |
+
if err := a.model.library.check("get target frame count", a.model.library.motionTargetFrames(handle, unsafe.Pointer(&targets.Frames), errorPointer(buffer), uint64(len(buffer))), buffer); err != nil {
|
| 392 |
+
return nil, err
|
| 393 |
+
}
|
| 394 |
+
if err := a.model.library.check("get target roots", a.model.library.motionTargetRoots(handle, unsafe.Pointer(&targetRootsPointer), unsafe.Pointer(&targetRootsCount), errorPointer(buffer), uint64(len(buffer))), buffer); err != nil {
|
| 395 |
+
return nil, err
|
| 396 |
+
}
|
| 397 |
+
if err := a.model.library.check("get target rotations", a.model.library.motionTargetRotations(handle, unsafe.Pointer(&targetRotationsPointer), unsafe.Pointer(&targetRotationsCount), errorPointer(buffer), uint64(len(buffer))), buffer); err != nil {
|
| 398 |
+
return nil, err
|
| 399 |
+
}
|
| 400 |
+
if rootsCount != motion.Frames*3 || rotationsCount != motion.Frames*motion.Joints*4 ||
|
| 401 |
+
targetRootsCount != targets.Frames*3 || targetRotationsCount != targets.Frames*targets.Joints*4 {
|
| 402 |
+
return nil, errors.New("native motion data has inconsistent dimensions")
|
| 403 |
+
}
|
| 404 |
+
motion.Roots = append([]float32(nil), unsafe.Slice((*float32)(unsafe.Pointer(rootsPointer)), rootsCount)...)
|
| 405 |
+
motion.Rotations = append([]float32(nil), unsafe.Slice((*float32)(unsafe.Pointer(rotationsPointer)), rotationsCount)...)
|
| 406 |
+
targets.Roots = append([]float32(nil), unsafe.Slice((*float32)(unsafe.Pointer(targetRootsPointer)), targetRootsCount)...)
|
| 407 |
+
targets.Rotations = append([]float32(nil), unsafe.Slice((*float32)(unsafe.Pointer(targetRotationsPointer)), targetRotationsCount)...)
|
| 408 |
+
motion.Targets = targets
|
| 409 |
+
return motion, nil
|
| 410 |
+
}
|
demo/go.mod
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
module github.com/localai/motion-bricks.cpp/demo
|
| 2 |
+
|
| 3 |
+
go 1.26
|
| 4 |
+
|
| 5 |
+
require (
|
| 6 |
+
github.com/chromedp/cdproto v0.0.0-20260714215040-dc233986426f
|
| 7 |
+
github.com/chromedp/chromedp v0.16.0
|
| 8 |
+
github.com/localai/motion-bricks.cpp/bindings/go v0.0.0
|
| 9 |
+
)
|
| 10 |
+
|
| 11 |
+
require (
|
| 12 |
+
github.com/chromedp/sysutil v1.1.0 // indirect
|
| 13 |
+
github.com/ebitengine/purego v0.10.0 // indirect
|
| 14 |
+
github.com/go-json-experiment/json v0.0.0-20260623181947-01eb4420fa68 // indirect
|
| 15 |
+
github.com/gobwas/httphead v0.1.0 // indirect
|
| 16 |
+
github.com/gobwas/pool v0.2.1 // indirect
|
| 17 |
+
github.com/gobwas/ws v1.4.0 // indirect
|
| 18 |
+
golang.org/x/sys v0.47.0 // indirect
|
| 19 |
+
)
|
| 20 |
+
|
| 21 |
+
replace github.com/localai/motion-bricks.cpp/bindings/go => ../bindings/go
|
demo/go.sum
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
github.com/chromedp/cdproto v0.0.0-20260714215040-dc233986426f h1:0Z1zcSLEmnj2c2CmJYBqewtS6pxhB39bNWUSEUAWjgk=
|
| 2 |
+
github.com/chromedp/cdproto v0.0.0-20260714215040-dc233986426f/go.mod h1:RwFsSODCtFExll+GhHM6R92SARHR3Z3oipaxLHj46C0=
|
| 3 |
+
github.com/chromedp/chromedp v0.16.0 h1:rOO4deOm4CbZgBCa8mD9g2rDyIoNs0BkgvNrlbp5ouk=
|
| 4 |
+
github.com/chromedp/chromedp v0.16.0/go.mod h1:rbuGKFT1vMcFcFqKfPIO1GpX/N+2s8onm2qMxZLbU5U=
|
| 5 |
+
github.com/chromedp/sysutil v1.1.0 h1:PUFNv5EcprjqXZD9nJb9b/c9ibAbxiYo4exNWZyipwM=
|
| 6 |
+
github.com/chromedp/sysutil v1.1.0/go.mod h1:WiThHUdltqCNKGc4gaU50XgYjwjYIhKWoHGPTUfWTJ8=
|
| 7 |
+
github.com/ebitengine/purego v0.10.0 h1:QIw4xfpWT6GWTzaW5XEKy3HXoqrJGx1ijYHzTF0/ISU=
|
| 8 |
+
github.com/ebitengine/purego v0.10.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
|
| 9 |
+
github.com/go-json-experiment/json v0.0.0-20260623181947-01eb4420fa68 h1:KZaTBSyshWX3MP5jukJcNSuXDQTO+rNpt0J564dX/eg=
|
| 10 |
+
github.com/go-json-experiment/json v0.0.0-20260623181947-01eb4420fa68/go.mod h1:tphK2c80bpPhMOI4v6bIc2xWywPfbqi1Z06+RcrMkDg=
|
| 11 |
+
github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU=
|
| 12 |
+
github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM=
|
| 13 |
+
github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og=
|
| 14 |
+
github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
|
| 15 |
+
github.com/gobwas/ws v1.4.0 h1:CTaoG1tojrh4ucGPcoJFiAQUAsEWekEWvLy7GsVNqGs=
|
| 16 |
+
github.com/gobwas/ws v1.4.0/go.mod h1:G3gNqMNtPppf5XUz7O4shetPpcZ1VJ7zt18dlUeakrc=
|
| 17 |
+
github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 h1:6Yzfa6GP0rIo/kULo2bwGEkFvCePZ3qHDDTC3/J9Swo=
|
| 18 |
+
github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs=
|
| 19 |
+
github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw=
|
| 20 |
+
github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0=
|
| 21 |
+
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
| 22 |
+
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
|
| 23 |
+
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
demo/main.go
ADDED
|
@@ -0,0 +1,371 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
package main
|
| 2 |
+
|
| 3 |
+
import (
|
| 4 |
+
"crypto/rand"
|
| 5 |
+
"embed"
|
| 6 |
+
"encoding/hex"
|
| 7 |
+
"encoding/json"
|
| 8 |
+
"errors"
|
| 9 |
+
"flag"
|
| 10 |
+
"fmt"
|
| 11 |
+
"io"
|
| 12 |
+
"io/fs"
|
| 13 |
+
"log"
|
| 14 |
+
"math"
|
| 15 |
+
"net/http"
|
| 16 |
+
"os"
|
| 17 |
+
"os/signal"
|
| 18 |
+
"path/filepath"
|
| 19 |
+
"sort"
|
| 20 |
+
"strings"
|
| 21 |
+
"sync"
|
| 22 |
+
"syscall"
|
| 23 |
+
"time"
|
| 24 |
+
|
| 25 |
+
mb "github.com/localai/motion-bricks.cpp/bindings/go"
|
| 26 |
+
)
|
| 27 |
+
|
| 28 |
+
//go:embed web/* web/vendor/*
|
| 29 |
+
var webFiles embed.FS
|
| 30 |
+
|
| 31 |
+
type styleInfo struct {
|
| 32 |
+
Name string `json:"name"`
|
| 33 |
+
Speed float32 `json:"speed"`
|
| 34 |
+
}
|
| 35 |
+
|
| 36 |
+
type session struct {
|
| 37 |
+
mu sync.Mutex
|
| 38 |
+
agent *mb.Agent
|
| 39 |
+
planned bool
|
| 40 |
+
}
|
| 41 |
+
|
| 42 |
+
type demoServer struct {
|
| 43 |
+
library *mb.Library
|
| 44 |
+
model *mb.Model
|
| 45 |
+
styles map[string]*mb.Style
|
| 46 |
+
ordered []styleInfo
|
| 47 |
+
joints []mb.Joint
|
| 48 |
+
planMu sync.Mutex
|
| 49 |
+
mu sync.Mutex
|
| 50 |
+
sessions map[string]*session
|
| 51 |
+
static http.Handler
|
| 52 |
+
}
|
| 53 |
+
|
| 54 |
+
type sessionRequest struct {
|
| 55 |
+
Style string `json:"style"`
|
| 56 |
+
}
|
| 57 |
+
type planRequest struct {
|
| 58 |
+
Session string `json:"session"`
|
| 59 |
+
Style string `json:"style"`
|
| 60 |
+
Move [2]float32 `json:"move"`
|
| 61 |
+
Facing [2]float32 `json:"facing"`
|
| 62 |
+
Speed *float32 `json:"speed,omitempty"`
|
| 63 |
+
Seed uint64 `json:"seed"`
|
| 64 |
+
Advance uint32 `json:"advance"`
|
| 65 |
+
}
|
| 66 |
+
type planResponse struct {
|
| 67 |
+
Session string `json:"session"`
|
| 68 |
+
Style string `json:"style"`
|
| 69 |
+
Motion *mb.Motion `json:"motion"`
|
| 70 |
+
Targets *mb.Keyframes `json:"targets"`
|
| 71 |
+
}
|
| 72 |
+
|
| 73 |
+
func parseDevice(value string) (mb.Device, error) {
|
| 74 |
+
switch strings.ToLower(value) {
|
| 75 |
+
case "auto":
|
| 76 |
+
return mb.DeviceAuto, nil
|
| 77 |
+
case "cpu":
|
| 78 |
+
return mb.DeviceCPU, nil
|
| 79 |
+
case "vulkan":
|
| 80 |
+
return mb.DeviceVulkan, nil
|
| 81 |
+
default:
|
| 82 |
+
return 0, fmt.Errorf("unknown device %q", value)
|
| 83 |
+
}
|
| 84 |
+
}
|
| 85 |
+
|
| 86 |
+
func loadDemoServer(libraryPath, modelPath, styleDirectory string, device mb.Device) (*demoServer, error) {
|
| 87 |
+
library, err := mb.Open(libraryPath)
|
| 88 |
+
if err != nil {
|
| 89 |
+
return nil, fmt.Errorf("open native library: %w", err)
|
| 90 |
+
}
|
| 91 |
+
model, err := library.LoadModel(modelPath, device)
|
| 92 |
+
if err != nil {
|
| 93 |
+
library.Close()
|
| 94 |
+
return nil, err
|
| 95 |
+
}
|
| 96 |
+
paths, err := filepath.Glob(filepath.Join(styleDirectory, "*.mbstyle"))
|
| 97 |
+
if err != nil || len(paths) == 0 {
|
| 98 |
+
model.Close()
|
| 99 |
+
library.Close()
|
| 100 |
+
return nil, fmt.Errorf("no .mbstyle files in %s", styleDirectory)
|
| 101 |
+
}
|
| 102 |
+
sort.Strings(paths)
|
| 103 |
+
server := &demoServer{library: library, model: model, styles: make(map[string]*mb.Style), sessions: make(map[string]*session)}
|
| 104 |
+
for _, path := range paths {
|
| 105 |
+
style, loadErr := model.LoadStyle(path)
|
| 106 |
+
if loadErr != nil {
|
| 107 |
+
server.Close()
|
| 108 |
+
return nil, fmt.Errorf("load %s: %w", path, loadErr)
|
| 109 |
+
}
|
| 110 |
+
if _, exists := server.styles[style.Name]; exists {
|
| 111 |
+
style.Close()
|
| 112 |
+
server.Close()
|
| 113 |
+
return nil, fmt.Errorf("duplicate style %q", style.Name)
|
| 114 |
+
}
|
| 115 |
+
server.styles[style.Name] = style
|
| 116 |
+
server.ordered = append(server.ordered, styleInfo{Name: style.Name, Speed: style.Speed})
|
| 117 |
+
}
|
| 118 |
+
sort.Slice(server.ordered, func(i, j int) bool { return server.ordered[i].Name < server.ordered[j].Name })
|
| 119 |
+
server.joints, err = model.Skeleton()
|
| 120 |
+
if err != nil {
|
| 121 |
+
server.Close()
|
| 122 |
+
return nil, err
|
| 123 |
+
}
|
| 124 |
+
root, err := fs.Sub(webFiles, "web")
|
| 125 |
+
if err != nil {
|
| 126 |
+
server.Close()
|
| 127 |
+
return nil, err
|
| 128 |
+
}
|
| 129 |
+
server.static = http.FileServer(http.FS(root))
|
| 130 |
+
return server, nil
|
| 131 |
+
}
|
| 132 |
+
|
| 133 |
+
func (s *demoServer) Close() {
|
| 134 |
+
if s == nil {
|
| 135 |
+
return
|
| 136 |
+
}
|
| 137 |
+
s.mu.Lock()
|
| 138 |
+
for _, item := range s.sessions {
|
| 139 |
+
item.agent.Close()
|
| 140 |
+
}
|
| 141 |
+
s.sessions = nil
|
| 142 |
+
s.mu.Unlock()
|
| 143 |
+
for _, style := range s.styles {
|
| 144 |
+
style.Close()
|
| 145 |
+
}
|
| 146 |
+
if s.model != nil {
|
| 147 |
+
s.model.Close()
|
| 148 |
+
}
|
| 149 |
+
if s.library != nil {
|
| 150 |
+
_ = s.library.Close()
|
| 151 |
+
}
|
| 152 |
+
}
|
| 153 |
+
|
| 154 |
+
func randomID() (string, error) {
|
| 155 |
+
var value [16]byte
|
| 156 |
+
if _, err := rand.Read(value[:]); err != nil {
|
| 157 |
+
return "", err
|
| 158 |
+
}
|
| 159 |
+
return hex.EncodeToString(value[:]), nil
|
| 160 |
+
}
|
| 161 |
+
|
| 162 |
+
func jsonResponse(writer http.ResponseWriter, status int, value any) {
|
| 163 |
+
writer.Header().Set("Content-Type", "application/json")
|
| 164 |
+
writer.Header().Set("Cache-Control", "no-store")
|
| 165 |
+
writer.WriteHeader(status)
|
| 166 |
+
_ = json.NewEncoder(writer).Encode(value)
|
| 167 |
+
}
|
| 168 |
+
func apiError(writer http.ResponseWriter, status int, err error) {
|
| 169 |
+
jsonResponse(writer, status, map[string]string{"error": err.Error()})
|
| 170 |
+
}
|
| 171 |
+
func decodeJSON(request *http.Request, output any) error {
|
| 172 |
+
decoder := json.NewDecoder(io.LimitReader(request.Body, 1<<20))
|
| 173 |
+
decoder.DisallowUnknownFields()
|
| 174 |
+
if err := decoder.Decode(output); err != nil {
|
| 175 |
+
return fmt.Errorf("invalid JSON: %w", err)
|
| 176 |
+
}
|
| 177 |
+
return nil
|
| 178 |
+
}
|
| 179 |
+
func finite(values ...float32) bool {
|
| 180 |
+
for _, value := range values {
|
| 181 |
+
if math.IsNaN(float64(value)) || math.IsInf(float64(value), 0) {
|
| 182 |
+
return false
|
| 183 |
+
}
|
| 184 |
+
}
|
| 185 |
+
return true
|
| 186 |
+
}
|
| 187 |
+
|
| 188 |
+
func (s *demoServer) style(name string) (*mb.Style, error) {
|
| 189 |
+
style := s.styles[name]
|
| 190 |
+
if style == nil {
|
| 191 |
+
return nil, fmt.Errorf("unknown style %q", name)
|
| 192 |
+
}
|
| 193 |
+
return style, nil
|
| 194 |
+
}
|
| 195 |
+
|
| 196 |
+
func (s *demoServer) commandPlan(item *session, style *mb.Style, request planRequest) (*mb.Motion, error) {
|
| 197 |
+
if !finite(request.Move[0], request.Move[1], request.Facing[0], request.Facing[1]) {
|
| 198 |
+
return nil, errors.New("control vector is not finite")
|
| 199 |
+
}
|
| 200 |
+
if math.Hypot(float64(request.Facing[0]), float64(request.Facing[1])) < 1e-6 {
|
| 201 |
+
return nil, errors.New("facing vector is zero")
|
| 202 |
+
}
|
| 203 |
+
item.mu.Lock()
|
| 204 |
+
defer item.mu.Unlock()
|
| 205 |
+
if item.planned && request.Advance > 0 {
|
| 206 |
+
if err := item.agent.Advance(request.Advance); err != nil {
|
| 207 |
+
return nil, err
|
| 208 |
+
}
|
| 209 |
+
}
|
| 210 |
+
command, err := s.library.NewCommand()
|
| 211 |
+
if err != nil {
|
| 212 |
+
return nil, err
|
| 213 |
+
}
|
| 214 |
+
defer command.Close()
|
| 215 |
+
if err = command.SetStyle(style); err != nil {
|
| 216 |
+
return nil, err
|
| 217 |
+
}
|
| 218 |
+
if err = command.SetMovement(request.Move[0], 0, request.Move[1]); err != nil {
|
| 219 |
+
return nil, err
|
| 220 |
+
}
|
| 221 |
+
if err = command.SetFacing(request.Facing[0], 0, request.Facing[1]); err != nil {
|
| 222 |
+
return nil, err
|
| 223 |
+
}
|
| 224 |
+
if request.Speed != nil {
|
| 225 |
+
if !finite(*request.Speed) || *request.Speed < 0 {
|
| 226 |
+
return nil, errors.New("speed is invalid")
|
| 227 |
+
}
|
| 228 |
+
if err = command.SetSpeed(*request.Speed); err != nil {
|
| 229 |
+
return nil, err
|
| 230 |
+
}
|
| 231 |
+
}
|
| 232 |
+
if err = command.SetSeed(request.Seed); err != nil {
|
| 233 |
+
return nil, err
|
| 234 |
+
}
|
| 235 |
+
s.planMu.Lock()
|
| 236 |
+
motion, err := item.agent.Plan(command)
|
| 237 |
+
s.planMu.Unlock()
|
| 238 |
+
if err == nil {
|
| 239 |
+
item.planned = true
|
| 240 |
+
}
|
| 241 |
+
return motion, err
|
| 242 |
+
}
|
| 243 |
+
|
| 244 |
+
func (s *demoServer) routes() http.Handler {
|
| 245 |
+
mux := http.NewServeMux()
|
| 246 |
+
mux.HandleFunc("GET /api/health", func(w http.ResponseWriter, _ *http.Request) {
|
| 247 |
+
jsonResponse(w, http.StatusOK, map[string]any{"ok": true})
|
| 248 |
+
})
|
| 249 |
+
mux.HandleFunc("GET /api/meta", func(w http.ResponseWriter, _ *http.Request) {
|
| 250 |
+
jsonResponse(w, http.StatusOK, map[string]any{"fps": 30, "joints": s.joints, "styles": s.ordered})
|
| 251 |
+
})
|
| 252 |
+
mux.HandleFunc("POST /api/session", s.createSession)
|
| 253 |
+
mux.HandleFunc("POST /api/plan", s.plan)
|
| 254 |
+
mux.Handle("/", s.static)
|
| 255 |
+
return securityHeaders(mux)
|
| 256 |
+
}
|
| 257 |
+
|
| 258 |
+
func securityHeaders(next http.Handler) http.Handler {
|
| 259 |
+
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
| 260 |
+
w.Header().Set("Cache-Control", "no-store")
|
| 261 |
+
w.Header().Set("X-Content-Type-Options", "nosniff")
|
| 262 |
+
w.Header().Set("Referrer-Policy", "no-referrer")
|
| 263 |
+
w.Header().Set("Content-Security-Policy", "default-src 'self'; script-src 'self'; style-src 'self'; img-src 'self' data:; connect-src 'self'")
|
| 264 |
+
next.ServeHTTP(w, r)
|
| 265 |
+
})
|
| 266 |
+
}
|
| 267 |
+
|
| 268 |
+
func (s *demoServer) createSession(w http.ResponseWriter, r *http.Request) {
|
| 269 |
+
var request sessionRequest
|
| 270 |
+
if err := decodeJSON(r, &request); err != nil {
|
| 271 |
+
apiError(w, http.StatusBadRequest, err)
|
| 272 |
+
return
|
| 273 |
+
}
|
| 274 |
+
if request.Style == "" {
|
| 275 |
+
if _, ok := s.styles["idle"]; ok {
|
| 276 |
+
request.Style = "idle"
|
| 277 |
+
} else {
|
| 278 |
+
request.Style = s.ordered[0].Name
|
| 279 |
+
}
|
| 280 |
+
}
|
| 281 |
+
style, err := s.style(request.Style)
|
| 282 |
+
if err != nil {
|
| 283 |
+
apiError(w, http.StatusBadRequest, err)
|
| 284 |
+
return
|
| 285 |
+
}
|
| 286 |
+
agent, err := s.model.NewAgent()
|
| 287 |
+
if err != nil {
|
| 288 |
+
apiError(w, http.StatusInternalServerError, err)
|
| 289 |
+
return
|
| 290 |
+
}
|
| 291 |
+
if err = agent.Reset(style); err != nil {
|
| 292 |
+
agent.Close()
|
| 293 |
+
apiError(w, http.StatusInternalServerError, err)
|
| 294 |
+
return
|
| 295 |
+
}
|
| 296 |
+
id, err := randomID()
|
| 297 |
+
if err != nil {
|
| 298 |
+
agent.Close()
|
| 299 |
+
apiError(w, http.StatusInternalServerError, err)
|
| 300 |
+
return
|
| 301 |
+
}
|
| 302 |
+
item := &session{agent: agent}
|
| 303 |
+
s.mu.Lock()
|
| 304 |
+
s.sessions[id] = item
|
| 305 |
+
s.mu.Unlock()
|
| 306 |
+
motion, err := s.commandPlan(item, style, planRequest{Move: [2]float32{0, 0}, Facing: [2]float32{0, 1}, Seed: 1})
|
| 307 |
+
if err != nil {
|
| 308 |
+
s.mu.Lock()
|
| 309 |
+
delete(s.sessions, id)
|
| 310 |
+
s.mu.Unlock()
|
| 311 |
+
agent.Close()
|
| 312 |
+
apiError(w, http.StatusInternalServerError, err)
|
| 313 |
+
return
|
| 314 |
+
}
|
| 315 |
+
jsonResponse(w, http.StatusOK, planResponse{Session: id, Style: style.Name, Motion: motion, Targets: motion.Targets})
|
| 316 |
+
}
|
| 317 |
+
|
| 318 |
+
func (s *demoServer) plan(w http.ResponseWriter, r *http.Request) {
|
| 319 |
+
var request planRequest
|
| 320 |
+
if err := decodeJSON(r, &request); err != nil {
|
| 321 |
+
apiError(w, http.StatusBadRequest, err)
|
| 322 |
+
return
|
| 323 |
+
}
|
| 324 |
+
s.mu.Lock()
|
| 325 |
+
item := s.sessions[request.Session]
|
| 326 |
+
s.mu.Unlock()
|
| 327 |
+
if item == nil {
|
| 328 |
+
apiError(w, http.StatusNotFound, errors.New("unknown session"))
|
| 329 |
+
return
|
| 330 |
+
}
|
| 331 |
+
style, err := s.style(request.Style)
|
| 332 |
+
if err != nil {
|
| 333 |
+
apiError(w, http.StatusBadRequest, err)
|
| 334 |
+
return
|
| 335 |
+
}
|
| 336 |
+
motion, err := s.commandPlan(item, style, request)
|
| 337 |
+
if err != nil {
|
| 338 |
+
apiError(w, http.StatusInternalServerError, err)
|
| 339 |
+
return
|
| 340 |
+
}
|
| 341 |
+
jsonResponse(w, http.StatusOK, planResponse{Session: request.Session, Style: style.Name, Motion: motion, Targets: motion.Targets})
|
| 342 |
+
}
|
| 343 |
+
|
| 344 |
+
func main() {
|
| 345 |
+
listen := flag.String("listen", "127.0.0.1:8080", "HTTP listen address")
|
| 346 |
+
libraryPath := flag.String("library", os.Getenv("MOTIONBRICKS_LIB"), "path to libmotionbricks")
|
| 347 |
+
modelPath := flag.String("model", os.Getenv("MOTIONBRICKS_MODEL"), "model bundle directory")
|
| 348 |
+
stylesPath := flag.String("styles", os.Getenv("MOTIONBRICKS_STYLES"), "style directory")
|
| 349 |
+
deviceName := flag.String("device", "cpu", "auto, cpu, or vulkan")
|
| 350 |
+
flag.Parse()
|
| 351 |
+
if *libraryPath == "" || *modelPath == "" || *stylesPath == "" {
|
| 352 |
+
log.Fatal("-library, -model, and -styles are required")
|
| 353 |
+
}
|
| 354 |
+
device, err := parseDevice(*deviceName)
|
| 355 |
+
if err != nil {
|
| 356 |
+
log.Fatal(err)
|
| 357 |
+
}
|
| 358 |
+
demo, err := loadDemoServer(*libraryPath, *modelPath, *stylesPath, device)
|
| 359 |
+
if err != nil {
|
| 360 |
+
log.Fatal(err)
|
| 361 |
+
}
|
| 362 |
+
defer demo.Close()
|
| 363 |
+
server := &http.Server{Addr: *listen, Handler: demo.routes(), ReadHeaderTimeout: 5 * time.Second}
|
| 364 |
+
stopped := make(chan os.Signal, 1)
|
| 365 |
+
signal.Notify(stopped, os.Interrupt, syscall.SIGTERM)
|
| 366 |
+
go func() { <-stopped; _ = server.Close() }()
|
| 367 |
+
log.Printf("MotionBricks demo: http://%s", *listen)
|
| 368 |
+
if err = server.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
|
| 369 |
+
log.Fatal(err)
|
| 370 |
+
}
|
| 371 |
+
}
|
demo/main_test.go
ADDED
|
@@ -0,0 +1,236 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
package main
|
| 2 |
+
|
| 3 |
+
import (
|
| 4 |
+
"bytes"
|
| 5 |
+
"context"
|
| 6 |
+
"encoding/json"
|
| 7 |
+
"errors"
|
| 8 |
+
"fmt"
|
| 9 |
+
"net/http"
|
| 10 |
+
"net/http/httptest"
|
| 11 |
+
"os"
|
| 12 |
+
"os/exec"
|
| 13 |
+
"path/filepath"
|
| 14 |
+
"strings"
|
| 15 |
+
"testing"
|
| 16 |
+
"time"
|
| 17 |
+
|
| 18 |
+
cdplog "github.com/chromedp/cdproto/log"
|
| 19 |
+
"github.com/chromedp/cdproto/runtime"
|
| 20 |
+
"github.com/chromedp/chromedp"
|
| 21 |
+
mb "github.com/localai/motion-bricks.cpp/bindings/go"
|
| 22 |
+
)
|
| 23 |
+
|
| 24 |
+
func TestParseDevice(t *testing.T) {
|
| 25 |
+
for name, expected := range map[string]mb.Device{"auto": mb.DeviceAuto, "cpu": mb.DeviceCPU, "vulkan": mb.DeviceVulkan, "CPU": mb.DeviceCPU} {
|
| 26 |
+
actual, err := parseDevice(name)
|
| 27 |
+
if err != nil || actual != expected {
|
| 28 |
+
t.Fatalf("parseDevice(%q)=(%d,%v), want %d", name, actual, err, expected)
|
| 29 |
+
}
|
| 30 |
+
}
|
| 31 |
+
if _, err := parseDevice("cuda"); err == nil {
|
| 32 |
+
t.Fatal("unknown device was accepted")
|
| 33 |
+
}
|
| 34 |
+
}
|
| 35 |
+
|
| 36 |
+
func TestNativeHTTPFlow(t *testing.T) {
|
| 37 |
+
libraryPath, modelPath, stylesPath := os.Getenv("MOTIONBRICKS_LIB"), os.Getenv("MOTIONBRICKS_MODEL"), os.Getenv("MOTIONBRICKS_STYLES")
|
| 38 |
+
if libraryPath == "" || modelPath == "" || stylesPath == "" {
|
| 39 |
+
t.Skip("native demo paths are not configured")
|
| 40 |
+
}
|
| 41 |
+
demo, err := loadDemoServer(libraryPath, modelPath, stylesPath, mb.DeviceCPU)
|
| 42 |
+
if err != nil {
|
| 43 |
+
t.Fatal(err)
|
| 44 |
+
}
|
| 45 |
+
defer demo.Close()
|
| 46 |
+
server := httptest.NewServer(demo.routes())
|
| 47 |
+
defer server.Close()
|
| 48 |
+
response, err := http.Get(server.URL + "/api/meta")
|
| 49 |
+
if err != nil {
|
| 50 |
+
t.Fatal(err)
|
| 51 |
+
}
|
| 52 |
+
var metadata struct {
|
| 53 |
+
FPS int `json:"fps"`
|
| 54 |
+
Joints []mb.Joint `json:"joints"`
|
| 55 |
+
Styles []styleInfo `json:"styles"`
|
| 56 |
+
}
|
| 57 |
+
if err = json.NewDecoder(response.Body).Decode(&metadata); err != nil {
|
| 58 |
+
t.Fatal(err)
|
| 59 |
+
}
|
| 60 |
+
response.Body.Close()
|
| 61 |
+
if response.StatusCode != http.StatusOK || metadata.FPS != 30 || len(metadata.Joints) != 34 || len(metadata.Styles) < 10 {
|
| 62 |
+
t.Fatalf("invalid metadata: status=%d fps=%d joints=%d styles=%d", response.StatusCode, metadata.FPS, len(metadata.Joints), len(metadata.Styles))
|
| 63 |
+
}
|
| 64 |
+
post := func(path string, value any, output any) {
|
| 65 |
+
t.Helper()
|
| 66 |
+
body, _ := json.Marshal(value)
|
| 67 |
+
reply, postErr := http.Post(server.URL+path, "application/json", bytes.NewReader(body))
|
| 68 |
+
if postErr != nil {
|
| 69 |
+
t.Fatal(postErr)
|
| 70 |
+
}
|
| 71 |
+
defer reply.Body.Close()
|
| 72 |
+
if reply.StatusCode != http.StatusOK {
|
| 73 |
+
var failure any
|
| 74 |
+
_ = json.NewDecoder(reply.Body).Decode(&failure)
|
| 75 |
+
t.Fatalf("%s: status=%d body=%v", path, reply.StatusCode, failure)
|
| 76 |
+
}
|
| 77 |
+
if err := json.NewDecoder(reply.Body).Decode(output); err != nil {
|
| 78 |
+
t.Fatal(err)
|
| 79 |
+
}
|
| 80 |
+
}
|
| 81 |
+
var initial planResponse
|
| 82 |
+
post("/api/session", sessionRequest{Style: "walk"}, &initial)
|
| 83 |
+
if initial.Session == "" || initial.Motion == nil || initial.Motion.Joints != 34 || initial.Motion.Frames < 24 ||
|
| 84 |
+
initial.Targets == nil || initial.Targets.Frames != 4 || initial.Targets.Joints != 34 ||
|
| 85 |
+
len(initial.Targets.Roots) != 12 || len(initial.Targets.Rotations) != 4*34*4 {
|
| 86 |
+
t.Fatalf("invalid initial response: %+v", initial)
|
| 87 |
+
}
|
| 88 |
+
turnedStyle := ""
|
| 89 |
+
for _, style := range metadata.Styles {
|
| 90 |
+
if style.Name == "walk_zombie" {
|
| 91 |
+
turnedStyle = style.Name
|
| 92 |
+
break
|
| 93 |
+
}
|
| 94 |
+
if style.Name != "walk" && turnedStyle == "" {
|
| 95 |
+
turnedStyle = style.Name
|
| 96 |
+
}
|
| 97 |
+
}
|
| 98 |
+
if turnedStyle == "" {
|
| 99 |
+
t.Fatal("no alternate upstream style is available")
|
| 100 |
+
}
|
| 101 |
+
var turned planResponse
|
| 102 |
+
post("/api/plan", planRequest{Session: initial.Session, Style: turnedStyle, Move: [2]float32{1, 0}, Facing: [2]float32{1, 0}, Advance: 3, Seed: 77}, &turned)
|
| 103 |
+
if turned.Style != turnedStyle || turned.Motion == nil || turned.Motion.Joints != 34 || len(turned.Motion.Rotations) != int(turned.Motion.Frames*34*4) ||
|
| 104 |
+
turned.Targets == nil || turned.Targets.Frames != 4 || len(turned.Targets.Rotations) != 4*34*4 {
|
| 105 |
+
t.Fatalf("invalid turned response: style=%q motion=%+v", turned.Style, turned.Motion)
|
| 106 |
+
}
|
| 107 |
+
}
|
| 108 |
+
|
| 109 |
+
func TestHeadlessChrome(t *testing.T) {
|
| 110 |
+
libraryPath, modelPath, stylesPath := os.Getenv("MOTIONBRICKS_LIB"), os.Getenv("MOTIONBRICKS_MODEL"), os.Getenv("MOTIONBRICKS_STYLES")
|
| 111 |
+
if libraryPath == "" || modelPath == "" || stylesPath == "" {
|
| 112 |
+
t.Skip("native demo paths are not configured")
|
| 113 |
+
}
|
| 114 |
+
chrome := os.Getenv("MOTIONBRICKS_CHROME")
|
| 115 |
+
if chrome == "" {
|
| 116 |
+
var err error
|
| 117 |
+
chrome, err = exec.LookPath("chromium")
|
| 118 |
+
if err != nil {
|
| 119 |
+
t.Skip("chromium is not installed")
|
| 120 |
+
}
|
| 121 |
+
}
|
| 122 |
+
demo, err := loadDemoServer(libraryPath, modelPath, stylesPath, mb.DeviceCPU)
|
| 123 |
+
if err != nil {
|
| 124 |
+
t.Fatal(err)
|
| 125 |
+
}
|
| 126 |
+
defer demo.Close()
|
| 127 |
+
server := httptest.NewServer(demo.routes())
|
| 128 |
+
defer server.Close()
|
| 129 |
+
|
| 130 |
+
options := append([]chromedp.ExecAllocatorOption{}, chromedp.DefaultExecAllocatorOptions[:]...)
|
| 131 |
+
options = append(options,
|
| 132 |
+
chromedp.ExecPath(chrome),
|
| 133 |
+
chromedp.Flag("no-sandbox", true),
|
| 134 |
+
chromedp.Flag("disable-dev-shm-usage", true),
|
| 135 |
+
chromedp.Flag("use-angle", "swiftshader"),
|
| 136 |
+
chromedp.Flag("enable-unsafe-swiftshader", true),
|
| 137 |
+
chromedp.WindowSize(1280, 800),
|
| 138 |
+
)
|
| 139 |
+
allocator, cancelAllocator := chromedp.NewExecAllocator(context.Background(), options...)
|
| 140 |
+
defer cancelAllocator()
|
| 141 |
+
browser, cancelBrowser := chromedp.NewContext(allocator)
|
| 142 |
+
defer cancelBrowser()
|
| 143 |
+
chromedp.ListenTarget(browser, func(event any) {
|
| 144 |
+
switch event := event.(type) {
|
| 145 |
+
case *cdplog.EventEntryAdded:
|
| 146 |
+
t.Logf("browser log: %s", event.Entry.Text)
|
| 147 |
+
case *runtime.EventConsoleAPICalled:
|
| 148 |
+
t.Logf("browser console: %v", event.Args)
|
| 149 |
+
case *runtime.EventExceptionThrown:
|
| 150 |
+
t.Logf("browser exception: %s", event.ExceptionDetails.Text)
|
| 151 |
+
}
|
| 152 |
+
})
|
| 153 |
+
ctx, cancel := context.WithTimeout(browser, 30*time.Second)
|
| 154 |
+
defer cancel()
|
| 155 |
+
|
| 156 |
+
waitFor := func(expression, description string) chromedp.Action {
|
| 157 |
+
return chromedp.ActionFunc(func(ctx context.Context) error {
|
| 158 |
+
deadline := time.Now().Add(25 * time.Second)
|
| 159 |
+
for time.Now().Before(deadline) {
|
| 160 |
+
var ready bool
|
| 161 |
+
if evaluateErr := chromedp.Evaluate(expression, &ready).Do(ctx); evaluateErr != nil {
|
| 162 |
+
return evaluateErr
|
| 163 |
+
}
|
| 164 |
+
if ready {
|
| 165 |
+
return nil
|
| 166 |
+
}
|
| 167 |
+
time.Sleep(100 * time.Millisecond)
|
| 168 |
+
}
|
| 169 |
+
var diagnostic any
|
| 170 |
+
_ = chromedp.Evaluate(`({status: document.documentElement.dataset.testStatus, sequence: document.documentElement.dataset.planSequence, moveX: document.documentElement.dataset.plannedMoveX, moveZ: document.documentElement.dataset.plannedMoveZ, scripts: [...document.scripts].map(s => ({src:s.src,type:s.type})), resources: performance.getEntriesByType("resource").map(r => r.name)})`, &diagnostic).Do(ctx)
|
| 171 |
+
return fmt.Errorf("%w for %s: %#v", errors.New("timeout waiting for browser"), description, diagnostic)
|
| 172 |
+
})
|
| 173 |
+
}
|
| 174 |
+
var initialScreenshot, movingScreenshot, screenshot []byte
|
| 175 |
+
var message string
|
| 176 |
+
err = chromedp.Run(ctx,
|
| 177 |
+
chromedp.ActionFunc(func(ctx context.Context) error {
|
| 178 |
+
if enableErr := cdplog.Enable().Do(ctx); enableErr != nil {
|
| 179 |
+
return enableErr
|
| 180 |
+
}
|
| 181 |
+
return runtime.Enable().Do(ctx)
|
| 182 |
+
}),
|
| 183 |
+
chromedp.Navigate(server.URL+"/"),
|
| 184 |
+
waitFor(`document.documentElement.dataset.testStatus === "ready"`, "initial plan"),
|
| 185 |
+
chromedp.FullScreenshot(&initialScreenshot, 90),
|
| 186 |
+
chromedp.Click(`.pad button[data-key="w"]`, chromedp.ByQuery),
|
| 187 |
+
waitFor(`Number(document.documentElement.dataset.planSequence) >= 2 && document.documentElement.dataset.plannedMoveZ === "1"`, "forward pad-button plan"),
|
| 188 |
+
chromedp.FullScreenshot(&movingScreenshot, 90),
|
| 189 |
+
chromedp.Click(`.pad button[data-key="w"]`, chromedp.ByQuery),
|
| 190 |
+
waitFor(`Number(document.documentElement.dataset.planSequence) >= 3 && document.documentElement.dataset.plannedMoveX === "0" && document.documentElement.dataset.plannedMoveZ === "0"`, "pad-button stop plan"),
|
| 191 |
+
chromedp.Evaluate(`dispatchEvent(new KeyboardEvent("keydown", {key:"d", bubbles:true}))`, nil),
|
| 192 |
+
waitFor(`Number(document.documentElement.dataset.planSequence) >= 4 && document.documentElement.dataset.plannedMoveX === "1"`, "keyboard-right plan"),
|
| 193 |
+
chromedp.Evaluate(`dispatchEvent(new KeyboardEvent("keyup", {key:"d", bubbles:true}))`, nil),
|
| 194 |
+
waitFor(`Number(document.documentElement.dataset.planSequence) >= 5 && document.documentElement.dataset.plannedMoveX === "0" && document.documentElement.dataset.plannedMoveZ === "0"`, "keyboard stop plan"),
|
| 195 |
+
chromedp.Navigate(server.URL+"/?test=1"),
|
| 196 |
+
waitFor(`document.documentElement.dataset.testStatus === "passed" || document.documentElement.dataset.testStatus === "failed"`, "style-and-turn self-test"),
|
| 197 |
+
chromedp.Text("#test-result", &message, chromedp.ByQuery),
|
| 198 |
+
chromedp.FullScreenshot(&screenshot, 90),
|
| 199 |
+
)
|
| 200 |
+
if err != nil {
|
| 201 |
+
t.Fatal(err)
|
| 202 |
+
}
|
| 203 |
+
var status string
|
| 204 |
+
if err = chromedp.Run(ctx, chromedp.Evaluate(`document.documentElement.dataset.testStatus`, &status)); err != nil {
|
| 205 |
+
t.Fatal(err)
|
| 206 |
+
}
|
| 207 |
+
if status != "passed" {
|
| 208 |
+
t.Fatalf("browser self-test status=%q: %s", status, message)
|
| 209 |
+
}
|
| 210 |
+
if len(initialScreenshot) < 10_000 || len(movingScreenshot) < 10_000 || len(screenshot) < 10_000 {
|
| 211 |
+
t.Fatalf("rendered screenshots are unexpectedly small: initial=%d moving=%d final=%d", len(initialScreenshot), len(movingScreenshot), len(screenshot))
|
| 212 |
+
}
|
| 213 |
+
artifact := os.Getenv("MOTIONBRICKS_SCREENSHOT")
|
| 214 |
+
if artifact == "" {
|
| 215 |
+
artifact = filepath.Join(t.TempDir(), "motionbricks-demo.png")
|
| 216 |
+
}
|
| 217 |
+
extension := filepath.Ext(artifact)
|
| 218 |
+
if extension == "" {
|
| 219 |
+
extension = ".png"
|
| 220 |
+
}
|
| 221 |
+
base := strings.TrimSuffix(artifact, filepath.Ext(artifact))
|
| 222 |
+
artifacts := []struct {
|
| 223 |
+
path string
|
| 224 |
+
data []byte
|
| 225 |
+
}{
|
| 226 |
+
{path: base + "-initial" + extension, data: initialScreenshot},
|
| 227 |
+
{path: base + "-moving" + extension, data: movingScreenshot},
|
| 228 |
+
{path: artifact, data: screenshot},
|
| 229 |
+
}
|
| 230 |
+
for _, item := range artifacts {
|
| 231 |
+
if err = os.WriteFile(item.path, item.data, 0o600); err != nil {
|
| 232 |
+
t.Fatal(err)
|
| 233 |
+
}
|
| 234 |
+
}
|
| 235 |
+
t.Logf("%s; screenshots: %s, %s, %s (final %s)", message, artifacts[0].path, artifacts[1].path, artifacts[2].path, fmt.Sprintf("%d bytes", len(screenshot)))
|
| 236 |
+
}
|
demo/web/app.js
ADDED
|
@@ -0,0 +1,499 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import * as THREE from './vendor/three.module.min.js';
|
| 2 |
+
|
| 3 |
+
const viewport = document.querySelector('#viewport');
|
| 4 |
+
const styleSelect = document.querySelector('#style-select');
|
| 5 |
+
const statusElement = document.querySelector('#status');
|
| 6 |
+
const planInfo = document.querySelector('#plan-info');
|
| 7 |
+
const targetInfo = document.querySelector('#target-info');
|
| 8 |
+
const testResult = document.querySelector('#test-result');
|
| 9 |
+
const showAllTargets = document.querySelector('#show-all-targets');
|
| 10 |
+
const targetFrame = document.querySelector('#target-frame');
|
| 11 |
+
const targetFrameLabel = document.querySelector('#target-frame-label');
|
| 12 |
+
const resetCamera = document.querySelector('#reset-camera');
|
| 13 |
+
const query = new URLSearchParams(location.search);
|
| 14 |
+
|
| 15 |
+
const state = {
|
| 16 |
+
meta: null, session: '', motion: null, targets: null, playhead: 0,
|
| 17 |
+
lastTime: performance.now(), move: [0, 0], facing: [0, 1], keys: new Set(),
|
| 18 |
+
padKey: '', pending: false, replanQueued: false, seed: 10, style: '', rig: null, targetRigs: [],
|
| 19 |
+
generatedPath: null, targetPath: null,
|
| 20 |
+
};
|
| 21 |
+
|
| 22 |
+
const scene = new THREE.Scene();
|
| 23 |
+
scene.fog = new THREE.FogExp2(0x090b10, 0.045);
|
| 24 |
+
const camera = new THREE.PerspectiveCamera(42, 1, 0.02, 100);
|
| 25 |
+
const renderer = new THREE.WebGLRenderer({antialias: true, alpha: true});
|
| 26 |
+
renderer.setPixelRatio(Math.min(devicePixelRatio, 2));
|
| 27 |
+
renderer.outputColorSpace = THREE.SRGBColorSpace;
|
| 28 |
+
renderer.shadowMap.enabled = true;
|
| 29 |
+
renderer.shadowMap.type = THREE.PCFSoftShadowMap;
|
| 30 |
+
viewport.append(renderer.domElement);
|
| 31 |
+
|
| 32 |
+
scene.add(new THREE.HemisphereLight(0xcde4ff, 0x253226, 2.5));
|
| 33 |
+
const keyLight = new THREE.DirectionalLight(0xffffff, 3.2);
|
| 34 |
+
keyLight.position.set(3, 7, 4);
|
| 35 |
+
keyLight.castShadow = true;
|
| 36 |
+
scene.add(keyLight);
|
| 37 |
+
const rimLight = new THREE.DirectionalLight(0x67e8c3, 1.2);
|
| 38 |
+
rimLight.position.set(-4, 3, -3);
|
| 39 |
+
scene.add(rimLight);
|
| 40 |
+
const floor = new THREE.Mesh(
|
| 41 |
+
new THREE.PlaneGeometry(40, 40),
|
| 42 |
+
new THREE.MeshStandardMaterial({color: 0x0d121a, roughness: 0.96, metalness: 0.02}),
|
| 43 |
+
);
|
| 44 |
+
floor.rotation.x = -Math.PI / 2;
|
| 45 |
+
floor.position.y = -0.012;
|
| 46 |
+
floor.receiveShadow = true;
|
| 47 |
+
scene.add(floor);
|
| 48 |
+
const grid = new THREE.GridHelper(24, 48, 0x3a4b5d, 0x202b38);
|
| 49 |
+
grid.position.y = 0;
|
| 50 |
+
scene.add(grid);
|
| 51 |
+
|
| 52 |
+
const cylinderGeometry = new THREE.CylinderGeometry(1, 1, 1, 8, 1, false);
|
| 53 |
+
const sphereGeometry = new THREE.SphereGeometry(1, 14, 10);
|
| 54 |
+
const diamondGeometry = new THREE.OctahedronGeometry(1, 0);
|
| 55 |
+
const up = new THREE.Vector3(0, 1, 0);
|
| 56 |
+
const startPoint = new THREE.Vector3();
|
| 57 |
+
const endPoint = new THREE.Vector3();
|
| 58 |
+
const direction = new THREE.Vector3();
|
| 59 |
+
const midpoint = new THREE.Vector3();
|
| 60 |
+
|
| 61 |
+
function labelSprite(text, color, opacity) {
|
| 62 |
+
const canvas = document.createElement('canvas');
|
| 63 |
+
canvas.width = 128;
|
| 64 |
+
canvas.height = 64;
|
| 65 |
+
const context = canvas.getContext('2d');
|
| 66 |
+
context.font = '700 30px system-ui';
|
| 67 |
+
context.textAlign = 'center';
|
| 68 |
+
context.textBaseline = 'middle';
|
| 69 |
+
context.fillStyle = color;
|
| 70 |
+
context.fillText(text, 64, 32);
|
| 71 |
+
const texture = new THREE.CanvasTexture(canvas);
|
| 72 |
+
texture.colorSpace = THREE.SRGBColorSpace;
|
| 73 |
+
const sprite = new THREE.Sprite(new THREE.SpriteMaterial({map: texture, transparent: true, opacity, depthTest: false}));
|
| 74 |
+
sprite.scale.set(0.32, 0.16, 1);
|
| 75 |
+
sprite.renderOrder = 20;
|
| 76 |
+
return sprite;
|
| 77 |
+
}
|
| 78 |
+
|
| 79 |
+
class SkeletonRig {
|
| 80 |
+
constructor(joints, options) {
|
| 81 |
+
this.bones = [];
|
| 82 |
+
this.jointMeshes = [];
|
| 83 |
+
this.segments = [];
|
| 84 |
+
this.group = new THREE.Group();
|
| 85 |
+
this.radius = options.radius;
|
| 86 |
+
this.jointRadius = options.jointRadius;
|
| 87 |
+
this.rootRadius = options.rootRadius;
|
| 88 |
+
this.labelHeight = options.labelHeight ?? 0.34;
|
| 89 |
+
this.boneMaterial = new THREE.MeshStandardMaterial({
|
| 90 |
+
color: options.color, emissive: options.emissive, emissiveIntensity: options.emissiveIntensity,
|
| 91 |
+
roughness: 0.42, transparent: options.opacity < 1, opacity: options.opacity,
|
| 92 |
+
depthWrite: options.opacity >= 0.95,
|
| 93 |
+
});
|
| 94 |
+
this.jointMaterial = new THREE.MeshStandardMaterial({
|
| 95 |
+
color: options.jointColor, emissive: options.emissive, emissiveIntensity: options.emissiveIntensity,
|
| 96 |
+
roughness: 0.35, transparent: options.opacity < 1, opacity: options.opacity,
|
| 97 |
+
depthWrite: options.opacity >= 0.95,
|
| 98 |
+
});
|
| 99 |
+
this.rootMaterial = new THREE.MeshStandardMaterial({
|
| 100 |
+
color: options.rootColor, emissive: options.rootEmissive, emissiveIntensity: 1.4,
|
| 101 |
+
transparent: options.opacity < 1, opacity: options.opacity, depthWrite: options.opacity >= 0.95,
|
| 102 |
+
});
|
| 103 |
+
for (let index = 0; index < joints.length; index++) {
|
| 104 |
+
const joint = joints[index];
|
| 105 |
+
const bone = new THREE.Bone();
|
| 106 |
+
bone.name = joint.name;
|
| 107 |
+
if (joint.parent < 0) {
|
| 108 |
+
bone.position.set(0, 0, 0);
|
| 109 |
+
} else {
|
| 110 |
+
const parent = joints[joint.parent].position;
|
| 111 |
+
bone.position.set(joint.position[0] - parent[0], joint.position[1] - parent[1], joint.position[2] - parent[2]);
|
| 112 |
+
this.bones[joint.parent].add(bone);
|
| 113 |
+
}
|
| 114 |
+
this.bones.push(bone);
|
| 115 |
+
const geometry = options.diamonds ? diamondGeometry : sphereGeometry;
|
| 116 |
+
const marker = new THREE.Mesh(geometry, index === 0 ? this.rootMaterial : this.jointMaterial);
|
| 117 |
+
marker.castShadow = !options.diamonds;
|
| 118 |
+
marker.renderOrder = options.renderOrder;
|
| 119 |
+
this.jointMeshes.push(marker);
|
| 120 |
+
this.group.add(marker);
|
| 121 |
+
if (joint.parent >= 0) {
|
| 122 |
+
const segment = new THREE.Mesh(cylinderGeometry, this.boneMaterial);
|
| 123 |
+
segment.castShadow = !options.diamonds;
|
| 124 |
+
segment.renderOrder = options.renderOrder;
|
| 125 |
+
this.segments.push({mesh: segment, child: index, parent: joint.parent});
|
| 126 |
+
this.group.add(segment);
|
| 127 |
+
}
|
| 128 |
+
}
|
| 129 |
+
scene.add(this.bones[0]);
|
| 130 |
+
scene.add(this.group);
|
| 131 |
+
this.label = options.label ? labelSprite(options.label, options.labelColor, options.opacity) : null;
|
| 132 |
+
if (this.label) scene.add(this.label);
|
| 133 |
+
}
|
| 134 |
+
|
| 135 |
+
pose(roots, rotations, frame, frameCount) {
|
| 136 |
+
const index = Math.min(frameCount - 1, Math.max(0, frame));
|
| 137 |
+
this.bones[0].position.fromArray(roots, index * 3);
|
| 138 |
+
for (let joint = 0; joint < this.bones.length; joint++) {
|
| 139 |
+
this.bones[joint].quaternion.fromArray(rotations, (index * this.bones.length + joint) * 4);
|
| 140 |
+
}
|
| 141 |
+
this.updateGeometry();
|
| 142 |
+
}
|
| 143 |
+
|
| 144 |
+
updateGeometry() {
|
| 145 |
+
this.bones[0].updateMatrixWorld(true);
|
| 146 |
+
for (let index = 0; index < this.bones.length; index++) {
|
| 147 |
+
this.bones[index].getWorldPosition(endPoint);
|
| 148 |
+
const marker = this.jointMeshes[index];
|
| 149 |
+
marker.position.copy(endPoint);
|
| 150 |
+
const radius = index === 0 ? this.rootRadius : this.jointRadius;
|
| 151 |
+
marker.scale.setScalar(radius);
|
| 152 |
+
}
|
| 153 |
+
for (const segment of this.segments) {
|
| 154 |
+
this.bones[segment.parent].getWorldPosition(startPoint);
|
| 155 |
+
this.bones[segment.child].getWorldPosition(endPoint);
|
| 156 |
+
direction.subVectors(endPoint, startPoint);
|
| 157 |
+
const length = direction.length();
|
| 158 |
+
midpoint.addVectors(startPoint, endPoint).multiplyScalar(0.5);
|
| 159 |
+
segment.mesh.position.copy(midpoint);
|
| 160 |
+
segment.mesh.quaternion.setFromUnitVectors(up, direction.normalize());
|
| 161 |
+
segment.mesh.scale.set(this.radius, length, this.radius);
|
| 162 |
+
}
|
| 163 |
+
if (this.label) {
|
| 164 |
+
this.bones[0].getWorldPosition(endPoint);
|
| 165 |
+
this.label.position.set(endPoint.x, endPoint.y + this.labelHeight, endPoint.z);
|
| 166 |
+
}
|
| 167 |
+
}
|
| 168 |
+
|
| 169 |
+
setVisible(visible) {
|
| 170 |
+
this.group.visible = visible;
|
| 171 |
+
this.bones[0].visible = visible;
|
| 172 |
+
if (this.label) this.label.visible = visible;
|
| 173 |
+
}
|
| 174 |
+
|
| 175 |
+
setOpacity(opacity) {
|
| 176 |
+
for (const material of [this.boneMaterial, this.jointMaterial, this.rootMaterial]) {
|
| 177 |
+
material.opacity = opacity;
|
| 178 |
+
material.transparent = opacity < 1;
|
| 179 |
+
material.depthWrite = opacity >= 0.95;
|
| 180 |
+
}
|
| 181 |
+
if (this.label) this.label.material.opacity = opacity;
|
| 182 |
+
}
|
| 183 |
+
}
|
| 184 |
+
|
| 185 |
+
function makeLine(color, dashed = false) {
|
| 186 |
+
const material = dashed
|
| 187 |
+
? new THREE.LineDashedMaterial({color, dashSize: 0.09, gapSize: 0.055, transparent: true, opacity: 0.9})
|
| 188 |
+
: new THREE.LineBasicMaterial({color, transparent: true, opacity: 0.72});
|
| 189 |
+
const line = new THREE.Line(new THREE.BufferGeometry(), material);
|
| 190 |
+
line.renderOrder = 1;
|
| 191 |
+
scene.add(line);
|
| 192 |
+
return line;
|
| 193 |
+
}
|
| 194 |
+
|
| 195 |
+
function setGroundPath(line, roots, frames) {
|
| 196 |
+
const points = [];
|
| 197 |
+
for (let frame = 0; frame < frames; frame++) {
|
| 198 |
+
points.push(new THREE.Vector3(roots[frame * 3], 0.018, roots[frame * 3 + 2]));
|
| 199 |
+
}
|
| 200 |
+
line.geometry.dispose();
|
| 201 |
+
line.geometry = new THREE.BufferGeometry().setFromPoints(points);
|
| 202 |
+
if (line.material.isLineDashedMaterial) line.computeLineDistances();
|
| 203 |
+
}
|
| 204 |
+
|
| 205 |
+
function makeSkeletons(joints) {
|
| 206 |
+
state.rig = new SkeletonRig(joints, {
|
| 207 |
+
color: 0x55efc4, jointColor: 0xd9fff3, rootColor: 0xffd166,
|
| 208 |
+
emissive: 0x0c5b49, rootEmissive: 0x6a3b00, emissiveIntensity: 0.8,
|
| 209 |
+
radius: 0.022, jointRadius: 0.034, rootRadius: 0.062,
|
| 210 |
+
opacity: 1, diamonds: false, renderOrder: 5,
|
| 211 |
+
});
|
| 212 |
+
for (let frame = 0; frame < 4; frame++) {
|
| 213 |
+
state.targetRigs.push(new SkeletonRig(joints, {
|
| 214 |
+
color: 0xff763b, jointColor: 0xffb06b, rootColor: 0xff4f8b,
|
| 215 |
+
emissive: 0x7b1f00, rootEmissive: 0x790025, emissiveIntensity: 1,
|
| 216 |
+
radius: 0.018, jointRadius: 0.032, rootRadius: 0.056, opacity: 0.9, diamonds: true,
|
| 217 |
+
renderOrder: 8 + frame, label: `T${frame}`, labelColor: frame === 3 ? '#ffb477' : '#c97354',
|
| 218 |
+
labelHeight: 0.34 + (3 - frame) * 0.11,
|
| 219 |
+
}));
|
| 220 |
+
}
|
| 221 |
+
state.generatedPath = makeLine(0x43e8bf);
|
| 222 |
+
state.targetPath = makeLine(0xff6b3d, true);
|
| 223 |
+
}
|
| 224 |
+
|
| 225 |
+
function updateTargetVisibility() {
|
| 226 |
+
const frames = state.targets?.frames ?? 0;
|
| 227 |
+
const maximum = Math.max(0, frames - 1);
|
| 228 |
+
targetFrame.max = String(maximum);
|
| 229 |
+
const selected = THREE.MathUtils.clamp(Number(targetFrame.value), 0, maximum);
|
| 230 |
+
targetFrame.value = String(selected);
|
| 231 |
+
targetFrameLabel.textContent = `T${selected}`;
|
| 232 |
+
const overlay = showAllTargets.checked;
|
| 233 |
+
const overlayOpacities = [0.16, 0.25, 0.42, 0.9];
|
| 234 |
+
for (let frame = 0; frame < state.targetRigs.length; frame++) {
|
| 235 |
+
const exists = frame < frames;
|
| 236 |
+
state.targetRigs[frame].setOpacity(overlay ? overlayOpacities[frame] : 0.9);
|
| 237 |
+
state.targetRigs[frame].setVisible(exists && (overlay || frame === selected));
|
| 238 |
+
}
|
| 239 |
+
state.targetPath.visible = frames > 0;
|
| 240 |
+
targetInfo.textContent = frames > 0
|
| 241 |
+
? (overlay ? `${frames} consecutive constraints overlaid` : `T${selected} of ${frames} consecutive constraints`)
|
| 242 |
+
: '—';
|
| 243 |
+
}
|
| 244 |
+
|
| 245 |
+
const cameraView = {yaw: 0.68, pitch: 0.24, distance: 4.8, dragging: false, x: 0, y: 0};
|
| 246 |
+
const focus = new THREE.Vector3();
|
| 247 |
+
const desiredCamera = new THREE.Vector3();
|
| 248 |
+
const visibleBounds = new THREE.Box3();
|
| 249 |
+
function resetCameraView() {
|
| 250 |
+
cameraView.yaw = 0.68;
|
| 251 |
+
cameraView.pitch = 0.24;
|
| 252 |
+
cameraView.distance = 4.8;
|
| 253 |
+
}
|
| 254 |
+
resetCamera.addEventListener('click', resetCameraView);
|
| 255 |
+
renderer.domElement.addEventListener('pointerdown', event => {
|
| 256 |
+
cameraView.dragging = true; cameraView.x = event.clientX; cameraView.y = event.clientY;
|
| 257 |
+
renderer.domElement.setPointerCapture(event.pointerId);
|
| 258 |
+
});
|
| 259 |
+
renderer.domElement.addEventListener('pointermove', event => {
|
| 260 |
+
if (!cameraView.dragging) return;
|
| 261 |
+
cameraView.yaw -= (event.clientX - cameraView.x) * 0.006;
|
| 262 |
+
cameraView.pitch = THREE.MathUtils.clamp(cameraView.pitch + (event.clientY - cameraView.y) * 0.004, -0.05, 1.05);
|
| 263 |
+
cameraView.x = event.clientX; cameraView.y = event.clientY;
|
| 264 |
+
});
|
| 265 |
+
renderer.domElement.addEventListener('pointerup', event => {
|
| 266 |
+
cameraView.dragging = false; renderer.domElement.releasePointerCapture(event.pointerId);
|
| 267 |
+
});
|
| 268 |
+
renderer.domElement.addEventListener('pointercancel', () => { cameraView.dragging = false; });
|
| 269 |
+
renderer.domElement.addEventListener('wheel', event => {
|
| 270 |
+
cameraView.distance = THREE.MathUtils.clamp(cameraView.distance * Math.exp(event.deltaY * 0.001), 2.1, 8);
|
| 271 |
+
event.preventDefault();
|
| 272 |
+
}, {passive: false});
|
| 273 |
+
renderer.domElement.addEventListener('dblclick', resetCameraView);
|
| 274 |
+
|
| 275 |
+
function resize() {
|
| 276 |
+
const width = Math.max(1, viewport.clientWidth), height = Math.max(1, viewport.clientHeight);
|
| 277 |
+
renderer.setSize(width, height, false);
|
| 278 |
+
camera.aspect = width / height;
|
| 279 |
+
camera.updateProjectionMatrix();
|
| 280 |
+
}
|
| 281 |
+
addEventListener('resize', resize);
|
| 282 |
+
resize();
|
| 283 |
+
|
| 284 |
+
async function api(path, body) {
|
| 285 |
+
const response = await fetch(path, {
|
| 286 |
+
method: body ? 'POST' : 'GET', headers: {'Content-Type': 'application/json'},
|
| 287 |
+
body: body ? JSON.stringify(body) : undefined,
|
| 288 |
+
});
|
| 289 |
+
const value = await response.json();
|
| 290 |
+
if (!response.ok) throw new Error(value.error || `${response.status} ${response.statusText}`);
|
| 291 |
+
return value;
|
| 292 |
+
}
|
| 293 |
+
|
| 294 |
+
function installStyles(styles) {
|
| 295 |
+
for (const style of styles) {
|
| 296 |
+
const option = document.createElement('option');
|
| 297 |
+
option.value = style.name;
|
| 298 |
+
option.textContent = `${style.name.replaceAll('_', ' ')} · ${style.speed.toFixed(1)} m/s`;
|
| 299 |
+
styleSelect.append(option);
|
| 300 |
+
}
|
| 301 |
+
state.style = styles.some(item => item.name === 'walk') ? 'walk' : styles[0].name;
|
| 302 |
+
styleSelect.value = state.style;
|
| 303 |
+
styleSelect.addEventListener('change', () => { state.style = styleSelect.value; void requestPlan(); });
|
| 304 |
+
}
|
| 305 |
+
|
| 306 |
+
function useMotion(response, plannedMove = state.move) {
|
| 307 |
+
state.session = response.session;
|
| 308 |
+
state.style = response.style;
|
| 309 |
+
styleSelect.value = response.style;
|
| 310 |
+
state.motion = response.motion;
|
| 311 |
+
state.targets = response.targets;
|
| 312 |
+
state.playhead = 0;
|
| 313 |
+
state.lastTime = performance.now();
|
| 314 |
+
planInfo.textContent = `${response.motion.frames} frames · ${response.style.replaceAll('_', ' ')}`;
|
| 315 |
+
statusElement.textContent = 'Playing';
|
| 316 |
+
document.documentElement.dataset.planSequence = String(Number(document.documentElement.dataset.planSequence || 0) + 1);
|
| 317 |
+
document.documentElement.dataset.plannedMoveX = String(plannedMove[0]);
|
| 318 |
+
document.documentElement.dataset.plannedMoveZ = String(plannedMove[1]);
|
| 319 |
+
setGroundPath(state.generatedPath, response.motion.roots, response.motion.frames);
|
| 320 |
+
setGroundPath(state.targetPath, response.targets.roots, response.targets.frames);
|
| 321 |
+
for (let frame = 0; frame < response.targets.frames; frame++) {
|
| 322 |
+
state.targetRigs[frame].pose(response.targets.roots, response.targets.rotations, frame, response.targets.frames);
|
| 323 |
+
}
|
| 324 |
+
updateTargetVisibility();
|
| 325 |
+
}
|
| 326 |
+
|
| 327 |
+
async function requestPlan(advance = Math.floor(state.playhead)) {
|
| 328 |
+
if (!state.session) return null;
|
| 329 |
+
if (state.pending) {
|
| 330 |
+
state.replanQueued = true;
|
| 331 |
+
return null;
|
| 332 |
+
}
|
| 333 |
+
state.pending = true;
|
| 334 |
+
statusElement.textContent = 'Planning…';
|
| 335 |
+
const plannedMove = [...state.move];
|
| 336 |
+
try {
|
| 337 |
+
const response = await api('/api/plan', {
|
| 338 |
+
session: state.session, style: state.style, move: state.move, facing: state.facing,
|
| 339 |
+
seed: state.seed++, advance,
|
| 340 |
+
});
|
| 341 |
+
useMotion(response, plannedMove);
|
| 342 |
+
return response;
|
| 343 |
+
} finally {
|
| 344 |
+
state.pending = false;
|
| 345 |
+
if (state.replanQueued) {
|
| 346 |
+
state.replanQueued = false;
|
| 347 |
+
queueMicrotask(() => void requestPlan(Math.floor(state.playhead)));
|
| 348 |
+
}
|
| 349 |
+
}
|
| 350 |
+
}
|
| 351 |
+
|
| 352 |
+
function updateControl() {
|
| 353 |
+
let x = 0, z = 0;
|
| 354 |
+
const active = key => state.keys.has(key) || state.padKey === key;
|
| 355 |
+
if (active('w')) z += 1;
|
| 356 |
+
if (active('s')) z -= 1;
|
| 357 |
+
if (active('a')) x -= 1;
|
| 358 |
+
if (active('d')) x += 1;
|
| 359 |
+
const length = Math.hypot(x, z);
|
| 360 |
+
if (length > 0) { x /= length; z /= length; state.facing = [x, z]; }
|
| 361 |
+
state.move = [x, z];
|
| 362 |
+
document.querySelectorAll('.pad button').forEach(button => {
|
| 363 |
+
const pressed = active(button.dataset.key);
|
| 364 |
+
button.classList.toggle('active', pressed);
|
| 365 |
+
button.setAttribute('aria-pressed', String(pressed));
|
| 366 |
+
});
|
| 367 |
+
}
|
| 368 |
+
|
| 369 |
+
let controlTimer = 0;
|
| 370 |
+
function schedulePlan(delay = 0) { clearTimeout(controlTimer); controlTimer = setTimeout(() => void requestPlan(), delay); }
|
| 371 |
+
addEventListener('keydown', event => {
|
| 372 |
+
const key = event.key.toLowerCase();
|
| 373 |
+
if (['w', 'a', 's', 'd'].includes(key) && !state.keys.has(key)) {
|
| 374 |
+
state.keys.add(key); updateControl(); schedulePlan(); event.preventDefault();
|
| 375 |
+
}
|
| 376 |
+
if (event.code === 'Space' || event.key === 'Escape') {
|
| 377 |
+
state.keys.clear(); state.padKey = ''; updateControl(); schedulePlan(); event.preventDefault();
|
| 378 |
+
}
|
| 379 |
+
if (event.key === 'ArrowLeft' || event.key === 'ArrowRight') {
|
| 380 |
+
const angle = Math.atan2(state.facing[0], state.facing[1]) + (event.key === 'ArrowLeft' ? 0.25 : -0.25);
|
| 381 |
+
state.facing = [Math.sin(angle), Math.cos(angle)]; schedulePlan(35); event.preventDefault();
|
| 382 |
+
}
|
| 383 |
+
});
|
| 384 |
+
addEventListener('keyup', event => {
|
| 385 |
+
const key = event.key.toLowerCase();
|
| 386 |
+
if (state.keys.delete(key)) { updateControl(); schedulePlan(); }
|
| 387 |
+
});
|
| 388 |
+
for (const button of document.querySelectorAll('.pad button')) {
|
| 389 |
+
button.setAttribute('aria-pressed', 'false');
|
| 390 |
+
button.addEventListener('click', event => {
|
| 391 |
+
state.padKey = state.padKey === button.dataset.key ? '' : button.dataset.key;
|
| 392 |
+
updateControl();
|
| 393 |
+
schedulePlan();
|
| 394 |
+
event.preventDefault();
|
| 395 |
+
});
|
| 396 |
+
}
|
| 397 |
+
showAllTargets.addEventListener('change', updateTargetVisibility);
|
| 398 |
+
targetFrame.addEventListener('input', updateTargetVisibility);
|
| 399 |
+
|
| 400 |
+
function updateCamera() {
|
| 401 |
+
visibleBounds.makeEmpty();
|
| 402 |
+
for (const marker of state.rig.jointMeshes) visibleBounds.expandByPoint(marker.position);
|
| 403 |
+
visibleBounds.getCenter(focus);
|
| 404 |
+
const horizontal = Math.cos(cameraView.pitch) * cameraView.distance;
|
| 405 |
+
desiredCamera.set(
|
| 406 |
+
focus.x + Math.sin(cameraView.yaw) * horizontal,
|
| 407 |
+
focus.y + Math.sin(cameraView.pitch) * cameraView.distance,
|
| 408 |
+
focus.z + Math.cos(cameraView.yaw) * horizontal,
|
| 409 |
+
);
|
| 410 |
+
if (camera.userData.positioned) camera.position.lerp(desiredCamera, 0.12);
|
| 411 |
+
else { camera.position.copy(desiredCamera); camera.userData.positioned = true; }
|
| 412 |
+
camera.lookAt(focus);
|
| 413 |
+
}
|
| 414 |
+
|
| 415 |
+
function renderMotion(frame) {
|
| 416 |
+
if (!state.motion) return;
|
| 417 |
+
const index = Math.min(state.motion.frames - 1, Math.max(0, frame));
|
| 418 |
+
state.rig.pose(state.motion.roots, state.motion.rotations, index, state.motion.frames);
|
| 419 |
+
updateCamera();
|
| 420 |
+
}
|
| 421 |
+
|
| 422 |
+
function animate(now) {
|
| 423 |
+
requestAnimationFrame(animate);
|
| 424 |
+
const delta = Math.min(0.1, (now - state.lastTime) / 1000);
|
| 425 |
+
state.lastTime = now;
|
| 426 |
+
if (state.motion) {
|
| 427 |
+
state.playhead += delta * state.meta.fps;
|
| 428 |
+
if (state.playhead >= state.motion.frames - 5 && !state.pending) void requestPlan(Math.floor(state.playhead));
|
| 429 |
+
renderMotion(Math.floor(state.playhead));
|
| 430 |
+
}
|
| 431 |
+
renderer.render(scene, camera);
|
| 432 |
+
}
|
| 433 |
+
|
| 434 |
+
async function selfTest() {
|
| 435 |
+
const alternate = state.meta.styles.find(item => item.name === 'walk_zombie') || state.meta.styles.find(item => item.name !== 'walk');
|
| 436 |
+
if (!alternate) throw new Error('no alternate upstream style available');
|
| 437 |
+
state.style = alternate.name;
|
| 438 |
+
styleSelect.value = alternate.name;
|
| 439 |
+
state.move = [1, 0];
|
| 440 |
+
state.facing = [1, 0];
|
| 441 |
+
const response = await requestPlan(3);
|
| 442 |
+
if (!response || response.style !== alternate.name) throw new Error('style change was not applied');
|
| 443 |
+
if (response.motion.joints !== 34 || response.motion.frames < 24 || response.motion.rotations.length !== response.motion.frames * 34 * 4) {
|
| 444 |
+
throw new Error('invalid skeletal animation response');
|
| 445 |
+
}
|
| 446 |
+
if (response.targets.frames !== 4 || response.targets.joints !== 34 || response.targets.roots.length !== 12 || response.targets.rotations.length !== 4 * 34 * 4) {
|
| 447 |
+
throw new Error('invalid placed target-keyframe response');
|
| 448 |
+
}
|
| 449 |
+
renderMotion(2);
|
| 450 |
+
renderer.render(scene, camera);
|
| 451 |
+
let visibleTargets = state.targetRigs.filter(rig => rig.group.visible).length;
|
| 452 |
+
if (!renderer.domElement.width || state.rig.bones.length !== 34 || visibleTargets !== 1 || !state.targetRigs[3].group.visible) {
|
| 453 |
+
throw new Error('animated and target skeletons were not rendered');
|
| 454 |
+
}
|
| 455 |
+
const expectedFocus = new THREE.Vector3();
|
| 456 |
+
const animatedBounds = new THREE.Box3();
|
| 457 |
+
for (const marker of state.rig.jointMeshes) animatedBounds.expandByPoint(marker.position);
|
| 458 |
+
animatedBounds.getCenter(expectedFocus);
|
| 459 |
+
if (focus.distanceTo(expectedFocus) > 1e-6) throw new Error('camera is not anchored to the animated skeleton');
|
| 460 |
+
targetFrame.value = '1';
|
| 461 |
+
updateTargetVisibility();
|
| 462 |
+
if (!state.targetRigs[1].group.visible || state.targetRigs.filter(rig => rig.group.visible).length !== 1) {
|
| 463 |
+
throw new Error('target-frame inspector did not select T1');
|
| 464 |
+
}
|
| 465 |
+
showAllTargets.checked = true;
|
| 466 |
+
updateTargetVisibility();
|
| 467 |
+
visibleTargets = state.targetRigs.filter(rig => rig.group.visible).length;
|
| 468 |
+
if (visibleTargets !== 4) throw new Error('four-pose target overlay did not render');
|
| 469 |
+
showAllTargets.checked = false;
|
| 470 |
+
targetFrame.value = '3';
|
| 471 |
+
updateTargetVisibility();
|
| 472 |
+
document.documentElement.dataset.animatedJoints = String(state.rig.bones.length);
|
| 473 |
+
document.documentElement.dataset.targetFrames = String(response.targets.frames);
|
| 474 |
+
document.documentElement.dataset.visibleTargets = '1';
|
| 475 |
+
document.documentElement.dataset.cameraSubject = 'animated';
|
| 476 |
+
document.documentElement.dataset.testStatus = 'passed';
|
| 477 |
+
testResult.textContent = `Headless check passed: animated-camera anchor + target inspector/overlay, ${alternate.name}, right turn`;
|
| 478 |
+
}
|
| 479 |
+
|
| 480 |
+
async function start() {
|
| 481 |
+
try {
|
| 482 |
+
state.meta = await api('/api/meta');
|
| 483 |
+
installStyles(state.meta.styles);
|
| 484 |
+
makeSkeletons(state.meta.joints);
|
| 485 |
+
const initial = await api('/api/session', {style: state.style});
|
| 486 |
+
useMotion(initial);
|
| 487 |
+
renderMotion(0);
|
| 488 |
+
requestAnimationFrame(animate);
|
| 489 |
+
if (query.get('test') === '1') await selfTest();
|
| 490 |
+
else document.documentElement.dataset.testStatus = 'ready';
|
| 491 |
+
} catch (error) {
|
| 492 |
+
console.error(error);
|
| 493 |
+
statusElement.textContent = 'Error';
|
| 494 |
+
testResult.textContent = error.message;
|
| 495 |
+
document.documentElement.dataset.testStatus = 'failed';
|
| 496 |
+
}
|
| 497 |
+
}
|
| 498 |
+
|
| 499 |
+
await start();
|
demo/web/index.html
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
<!doctype html>
|
| 2 |
+
<html lang="en" data-test-status="starting">
|
| 3 |
+
<head>
|
| 4 |
+
<meta charset="utf-8">
|
| 5 |
+
<meta name="viewport" content="width=device-width,initial-scale=1">
|
| 6 |
+
<link rel="icon" href="data:,">
|
| 7 |
+
<title>MotionBricks G1</title>
|
| 8 |
+
<link rel="stylesheet" href="style.css">
|
| 9 |
+
</head>
|
| 10 |
+
<body>
|
| 11 |
+
<main>
|
| 12 |
+
<section id="viewport" aria-label="Animated G1 skeleton"></section>
|
| 13 |
+
<aside>
|
| 14 |
+
<p class="eyebrow">MOTIONBRICKS / G1</p>
|
| 15 |
+
<h1>Whole-body motion</h1>
|
| 16 |
+
<p class="lede">Move with WASD. The character faces its travel direction; arrow keys turn independently.</p>
|
| 17 |
+
<div class="legend" aria-label="Skeleton legend">
|
| 18 |
+
<span><i class="animated-swatch"></i>Animated model</span>
|
| 19 |
+
<span><i class="target-swatch"></i>Target keyframes</span>
|
| 20 |
+
</div>
|
| 21 |
+
<label for="style-select">Upstream pose style</label>
|
| 22 |
+
<select id="style-select" aria-label="Motion style"></select>
|
| 23 |
+
<div class="target-frame-control">
|
| 24 |
+
<label for="target-frame">Inspect placed target <output id="target-frame-label" for="target-frame">T3</output></label>
|
| 25 |
+
<input id="target-frame" type="range" min="0" max="3" step="1" value="3" aria-label="Target frame">
|
| 26 |
+
</div>
|
| 27 |
+
<div class="view-options">
|
| 28 |
+
<label><input id="show-all-targets" type="checkbox"> Overlay all four consecutive poses</label>
|
| 29 |
+
<button id="reset-camera" type="button">Reset camera</button>
|
| 30 |
+
</div>
|
| 31 |
+
<p class="target-help">T0–T3 are consecutive 30 FPS constraints, so their exact positions are intentionally close.</p>
|
| 32 |
+
<p class="camera-help">Drag to orbit · wheel to zoom · tap a pad direction to move, tap it again to stop</p>
|
| 33 |
+
<div class="pad" aria-label="Directional controls">
|
| 34 |
+
<button data-key="w" data-x="0" data-z="1" aria-label="Walk forward">W</button>
|
| 35 |
+
<button data-key="a" data-x="-1" data-z="0" aria-label="Walk left">A</button>
|
| 36 |
+
<button data-key="s" data-x="0" data-z="-1" aria-label="Walk backward">S</button>
|
| 37 |
+
<button data-key="d" data-x="1" data-z="0" aria-label="Walk right">D</button>
|
| 38 |
+
</div>
|
| 39 |
+
<dl>
|
| 40 |
+
<div><dt>Status</dt><dd id="status">Loading model…</dd></div>
|
| 41 |
+
<div><dt>Plan</dt><dd id="plan-info">—</dd></div>
|
| 42 |
+
<div><dt>Targets</dt><dd id="target-info">—</dd></div>
|
| 43 |
+
<div><dt>Backend</dt><dd>native GGML</dd></div>
|
| 44 |
+
</dl>
|
| 45 |
+
<p id="test-result" role="status"></p>
|
| 46 |
+
</aside>
|
| 47 |
+
</main>
|
| 48 |
+
<script type="module" src="app.js"></script>
|
| 49 |
+
</body>
|
| 50 |
+
</html>
|
demo/web/style.css
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
:root { color-scheme: dark; font-family: Inter, ui-sans-serif, system-ui, sans-serif; background:#090b10; color:#edf1f7; }
|
| 2 |
+
* { box-sizing:border-box; }
|
| 3 |
+
html, body, main { width:100%; height:100%; margin:0; overflow:hidden; }
|
| 4 |
+
main { display:grid; grid-template-columns:minmax(0,1fr) 320px; }
|
| 5 |
+
#viewport { min-width:0; background:radial-gradient(circle at 50% 36%,#20293a 0,#10141d 38%,#080a0f 76%); }
|
| 6 |
+
#viewport canvas { display:block; width:100%; height:100%; }
|
| 7 |
+
aside { padding:36px 28px; border-left:1px solid #2a3240; background:rgba(11,14,20,.96); box-shadow:-18px 0 60px #0008; }
|
| 8 |
+
.eyebrow { margin:0 0 10px; color:#6be6ba; letter-spacing:.18em; font-size:11px; font-weight:750; }
|
| 9 |
+
h1 { margin:0 0 12px; font-size:30px; letter-spacing:-.04em; }
|
| 10 |
+
.lede { color:#9eabbc; line-height:1.45; margin:0 0 18px; }
|
| 11 |
+
.legend { display:flex; gap:16px; margin:0 0 24px; color:#aeb9c8; font-size:12px; }
|
| 12 |
+
.legend span { display:flex; align-items:center; gap:7px; }
|
| 13 |
+
.legend i { display:block; width:20px; height:5px; border-radius:4px; box-shadow:0 0 10px currentColor; }
|
| 14 |
+
.animated-swatch { color:#55efc4; background:#55efc4; }
|
| 15 |
+
.target-swatch { color:#ff763b; background:#ff763b; transform:rotate(-8deg); }
|
| 16 |
+
label { display:block; color:#c8d1dd; font-size:12px; font-weight:700; margin-bottom:8px; }
|
| 17 |
+
select { width:100%; color:#f2f6fb; background:#171c26; border:1px solid #343d4d; border-radius:8px; padding:11px; font:inherit; }
|
| 18 |
+
.target-frame-control { margin-top:16px; }
|
| 19 |
+
.target-frame-control label { display:flex; justify-content:space-between; margin-bottom:6px; }
|
| 20 |
+
.target-frame-control output { color:#ff9a68; font-variant-numeric:tabular-nums; }
|
| 21 |
+
#target-frame { display:block; width:100%; margin:0; accent-color:#ff763b; }
|
| 22 |
+
.view-options { display:flex; align-items:center; justify-content:space-between; gap:10px; margin-top:14px; }
|
| 23 |
+
.view-options label { display:flex; align-items:center; gap:7px; margin:0; font-weight:600; color:#b7c2d0; }
|
| 24 |
+
.view-options input { accent-color:#ff763b; }
|
| 25 |
+
#reset-camera { color:#dbe4ef; background:#171c26; border:1px solid #343d4d; border-radius:7px; padding:7px 9px; font:inherit; font-size:11px; white-space:nowrap; }
|
| 26 |
+
#reset-camera:hover { border-color:#59677b; background:#202735; }
|
| 27 |
+
.target-help, .camera-help { margin:9px 0 0; color:#647286; font-size:11px; line-height:1.35; }
|
| 28 |
+
.pad { margin:28px auto; width:176px; display:grid; grid-template:52px 52px / 52px 52px 52px; gap:8px; }
|
| 29 |
+
.pad button { border:1px solid #3b4658; border-radius:10px; background:#181e29; color:#dae3ed; font-weight:800; font-size:16px; box-shadow:0 4px 12px #0005; }
|
| 30 |
+
.pad button:first-child { grid-column:2; }
|
| 31 |
+
.pad button:nth-child(2) { grid-row:2; grid-column:1; }
|
| 32 |
+
.pad button:nth-child(3) { grid-row:2; grid-column:2; }
|
| 33 |
+
.pad button:nth-child(4) { grid-row:2; grid-column:3; }
|
| 34 |
+
.pad button.active { background:#2e735f; border-color:#77f0c4; transform:translateY(1px); }
|
| 35 |
+
dl { border-top:1px solid #2b3341; margin-top:30px; padding-top:10px; }
|
| 36 |
+
dl div { display:flex; justify-content:space-between; gap:10px; padding:8px 0; }
|
| 37 |
+
dt { color:#778599; } dd { margin:0; text-align:right; color:#dce4ee; }
|
| 38 |
+
#test-result { min-height:1.3em; color:#6be6ba; font-size:12px; }
|
| 39 |
+
@media (max-width:760px) { main { grid-template-columns:1fr; grid-template-rows:minmax(0,1fr) auto; } aside { padding:16px 20px; border-left:0; border-top:1px solid #2a3240; } .lede, dl, .target-help, .camera-help { display:none; } .legend { margin-bottom:12px; } .view-options { max-width:420px; } .pad { position:absolute; right:18px; bottom:12px; margin:0; transform:scale(.8); transform-origin:bottom right; } }
|
demo/web/vendor/THREE-LICENSE.txt
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
The MIT License
|
| 2 |
+
|
| 3 |
+
Copyright © 2010-2025 three.js authors
|
| 4 |
+
|
| 5 |
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
| 6 |
+
of this software and associated documentation files (the "Software"), to deal
|
| 7 |
+
in the Software without restriction, including without limitation the rights
|
| 8 |
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
| 9 |
+
copies of the Software, and to permit persons to whom the Software is
|
| 10 |
+
furnished to do so, subject to the following conditions:
|
| 11 |
+
|
| 12 |
+
The above copyright notice and this permission notice shall be included in
|
| 13 |
+
all copies or substantial portions of the Software.
|
| 14 |
+
|
| 15 |
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
| 16 |
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
| 17 |
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
| 18 |
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
| 19 |
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
| 20 |
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
| 21 |
+
THE SOFTWARE.
|
demo/web/vendor/three.core.min.js
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
demo/web/vendor/three.module.min.js
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
docs/DEMO.md
ADDED
|
@@ -0,0 +1,120 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Go and Three.js demo
|
| 2 |
+
|
| 3 |
+
The initial demo is a local Go application that calls `libmotionbricks`
|
| 4 |
+
through the PureGo binding and serves an embedded Three.js viewer. It renders
|
| 5 |
+
the released 34-joint G1 hierarchy directly from MotionBricks root
|
| 6 |
+
translations and local XYZW joint rotations; it does not require MuJoCo or a
|
| 7 |
+
skinned mesh. Solid cyan cylinders and round joints identify the generated
|
| 8 |
+
character. An orange diamond-jointed ghost skeleton identifies a selected
|
| 9 |
+
placed style-pose constraint supplied to the planner; all four constraints can
|
| 10 |
+
be overlaid for inspection.
|
| 11 |
+
|
| 12 |
+
## Build and run
|
| 13 |
+
|
| 14 |
+
First build the native project and create the model/style assets described in
|
| 15 |
+
the main README. Then build the Go application:
|
| 16 |
+
|
| 17 |
+
```sh
|
| 18 |
+
cmake --build --preset debug
|
| 19 |
+
cd demo
|
| 20 |
+
CGO_ENABLED=0 go build -o ../build/debug/bin/motionbricks-demo .
|
| 21 |
+
cd ..
|
| 22 |
+
```
|
| 23 |
+
|
| 24 |
+
The demo and reusable Go binding call the native shared library through
|
| 25 |
+
PureGo. They contain no cgo bridge and do not require a C compiler during the
|
| 26 |
+
Go build, so cgo can be disabled. This does not remove the separate C++/CMake
|
| 27 |
+
build that produces `libmotionbricks`.
|
| 28 |
+
|
| 29 |
+
Run it from the repository root:
|
| 30 |
+
|
| 31 |
+
```sh
|
| 32 |
+
./build/debug/bin/motionbricks-demo \
|
| 33 |
+
-listen 127.0.0.1:8080 \
|
| 34 |
+
-library ./build/debug/libmotionbricks.so \
|
| 35 |
+
-model ./generated/g1-f32 \
|
| 36 |
+
-styles ./generated/styles \
|
| 37 |
+
-device cpu
|
| 38 |
+
```
|
| 39 |
+
|
| 40 |
+
Open `http://127.0.0.1:8080/`. Hold physical W/A/S/D keys to move, or tap an
|
| 41 |
+
on-screen direction to latch it; tap the active pad direction again to stop.
|
| 42 |
+
Space or Escape also stops movement. Left/right arrow keys rotate the facing
|
| 43 |
+
direction without changing the current travel vector. The selector switches
|
| 44 |
+
among all `.mbstyle` files found in the style directory, including the 15
|
| 45 |
+
converted upstream styles. Drag over the viewport to orbit, use the wheel to
|
| 46 |
+
zoom, and use **Reset camera** to restore the automatically framed view. The
|
| 47 |
+
camera follows only the animated skeleton, so target placement never pulls the
|
| 48 |
+
view away from the character. The T0–T3 slider selects one fully visible target
|
| 49 |
+
pose. **Overlay all four consecutive poses** reveals the complete constraint
|
| 50 |
+
window. These are adjacent 30 FPS constraint frames rather than four distant
|
| 51 |
+
waypoints, so their exact world positions are intentionally close together.
|
| 52 |
+
|
| 53 |
+
`-device` accepts `cpu`, `vulkan`, or `auto`. The server deliberately binds to
|
| 54 |
+
localhost by default. Model inference is serialized while sessions keep
|
| 55 |
+
independent agent/context state.
|
| 56 |
+
|
| 57 |
+
## Runtime shape
|
| 58 |
+
|
| 59 |
+
The browser creates a session, receives a 30 FPS animation chunk, and asks for
|
| 60 |
+
a replacement chunk when controls change or playback approaches the end.
|
| 61 |
+
Each request contains movement, facing, style, seed, and the number of frames
|
| 62 |
+
already consumed. The Go server advances that session's native agent and
|
| 63 |
+
returns owned animation data and target constraints as JSON:
|
| 64 |
+
|
| 65 |
+
- root translations: `[frames, 3]`;
|
| 66 |
+
- local joint rotations: `[frames, 34, 4]`, XYZW;
|
| 67 |
+
- placed target roots: `[4, 3]`;
|
| 68 |
+
- placed target local rotations: `[4, 34, 4]`, XYZW;
|
| 69 |
+
- G1 joint names, parent indices, and neutral positions from the loaded model.
|
| 70 |
+
|
| 71 |
+
The native target data is captured after style-frame sampling, spring-based
|
| 72 |
+
world placement, and heading correction. The ghosts therefore visualize the
|
| 73 |
+
actual planner inputs. The viewer does not force them in front: forward travel
|
| 74 |
+
normally places them ahead, while stops and turns can make them overlap the
|
| 75 |
+
character or move sideways.
|
| 76 |
+
|
| 77 |
+
The browser builds `THREE.Bone` objects from the returned hierarchy and draws
|
| 78 |
+
solid cylinders/spheres for the generated skeleton and translucent
|
| 79 |
+
cylinders/diamonds for target poses. It also draws generated and target root
|
| 80 |
+
paths on the floor. Animation chunks are immutable in JavaScript; a later
|
| 81 |
+
version can replace JSON with a binary streaming protocol without changing
|
| 82 |
+
the native API.
|
| 83 |
+
|
| 84 |
+
Three.js r180 is vendored under `demo/web/vendor` so the demo has no runtime
|
| 85 |
+
CDN dependency.
|
| 86 |
+
|
| 87 |
+
## Tests
|
| 88 |
+
|
| 89 |
+
With the generated assets present, CTest registers `motionbricks-go-demo` when
|
| 90 |
+
Go and Chromium are available. The test starts an in-process HTTP server,
|
| 91 |
+
loads the real native model, plans an initial `walk` chunk, then uses headless
|
| 92 |
+
Chromium to select `walk_zombie`, turn right, plan another chunk, render the
|
| 93 |
+
34-joint generated hierarchy plus the target inspector, and capture initial,
|
| 94 |
+
forward-motion, and style-and-turn screenshots. It uses real Chrome click and
|
| 95 |
+
keyboard events and asserts forward-pad movement, pad stop, keyboard movement,
|
| 96 |
+
keyboard stop, animated-skeleton camera anchoring, individual target selection,
|
| 97 |
+
and the four-pose overlay before the visual self-test.
|
| 98 |
+
|
| 99 |
+
The Go tests can also be run directly:
|
| 100 |
+
|
| 101 |
+
```sh
|
| 102 |
+
cd demo
|
| 103 |
+
MOTIONBRICKS_LIB=../build/debug/libmotionbricks.so \
|
| 104 |
+
MOTIONBRICKS_MODEL=../generated/g1-f32 \
|
| 105 |
+
MOTIONBRICKS_STYLES=../generated/styles \
|
| 106 |
+
MOTIONBRICKS_CHROME="$(command -v chromium)" \
|
| 107 |
+
CGO_ENABLED=0 go test -v ./...
|
| 108 |
+
```
|
| 109 |
+
|
| 110 |
+
Without the native asset environment variables, the parser test still runs
|
| 111 |
+
and the native/browser integration cases are skipped.
|
| 112 |
+
|
| 113 |
+
## Initial limitations
|
| 114 |
+
|
| 115 |
+
- G1 is the only skeleton supported by the released model.
|
| 116 |
+
- The viewer intentionally shows a bone skeleton, not a skinned avatar.
|
| 117 |
+
- HTTP JSON carries whole planned chunks; binary streaming and client-side
|
| 118 |
+
overlap blending are future work.
|
| 119 |
+
- Sessions are in-memory and intended for a trusted local demo, not an
|
| 120 |
+
internet-facing multi-user service.
|
docs/FORMATS.md
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Versioned format registry
|
| 2 |
+
|
| 3 |
+
A reader must reject an unknown major version rather than guessing.
|
| 4 |
+
|
| 5 |
+
| Format | Version | Status |
|
| 6 |
+
|---|---:|---|
|
| 7 |
+
| Installed C ABI | 1 | Model, style, command, agent, and motion contract implemented |
|
| 8 |
+
| GGUF model-bundle manifest | 1 | F32 inference components implemented |
|
| 9 |
+
| MotionBricks style asset | 1 | Upstream G1 exemplar schema implemented |
|
| 10 |
+
| Reference fixture manifest | 1 | Root, pose, and decoder fixtures implemented |
|
| 11 |
+
| WebSocket animation protocol | 1 | Reserved; schema not implemented |
|
| 12 |
+
|
| 13 |
+
Minor, backwards-compatible additions are represented by optional keys or by
|
| 14 |
+
new API functions. Existing function signatures and field meanings do not
|
| 15 |
+
change within one C ABI version.
|
| 16 |
+
|
| 17 |
+
## GGUF model bundle v1
|
| 18 |
+
|
| 19 |
+
A bundle is a directory containing `manifest.json` and four GGUF v3 files:
|
| 20 |
+
|
| 21 |
+
| Component | Runtime contents | Tensors | Learned parameters |
|
| 22 |
+
|---|---|---:|---:|
|
| 23 |
+
| `pose.gguf` | Pose-token planner | 209 | 136,588,272 |
|
| 24 |
+
| `root.gguf` | Root-trajectory and duration planner | 150 | 34,122,833 |
|
| 25 |
+
| `vq-decoder.gguf` | Pose codebook and convolutional decoder | 51 | 12,437,277 |
|
| 26 |
+
| `support.gguf` | G1 skeleton, parents, mean, and standard deviation | 4 | 0 |
|
| 27 |
+
|
| 28 |
+
The learned inference total is 183,148,382 F32 parameters. The support file
|
| 29 |
+
contains 972 non-learned scalar values. Training-only VQ encoder tensors,
|
| 30 |
+
codebook EMA state, and initialization flags are deliberately omitted.
|
| 31 |
+
|
| 32 |
+
Every component carries `general.architecture=motionbricks`, format version,
|
| 33 |
+
component role, `g1skel34` skeleton identity, pinned upstream revision, source
|
| 34 |
+
safetensors hash, and exact scalar count. PyTorch tensor dimensions are stored
|
| 35 |
+
in reversed GGML order. Names at or above GGML's 64-byte limit are compacted
|
| 36 |
+
deterministically and collision-checked by the converter.
|
| 37 |
+
|
| 38 |
+
## MotionBricks style asset v1
|
| 39 |
+
|
| 40 |
+
A `.mbstyle` is a GGUF v3 file with `component=style`, the `g1skel34`
|
| 41 |
+
skeleton identity, a source SHA-256, name, configured speed, and frame count.
|
| 42 |
+
It contains checked F32 tensors for global joint positions
|
| 43 |
+
`[frames,34,3]`, flattened global rotation matrices `[frames,34,9]`, root
|
| 44 |
+
positions `[frames,3]`, and headings `[frames]`, plus an I32 allowed-duration
|
| 45 |
+
mask `[11]` corresponding to 6--16 tokens. The runtime validates all metadata,
|
| 46 |
+
shapes, finite values, frame limits, and the binary mask before accepting it.
|
| 47 |
+
|
| 48 |
+
The source identity is not restricted to NVIDIA's clip archive, so the same
|
| 49 |
+
schema can hold a future checked Kimodo/G1 conversion. Skeleton and coordinate
|
| 50 |
+
compatibility must still be established by the converter.
|
docs/IMPLEMENTATION.md
ADDED
|
@@ -0,0 +1,806 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# motion-bricks.cpp implementation sketch and plan
|
| 2 |
+
|
| 3 |
+
This document proposes an implementation derived from `motions-bricks.md` and
|
| 4 |
+
the checked-out NVIDIA MotionBricks source. It does not replace the human-led
|
| 5 |
+
design document.
|
| 6 |
+
|
| 7 |
+
## Goal and first supported slice
|
| 8 |
+
|
| 9 |
+
`motion-bricks.cpp` will be a C++23/GGML inference port of the released G1
|
| 10 |
+
MotionBricks planner. It will accept a recent G1 motion context plus a
|
| 11 |
+
movement/facing command and a short reference-style clip, then generate the
|
| 12 |
+
next continuous section of 30 FPS skeletal animation. The initial product
|
| 13 |
+
slice is:
|
| 14 |
+
|
| 15 |
+
- batch size one;
|
| 16 |
+
- the released 34-joint G1 model only;
|
| 17 |
+
- the same 4-frame context and 4-frame target-pose convention as upstream;
|
| 18 |
+
- 24--64 generated frames (6--16 tokens, 4 frames per token);
|
| 19 |
+
- F32 CPU inference with reference parity;
|
| 20 |
+
- root translations and 34 local joint quaternions as the public animation
|
| 21 |
+
result;
|
| 22 |
+
- a Go/PureGo server and a plain HTML/CSS/Three.js interactive viewer.
|
| 23 |
+
|
| 24 |
+
Vulkan follows CPU parity. F16 and quantization are optimizations after both
|
| 25 |
+
backends pass the F32 reference suite.
|
| 26 |
+
|
| 27 |
+
Kimodo is an authoring input, not a runtime dependency of the MotionBricks
|
| 28 |
+
neural network. A Kimodo G1 animation supplies characteristic poses for a
|
| 29 |
+
named style. The controller positions and rotates those poses in the world,
|
| 30 |
+
and MotionBricks generates the transition from the character's actual current
|
| 31 |
+
motion. The original upstream demo clips use the same style-asset boundary.
|
| 32 |
+
|
| 33 |
+
## Current implementation status
|
| 34 |
+
|
| 35 |
+
The native inference slice described above is operational. The strict bundle
|
| 36 |
+
loader, all three released neural graphs, normalization and motion
|
| 37 |
+
representation conversion, original G1 style assets, spring-based target
|
| 38 |
+
construction, stateful agent, and public skeletal outputs are implemented.
|
| 39 |
+
The reusable PureGo binding, local Go session server, and embedded Three.js G1
|
| 40 |
+
skeleton viewer are also implemented. The viewer supports directional
|
| 41 |
+
steering, independent facing changes, and all converted upstream styles.
|
| 42 |
+
|
| 43 |
+
The deterministic component fixtures pass against upstream PyTorch on CPU;
|
| 44 |
+
CPU and Vulkan preserve duration and pose-token choices and agree end to end
|
| 45 |
+
within small F32 tolerances. The demo has native HTTP integration coverage and
|
| 46 |
+
a headless-Chromium test that performs real inference, changes style and
|
| 47 |
+
direction, renders all 34 bones, and captures a screenshot.
|
| 48 |
+
|
| 49 |
+
Still outstanding from the wider plan are direct Kimodo GLB style import,
|
| 50 |
+
recorded upstream controller/session fixtures, context-output blending,
|
| 51 |
+
Gumbel sampling, graph/buffer caching and performance work, binary browser
|
| 52 |
+
streaming, fuzzing, and optimized weight formats. The human-led
|
| 53 |
+
`motions-bricks.md` remains unchanged.
|
| 54 |
+
|
| 55 |
+
## System shape
|
| 56 |
+
|
| 57 |
+
```text
|
| 58 |
+
offline / authoring
|
| 59 |
+
|
| 60 |
+
upstream .ckpt + config + stats Kimodo G1 GLB / original demo clips
|
| 61 |
+
| |
|
| 62 |
+
trusted reference container |
|
| 63 |
+
| |
|
| 64 |
+
safetensors + manifest checked style conversion
|
| 65 |
+
| |
|
| 66 |
+
+---------------> GGUF model bundle <-------+
|
| 67 |
+
|
| 68 |
+
runtime
|
| 69 |
+
|
| 70 |
+
keyboard/game input
|
| 71 |
+
|
|
| 72 |
+
Go session controller -- movement, facing, speed, selected style
|
| 73 |
+
|
|
| 74 |
+
PureGo -> stable C ABI -> stateful C++ agent
|
| 75 |
+
|
|
| 76 |
+
recent 4 frames -> target construction and normalization
|
| 77 |
+
|
|
| 78 |
+
root transformer
|
| 79 |
+
|
|
| 80 |
+
predicted root path and predicted duration
|
| 81 |
+
|
|
| 82 |
+
pose transformer
|
| 83 |
+
|
|
| 84 |
+
VQ pose decoder
|
| 85 |
+
|
|
| 86 |
+
representation inverse / FK
|
| 87 |
+
|
|
| 88 |
+
root translations + local joint rotations
|
| 89 |
+
|
|
| 90 |
+
HTTP JSON chunks -> Three.js
|
| 91 |
+
```
|
| 92 |
+
|
| 93 |
+
There are three deliberately separate layers:
|
| 94 |
+
|
| 95 |
+
1. **Neural inference** reproduces the root transformer, pose-token
|
| 96 |
+
transformer, and VQ decoder.
|
| 97 |
+
2. **Planner/controller** reproduces context management, the critically
|
| 98 |
+
damped spring target, style-pose sampling/alignment, replanning, and output
|
| 99 |
+
blending.
|
| 100 |
+
3. **Application** maps keyboard input to commands, owns user sessions, and
|
| 101 |
+
renders results. It does not implement model math.
|
| 102 |
+
|
| 103 |
+
This separation lets network parity be established without the interactive
|
| 104 |
+
demo and lets controller behavior be tested with recorded neural outputs.
|
| 105 |
+
|
| 106 |
+
## Proposed repository layout
|
| 107 |
+
|
| 108 |
+
```text
|
| 109 |
+
CMakeLists.txt
|
| 110 |
+
CMakePresets.json
|
| 111 |
+
flake.nix
|
| 112 |
+
include/motionbricks/
|
| 113 |
+
motionbricks.h stable C ABI; opaque handles only
|
| 114 |
+
motionbricks.hpp optional RAII C++ wrapper
|
| 115 |
+
src/
|
| 116 |
+
backend.{hpp,cpp} GGML CPU/Vulkan selection and buffers
|
| 117 |
+
error.{hpp,cpp} status mapping and exception firewall
|
| 118 |
+
gguf.{hpp,cpp} checked metadata/tensor loading
|
| 119 |
+
model.{hpp,cpp} immutable model bundle
|
| 120 |
+
transformer.{hpp,cpp} MHA, FFN, norms, masks, embeddings
|
| 121 |
+
root_model.{hpp,cpp} duration and root-trajectory graph
|
| 122 |
+
pose_model.{hpp,cpp} masked pose-token prediction graph
|
| 123 |
+
vq_decoder.{hpp,cpp} multi-head code lookup and conv decoder
|
| 124 |
+
sampling.{hpp,cpp} argmax/Gumbel and local deterministic PRNG
|
| 125 |
+
skeleton_g1.{hpp,cpp} G1 hierarchy, rest pose, FK and mappings
|
| 126 |
+
motion_rep.{hpp,cpp} 414/413/418 feature conversion and stats
|
| 127 |
+
clip.{hpp,cpp} checked skeletal clip/style assets
|
| 128 |
+
controller.{hpp,cpp} spring target and exemplar alignment
|
| 129 |
+
agent.{hpp,cpp} stateful replanning and frame buffer
|
| 130 |
+
capi.cpp PureGo-facing ownership API
|
| 131 |
+
cli.cpp inspect, plan, convert-style, benchmark
|
| 132 |
+
scripts/
|
| 133 |
+
convert_to_gguf.py safetensors/config -> GGUF
|
| 134 |
+
inspect_bundle.py independent metadata/tensor inspection
|
| 135 |
+
reference/
|
| 136 |
+
Dockerfile pinned trusted PyTorch environment
|
| 137 |
+
extract_safe.py legacy Lightning checkpoint -> safetensors
|
| 138 |
+
capture_fixture.py layer/stage/end-to-end reference capture
|
| 139 |
+
capture_demo_session.py recorded upstream controller interaction
|
| 140 |
+
tests/
|
| 141 |
+
unit/ deterministic C++ math and validation
|
| 142 |
+
parity/ F32 CPU and Vulkan fixture runners
|
| 143 |
+
integration/ C ABI, agent, GLB/style and demo protocol
|
| 144 |
+
fuzz/
|
| 145 |
+
capi_fuzz.cpp public non-GGUF inputs and call sequences
|
| 146 |
+
clip_fuzz.cpp GLB/style parsing and validation
|
| 147 |
+
demo/
|
| 148 |
+
main.go PureGo server and per-client sessions
|
| 149 |
+
web/index.html
|
| 150 |
+
web/app.js
|
| 151 |
+
web/style.css
|
| 152 |
+
web/vendor/ pinned Three.js distribution or import map
|
| 153 |
+
```
|
| 154 |
+
|
| 155 |
+
GGML should be a pinned submodule as in the two reference projects. CMake
|
| 156 |
+
builds shared and static library variants, a CLI, tests, and optional fuzzers.
|
| 157 |
+
No source path, model path, GPU index, or backend path is compiled in.
|
| 158 |
+
|
| 159 |
+
## Runtime data contracts
|
| 160 |
+
|
| 161 |
+
### Canonical public animation
|
| 162 |
+
|
| 163 |
+
The public animation boundary should be independent of MuJoCo:
|
| 164 |
+
|
| 165 |
+
- `root_translation`: row-major F32 `[frames, 3]`, Y-up motion space;
|
| 166 |
+
- `local_rotation_xyzw`: row-major F32 `[frames, 34, 4]`;
|
| 167 |
+
- debug target roots: row-major F32 `[4, 3]`;
|
| 168 |
+
- debug target local rotations: row-major F32 `[4, 34, 4]`;
|
| 169 |
+
- frame rate: 30 FPS for the released model;
|
| 170 |
+
- joint names, parents, and rest offsets: queried from model metadata;
|
| 171 |
+
- optional contacts: row-major F32 `[frames, 4]`.
|
| 172 |
+
|
| 173 |
+
MuJoCo qpos is an adapter/debug output, not the primary animation format. The
|
| 174 |
+
29-DOF G1 qpos form is useful for exact upstream demo parity but loses the
|
| 175 |
+
generic skeletal-animation boundary needed by Three.js and Kimodo.
|
| 176 |
+
|
| 177 |
+
### Neural input
|
| 178 |
+
|
| 179 |
+
Internally the planner builds the same unnormalized sparse constraints as
|
| 180 |
+
upstream:
|
| 181 |
+
|
| 182 |
+
- 4 recent context frames and up to 4 target frames;
|
| 183 |
+
- global root features `[constraint_frames, 5]`;
|
| 184 |
+
- local root features `[constraint_frames, 4]`;
|
| 185 |
+
- local-pose constraint features;
|
| 186 |
+
- masks saying which sparse constraints are present;
|
| 187 |
+
- a requested duration or an allowed-duration mask;
|
| 188 |
+
- optional text embeddings kept absent for the released demo path.
|
| 189 |
+
|
| 190 |
+
The released inference path recenters root X/Z, normalizes features, predicts
|
| 191 |
+
the root trajectory and duration, predicts 8-head pose codes, decodes four
|
| 192 |
+
frames per token, converts local-root features back to global motion, and
|
| 193 |
+
restores the original world offset.
|
| 194 |
+
|
| 195 |
+
### Style assets
|
| 196 |
+
|
| 197 |
+
A style is application/controller data rather than a learned model selector.
|
| 198 |
+
It contains:
|
| 199 |
+
|
| 200 |
+
- a name and optional command binding;
|
| 201 |
+
- one or more G1 reference clips;
|
| 202 |
+
- an allowed-duration mask;
|
| 203 |
+
- average travel speed or an explicit override;
|
| 204 |
+
- optional sampling range and heading correction;
|
| 205 |
+
- provenance, skeleton identity, FPS, and source hash.
|
| 206 |
+
|
| 207 |
+
Original demo styles are extracted from `G1-clip.ckpt` in the trusted reference
|
| 208 |
+
container. User styles are imported from G1 skeletal GLB, including GLB output
|
| 209 |
+
from `kimodo.cpp`. Both are converted to the same checked style format. The
|
| 210 |
+
controller samples four adjacent pose frames, applies the requested world
|
| 211 |
+
heading and spring-produced root target, and supplies them as ending
|
| 212 |
+
constraints. A style can therefore be changed or added without changing
|
| 213 |
+
MotionBricks weights.
|
| 214 |
+
|
| 215 |
+
Before claiming direct Kimodo compatibility, a fixture must verify the G1
|
| 216 |
+
joint order, rest pose, local-rotation convention, coordinate system, and FPS.
|
| 217 |
+
The adapter will reject rather than guess when metadata is incompatible.
|
| 218 |
+
|
| 219 |
+
## PureGo-compatible C API sketch
|
| 220 |
+
|
| 221 |
+
All configurable records and returned objects are opaque heap handles. Public
|
| 222 |
+
integer types have explicit widths; booleans and enum-like values are
|
| 223 |
+
`uint32_t`; no C enum or by-value struct is part of the ABI. Every constructor
|
| 224 |
+
has a matching free function. Functions return a status and use a caller-owned
|
| 225 |
+
error buffer.
|
| 226 |
+
|
| 227 |
+
```c
|
| 228 |
+
typedef uint32_t mb_status;
|
| 229 |
+
typedef uint32_t mb_device;
|
| 230 |
+
|
| 231 |
+
typedef struct mb_runtime_options mb_runtime_options;
|
| 232 |
+
typedef struct mb_model mb_model;
|
| 233 |
+
typedef struct mb_style mb_style;
|
| 234 |
+
typedef struct mb_command mb_command;
|
| 235 |
+
typedef struct mb_agent mb_agent;
|
| 236 |
+
typedef struct mb_motion mb_motion;
|
| 237 |
+
|
| 238 |
+
uint32_t mb_abi_version(void);
|
| 239 |
+
|
| 240 |
+
mb_status mb_runtime_options_create(mb_runtime_options **out,
|
| 241 |
+
char *err, uint64_t err_cap);
|
| 242 |
+
void mb_runtime_options_free(mb_runtime_options *value);
|
| 243 |
+
mb_status mb_runtime_options_set_device(mb_runtime_options *, mb_device,
|
| 244 |
+
char *, uint64_t);
|
| 245 |
+
mb_status mb_runtime_options_set_threads(mb_runtime_options *, uint32_t,
|
| 246 |
+
char *, uint64_t);
|
| 247 |
+
mb_status mb_runtime_options_set_backend_directory(mb_runtime_options *,
|
| 248 |
+
const char *, char *, uint64_t);
|
| 249 |
+
mb_status mb_runtime_options_get_device(const mb_runtime_options *, mb_device *,
|
| 250 |
+
char *, uint64_t);
|
| 251 |
+
|
| 252 |
+
mb_status mb_model_load(const char *bundle_directory,
|
| 253 |
+
const mb_runtime_options *, mb_model **out,
|
| 254 |
+
char *err, uint64_t err_cap);
|
| 255 |
+
void mb_model_free(mb_model *);
|
| 256 |
+
mb_status mb_model_get_joint_count(const mb_model *, uint32_t *out,
|
| 257 |
+
char *, uint64_t);
|
| 258 |
+
mb_status mb_model_get_joint_name(const mb_model *, uint32_t joint,
|
| 259 |
+
const char **borrowed, char *, uint64_t);
|
| 260 |
+
mb_status mb_model_get_joint_parent(const mb_model *, uint32_t joint,
|
| 261 |
+
int32_t *out, char *, uint64_t);
|
| 262 |
+
mb_status mb_model_get_neutral_joint_position(const mb_model *, uint32_t joint,
|
| 263 |
+
float *x, float *y, float *z,
|
| 264 |
+
char *, uint64_t);
|
| 265 |
+
|
| 266 |
+
mb_status mb_style_load(const mb_model *, const char *style_path,
|
| 267 |
+
mb_style **out, char *err, uint64_t err_cap);
|
| 268 |
+
void mb_style_free(mb_style *);
|
| 269 |
+
mb_status mb_style_get_name(const mb_style *, const char **borrowed,
|
| 270 |
+
char *, uint64_t);
|
| 271 |
+
mb_status mb_style_set_speed(mb_style *, float metres_per_second,
|
| 272 |
+
char *, uint64_t);
|
| 273 |
+
|
| 274 |
+
mb_status mb_command_create(mb_command **out, char *err, uint64_t err_cap);
|
| 275 |
+
void mb_command_free(mb_command *);
|
| 276 |
+
mb_status mb_command_set_style(mb_command *, const mb_style *, char *, uint64_t);
|
| 277 |
+
mb_status mb_command_set_movement_direction(mb_command *, float x, float y, float z,
|
| 278 |
+
char *, uint64_t);
|
| 279 |
+
mb_status mb_command_set_facing_direction(mb_command *, float x, float y, float z,
|
| 280 |
+
char *, uint64_t);
|
| 281 |
+
mb_status mb_command_set_target_speed(mb_command *, float, char *, uint64_t);
|
| 282 |
+
mb_status mb_command_set_world_target(mb_command *, float x, float y, float z,
|
| 283 |
+
float heading_radians, uint32_t enabled,
|
| 284 |
+
char *, uint64_t);
|
| 285 |
+
mb_status mb_command_set_seed(mb_command *, uint64_t, char *, uint64_t);
|
| 286 |
+
|
| 287 |
+
mb_status mb_agent_create(const mb_model *, mb_agent **out,
|
| 288 |
+
char *err, uint64_t err_cap);
|
| 289 |
+
void mb_agent_free(mb_agent *);
|
| 290 |
+
mb_status mb_agent_reset(mb_agent *, const mb_style *initial_style,
|
| 291 |
+
char *err, uint64_t err_cap);
|
| 292 |
+
mb_status mb_agent_set_context(mb_agent *, const float *root_xyz,
|
| 293 |
+
const float *local_xyzw, uint64_t frames,
|
| 294 |
+
uint64_t joints, char *err, uint64_t err_cap);
|
| 295 |
+
mb_status mb_agent_plan(mb_agent *, const mb_command *, mb_motion **out,
|
| 296 |
+
char *err, uint64_t err_cap);
|
| 297 |
+
mb_status mb_agent_advance(mb_agent *, uint32_t frames, char *, uint64_t);
|
| 298 |
+
|
| 299 |
+
void mb_motion_free(mb_motion *);
|
| 300 |
+
mb_status mb_motion_get_frame_count(const mb_motion *, uint64_t *out,
|
| 301 |
+
char *, uint64_t);
|
| 302 |
+
mb_status mb_motion_get_joint_count(const mb_motion *, uint64_t *out,
|
| 303 |
+
char *, uint64_t);
|
| 304 |
+
mb_status mb_motion_get_root_translations(const mb_motion *,
|
| 305 |
+
const float **borrowed, uint64_t *values,
|
| 306 |
+
char *, uint64_t);
|
| 307 |
+
mb_status mb_motion_get_local_rotations_xyzw(const mb_motion *,
|
| 308 |
+
const float **borrowed, uint64_t *values,
|
| 309 |
+
char *, uint64_t);
|
| 310 |
+
mb_status mb_motion_get_target_frame_count(const mb_motion *, uint64_t *out,
|
| 311 |
+
char *, uint64_t);
|
| 312 |
+
mb_status mb_motion_get_target_root_translations(const mb_motion *,
|
| 313 |
+
const float **borrowed, uint64_t *values,
|
| 314 |
+
char *, uint64_t);
|
| 315 |
+
mb_status mb_motion_get_target_local_rotations_xyzw(const mb_motion *,
|
| 316 |
+
const float **borrowed, uint64_t *values,
|
| 317 |
+
char *, uint64_t);
|
| 318 |
+
```
|
| 319 |
+
|
| 320 |
+
The final header will add getters for every option setter and explicit lifetime
|
| 321 |
+
documentation. Borrowed strings and arrays remain valid until their owning
|
| 322 |
+
handle is freed or mutated. `mb_model` is immutable and may be shared;
|
| 323 |
+
`mb_agent`, `mb_command`, and `mb_motion` are not concurrently mutable. No C++
|
| 324 |
+
exception crosses `capi.cpp`.
|
| 325 |
+
|
| 326 |
+
The CLI uses this same C API for end-to-end operations so the ABI is exercised
|
| 327 |
+
outside unit tests. Internal parity tools may access raw tensors through a
|
| 328 |
+
non-installed test header; raw GGML objects and 413/414-dimensional model
|
| 329 |
+
features are not exposed as stable public API.
|
| 330 |
+
|
| 331 |
+
## Model conversion and bundle
|
| 332 |
+
|
| 333 |
+
Use a directory manifest with separate memory-mappable component files:
|
| 334 |
+
|
| 335 |
+
```text
|
| 336 |
+
motionbricks-g1-f32/
|
| 337 |
+
manifest.json
|
| 338 |
+
root-f32.gguf
|
| 339 |
+
pose-f32.gguf
|
| 340 |
+
vq-decoder-f32.gguf
|
| 341 |
+
styles/
|
| 342 |
+
upstream-defaults.mbstyle
|
| 343 |
+
```
|
| 344 |
+
|
| 345 |
+
The manifest records format version, model identity, source revision, source
|
| 346 |
+
and intermediate hashes, skeleton key, FPS, component dtype, GGML revision,
|
| 347 |
+
and mutually compatible component identities. Loading rejects mixed bundles.
|
| 348 |
+
|
| 349 |
+
GGUF metadata records all architecture values rather than relying on C++
|
| 350 |
+
defaults, including the released 512-wide root transformer, 1024-wide pose
|
| 351 |
+
transformer, 16 attention heads, layer counts, token range, eight pose-code
|
| 352 |
+
heads, four frames per token, normalization statistics, feature-index tables,
|
| 353 |
+
skeleton data, and sampling defaults. Every tensor lookup is named, typed,
|
| 354 |
+
shape checked, and bounds checked.
|
| 355 |
+
|
| 356 |
+
The ordinary converter accepts safetensors plus JSON only. NVIDIA's Lightning
|
| 357 |
+
checkpoints and the serialized demo-clip checkpoint are Python pickle formats;
|
| 358 |
+
only the pinned, network-disabled reference container loads those files. It
|
| 359 |
+
then writes safetensors/JSON with hashes for the normal converter. The VQ
|
| 360 |
+
encoder is omitted from runtime bundles because inference only needs the code
|
| 361 |
+
embeddings and decoder; a reference/debug bundle may retain it for tokenizer
|
| 362 |
+
parity.
|
| 363 |
+
|
| 364 |
+
## Neural implementation order
|
| 365 |
+
|
| 366 |
+
### Shared operators
|
| 367 |
+
|
| 368 |
+
Implement the smallest operation set used by the released checkpoints:
|
| 369 |
+
|
| 370 |
+
- checked embedding lookup, linear layers, ReLU, LayerNorm, residual add;
|
| 371 |
+
- PyTorch-compatible multi-head self-attention and key-padding masks;
|
| 372 |
+
- fixed/learned position embeddings;
|
| 373 |
+
- 1D convolution, transpose convolution, and dilated residual blocks;
|
| 374 |
+
- reshape, permute, concatenate, gather/scatter, softmax and argmax.
|
| 375 |
+
|
| 376 |
+
An operation audit against the actual `state_dict` and forward hooks precedes
|
| 377 |
+
graph work. PyTorch defaults such as post-norm transformer order, bias,
|
| 378 |
+
activation, epsilon, and attention scaling must be captured explicitly, not
|
| 379 |
+
reconstructed from memory.
|
| 380 |
+
|
| 381 |
+
### Root model
|
| 382 |
+
|
| 383 |
+
The root graph embeds sparse start/end pose and root constraints, duration,
|
| 384 |
+
and position; runs shared and output transformer stacks; predicts duration;
|
| 385 |
+
and expands transformer features through the conditioned convolutional decoder
|
| 386 |
+
to the global root trajectory. Duration filtering uses the caller/style's
|
| 387 |
+
allowed-token mask exactly as upstream.
|
| 388 |
+
|
| 389 |
+
### Pose model
|
| 390 |
+
|
| 391 |
+
The pose graph combines masked multi-head VQ code embeddings, predicted root
|
| 392 |
+
features, sparse pose constraints, duration embedding, and position embedding;
|
| 393 |
+
runs the 16-layer transformer; and produces per-head code logits. One sampling
|
| 394 |
+
iteration is the released demo default. Argmax is supported for deterministic
|
| 395 |
+
diagnostics; production Gumbel sampling uses an agent-local PRNG and seed.
|
| 396 |
+
Reference fixtures store the random uniforms/Gumbel values explicitly instead
|
| 397 |
+
of assuming PyTorch and C++ generators produce identical streams.
|
| 398 |
+
|
| 399 |
+
### VQ decoder and representation inverse
|
| 400 |
+
|
| 401 |
+
Selected codes are expanded through the eight code heads and conditioned
|
| 402 |
+
decoder. Sparse start/end constraints and the predicted root path are applied
|
| 403 |
+
as upstream. The resulting normalized local representation is converted to
|
| 404 |
+
global features, denormalized, and transformed into skeletal animation. These
|
| 405 |
+
ordinary math stages are implemented in C++ and tested independently of GGML.
|
| 406 |
+
|
| 407 |
+
### Stateful controller
|
| 408 |
+
|
| 409 |
+
An agent owns its current generated buffer and frame cursor. At a replan it:
|
| 410 |
+
|
| 411 |
+
1. gathers the previous four actual frames;
|
| 412 |
+
2. derives current root velocity and heading velocity;
|
| 413 |
+
3. applies the upstream critically damped spring to the requested movement,
|
| 414 |
+
facing, speed, or explicit world target;
|
| 415 |
+
4. samples and aligns four frames from the selected style clip;
|
| 416 |
+
5. constructs sparse model constraints and runs inference;
|
| 417 |
+
6. restores the world transform, converts the output to local rotations, and
|
| 418 |
+
applies the upstream context blend;
|
| 419 |
+
7. exposes the valid predicted prefix and retains it as the next context.
|
| 420 |
+
|
| 421 |
+
The controller uses seconds internally and derives frame counts from model FPS.
|
| 422 |
+
It must not silently enable canonicalization, target skipping, alternate root
|
| 423 |
+
selection, extra pose iterations, or any other non-default upstream option.
|
| 424 |
+
|
| 425 |
+
## Web demo
|
| 426 |
+
|
| 427 |
+
The Go server dynamically loads the shared library with PureGo. It owns one
|
| 428 |
+
`mb_agent` per WebSocket session and shares one immutable `mb_model`. Initially
|
| 429 |
+
planning requests are serialized through a worker so Vulkan ownership and peak
|
| 430 |
+
memory are predictable; parallel sessions can be added after profiling.
|
| 431 |
+
|
| 432 |
+
The browser sends compact command state only: pressed controls, movement and
|
| 433 |
+
camera-facing vectors, selected style, and monotonically increasing sequence
|
| 434 |
+
number. The server returns a binary animation chunk containing a small
|
| 435 |
+
versioned header, frame count, root translations, and local quaternions.
|
| 436 |
+
Three.js keeps a short playback buffer and samples at render time. The server,
|
| 437 |
+
not the browser, owns authoritative agent state and drops stale commands.
|
| 438 |
+
|
| 439 |
+
HTTP endpoints provide model/style metadata and health information. A
|
| 440 |
+
WebSocket carries interactive commands and generated chunks. The frontend
|
| 441 |
+
includes:
|
| 442 |
+
|
| 443 |
+
- WASD movement relative to the camera;
|
| 444 |
+
- independently controllable facing direction;
|
| 445 |
+
- style/action buttons populated from the style manifest;
|
| 446 |
+
- a skeleton/mesh viewer, pause/reset, latency and buffered-frame display;
|
| 447 |
+
- an optional debug view for target position, facing vector, contacts, and
|
| 448 |
+
replan boundaries.
|
| 449 |
+
|
| 450 |
+
The initial viewer can render a simple G1 skeleton. A skinned character and
|
| 451 |
+
retargeting are later integrations, potentially using `skin-tokens.cpp`.
|
| 452 |
+
|
| 453 |
+
## Reference and parity strategy
|
| 454 |
+
|
| 455 |
+
Every fixture has a JSON manifest containing the upstream commit, checkpoint
|
| 456 |
+
and safetensors hashes, config hash, PyTorch version, dtype/device, tensor
|
| 457 |
+
shapes, seed/random inputs, selected style, command stream, and tolerances.
|
| 458 |
+
Binary tensors are little-endian and versioned. Tests refuse mismatched
|
| 459 |
+
fixtures.
|
| 460 |
+
|
| 461 |
+
Capture and compare in this order:
|
| 462 |
+
|
| 463 |
+
| Boundary | Reference values |
|
| 464 |
+
|---|---|
|
| 465 |
+
| checkpoint extraction | every tensor name, dtype, shape, hash |
|
| 466 |
+
| skeletal math | qpos/local rotations, FK positions, global rotations |
|
| 467 |
+
| motion representation | raw, global 414, local 413, dual conversion, normalization |
|
| 468 |
+
| style/controller | sampled frames, average speed, spring roots/headings, aligned targets |
|
| 469 |
+
| shared neural ops | linear, LayerNorm, attention, masks, conv/residual blocks |
|
| 470 |
+
| root network | every projection, transformer layer, duration logits, decoder layer, root path |
|
| 471 |
+
| pose network | every embedding/projection, transformer layer, logits, sampled codes |
|
| 472 |
+
| VQ decoder | code embeddings and every decoder layer |
|
| 473 |
+
| inference composition | recenter, constraints, root, pose, decode, restore world transform |
|
| 474 |
+
| agent session | recorded command changes, replans, valid lengths, final skeletal frames |
|
| 475 |
+
|
| 476 |
+
The first exemplar set should be small but discriminating:
|
| 477 |
+
|
| 478 |
+
- idle continuation with a fixed duration;
|
| 479 |
+
- walk forward, then a 90-degree turn during playback;
|
| 480 |
+
- switch from walk to one strongly recognizable upstream style;
|
| 481 |
+
- explicit world target behind the character;
|
| 482 |
+
- a fixed Kimodo G1 reference clip used as a style;
|
| 483 |
+
- minimum and maximum allowed duration;
|
| 484 |
+
- both argmax and stored-Gumbel-token inference.
|
| 485 |
+
|
| 486 |
+
F32 CPU tolerances are set per stage from observed error, with maximum absolute
|
| 487 |
+
and relative L2 limits. End-to-end animation also checks root error, joint
|
| 488 |
+
position error after FK, quaternion angular error, token equality where
|
| 489 |
+
applicable, duration equality, and contact equality. Vulkan has its own
|
| 490 |
+
documented tolerances but must preserve duration and discrete code decisions
|
| 491 |
+
on the exemplar suite unless an explicitly reviewed near-tie is present.
|
| 492 |
+
|
| 493 |
+
### Automated upstream sessions and observational tracing
|
| 494 |
+
|
| 495 |
+
The reference harness should drive the released upstream demo without a human
|
| 496 |
+
keyboard and record both numerical traces and a MuJoCo preview. Numerical
|
| 497 |
+
traces are the parity authority. Video is a synchronized diagnostic artifact,
|
| 498 |
+
not a pixel-exact test oracle.
|
| 499 |
+
|
| 500 |
+
Instrumentation must remain observational. In particular, adding target-pose
|
| 501 |
+
visualization must not alter inference inputs, controller state, random-number
|
| 502 |
+
consumption, playback timing, MuJoCo state, or final numerical output. Use the
|
| 503 |
+
following intervention order, stopping as soon as the required value is
|
| 504 |
+
available:
|
| 505 |
+
|
| 506 |
+
1. call an existing public method and copy its returned value in the capture
|
| 507 |
+
script;
|
| 508 |
+
2. wrap an existing method or register a PyTorch forward hook from the capture
|
| 509 |
+
script;
|
| 510 |
+
3. add an optional no-op trace callback at a stable method boundary;
|
| 511 |
+
4. make a small upstream source edit only when a value is otherwise confined
|
| 512 |
+
to a local variable;
|
| 513 |
+
5. never refactor or reorder model/controller calculations merely to expose a
|
| 514 |
+
trace.
|
| 515 |
+
|
| 516 |
+
Before adding internal taps, run the unmodified pinned upstream revision at
|
| 517 |
+
least twice with the same scripted controls and seeds. Record the final motion
|
| 518 |
+
features, MuJoCo qpos, valid lengths, modes, and replan frame indices using
|
| 519 |
+
only values already returned by the demo classes. This establishes both the
|
| 520 |
+
baseline and any numerical nondeterminism inherent to the reference CUDA
|
| 521 |
+
environment. Discrete outputs must repeat exactly. Float tolerances for the
|
| 522 |
+
instrumentation check must be no wider than the variation observed between
|
| 523 |
+
the two unmodified runs.
|
| 524 |
+
|
| 525 |
+
After each instrumentation change, repeat the same session with tracing off
|
| 526 |
+
and on. Both runs must match the unmodified baseline: exact for masks, modes,
|
| 527 |
+
seeds, duration choices and pose codes; bitwise for deterministic CPU values;
|
| 528 |
+
and within the measured baseline envelope for CUDA floats. Reject or redesign
|
| 529 |
+
any tap that changes those results. The recorder must not call random
|
| 530 |
+
functions, mutate a tensor or dictionary owned by upstream, change dtype or
|
| 531 |
+
device, enable gradients, or retain a view that upstream later modifies. It
|
| 532 |
+
copies with `detach().clone()` and performs CPU transfer only after the value
|
| 533 |
+
has been consumed by the upstream calculation.
|
| 534 |
+
|
| 535 |
+
The initial taps are:
|
| 536 |
+
|
| 537 |
+
| Boundary | Upstream location | Trace values |
|
| 538 |
+
|---|---|---|
|
| 539 |
+
| playback context | `full_navigation_agent.get_context_*` | four qpos frames and motion features |
|
| 540 |
+
| controller | `WASD_controller.generate_control_signals` | scripted keys, mode, movement/facing vectors, allowed durations |
|
| 541 |
+
| spring | `_generate_spring_model_position_and_heading` | start and target root positions/headings |
|
| 542 |
+
| placed target poses | `_generate_target_joint_transforms` | four global joint positions/rotations and root positions |
|
| 543 |
+
| sparse model input | `_generate_inbetween_frames` | global/local root values, poses, masks and requested/allowed durations |
|
| 544 |
+
| root network | root-backbone forward hook | duration logits, selected duration and predicted root path |
|
| 545 |
+
| pose network | pose-backbone forward hook | pose logits, selected codes and conditioning mask |
|
| 546 |
+
| VQ decoder | decoder forward hook | quantized input, conditions, masks and normalized decoded motion |
|
| 547 |
+
| composed inference | `motion_inference.predict` | valid length and global/local motion representations |
|
| 548 |
+
| playback output | `generate_new_frames`/`get_next_frame` | raw qpos, blended qpos, cursor and replan event |
|
| 549 |
+
|
| 550 |
+
Prefer explicit trace calls for controller-local values and forward hooks for
|
| 551 |
+
neural modules. The existing `info` argument to `motion_inference.predict` may
|
| 552 |
+
carry copied debug values, but normal inference must continue to work with it
|
| 553 |
+
absent. Large layer-by-layer captures are enabled for one planning event at a
|
| 554 |
+
time rather than every playback frame.
|
| 555 |
+
|
| 556 |
+
The scripted session driver supplies `control_info["key_pressed"]`, so it does
|
| 557 |
+
not require an OS keyboard listener. It uses a fixed camera proxy because the
|
| 558 |
+
WASD controller derives movement from viewer-camera orientation. NumPy,
|
| 559 |
+
PyTorch and per-plan MotionBricks seeds are explicit. An initial scenario is:
|
| 560 |
+
|
| 561 |
+
- 60 frames idle;
|
| 562 |
+
- 60 frames walking forward;
|
| 563 |
+
- 45 frames walking through a right turn;
|
| 564 |
+
- 60 frames released to idle;
|
| 565 |
+
- 60 frames walking with a distinctive style such as `walk_zombie`;
|
| 566 |
+
- 60 frames released and allowed to settle.
|
| 567 |
+
|
| 568 |
+
Each scenario directory contains a manifest, JSONL control/replan events,
|
| 569 |
+
Safetensors files for planning boundaries and playback, and optional media:
|
| 570 |
+
|
| 571 |
+
```text
|
| 572 |
+
session-idle-walk-turn/
|
| 573 |
+
manifest.json
|
| 574 |
+
controls.jsonl
|
| 575 |
+
events.jsonl
|
| 576 |
+
plan-000.safetensors
|
| 577 |
+
plan-001.safetensors
|
| 578 |
+
playback.safetensors
|
| 579 |
+
preview.mp4
|
| 580 |
+
snapshots/
|
| 581 |
+
```
|
| 582 |
+
|
| 583 |
+
The manifest records the upstream commit, checkpoint/config hashes, scenario
|
| 584 |
+
version, all relevant demo flags, dtype/device, Python/PyTorch/CUDA/MuJoCo
|
| 585 |
+
versions, seeds, FPS and coordinate conventions. Do not use pickle for fixture
|
| 586 |
+
data. Each planning file identifies tensors by semantic name, shape and dtype.
|
| 587 |
+
|
| 588 |
+
Generate video in a second, offline pass from the stored qpos and target-pose
|
| 589 |
+
traces. Use `mujoco.Renderer`, assign the captured qpos, call `mj_forward`, and
|
| 590 |
+
render with a deterministic camera at 30 FPS. On Linux use EGL when available
|
| 591 |
+
and OSMesa as a software fallback. Save selected lossless PNG frames alongside
|
| 592 |
+
the MP4. This separation ensures rendering load cannot affect controller
|
| 593 |
+
cadence or inference.
|
| 594 |
+
|
| 595 |
+
Visible target keyframes are also an offline rendering concern. Draw the four
|
| 596 |
+
captured target skeletons as orange/magenta ghost geometry in an `MjvScene` or
|
| 597 |
+
as a post-render overlay. Do not add bodies or constraints to the physics XML,
|
| 598 |
+
write target poses into `mjData`, or reuse mutable tensors from inference. The
|
| 599 |
+
animated G1 remains opaque and visually distinct. Overlay the absolute frame,
|
| 600 |
+
plan number, mode/style, movement/facing vectors, selected duration and replan
|
| 601 |
+
markers. If an interactive upstream target view is later useful, it is a
|
| 602 |
+
separate optional patch and must pass the same trace-off/trace-on
|
| 603 |
+
non-interference gate.
|
| 604 |
+
|
| 605 |
+
The port replays each captured planning event at the same public boundaries.
|
| 606 |
+
It compares integer decisions exactly, floats with boundary-specific
|
| 607 |
+
tolerances, quaternions by sign-invariant angular distance, and joint positions
|
| 608 |
+
after FK. Session checks cover replan timing, valid buffer lengths, root and
|
| 609 |
+
joint velocity continuity, and style transitions. The idle scenario must also
|
| 610 |
+
confirm that upstream performs only the expected transition into idle, then
|
| 611 |
+
ceases idle-to-idle replanning and eventually holds a numerically constant
|
| 612 |
+
frame.
|
| 613 |
+
|
| 614 |
+
Implement this work in gates:
|
| 615 |
+
|
| 616 |
+
1. create the external scripted driver and capture unmodified final outputs;
|
| 617 |
+
2. prove same-seed repeatability and write the baseline manifest;
|
| 618 |
+
3. add controller/model-input taps one boundary at a time, running the
|
| 619 |
+
non-interference comparison after each change;
|
| 620 |
+
4. add neural forward hooks and capture one complete planning event;
|
| 621 |
+
5. capture the full idle/walk/turn/style session and import it into the C++
|
| 622 |
+
parity runner;
|
| 623 |
+
6. render MuJoCo video and target ghosts offline from the accepted traces;
|
| 624 |
+
7. add the scenario to CPU CI and keep video generation as an optional local
|
| 625 |
+
or artifact-producing job.
|
| 626 |
+
|
| 627 |
+
**Exit:** a fresh pinned upstream environment reproduces the accepted session;
|
| 628 |
+
trace-off and trace-on outputs match the pre-instrumentation baseline; the C++
|
| 629 |
+
runtime passes every captured boundary; and the synchronized MuJoCo preview
|
| 630 |
+
shows playback and target keyframes without participating in inference.
|
| 631 |
+
|
| 632 |
+
## Safety, fuzzing, and validation
|
| 633 |
+
|
| 634 |
+
GGUF parsing remains GGML's responsibility, but motion-bricks validates bundle
|
| 635 |
+
metadata and every expected tensor. Fuzz targets cover:
|
| 636 |
+
|
| 637 |
+
- the installed C ABI with generated valid/invalid call sequences;
|
| 638 |
+
- opaque-handle nulls, ownership order, repeated free attempts in the harness,
|
| 639 |
+
lengths, finite values, dimensions, and invalid enums;
|
| 640 |
+
- GLB/style assets, skeleton metadata, malformed animation tracks, NaN/Inf,
|
| 641 |
+
huge frame counts, and incompatible joint orders;
|
| 642 |
+
- command normalization, degenerate direction vectors, duration masks, spring
|
| 643 |
+
math, representation conversions, quaternion/matrix conversions, and FK;
|
| 644 |
+
- the versioned WebSocket binary decoder independently in Go.
|
| 645 |
+
|
| 646 |
+
Run C++ fuzzers with ASan and UBSan. Unit and integration tests run under the
|
| 647 |
+
same sanitizers. File parsers impose explicit byte, frame, joint, accessor, and
|
| 648 |
+
allocation limits before allocating. No user-provided path is interpreted by
|
| 649 |
+
the browser; the server uses configured style/model roots.
|
| 650 |
+
|
| 651 |
+
## Build and configuration
|
| 652 |
+
|
| 653 |
+
The normal build is distro-agnostic CMake. Nix supplies a reproducible
|
| 654 |
+
developer shell containing Clang, CMake, Ninja, Go, Python fixture tools,
|
| 655 |
+
Vulkan headers/loader/tools, shaderc, and sanitizers. Docker contains upstream
|
| 656 |
+
Python/CUDA extraction and fixture capture. Neither is required by installed
|
| 657 |
+
runtime consumers.
|
| 658 |
+
|
| 659 |
+
Suggested configuration inputs:
|
| 660 |
+
|
| 661 |
+
- CMake cache: GGML source directory, tests/fuzzers, Vulkan enablement;
|
| 662 |
+
- environment/CLI: model bundle, style directory, backend directory, device,
|
| 663 |
+
CPU thread count, server address, fixture directory;
|
| 664 |
+
- Vulkan device selection: documented loader variables or a CLI-selected
|
| 665 |
+
GGML Vulkan device after confirming backend support. The Nix shell supplies
|
| 666 |
+
loader paths but never assumes an NVIDIA device index.
|
| 667 |
+
|
| 668 |
+
CI lanes are formatting/static checks, CPU unit tests, ASan/UBSan tests, CPU
|
| 669 |
+
parity with optional fixture artifacts, and a hardware-tagged Vulkan parity
|
| 670 |
+
lane. Large weights and fixtures are downloaded by hash and never committed.
|
| 671 |
+
|
| 672 |
+
## Implementation milestones
|
| 673 |
+
|
| 674 |
+
### 0. Lock the reference
|
| 675 |
+
|
| 676 |
+
- Record the upstream MotionBricks commit and GGML revision.
|
| 677 |
+
- Fetch Git LFS checkpoints/assets and record their hashes/licences.
|
| 678 |
+
- Build a pinned Python reference environment and reproduce the interactive
|
| 679 |
+
demo unchanged.
|
| 680 |
+
- Capture one deterministic end-to-end upstream exemplar.
|
| 681 |
+
|
| 682 |
+
**Exit:** the reference script reproduces a stored animation and manifest on a
|
| 683 |
+
fresh environment.
|
| 684 |
+
|
| 685 |
+
### 1. Scaffold and define contracts
|
| 686 |
+
|
| 687 |
+
- Add CMake, presets, Nix flake, pinned GGML, library/CLI/test targets.
|
| 688 |
+
- Land the opaque C header, C++ wrapper, status/error rules, and stub handles.
|
| 689 |
+
- Define fixture, GGUF bundle, style, and WebSocket format versions.
|
| 690 |
+
- Add ABI lifecycle tests from C and PureGo.
|
| 691 |
+
|
| 692 |
+
**Exit:** CPU-only stubs build, install, and are called successfully through
|
| 693 |
+
PureGo without any mirrored struct.
|
| 694 |
+
|
| 695 |
+
### 2. Safe extraction and exemplar capture
|
| 696 |
+
|
| 697 |
+
- Extract root, pose, VQ, stats, skeleton, and original style tensors to
|
| 698 |
+
safetensors/JSON in the trusted container.
|
| 699 |
+
- Capture the unmodified scripted upstream session before adding internal
|
| 700 |
+
instrumentation, then apply the observational tracing and non-interference
|
| 701 |
+
gates defined above.
|
| 702 |
+
- Add forward hooks/capture for every required layer and preprocessing stage.
|
| 703 |
+
- Capture the exemplar matrix above, including random sampling inputs.
|
| 704 |
+
- Render synchronized MuJoCo and target-keyframe previews offline from the
|
| 705 |
+
accepted numerical traces.
|
| 706 |
+
- Build the safetensors-to-GGUF converter and bundle inspector.
|
| 707 |
+
|
| 708 |
+
**Exit:** source and converted tensor inventories match exactly, all bundle
|
| 709 |
+
identities are validated, and trace-enabled upstream output matches the
|
| 710 |
+
unmodified baseline.
|
| 711 |
+
|
| 712 |
+
### 3. Deterministic motion/controller math
|
| 713 |
+
|
| 714 |
+
- Implement G1 metadata, quaternion/matrix conversions, FK, feature extraction,
|
| 715 |
+
414/413 representation conversion, stats, and skeletal output conversion.
|
| 716 |
+
- Implement style import, clip alignment, spring trajectory, duration masks,
|
| 717 |
+
context selection, and blending without neural inference.
|
| 718 |
+
- Add unit, fixture parity, and sanitizer tests for each stage.
|
| 719 |
+
|
| 720 |
+
**Exit:** C++ matches all non-neural upstream fixture boundaries.
|
| 721 |
+
|
| 722 |
+
### 4. VQ decoder on CPU
|
| 723 |
+
|
| 724 |
+
- Audit and implement required GGML convolution/residual operations.
|
| 725 |
+
- Load codebooks and decoder tensors from GGUF.
|
| 726 |
+
- Match each decoder layer and decoded local-motion features in F32.
|
| 727 |
+
|
| 728 |
+
**Exit:** CPU VQ decoder fixture passes end to end.
|
| 729 |
+
|
| 730 |
+
### 5. Root model on CPU
|
| 731 |
+
|
| 732 |
+
- Implement shared transformer operators and root embeddings/masks.
|
| 733 |
+
- Match each transformer layer, duration selection, conditioned decoder, and
|
| 734 |
+
predicted global/local root trajectories.
|
| 735 |
+
|
| 736 |
+
**Exit:** CPU root fixtures pass for fixed, predicted, minimum, and maximum
|
| 737 |
+
durations.
|
| 738 |
+
|
| 739 |
+
### 6. Pose model and composed inference on CPU
|
| 740 |
+
|
| 741 |
+
- Implement pose embeddings, transformer, logits, argmax/Gumbel sampling.
|
| 742 |
+
- Compose root -> pose -> VQ -> representation inverse.
|
| 743 |
+
- Implement the stateful agent and verify recorded replanning sessions.
|
| 744 |
+
|
| 745 |
+
**Exit:** F32 CPU passes every layer and end-to-end skeletal exemplar.
|
| 746 |
+
|
| 747 |
+
### 7. C API, Kimodo styles, and web demo
|
| 748 |
+
|
| 749 |
+
- Complete opaque getters/setters, ownership, error paths, and installed C API.
|
| 750 |
+
- Verify Kimodo G1 GLB import and add a style conversion workflow.
|
| 751 |
+
- Implement Go/PureGo server, binary WebSocket protocol, and Three.js viewer.
|
| 752 |
+
- Reproduce upstream styles and interactive turn/style-switch scenarios.
|
| 753 |
+
|
| 754 |
+
**Exit:** a fresh build runs the interactive demo using both an original and a
|
| 755 |
+
Kimodo-authored style, with no Python process at runtime.
|
| 756 |
+
|
| 757 |
+
### 8. Vulkan parity and performance
|
| 758 |
+
|
| 759 |
+
- Enable GGML Vulkan and select the intended GPU through runtime configuration.
|
| 760 |
+
- Match every component and end-to-end fixture using Vulkan-specific
|
| 761 |
+
tolerances.
|
| 762 |
+
- Profile allocation, upload, graph construction, inference, and WebSocket
|
| 763 |
+
latency; cache graphs/buffers where safe.
|
| 764 |
+
- Establish a replan cadence the machine sustains while playback remains
|
| 765 |
+
buffered.
|
| 766 |
+
|
| 767 |
+
**Exit:** Vulkan preserves exemplar behavior and meets the documented
|
| 768 |
+
interactive latency/buffer target on the test machine.
|
| 769 |
+
|
| 770 |
+
### 9. Hardening and optimized weights
|
| 771 |
+
|
| 772 |
+
- Run long ASan/UBSan fuzz campaigns and multi-session soak tests.
|
| 773 |
+
- Add F16 conversion/parity, then evaluate quantization component by component.
|
| 774 |
+
- Package install targets, model downloader, licences, deployment docs, and
|
| 775 |
+
reproducible release manifests.
|
| 776 |
+
|
| 777 |
+
**Exit:** release artifacts are reproducible, validated, and contain no hidden
|
| 778 |
+
machine-specific paths or upstream Python dependency.
|
| 779 |
+
|
| 780 |
+
## Decisions to revisit after the first fixtures
|
| 781 |
+
|
| 782 |
+
- Whether the public style format should be a small GGUF, GLB plus JSON, or a
|
| 783 |
+
custom binary. GLB plus JSON is easiest for user-authored Kimodo assets;
|
| 784 |
+
GGUF is likely best for exact preprocessed upstream tensors.
|
| 785 |
+
- Whether `mb_agent_plan` should retain output internally or return an owning
|
| 786 |
+
immutable copy. The owning result above is simpler and safer for PureGo;
|
| 787 |
+
profiling may justify an explicit caller-buffer API later.
|
| 788 |
+
- Whether to expose a generic sparse-keyframe API in ABI version 1. It is more
|
| 789 |
+
capable than the interactive controller but substantially enlarges the
|
| 790 |
+
validation surface. The first slice can support movement/facing/style/world
|
| 791 |
+
target while keeping the generic constraint builder internal.
|
| 792 |
+
- Whether to link a small shared GLB module from `kimodo.cpp`/`skin-tokens.cpp`
|
| 793 |
+
or keep style conversion in the CLI. Reuse should happen only through a
|
| 794 |
+
clean library boundary, not by copying diverging parser code.
|
| 795 |
+
- Exact interactive latency, memory, and browser buffer targets. Measure the
|
| 796 |
+
F32 CPU baseline and F32 Vulkan implementation before promising numbers.
|
| 797 |
+
|
| 798 |
+
## Explicitly deferred
|
| 799 |
+
|
| 800 |
+
- training or fine-tuning MotionBricks;
|
| 801 |
+
- arbitrary skeleton inference or automatic retargeting;
|
| 802 |
+
- text input to MotionBricks itself;
|
| 803 |
+
- robot torque/control-policy output;
|
| 804 |
+
- WebAssembly inference in the browser;
|
| 805 |
+
- F16/quantized defaults before F32 parity;
|
| 806 |
+
- multi-batch inference and concurrent mutation of one agent.
|
docs/motions-bricks.md
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Human led design document
|
| 2 |
+
|
| 3 |
+
Don't modify this unless instructed.
|
| 4 |
+
|
| 5 |
+
# Questions to loop back to
|
| 6 |
+
|
| 7 |
+
- camera controls, zoom pan etc
|
| 8 |
+
- Set a new target for parity, exactly what components should we select for that?
|
| 9 |
+
- where is the C API documentation
|
| 10 |
+
- Are low-level functions exposed so things like styles can be handle externally? could someone just use it for inference?
|
| 11 |
+
- camera relative WASD keys
|
| 12 |
+
|
| 13 |
+
# End goal
|
| 14 |
+
|
| 15 |
+
We want to use motion-bricks.cpp with kimodo.cpp animation key-frames and we want to be able to use these in a web demo.
|
| 16 |
+
|
| 17 |
+
So we have a web demo that takes some key-frames for particular movements and styles of movements, like in the upstream demo, including the original key-frames. Like in the upstream demo we are going to have a bunch of movements we can do with the WASD keys and jump or some other user definable motion.
|
| 18 |
+
|
| 19 |
+
Eventually, we want to have have a configurable list of movements, that include a keyframe, an action (e.g. move forward at 1m/s, move up (jump)), so we could define a walking action with a keyframe from someone walking and an action of moving forwards, and we could define a forward roll action by specifying one or more keyframes from rolling forwards and also a moving forward action (but perhaps faster forward motion).
|
| 20 |
+
|
| 21 |
+
The motion-bricks.cpp library should expose a C API for FFI which is compatible with libraries like purego. So purego can't predict struct layouts, so struct based interfaces should have a constructor for allocating structs and getters and setters.
|
| 22 |
+
|
| 23 |
+
The demo server should be written in Go and use purego (or similar). The frontend should use plain HTML/CSS and three.js.
|
| 24 |
+
|
| 25 |
+
# Bonus Goal
|
| 26 |
+
|
| 27 |
+
Use motion-bricks.cpp with a physically simulated robot in MuJoco or a web browser simulator.
|
| 28 |
+
|
| 29 |
+
# Process
|
| 30 |
+
|
| 31 |
+
- First we want to produce exemplar outputs from upstream
|
| 32 |
+
- We want to create parity tests for each component, including every layer of the neural network.
|
| 33 |
+
- We want to ensure that components feeding and consuming input and output from the neural network also match upstream in terms of functionality
|
| 34 |
+
- We need to ensure we are not using settings in our implementation by default which are not set as default in upstream
|
| 35 |
+
- We should use GGML with Vulkan and CPU backends, first CPU and then Vulkan parity should be achieved
|
| 36 |
+
- We can use ../kimodo.cpp and ../skin-tokens.cpp/ as a reference, you should scan these projects before designing and check them when trying to solve an issue for an existing solution
|
| 37 |
+
- We are on NixOS and can use a Nix flake to get depenencies for GGML/C++ and Vulkan, the machine we are on has a functioning NVIDIA Vulkan GPU and an AMD one. We want to use the NVIDIA gfx card.
|
| 38 |
+
- For the upstream depencies, if we can use Nix that is great, but otherwise we can use Docker
|
| 39 |
+
- Don't rely on the current machine's paths, use distro agnostic paths and env vars. parameterise anything that we need to specify in the CMake file or a .env
|
| 40 |
+
- We need to fuzz the C++ components except for GGUF input (we trust that GGML has that covered). We should do that with the ASan and UBSAN enabled.
|
| 41 |
+
- We should use GGML as git submodule and if we need to patch it for any reason we can apply patches during the project configuration phase
|
| 42 |
+
- Use headless chrome (from nix) to perform QA after changes. Use a few different actions and take screenshots before during and after
|
flake.lock
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"nodes": {
|
| 3 |
+
"nixpkgs": {
|
| 4 |
+
"locked": {
|
| 5 |
+
"lastModified": 1776169885,
|
| 6 |
+
"narHash": "sha256-l/iNYDZ4bGOAFQY2q8y5OAfBBtrDAaPuRQqWaFHVRXM=",
|
| 7 |
+
"owner": "NixOS",
|
| 8 |
+
"repo": "nixpkgs",
|
| 9 |
+
"rev": "4bd9165a9165d7b5e33ae57f3eecbcb28fb231c9",
|
| 10 |
+
"type": "github"
|
| 11 |
+
},
|
| 12 |
+
"original": {
|
| 13 |
+
"owner": "NixOS",
|
| 14 |
+
"repo": "nixpkgs",
|
| 15 |
+
"rev": "4bd9165a9165d7b5e33ae57f3eecbcb28fb231c9",
|
| 16 |
+
"type": "github"
|
| 17 |
+
}
|
| 18 |
+
},
|
| 19 |
+
"root": {
|
| 20 |
+
"inputs": {
|
| 21 |
+
"nixpkgs": "nixpkgs"
|
| 22 |
+
}
|
| 23 |
+
}
|
| 24 |
+
},
|
| 25 |
+
"root": "root",
|
| 26 |
+
"version": 7
|
| 27 |
+
}
|
flake.nix
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
description = "motion-bricks.cpp — GGML CPU/Vulkan motion planning";
|
| 3 |
+
|
| 4 |
+
inputs.nixpkgs.url = "github:NixOS/nixpkgs/4bd9165a9165d7b5e33ae57f3eecbcb28fb231c9";
|
| 5 |
+
|
| 6 |
+
outputs = { self, nixpkgs }:
|
| 7 |
+
let
|
| 8 |
+
systems = [ "x86_64-linux" "aarch64-linux" ];
|
| 9 |
+
eachSystem = nixpkgs.lib.genAttrs systems;
|
| 10 |
+
in {
|
| 11 |
+
devShells = eachSystem (system:
|
| 12 |
+
let pkgs = import nixpkgs { inherit system; };
|
| 13 |
+
in {
|
| 14 |
+
default = pkgs.mkShell {
|
| 15 |
+
packages = with pkgs; [
|
| 16 |
+
cmake
|
| 17 |
+
ninja
|
| 18 |
+
clang
|
| 19 |
+
pkg-config
|
| 20 |
+
git
|
| 21 |
+
git-lfs
|
| 22 |
+
go
|
| 23 |
+
python3
|
| 24 |
+
python3Packages.huggingface-hub
|
| 25 |
+
python3Packages.numpy
|
| 26 |
+
python3Packages.safetensors
|
| 27 |
+
shaderc
|
| 28 |
+
spirv-tools
|
| 29 |
+
vulkan-headers
|
| 30 |
+
vulkan-loader
|
| 31 |
+
vulkan-tools
|
| 32 |
+
];
|
| 33 |
+
shellHook = ''
|
| 34 |
+
export LD_LIBRARY_PATH="${pkgs.vulkan-loader}/lib:/run/opengl-driver/lib''${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
|
| 35 |
+
'';
|
| 36 |
+
};
|
| 37 |
+
});
|
| 38 |
+
};
|
| 39 |
+
}
|
include/motionbricks/motionbricks.h
ADDED
|
@@ -0,0 +1,178 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#ifndef MOTIONBRICKS_MOTIONBRICKS_H
|
| 2 |
+
#define MOTIONBRICKS_MOTIONBRICKS_H
|
| 3 |
+
|
| 4 |
+
#include <stdint.h>
|
| 5 |
+
|
| 6 |
+
#define MB_ABI_VERSION UINT32_C(1)
|
| 7 |
+
|
| 8 |
+
#if defined(_WIN32) && defined(MOTIONBRICKS_SHARED)
|
| 9 |
+
# if defined(MOTIONBRICKS_BUILD)
|
| 10 |
+
# define MB_API __declspec(dllexport)
|
| 11 |
+
# else
|
| 12 |
+
# define MB_API __declspec(dllimport)
|
| 13 |
+
# endif
|
| 14 |
+
#elif defined(__GNUC__) && defined(MOTIONBRICKS_SHARED)
|
| 15 |
+
# define MB_API __attribute__((visibility("default")))
|
| 16 |
+
#else
|
| 17 |
+
# define MB_API
|
| 18 |
+
#endif
|
| 19 |
+
|
| 20 |
+
#ifdef __cplusplus
|
| 21 |
+
extern "C" {
|
| 22 |
+
#endif
|
| 23 |
+
|
| 24 |
+
typedef uint32_t mb_status;
|
| 25 |
+
#define MB_OK UINT32_C(0)
|
| 26 |
+
#define MB_INVALID_ARGUMENT UINT32_C(1)
|
| 27 |
+
#define MB_OUT_OF_MEMORY UINT32_C(2)
|
| 28 |
+
#define MB_IO_ERROR UINT32_C(3)
|
| 29 |
+
#define MB_INVALID_FORMAT UINT32_C(4)
|
| 30 |
+
#define MB_INCOMPATIBLE_MODEL UINT32_C(5)
|
| 31 |
+
#define MB_BACKEND_UNAVAILABLE UINT32_C(6)
|
| 32 |
+
#define MB_COMPUTE_FAILED UINT32_C(7)
|
| 33 |
+
#define MB_NOT_IMPLEMENTED UINT32_C(8)
|
| 34 |
+
#define MB_INTERNAL_ERROR UINT32_C(9)
|
| 35 |
+
|
| 36 |
+
typedef uint32_t mb_device;
|
| 37 |
+
#define MB_DEVICE_AUTO UINT32_C(0)
|
| 38 |
+
#define MB_DEVICE_CPU UINT32_C(1)
|
| 39 |
+
#define MB_DEVICE_VULKAN UINT32_C(2)
|
| 40 |
+
|
| 41 |
+
typedef struct mb_runtime_options mb_runtime_options;
|
| 42 |
+
typedef struct mb_model mb_model;
|
| 43 |
+
typedef struct mb_style mb_style;
|
| 44 |
+
typedef struct mb_command mb_command;
|
| 45 |
+
typedef struct mb_agent mb_agent;
|
| 46 |
+
typedef struct mb_motion mb_motion;
|
| 47 |
+
|
| 48 |
+
MB_API uint32_t mb_abi_version(void);
|
| 49 |
+
MB_API const char * mb_status_string(mb_status status);
|
| 50 |
+
|
| 51 |
+
/* Runtime options are opaque so FFI callers never reproduce a struct layout. */
|
| 52 |
+
MB_API mb_status mb_runtime_options_create(mb_runtime_options ** output,
|
| 53 |
+
char * error, uint64_t error_capacity);
|
| 54 |
+
MB_API void mb_runtime_options_free(mb_runtime_options * value);
|
| 55 |
+
MB_API mb_status mb_runtime_options_set_device(mb_runtime_options * value, mb_device device,
|
| 56 |
+
char * error, uint64_t error_capacity);
|
| 57 |
+
MB_API mb_status mb_runtime_options_get_device(const mb_runtime_options * value, mb_device * output,
|
| 58 |
+
char * error, uint64_t error_capacity);
|
| 59 |
+
MB_API mb_status mb_runtime_options_set_threads(mb_runtime_options * value, uint32_t threads,
|
| 60 |
+
char * error, uint64_t error_capacity);
|
| 61 |
+
MB_API mb_status mb_runtime_options_get_threads(const mb_runtime_options * value, uint32_t * output,
|
| 62 |
+
char * error, uint64_t error_capacity);
|
| 63 |
+
/* Passing NULL clears the override. The returned string is borrowed. */
|
| 64 |
+
MB_API mb_status mb_runtime_options_set_backend_directory(mb_runtime_options * value,
|
| 65 |
+
const char * directory,
|
| 66 |
+
char * error, uint64_t error_capacity);
|
| 67 |
+
MB_API mb_status mb_runtime_options_get_backend_directory(const mb_runtime_options * value,
|
| 68 |
+
const char ** output,
|
| 69 |
+
char * error, uint64_t error_capacity);
|
| 70 |
+
|
| 71 |
+
/* A command is independent of compiler layout and may be reused between plans. */
|
| 72 |
+
MB_API mb_status mb_command_create(mb_command ** output,
|
| 73 |
+
char * error, uint64_t error_capacity);
|
| 74 |
+
MB_API void mb_command_free(mb_command * value);
|
| 75 |
+
MB_API mb_status mb_command_set_style(mb_command * value, const mb_style * style,
|
| 76 |
+
char * error, uint64_t error_capacity);
|
| 77 |
+
MB_API mb_status mb_command_get_style(const mb_command * value, const mb_style ** output,
|
| 78 |
+
char * error, uint64_t error_capacity);
|
| 79 |
+
MB_API mb_status mb_command_set_movement_direction(mb_command * value, float x, float y, float z,
|
| 80 |
+
char * error, uint64_t error_capacity);
|
| 81 |
+
MB_API mb_status mb_command_get_movement_direction(const mb_command * value,
|
| 82 |
+
float * x, float * y, float * z,
|
| 83 |
+
char * error, uint64_t error_capacity);
|
| 84 |
+
MB_API mb_status mb_command_set_facing_direction(mb_command * value, float x, float y, float z,
|
| 85 |
+
char * error, uint64_t error_capacity);
|
| 86 |
+
MB_API mb_status mb_command_get_facing_direction(const mb_command * value,
|
| 87 |
+
float * x, float * y, float * z,
|
| 88 |
+
char * error, uint64_t error_capacity);
|
| 89 |
+
/* A negative speed selects the style's configured speed. */
|
| 90 |
+
MB_API mb_status mb_command_set_target_speed(mb_command * value, float metres_per_second,
|
| 91 |
+
char * error, uint64_t error_capacity);
|
| 92 |
+
MB_API mb_status mb_command_get_target_speed(const mb_command * value, float * output,
|
| 93 |
+
char * error, uint64_t error_capacity);
|
| 94 |
+
MB_API mb_status mb_command_set_world_target(mb_command * value,
|
| 95 |
+
float x, float y, float z,
|
| 96 |
+
float heading_radians, uint32_t enabled,
|
| 97 |
+
char * error, uint64_t error_capacity);
|
| 98 |
+
MB_API mb_status mb_command_get_world_target(const mb_command * value,
|
| 99 |
+
float * x, float * y, float * z,
|
| 100 |
+
float * heading_radians, uint32_t * enabled,
|
| 101 |
+
char * error, uint64_t error_capacity);
|
| 102 |
+
MB_API mb_status mb_command_set_seed(mb_command * value, uint64_t seed,
|
| 103 |
+
char * error, uint64_t error_capacity);
|
| 104 |
+
MB_API mb_status mb_command_get_seed(const mb_command * value, uint64_t * output,
|
| 105 |
+
char * error, uint64_t error_capacity);
|
| 106 |
+
|
| 107 |
+
/* Neural model, style, and stateful animation planner. */
|
| 108 |
+
MB_API mb_status mb_model_load(const char * bundle_directory,
|
| 109 |
+
const mb_runtime_options * options, mb_model ** output,
|
| 110 |
+
char * error, uint64_t error_capacity);
|
| 111 |
+
MB_API void mb_model_free(mb_model * value);
|
| 112 |
+
MB_API mb_status mb_model_get_parameter_count(const mb_model * value, uint64_t * output,
|
| 113 |
+
char * error, uint64_t error_capacity);
|
| 114 |
+
MB_API mb_status mb_model_get_joint_count(const mb_model * value, uint32_t * output,
|
| 115 |
+
char * error, uint64_t error_capacity);
|
| 116 |
+
MB_API mb_status mb_model_get_joint_name(const mb_model * value, uint32_t joint,
|
| 117 |
+
const char ** output,
|
| 118 |
+
char * error, uint64_t error_capacity);
|
| 119 |
+
MB_API mb_status mb_model_get_joint_parent(const mb_model * value, uint32_t joint,
|
| 120 |
+
int32_t * output,
|
| 121 |
+
char * error, uint64_t error_capacity);
|
| 122 |
+
MB_API mb_status mb_model_get_neutral_joint_position(const mb_model * value, uint32_t joint,
|
| 123 |
+
float * x, float * y, float * z,
|
| 124 |
+
char * error, uint64_t error_capacity);
|
| 125 |
+
|
| 126 |
+
MB_API mb_status mb_style_load(const mb_model * model, const char * style_path,
|
| 127 |
+
mb_style ** output, char * error, uint64_t error_capacity);
|
| 128 |
+
MB_API void mb_style_free(mb_style * value);
|
| 129 |
+
MB_API mb_status mb_style_get_name(const mb_style * value, const char ** output,
|
| 130 |
+
char * error, uint64_t error_capacity);
|
| 131 |
+
MB_API mb_status mb_style_set_speed(mb_style * value, float metres_per_second,
|
| 132 |
+
char * error, uint64_t error_capacity);
|
| 133 |
+
MB_API mb_status mb_style_get_speed(const mb_style * value, float * output,
|
| 134 |
+
char * error, uint64_t error_capacity);
|
| 135 |
+
|
| 136 |
+
MB_API mb_status mb_agent_create(const mb_model * model, mb_agent ** output,
|
| 137 |
+
char * error, uint64_t error_capacity);
|
| 138 |
+
MB_API void mb_agent_free(mb_agent * value);
|
| 139 |
+
MB_API mb_status mb_agent_reset(mb_agent * value, const mb_style * initial_style,
|
| 140 |
+
char * error, uint64_t error_capacity);
|
| 141 |
+
MB_API mb_status mb_agent_set_context(mb_agent * value,
|
| 142 |
+
const float * root_xyz,
|
| 143 |
+
const float * local_rotation_xyzw,
|
| 144 |
+
uint64_t frames, uint64_t joints,
|
| 145 |
+
char * error, uint64_t error_capacity);
|
| 146 |
+
MB_API mb_status mb_agent_plan(mb_agent * value, const mb_command * command,
|
| 147 |
+
mb_motion ** output,
|
| 148 |
+
char * error, uint64_t error_capacity);
|
| 149 |
+
MB_API mb_status mb_agent_advance(mb_agent * value, uint32_t frames,
|
| 150 |
+
char * error, uint64_t error_capacity);
|
| 151 |
+
|
| 152 |
+
MB_API void mb_motion_free(mb_motion * value);
|
| 153 |
+
MB_API mb_status mb_motion_get_frame_count(const mb_motion * value, uint64_t * output,
|
| 154 |
+
char * error, uint64_t error_capacity);
|
| 155 |
+
MB_API mb_status mb_motion_get_joint_count(const mb_motion * value, uint64_t * output,
|
| 156 |
+
char * error, uint64_t error_capacity);
|
| 157 |
+
/* Borrowed row-major F32 buffers valid until mb_motion_free. */
|
| 158 |
+
MB_API mb_status mb_motion_get_root_translations(const mb_motion * value,
|
| 159 |
+
const float ** output, uint64_t * values,
|
| 160 |
+
char * error, uint64_t error_capacity);
|
| 161 |
+
MB_API mb_status mb_motion_get_local_rotations_xyzw(const mb_motion * value,
|
| 162 |
+
const float ** output, uint64_t * values,
|
| 163 |
+
char * error, uint64_t error_capacity);
|
| 164 |
+
/* The four placed style-pose constraints actually supplied to the planner. */
|
| 165 |
+
MB_API mb_status mb_motion_get_target_frame_count(const mb_motion * value, uint64_t * output,
|
| 166 |
+
char * error, uint64_t error_capacity);
|
| 167 |
+
MB_API mb_status mb_motion_get_target_root_translations(const mb_motion * value,
|
| 168 |
+
const float ** output, uint64_t * values,
|
| 169 |
+
char * error, uint64_t error_capacity);
|
| 170 |
+
MB_API mb_status mb_motion_get_target_local_rotations_xyzw(const mb_motion * value,
|
| 171 |
+
const float ** output, uint64_t * values,
|
| 172 |
+
char * error, uint64_t error_capacity);
|
| 173 |
+
|
| 174 |
+
#ifdef __cplusplus
|
| 175 |
+
}
|
| 176 |
+
#endif
|
| 177 |
+
|
| 178 |
+
#endif
|
include/motionbricks/motionbricks.hpp
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#pragma once
|
| 2 |
+
|
| 3 |
+
#include <motionbricks/motionbricks.h>
|
| 4 |
+
|
| 5 |
+
#include <array>
|
| 6 |
+
#include <cstdint>
|
| 7 |
+
#include <stdexcept>
|
| 8 |
+
#include <string>
|
| 9 |
+
|
| 10 |
+
namespace motionbricks {
|
| 11 |
+
|
| 12 |
+
class api_error final : public std::runtime_error {
|
| 13 |
+
public:
|
| 14 |
+
api_error(mb_status status, const char * message)
|
| 15 |
+
: std::runtime_error(message != nullptr ? message : mb_status_string(status)), status_(status) {}
|
| 16 |
+
|
| 17 |
+
[[nodiscard]] mb_status status() const noexcept { return status_; }
|
| 18 |
+
|
| 19 |
+
private:
|
| 20 |
+
mb_status status_;
|
| 21 |
+
};
|
| 22 |
+
|
| 23 |
+
class runtime_options final {
|
| 24 |
+
public:
|
| 25 |
+
runtime_options() {
|
| 26 |
+
std::array<char, 512> error{};
|
| 27 |
+
const auto status = mb_runtime_options_create(&value_, error.data(), error.size());
|
| 28 |
+
if (status != MB_OK) throw api_error(status, error.data());
|
| 29 |
+
}
|
| 30 |
+
|
| 31 |
+
~runtime_options() { mb_runtime_options_free(value_); }
|
| 32 |
+
runtime_options(const runtime_options &) = delete;
|
| 33 |
+
runtime_options & operator=(const runtime_options &) = delete;
|
| 34 |
+
|
| 35 |
+
runtime_options(runtime_options && other) noexcept : value_(other.value_) {
|
| 36 |
+
other.value_ = nullptr;
|
| 37 |
+
}
|
| 38 |
+
|
| 39 |
+
runtime_options & operator=(runtime_options && other) noexcept {
|
| 40 |
+
if (this != &other) {
|
| 41 |
+
mb_runtime_options_free(value_);
|
| 42 |
+
value_ = other.value_;
|
| 43 |
+
other.value_ = nullptr;
|
| 44 |
+
}
|
| 45 |
+
return *this;
|
| 46 |
+
}
|
| 47 |
+
|
| 48 |
+
[[nodiscard]] mb_runtime_options * get() noexcept { return value_; }
|
| 49 |
+
[[nodiscard]] const mb_runtime_options * get() const noexcept { return value_; }
|
| 50 |
+
|
| 51 |
+
private:
|
| 52 |
+
mb_runtime_options * value_ = nullptr;
|
| 53 |
+
};
|
| 54 |
+
|
| 55 |
+
} // namespace motionbricks
|
patches/ggml/README.md
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# GGML compatibility patches
|
| 2 |
+
|
| 3 |
+
The initial GGML revision is
|
| 4 |
+
`8c63e70982c95ceb862e3a1073a2c1beef75d60a` (`v0.20.2`). No compatibility
|
| 5 |
+
patch is currently required.
|
| 6 |
+
|
| 7 |
+
If a patch becomes necessary, add numbered patch files here and update CMake
|
| 8 |
+
to copy GGML into the build directory and apply them in lexical order. Never
|
| 9 |
+
modify the pinned submodule worktree during configuration.
|
reference/Dockerfile
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
FROM python:3.10.14-slim-bookworm@sha256:2407c61b1a18067393fecd8a22cf6fceede893b6aaca817bf9fbfe65e33614a3
|
| 2 |
+
|
| 3 |
+
RUN python -m pip install --no-cache-dir --upgrade pip==24.3.1 \
|
| 4 |
+
&& python -m pip install --no-cache-dir \
|
| 5 |
+
--index-url https://download.pytorch.org/whl/cpu \
|
| 6 |
+
torch==2.4.0 \
|
| 7 |
+
&& python -m pip install --no-cache-dir \
|
| 8 |
+
numpy==1.26.4 \
|
| 9 |
+
PyYAML==6.0.2 \
|
| 10 |
+
safetensors==0.4.5
|
| 11 |
+
|
| 12 |
+
WORKDIR /work
|
| 13 |
+
ENTRYPOINT ["python", "/work/reference/extract_safe.py"]
|
reference/README.md
ADDED
|
@@ -0,0 +1,117 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Upstream reference
|
| 2 |
+
|
| 3 |
+
The native implementation is validated against a pinned, unmodified NVIDIA
|
| 4 |
+
MotionBricks checkout.
|
| 5 |
+
|
| 6 |
+
| Component | Repository | Revision |
|
| 7 |
+
|---|---|---|
|
| 8 |
+
| MotionBricks | `https://github.com/NVlabs/GR00T-WholeBodyControl.git` | `a0732b642c0333077e127a2f56ab0014c196bca4` |
|
| 9 |
+
| GGML | `https://github.com/ggml-org/ggml.git` | `8c63e70982c95ceb862e3a1073a2c1beef75d60a` (`v0.20.2`) |
|
| 10 |
+
|
| 11 |
+
The `ggml/` submodule is pinned to the same revision used by the local
|
| 12 |
+
`kimodo.cpp` and `skin-tokens.cpp` reference ports. Compatibility patches, if
|
| 13 |
+
required, belong under `patches/ggml/` and will be applied to a build-directory
|
| 14 |
+
copy during CMake configuration; the submodule worktree will remain untouched.
|
| 15 |
+
|
| 16 |
+
## Checkpoint identities
|
| 17 |
+
|
| 18 |
+
The required MotionBricks checkpoints and G1 meshes have been fetched through
|
| 19 |
+
Git LFS. These identities are the initial trusted reference inputs:
|
| 20 |
+
|
| 21 |
+
| Artifact | Bytes | SHA-256 |
|
| 22 |
+
|---|---:|---|
|
| 23 |
+
| `G1-clip.ckpt` | 7,761,041 | `84afc7c229473351a24b0a7d79fc47be9dbb81bd12774285f2f60a3c0e9028df` |
|
| 24 |
+
| VQ-VAE `model-step=2000000.ckpt` | 285,607,244 | `f12a09d46ad390a8e2eecbe7219b2472fcab6b59df0a13f6a40c35cb6da4d99a` |
|
| 25 |
+
| pose `model-step=2000000.ckpt` | 1,639,126,476 | `0223c352b308ba638a499cc5c92104da36cb1d04cea2f8ce61d54a2489f853f1` |
|
| 26 |
+
| root `model-step=2000000.ckpt` | 409,551,754 | `d7299a9b1f5aca35730c36dfe7ea28075708ac8266bf384fe8e2ef9c9aee69c7` |
|
| 27 |
+
|
| 28 |
+
Saved configuration identities:
|
| 29 |
+
|
| 30 |
+
| Configuration | SHA-256 |
|
| 31 |
+
|---|---|
|
| 32 |
+
| VQ-VAE `config.yaml` | `027a2d7ba5f49cadeadbcc9a6b0c6784d657d5a842e41ff820c5233f1cd6c1f3` |
|
| 33 |
+
| pose `config.yaml` | `273af770d328b458510ee6049fac8e38fa486c8792cc27b3515dddc094a7cd1f` |
|
| 34 |
+
| root `config.yaml` | `b174f03a333f7f7857c3e2b8a32da517caddd198f228efeea2e203d046ce212a` |
|
| 35 |
+
|
| 36 |
+
The 29-DOF G1 XML identities used by the initial interactive reference are:
|
| 37 |
+
|
| 38 |
+
- `g1_29dof.xml`: `58660a6f1d0d33ffd8ee967ab3860def53e3327d956cb009dd1385ddaf430f56`
|
| 39 |
+
- `scene_29dof.xml`: `e254f11acce2ec6f6efa5bf9b15e288bbd0ca29aeeabb1e1d0fea92f65436bbf`
|
| 40 |
+
|
| 41 |
+
Legacy Lightning `.ckpt` and demo-clip files are deserialized only inside the
|
| 42 |
+
reference container. Normal conversion accepts safetensors and JSON, never
|
| 43 |
+
pickle-bearing checkpoints.
|
| 44 |
+
|
| 45 |
+
Build and run the trusted extractor from the project root:
|
| 46 |
+
|
| 47 |
+
```sh
|
| 48 |
+
docker build -t motionbricks-reference:torch2.4 reference
|
| 49 |
+
docker run --rm --user "$(id -u):$(id -g)" \
|
| 50 |
+
-v "$PWD:/work" \
|
| 51 |
+
-v /path/to/GR00T-WholeBodyControl:/upstream:ro \
|
| 52 |
+
motionbricks-reference:torch2.4 \
|
| 53 |
+
--upstream-root /upstream \
|
| 54 |
+
--output /work/generated/safe
|
| 55 |
+
```
|
| 56 |
+
|
| 57 |
+
The extractor verifies every pinned input hash and the small allowlist of
|
| 58 |
+
pickle globals before loading. It writes independent safetensors for the three
|
| 59 |
+
model checkpoints, original demo clips, and shared skeleton/statistics, plus a
|
| 60 |
+
manifest containing every tensor name, shape, dtype, value count, and hash.
|
| 61 |
+
|
| 62 |
+
## Native inference bundle
|
| 63 |
+
|
| 64 |
+
The normal converter reads only those verified safetensors and JSON files:
|
| 65 |
+
|
| 66 |
+
```sh
|
| 67 |
+
nix develop --command python scripts/convert_to_gguf.py \
|
| 68 |
+
--safe-directory generated/safe \
|
| 69 |
+
--output generated/g1-f32
|
| 70 |
+
```
|
| 71 |
+
|
| 72 |
+
It emits three inference-weight components plus one small skeleton/statistics
|
| 73 |
+
component. The VQ-VAE encoder and optimizer/training state are excluded. The
|
| 74 |
+
resulting learned parameter inventory is:
|
| 75 |
+
|
| 76 |
+
- pose planner: 136,588,272
|
| 77 |
+
- root planner: 34,122,833
|
| 78 |
+
- VQ pose decoder and codebook: 12,437,277
|
| 79 |
+
- total: 183,148,382
|
| 80 |
+
|
| 81 |
+
Validate the result through the same public model-loading path applications
|
| 82 |
+
use:
|
| 83 |
+
|
| 84 |
+
```sh
|
| 85 |
+
./build/debug/bin/motionbricks-cli inspect generated/g1-f32
|
| 86 |
+
```
|
| 87 |
+
|
| 88 |
+
## Reference fixtures and styles
|
| 89 |
+
|
| 90 |
+
Generate deterministic PyTorch layer fixtures inside the same pinned
|
| 91 |
+
container, then package them as small GGUF test inputs:
|
| 92 |
+
|
| 93 |
+
```sh
|
| 94 |
+
docker run --rm --user "$(id -u):$(id -g)" \
|
| 95 |
+
--entrypoint python \
|
| 96 |
+
-v "$PWD:/work" \
|
| 97 |
+
-v /path/to/GR00T-WholeBodyControl:/upstream:ro \
|
| 98 |
+
motionbricks-reference:torch2.4 \
|
| 99 |
+
/work/reference/generate_fixtures.py \
|
| 100 |
+
--upstream-root /upstream \
|
| 101 |
+
--safe-directory /work/generated/safe \
|
| 102 |
+
--output /work/generated/fixtures
|
| 103 |
+
|
| 104 |
+
python scripts/convert_fixtures_to_gguf.py \
|
| 105 |
+
--fixtures generated/fixtures \
|
| 106 |
+
--output generated/fixtures-gguf
|
| 107 |
+
|
| 108 |
+
python scripts/convert_styles.py \
|
| 109 |
+
--safe-directory generated/safe \
|
| 110 |
+
--output generated/styles
|
| 111 |
+
```
|
| 112 |
+
|
| 113 |
+
The native suite checks every released neural component against these actual
|
| 114 |
+
upstream PyTorch forwards. On the current reference machine, complete
|
| 115 |
+
style-to-animation output also matches between CPU and strict-F32 Vulkan with
|
| 116 |
+
the same 44-frame duration; observed maximum absolute differences were
|
| 117 |
+
`2.19e-5` for root translations and `1.02e-4` for local quaternion components.
|
reference/extract_safe.py
ADDED
|
@@ -0,0 +1,290 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""One-time extraction of pinned MotionBricks pickle checkpoints.
|
| 3 |
+
|
| 4 |
+
Run this only inside the disposable reference container. Normal conversion and
|
| 5 |
+
native inference consume the resulting safetensors and JSON and never open a
|
| 6 |
+
PyTorch pickle container.
|
| 7 |
+
"""
|
| 8 |
+
|
| 9 |
+
from __future__ import annotations
|
| 10 |
+
|
| 11 |
+
import argparse
|
| 12 |
+
import gc
|
| 13 |
+
import hashlib
|
| 14 |
+
import json
|
| 15 |
+
import os
|
| 16 |
+
import pickletools
|
| 17 |
+
from pathlib import Path
|
| 18 |
+
from typing import Any
|
| 19 |
+
from zipfile import ZipFile
|
| 20 |
+
|
| 21 |
+
import numpy as np
|
| 22 |
+
import torch
|
| 23 |
+
import yaml
|
| 24 |
+
from safetensors.torch import save_file
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
UPSTREAM_REVISION = "a0732b642c0333077e127a2f56ab0014c196bca4"
|
| 28 |
+
|
| 29 |
+
INPUTS = {
|
| 30 |
+
"clips": (
|
| 31 |
+
"motionbricks/out/G1-clip.ckpt",
|
| 32 |
+
"84afc7c229473351a24b0a7d79fc47be9dbb81bd12774285f2f60a3c0e9028df",
|
| 33 |
+
),
|
| 34 |
+
"vqvae": (
|
| 35 |
+
"motionbricks/out/motionbricks_vqvae/version_1/checkpoints/model-step=2000000.ckpt",
|
| 36 |
+
"f12a09d46ad390a8e2eecbe7219b2472fcab6b59df0a13f6a40c35cb6da4d99a",
|
| 37 |
+
),
|
| 38 |
+
"pose": (
|
| 39 |
+
"motionbricks/out/motionbricks_pose/version_1/checkpoints/model-step=2000000.ckpt",
|
| 40 |
+
"0223c352b308ba638a499cc5c92104da36cb1d04cea2f8ce61d54a2489f853f1",
|
| 41 |
+
),
|
| 42 |
+
"root": (
|
| 43 |
+
"motionbricks/out/motionbricks_root/version_1/checkpoints/model-step=2000000.ckpt",
|
| 44 |
+
"d7299a9b1f5aca35730c36dfe7ea28075708ac8266bf384fe8e2ef9c9aee69c7",
|
| 45 |
+
),
|
| 46 |
+
"joints": (
|
| 47 |
+
"motionbricks/out/motionbricks_pose/version_1/skeleton/joints.p",
|
| 48 |
+
"8a582b7020d1609a34a9ea5ddfa597c8727b3e726e38cc80ecefe68171f43ecd",
|
| 49 |
+
),
|
| 50 |
+
"parents": (
|
| 51 |
+
"motionbricks/out/motionbricks_pose/version_1/skeleton/parents.p",
|
| 52 |
+
"4ba0237379480ef33e64b7b8564f1d38dcb3e1ad5bfe01652ea1b7c97ccacb71",
|
| 53 |
+
),
|
| 54 |
+
"mean": (
|
| 55 |
+
"motionbricks/out/motionbricks_pose/version_1/stats/motion/mean.npy",
|
| 56 |
+
"ca390f0081e2373ab71e860a3546cb70cc11fdfac6ce525155403e047b66fdea",
|
| 57 |
+
),
|
| 58 |
+
"std": (
|
| 59 |
+
"motionbricks/out/motionbricks_pose/version_1/stats/motion/std.npy",
|
| 60 |
+
"fca7dd6135cbe96504b1307a7db97e9004556937a2fc72d139077b962cee6bd7",
|
| 61 |
+
),
|
| 62 |
+
}
|
| 63 |
+
|
| 64 |
+
CONFIGS = {
|
| 65 |
+
"vqvae": (
|
| 66 |
+
"motionbricks/out/motionbricks_vqvae/version_1/config.yaml",
|
| 67 |
+
"027a2d7ba5f49cadeadbcc9a6b0c6784d657d5a842e41ff820c5233f1cd6c1f3",
|
| 68 |
+
),
|
| 69 |
+
"pose": (
|
| 70 |
+
"motionbricks/out/motionbricks_pose/version_1/config.yaml",
|
| 71 |
+
"273af770d328b458510ee6049fac8e38fa486c8792cc27b3515dddc094a7cd1f",
|
| 72 |
+
),
|
| 73 |
+
"root": (
|
| 74 |
+
"motionbricks/out/motionbricks_root/version_1/config.yaml",
|
| 75 |
+
"b174f03a333f7f7857c3e2b8a32da517caddd198f228efeea2e203d046ce212a",
|
| 76 |
+
),
|
| 77 |
+
}
|
| 78 |
+
|
| 79 |
+
ALLOWED_PICKLE_GLOBALS = {
|
| 80 |
+
"collections OrderedDict",
|
| 81 |
+
"torch BoolStorage",
|
| 82 |
+
"torch DoubleStorage",
|
| 83 |
+
"torch FloatStorage",
|
| 84 |
+
"torch IntStorage",
|
| 85 |
+
"torch LongStorage",
|
| 86 |
+
"torch._utils _rebuild_tensor_v2",
|
| 87 |
+
}
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
def sha256(path: Path) -> str:
|
| 91 |
+
digest = hashlib.sha256()
|
| 92 |
+
with path.open("rb") as stream:
|
| 93 |
+
while block := stream.read(8 << 20):
|
| 94 |
+
digest.update(block)
|
| 95 |
+
return digest.hexdigest()
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
def verify(path: Path, expected: str) -> None:
|
| 99 |
+
actual = sha256(path)
|
| 100 |
+
if actual != expected:
|
| 101 |
+
raise ValueError(f"SHA-256 mismatch for {path}: {actual}, expected {expected}")
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
def inspect_pickle_globals(path: Path) -> None:
|
| 105 |
+
with ZipFile(path) as archive:
|
| 106 |
+
names = [name for name in archive.namelist() if name.endswith("/data.pkl")]
|
| 107 |
+
if len(names) != 1:
|
| 108 |
+
raise ValueError(f"{path} has {len(names)} data.pkl entries")
|
| 109 |
+
data = archive.read(names[0])
|
| 110 |
+
globals_found = {
|
| 111 |
+
str(argument)
|
| 112 |
+
for opcode, argument, _ in pickletools.genops(data)
|
| 113 |
+
if opcode.name == "GLOBAL"
|
| 114 |
+
}
|
| 115 |
+
unexpected = globals_found - ALLOWED_PICKLE_GLOBALS
|
| 116 |
+
if unexpected:
|
| 117 |
+
raise ValueError(f"unexpected pickle globals in {path}: {sorted(unexpected)}")
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
def plain(value: Any) -> Any:
|
| 121 |
+
if isinstance(value, dict):
|
| 122 |
+
return {str(key): plain(item) for key, item in value.items()}
|
| 123 |
+
if isinstance(value, (list, tuple)):
|
| 124 |
+
return [plain(item) for item in value]
|
| 125 |
+
if value is None or isinstance(value, (str, int, float, bool)):
|
| 126 |
+
return value
|
| 127 |
+
return repr(value)
|
| 128 |
+
|
| 129 |
+
|
| 130 |
+
def tensor_bytes(value: torch.Tensor) -> bytes:
|
| 131 |
+
return value.detach().cpu().contiguous().reshape(-1).view(torch.uint8).numpy().tobytes()
|
| 132 |
+
|
| 133 |
+
|
| 134 |
+
def tensor_record(name: str, value: torch.Tensor) -> dict[str, Any]:
|
| 135 |
+
return {
|
| 136 |
+
"name": name,
|
| 137 |
+
"dtype": str(value.dtype).removeprefix("torch."),
|
| 138 |
+
"shape": [int(item) for item in value.shape],
|
| 139 |
+
"numel": int(value.numel()),
|
| 140 |
+
"sha256": hashlib.sha256(tensor_bytes(value)).hexdigest(),
|
| 141 |
+
}
|
| 142 |
+
|
| 143 |
+
|
| 144 |
+
def write_safetensors(path: Path, tensors: dict[str, torch.Tensor], source_hash: str) -> dict[str, Any]:
|
| 145 |
+
if not tensors:
|
| 146 |
+
raise ValueError(f"refusing to write empty tensor set to {path}")
|
| 147 |
+
ordered = {
|
| 148 |
+
name: value.detach().cpu().contiguous().clone()
|
| 149 |
+
for name, value in sorted(tensors.items())
|
| 150 |
+
}
|
| 151 |
+
temporary = path.with_suffix(path.suffix + ".tmp")
|
| 152 |
+
save_file(
|
| 153 |
+
ordered,
|
| 154 |
+
temporary,
|
| 155 |
+
metadata={
|
| 156 |
+
"format": "motionbricks-safe-v1",
|
| 157 |
+
"upstream_revision": UPSTREAM_REVISION,
|
| 158 |
+
"source_sha256": source_hash,
|
| 159 |
+
},
|
| 160 |
+
)
|
| 161 |
+
os.replace(temporary, path)
|
| 162 |
+
inventory = {
|
| 163 |
+
"path": path.name,
|
| 164 |
+
"sha256": sha256(path),
|
| 165 |
+
"tensor_count": len(ordered),
|
| 166 |
+
"parameter_count": sum(int(value.numel()) for value in ordered.values()),
|
| 167 |
+
"tensors": [tensor_record(name, value) for name, value in ordered.items()],
|
| 168 |
+
}
|
| 169 |
+
del ordered
|
| 170 |
+
return inventory
|
| 171 |
+
|
| 172 |
+
|
| 173 |
+
def load_lightning(path: Path) -> tuple[dict[str, torch.Tensor], dict[str, Any], str]:
|
| 174 |
+
inspect_pickle_globals(path)
|
| 175 |
+
checkpoint = torch.load(path, map_location="cpu", weights_only=False)
|
| 176 |
+
if not isinstance(checkpoint, dict) or "state_dict" not in checkpoint:
|
| 177 |
+
raise ValueError(f"{path} is not a Lightning state dictionary")
|
| 178 |
+
state = checkpoint["state_dict"]
|
| 179 |
+
if not isinstance(state, dict) or any(not isinstance(value, torch.Tensor) for value in state.values()):
|
| 180 |
+
raise ValueError(f"{path} contains an invalid state_dict")
|
| 181 |
+
metadata = {
|
| 182 |
+
"global_step": plain(checkpoint.get("global_step")),
|
| 183 |
+
"epoch": plain(checkpoint.get("epoch")),
|
| 184 |
+
"pytorch_lightning_version": plain(checkpoint.get("pytorch-lightning_version")),
|
| 185 |
+
"hyper_parameters": plain(checkpoint.get("hyper_parameters", {})),
|
| 186 |
+
}
|
| 187 |
+
return state, metadata, sha256(path)
|
| 188 |
+
|
| 189 |
+
|
| 190 |
+
def load_tensor_mapping(path: Path) -> tuple[dict[str, torch.Tensor], str]:
|
| 191 |
+
inspect_pickle_globals(path)
|
| 192 |
+
value = torch.load(path, map_location="cpu", weights_only=False)
|
| 193 |
+
if not isinstance(value, dict) or any(not isinstance(item, torch.Tensor) for item in value.values()):
|
| 194 |
+
raise ValueError(f"{path} is not a tensor mapping")
|
| 195 |
+
return value, sha256(path)
|
| 196 |
+
|
| 197 |
+
|
| 198 |
+
def main() -> None:
|
| 199 |
+
parser = argparse.ArgumentParser()
|
| 200 |
+
parser.add_argument("--upstream-root", type=Path, required=True)
|
| 201 |
+
parser.add_argument("--output", type=Path, required=True)
|
| 202 |
+
args = parser.parse_args()
|
| 203 |
+
|
| 204 |
+
args.output.mkdir(parents=True, exist_ok=True)
|
| 205 |
+
resolved: dict[str, tuple[Path, str]] = {}
|
| 206 |
+
for key, (relative, expected) in INPUTS.items():
|
| 207 |
+
path = args.upstream_root / relative
|
| 208 |
+
verify(path, expected)
|
| 209 |
+
resolved[key] = (path, expected)
|
| 210 |
+
for relative, expected in CONFIGS.values():
|
| 211 |
+
verify(args.upstream_root / relative, expected)
|
| 212 |
+
|
| 213 |
+
manifest: dict[str, Any] = {
|
| 214 |
+
"format": "motionbricks-safe-manifest-v1",
|
| 215 |
+
"upstream_revision": UPSTREAM_REVISION,
|
| 216 |
+
"extractor": {
|
| 217 |
+
"python": os.sys.version,
|
| 218 |
+
"torch": torch.__version__,
|
| 219 |
+
"numpy": np.__version__,
|
| 220 |
+
},
|
| 221 |
+
"inputs": {
|
| 222 |
+
key: {"path": relative, "sha256": expected}
|
| 223 |
+
for key, (relative, expected) in INPUTS.items()
|
| 224 |
+
},
|
| 225 |
+
"configs": {},
|
| 226 |
+
"components": {},
|
| 227 |
+
"checkpoint_metadata": {},
|
| 228 |
+
}
|
| 229 |
+
|
| 230 |
+
for component in ("vqvae", "pose", "root"):
|
| 231 |
+
path, _ = resolved[component]
|
| 232 |
+
state, checkpoint_metadata, source_hash = load_lightning(path)
|
| 233 |
+
manifest["components"][component] = write_safetensors(
|
| 234 |
+
args.output / f"{component}.safetensors", state, source_hash
|
| 235 |
+
)
|
| 236 |
+
manifest["checkpoint_metadata"][component] = checkpoint_metadata
|
| 237 |
+
del state
|
| 238 |
+
gc.collect()
|
| 239 |
+
|
| 240 |
+
clip_path, clip_hash = resolved["clips"]
|
| 241 |
+
clips, _ = load_tensor_mapping(clip_path)
|
| 242 |
+
manifest["components"]["clips"] = write_safetensors(
|
| 243 |
+
args.output / "clips.safetensors", clips, clip_hash
|
| 244 |
+
)
|
| 245 |
+
del clips
|
| 246 |
+
gc.collect()
|
| 247 |
+
|
| 248 |
+
joints_path, joints_hash = resolved["joints"]
|
| 249 |
+
parents_path, _ = resolved["parents"]
|
| 250 |
+
inspect_pickle_globals(joints_path)
|
| 251 |
+
inspect_pickle_globals(parents_path)
|
| 252 |
+
joints = torch.load(joints_path, map_location="cpu", weights_only=False)
|
| 253 |
+
parents = torch.load(parents_path, map_location="cpu", weights_only=False)
|
| 254 |
+
mean = torch.from_numpy(np.load(resolved["mean"][0], allow_pickle=False))
|
| 255 |
+
std = torch.from_numpy(np.load(resolved["std"][0], allow_pickle=False))
|
| 256 |
+
support = {
|
| 257 |
+
"neutral_joints": joints,
|
| 258 |
+
"joint_parents": parents,
|
| 259 |
+
"motion_mean": mean,
|
| 260 |
+
"motion_std": std,
|
| 261 |
+
}
|
| 262 |
+
if any(not isinstance(value, torch.Tensor) for value in support.values()):
|
| 263 |
+
raise ValueError("support inputs did not decode to tensors")
|
| 264 |
+
manifest["components"]["support"] = write_safetensors(
|
| 265 |
+
args.output / "support.safetensors", support, joints_hash
|
| 266 |
+
)
|
| 267 |
+
|
| 268 |
+
for name, (relative, expected) in CONFIGS.items():
|
| 269 |
+
path = args.upstream_root / relative
|
| 270 |
+
value = yaml.safe_load(path.read_text(encoding="utf-8"))
|
| 271 |
+
manifest["configs"][name] = {
|
| 272 |
+
"path": relative,
|
| 273 |
+
"sha256": expected,
|
| 274 |
+
"value": value,
|
| 275 |
+
}
|
| 276 |
+
|
| 277 |
+
manifest_path = args.output / "manifest.json"
|
| 278 |
+
temporary = manifest_path.with_suffix(".json.tmp")
|
| 279 |
+
temporary.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
| 280 |
+
os.replace(temporary, manifest_path)
|
| 281 |
+
|
| 282 |
+
for name, component in manifest["components"].items():
|
| 283 |
+
print(
|
| 284 |
+
f"{name}: {component['tensor_count']} tensors, "
|
| 285 |
+
f"{component['parameter_count']:,} values, {component['sha256']}"
|
| 286 |
+
)
|
| 287 |
+
|
| 288 |
+
|
| 289 |
+
if __name__ == "__main__":
|
| 290 |
+
main()
|
reference/generate_fixtures.py
ADDED
|
@@ -0,0 +1,338 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Generate deterministic PyTorch fixtures from the pinned upstream modules.
|
| 3 |
+
|
| 4 |
+
Run only in the reference container. This script imports NVIDIA's actual
|
| 5 |
+
network classes, loads the verified safetensors intermediates, and records
|
| 6 |
+
inputs, selected layer outputs, and final outputs without opening a pickle.
|
| 7 |
+
"""
|
| 8 |
+
|
| 9 |
+
from __future__ import annotations
|
| 10 |
+
|
| 11 |
+
import argparse
|
| 12 |
+
import hashlib
|
| 13 |
+
import json
|
| 14 |
+
import os
|
| 15 |
+
import struct
|
| 16 |
+
import sys
|
| 17 |
+
from pathlib import Path
|
| 18 |
+
from types import SimpleNamespace
|
| 19 |
+
from typing import Any
|
| 20 |
+
|
| 21 |
+
import numpy as np
|
| 22 |
+
import torch
|
| 23 |
+
from safetensors import safe_open
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
UPSTREAM_REVISION = "a0732b642c0333077e127a2f56ab0014c196bca4"
|
| 27 |
+
SAFE_HASHES = {
|
| 28 |
+
"pose": "01327768be7413111dc927947a95cfb3e9ee7c52acfa35a054e8c2f3b838b888",
|
| 29 |
+
"root": "d1529a8c9da915cb7dd0499272baba61db480660c0aae92b67fbe69828e83c5a",
|
| 30 |
+
"vqvae": "544782e605ed96d60bf999243ef8f44640ea021a5655ef20af2d2853c3e18b5b",
|
| 31 |
+
}
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def sha256(path: Path) -> str:
|
| 35 |
+
digest = hashlib.sha256()
|
| 36 |
+
with path.open("rb") as stream:
|
| 37 |
+
while block := stream.read(8 << 20):
|
| 38 |
+
digest.update(block)
|
| 39 |
+
return digest.hexdigest()
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def verify(path: Path, expected: str) -> None:
|
| 43 |
+
actual = sha256(path)
|
| 44 |
+
if actual != expected:
|
| 45 |
+
raise ValueError(f"SHA-256 mismatch for {path}: {actual}, expected {expected}")
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
class Rep:
|
| 49 |
+
pass
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def make_motion_rep() -> Rep:
|
| 53 |
+
"""Minimal shape-compatible rep used only by upstream constructors."""
|
| 54 |
+
global_rep, local_rep, combined = Rep(), Rep(), Rep()
|
| 55 |
+
global_rep.root_mode = "global"
|
| 56 |
+
global_rep.indices = {
|
| 57 |
+
"all": np.arange(418),
|
| 58 |
+
"root": np.arange(5),
|
| 59 |
+
"global_root_pos": np.arange(3),
|
| 60 |
+
"global_root_pos_2d": np.arange(2),
|
| 61 |
+
"global_root_heading": np.arange(2, 4),
|
| 62 |
+
"ric_data": np.arange(99),
|
| 63 |
+
"global_rot_data": np.arange(99, 303),
|
| 64 |
+
}
|
| 65 |
+
local_rep.root_mode = "local"
|
| 66 |
+
local_rep.indices = {
|
| 67 |
+
"all": np.arange(413),
|
| 68 |
+
"root": np.arange(4),
|
| 69 |
+
"local_root_rot_vel": np.arange(2),
|
| 70 |
+
"local_root_vel": np.arange(2, 4),
|
| 71 |
+
"global_root_y": np.arange(1),
|
| 72 |
+
"ric_data": np.arange(99),
|
| 73 |
+
"global_rot_data": np.arange(99, 303),
|
| 74 |
+
}
|
| 75 |
+
dual = SimpleNamespace(global_motion_rep=global_rep, local_motion_rep=local_rep)
|
| 76 |
+
combined.dual_rep = dual
|
| 77 |
+
combined.root_mode = "global"
|
| 78 |
+
combined.indices = global_rep.indices
|
| 79 |
+
return combined
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
def state(path: Path, prefix: str, select=lambda _name: True) -> dict[str, torch.Tensor]:
|
| 83 |
+
result: dict[str, torch.Tensor] = {}
|
| 84 |
+
with safe_open(path, framework="pt", device="cpu") as source:
|
| 85 |
+
for name in source.keys():
|
| 86 |
+
if name.startswith(prefix) and select(name):
|
| 87 |
+
result[name.removeprefix(prefix)] = source.get_tensor(name)
|
| 88 |
+
return result
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
def ramp(shape: tuple[int, ...], scale: float, phase: float = 0.0) -> torch.Tensor:
|
| 92 |
+
count = int(np.prod(shape))
|
| 93 |
+
values = torch.arange(count, dtype=torch.float32)
|
| 94 |
+
return (torch.sin(values * 0.017 + phase) * scale).reshape(shape)
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
def trace_modules(model: torch.nn.Module, output: dict[str, torch.Tensor]) -> list[Any]:
|
| 98 |
+
handles = []
|
| 99 |
+
traced_types = (torch.nn.Linear, torch.nn.Conv1d, torch.nn.TransformerEncoderLayer)
|
| 100 |
+
for name, module in model.named_modules():
|
| 101 |
+
if not name or not isinstance(module, traced_types):
|
| 102 |
+
continue
|
| 103 |
+
|
| 104 |
+
def capture(_module, _inputs, value, trace_name=name):
|
| 105 |
+
if isinstance(value, torch.Tensor):
|
| 106 |
+
if value.is_nested:
|
| 107 |
+
value = value.to_padded_tensor(0.0)
|
| 108 |
+
output[f"trace.{trace_name}"] = value.detach().cpu().contiguous()
|
| 109 |
+
|
| 110 |
+
handles.append(module.register_forward_hook(capture))
|
| 111 |
+
return handles
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
def pose_fixture(pose_type, safe: Path) -> dict[str, torch.Tensor]:
|
| 115 |
+
args = {
|
| 116 |
+
"cond_root_feature": "root_without_hip_height",
|
| 117 |
+
"cond_root_feature_is_from_motion_rep": "global",
|
| 118 |
+
"down_t": 2,
|
| 119 |
+
"local_pose_feature": "joint_positions_and_rotations_and_hip_height",
|
| 120 |
+
"max_tokens": 16,
|
| 121 |
+
"min_tokens": 6,
|
| 122 |
+
"n_embd": 1024,
|
| 123 |
+
"n_head": 16,
|
| 124 |
+
"n_layers": 16,
|
| 125 |
+
"pose_feat_width": 640,
|
| 126 |
+
"pose_root_mode": "pose",
|
| 127 |
+
"pose_token_mlp_num_layers": 2,
|
| 128 |
+
"pose_vqvae": {"code_dim": 256, "has_codebook": True, "nb_code": 100_000_000, "num_heads": 8},
|
| 129 |
+
"root_feat_width": 256,
|
| 130 |
+
"root_vqvae": {"code_dim": 60, "nb_code": 1024, "num_heads": 1},
|
| 131 |
+
"text_emb_dim": 4096,
|
| 132 |
+
"text_embeddings": None,
|
| 133 |
+
"token_length_feat_width": 128,
|
| 134 |
+
}
|
| 135 |
+
model = pose_type(make_motion_rep(), args).eval()
|
| 136 |
+
missing, unexpected = model.load_state_dict(state(safe / "pose.safetensors", "backbone_net."), strict=True)
|
| 137 |
+
if missing or unexpected:
|
| 138 |
+
raise ValueError(f"pose state mismatch: missing={missing}, unexpected={unexpected}")
|
| 139 |
+
|
| 140 |
+
result: dict[str, torch.Tensor] = {}
|
| 141 |
+
pose_tokens = (torch.arange(48).reshape(1, 6, 8) * 7 % 11).to(torch.int64)
|
| 142 |
+
local_root = ramp((1, 24, 4), 0.35, 0.1)
|
| 143 |
+
pose_cond = ramp((1, 24, 304), 0.2, 0.7)
|
| 144 |
+
has_pose = torch.zeros((1, 24), dtype=torch.bool)
|
| 145 |
+
has_pose[:, [0, 1, 22, 23]] = True
|
| 146 |
+
num_tokens = torch.tensor([[6]], dtype=torch.int64)
|
| 147 |
+
result.update({
|
| 148 |
+
"input.pose_tokens": pose_tokens,
|
| 149 |
+
"input.local_root_values": local_root,
|
| 150 |
+
"input.pose_cond": pose_cond,
|
| 151 |
+
"input.has_pose_cond": has_pose,
|
| 152 |
+
"input.num_tokens": num_tokens,
|
| 153 |
+
})
|
| 154 |
+
handles = trace_modules(model, result)
|
| 155 |
+
with torch.inference_mode():
|
| 156 |
+
output = model(pose_tokens, local_root, pose_cond, has_pose, num_tokens)
|
| 157 |
+
result["output.pose_logits"] = output["pose_logits"].contiguous()
|
| 158 |
+
for handle in handles:
|
| 159 |
+
handle.remove()
|
| 160 |
+
return result
|
| 161 |
+
|
| 162 |
+
|
| 163 |
+
def root_fixture(root_type, safe: Path) -> dict[str, torch.Tensor]:
|
| 164 |
+
args = {
|
| 165 |
+
"activation": "relu", "depth": 4, "dilation_growth_rate": 3, "down_t": 2,
|
| 166 |
+
"global_root_feat_dim": 64, "global_root_feature": "root",
|
| 167 |
+
"input_feat_mlp_num_layers": 2,
|
| 168 |
+
"local_pose_feature": "joint_positions_and_rotations_and_hip_height",
|
| 169 |
+
"local_root_feat_dim": 64, "local_root_feature": "root",
|
| 170 |
+
"max_tokens": 16, "min_tokens": 6, "n_embd": 512, "n_head": 16,
|
| 171 |
+
"n_layers_root_token": 3, "n_layers_shared": 3, "norm": "None",
|
| 172 |
+
"pose_feat_dim": 256,
|
| 173 |
+
"pose_vqvae": {"code_dim": 60, "nb_code": 1024, "num_heads": 1},
|
| 174 |
+
"root_vqvae": {"code_dim": 60, "nb_code": 1024, "num_heads": 1},
|
| 175 |
+
"text_emb_dim": 4096, "text_embeddings": None,
|
| 176 |
+
"use_hard_num_token_emb_for_root_prediction": True, "width": 512,
|
| 177 |
+
}
|
| 178 |
+
model = root_type(args, make_motion_rep()).eval()
|
| 179 |
+
missing, unexpected = model.load_state_dict(state(safe / "root.safetensors", "backbone_net."), strict=True)
|
| 180 |
+
if missing or unexpected:
|
| 181 |
+
raise ValueError(f"root state mismatch: missing={missing}, unexpected={unexpected}")
|
| 182 |
+
|
| 183 |
+
result: dict[str, torch.Tensor] = {}
|
| 184 |
+
global_root = ramp((1, 8, 5), 0.3, 0.2)
|
| 185 |
+
local_root = ramp((1, 8, 4), 0.25, 0.9)
|
| 186 |
+
poses = ramp((1, 8, 304), 0.15, 1.7)
|
| 187 |
+
has_global = torch.tensor([[1, 1, 1, 1, 0, 0, 1, 1]], dtype=torch.bool)
|
| 188 |
+
has_local = torch.tensor([[1, 1, 1, 1, 0, 0, 1, 1]], dtype=torch.bool)
|
| 189 |
+
has_poses = torch.tensor([[1, 1, 1, 1, 0, 0, 1, 1]], dtype=torch.bool)
|
| 190 |
+
num_tokens = torch.tensor([[6]], dtype=torch.int64)
|
| 191 |
+
result.update({
|
| 192 |
+
"input.global_root_values": global_root, "input.has_global_root_values": has_global,
|
| 193 |
+
"input.local_root_values": local_root, "input.has_local_root_values": has_local,
|
| 194 |
+
"input.poses": poses, "input.has_poses": has_poses, "input.num_tokens": num_tokens,
|
| 195 |
+
})
|
| 196 |
+
handles = trace_modules(model, result)
|
| 197 |
+
with torch.inference_mode():
|
| 198 |
+
output = model(global_root, has_global, local_root, has_local, poses, has_poses, num_tokens)
|
| 199 |
+
result["output.num_token_logits"] = output["num_token_logits"].contiguous()
|
| 200 |
+
result["output.pred_num_tokens"] = output["pred_num_tokens"].contiguous()
|
| 201 |
+
result["output.pred_global_root_values"] = output["pred_global_root_values"].contiguous()
|
| 202 |
+
for handle in handles:
|
| 203 |
+
handle.remove()
|
| 204 |
+
return result
|
| 205 |
+
|
| 206 |
+
|
| 207 |
+
def decoder_fixture(decoder_type, safe: Path) -> dict[str, torch.Tensor]:
|
| 208 |
+
model = decoder_type(
|
| 209 |
+
input_emb_width=413, output_emb_width=256, down_t=2, width=512, depth=4,
|
| 210 |
+
dilation_growth_rate=3, activation="relu", norm="None",
|
| 211 |
+
target_cond_dim=304, external_cond_dim=2, cond_fusion_last_layer=False,
|
| 212 |
+
).eval()
|
| 213 |
+
decoder_state = state(safe / "vqvae.safetensors", "pose_net.decoder.")
|
| 214 |
+
missing, unexpected = model.load_state_dict(decoder_state, strict=True)
|
| 215 |
+
if missing or unexpected:
|
| 216 |
+
raise ValueError(f"decoder state mismatch: missing={missing}, unexpected={unexpected}")
|
| 217 |
+
codebook = state(
|
| 218 |
+
safe / "vqvae.safetensors", "pose_net.",
|
| 219 |
+
lambda name: name == "pose_net.quantizer.vq._codebook.embed",
|
| 220 |
+
)["quantizer.vq._codebook.embed"]
|
| 221 |
+
|
| 222 |
+
result: dict[str, torch.Tensor] = {}
|
| 223 |
+
indices = (torch.arange(48).reshape(1, 6, 8) * 3 + 1).remainder(10).to(torch.int64)
|
| 224 |
+
batch_index = torch.arange(indices.shape[0])[:, None, None]
|
| 225 |
+
head_index = torch.arange(8)[None, None, :]
|
| 226 |
+
quantized = codebook[head_index, indices].reshape(1, 6, 256).transpose(1, 2).contiguous()
|
| 227 |
+
target = ramp((1, 24, 304), 0.18, 0.4)
|
| 228 |
+
has_target = torch.zeros((1, 24), dtype=torch.bool)
|
| 229 |
+
has_target[:, [0, 1, 22, 23]] = True
|
| 230 |
+
external = ramp((1, 24, 2), 0.22, 1.1)
|
| 231 |
+
token_mask = torch.ones((1, 6), dtype=torch.bool)
|
| 232 |
+
result.update({
|
| 233 |
+
"input.indices": indices, "input.quantized": quantized,
|
| 234 |
+
"input.target_cond": target, "input.has_target_cond": has_target,
|
| 235 |
+
"input.external_cond": external, "input.token_mask": token_mask,
|
| 236 |
+
})
|
| 237 |
+
handles = trace_modules(model, result)
|
| 238 |
+
with torch.inference_mode():
|
| 239 |
+
result["output.motion"] = model(
|
| 240 |
+
quantized, external_cond=external, target_cond=target,
|
| 241 |
+
has_target_cond=has_target, token_mask=token_mask,
|
| 242 |
+
).transpose(1, 2).contiguous()
|
| 243 |
+
for handle in handles:
|
| 244 |
+
handle.remove()
|
| 245 |
+
return result
|
| 246 |
+
|
| 247 |
+
|
| 248 |
+
def tensor_record(name: str, value: torch.Tensor) -> dict[str, Any]:
|
| 249 |
+
raw = value.detach().cpu().contiguous().reshape(-1).view(torch.uint8).numpy().tobytes()
|
| 250 |
+
return {
|
| 251 |
+
"name": name, "dtype": str(value.dtype).removeprefix("torch."),
|
| 252 |
+
"shape": list(value.shape), "numel": value.numel(),
|
| 253 |
+
"sha256": hashlib.sha256(raw).hexdigest(),
|
| 254 |
+
}
|
| 255 |
+
|
| 256 |
+
|
| 257 |
+
def write_fixture(path: Path, tensors: dict[str, torch.Tensor]) -> dict[str, Any]:
|
| 258 |
+
ordered = {name: value.detach().cpu().contiguous().clone() for name, value in sorted(tensors.items())}
|
| 259 |
+
dtype_names = {
|
| 260 |
+
torch.float32: "F32", torch.int64: "I64", torch.int32: "I32", torch.bool: "BOOL",
|
| 261 |
+
}
|
| 262 |
+
header: dict[str, Any] = {
|
| 263 |
+
"__metadata__": {
|
| 264 |
+
"format": "motionbricks-reference-fixture-v1",
|
| 265 |
+
"torch": torch.__version__,
|
| 266 |
+
"upstream_revision": UPSTREAM_REVISION,
|
| 267 |
+
}
|
| 268 |
+
}
|
| 269 |
+
payloads: list[bytes] = []
|
| 270 |
+
offset = 0
|
| 271 |
+
for name, value in ordered.items():
|
| 272 |
+
if value.dtype not in dtype_names:
|
| 273 |
+
raise ValueError(f"unsupported fixture dtype for {name}: {value.dtype}")
|
| 274 |
+
payload = value.reshape(-1).view(torch.uint8).numpy().tobytes()
|
| 275 |
+
header[name] = {
|
| 276 |
+
"dtype": dtype_names[value.dtype], "shape": list(value.shape),
|
| 277 |
+
"data_offsets": [offset, offset + len(payload)],
|
| 278 |
+
}
|
| 279 |
+
payloads.append(payload)
|
| 280 |
+
offset += len(payload)
|
| 281 |
+
encoded_header = json.dumps(header, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
|
| 282 |
+
encoded_header += b" " * (-len(encoded_header) % 8)
|
| 283 |
+
temporary = path.with_suffix(path.suffix + ".tmp")
|
| 284 |
+
with temporary.open("wb") as stream:
|
| 285 |
+
stream.write(struct.pack("<Q", len(encoded_header)))
|
| 286 |
+
stream.write(encoded_header)
|
| 287 |
+
for payload in payloads:
|
| 288 |
+
stream.write(payload)
|
| 289 |
+
stream.flush()
|
| 290 |
+
os.fsync(stream.fileno())
|
| 291 |
+
os.replace(temporary, path)
|
| 292 |
+
return {
|
| 293 |
+
"path": path.name, "sha256": sha256(path), "tensor_count": len(ordered),
|
| 294 |
+
"tensors": [tensor_record(name, value) for name, value in ordered.items()],
|
| 295 |
+
}
|
| 296 |
+
|
| 297 |
+
|
| 298 |
+
def main() -> None:
|
| 299 |
+
parser = argparse.ArgumentParser()
|
| 300 |
+
parser.add_argument("--upstream-root", type=Path, required=True)
|
| 301 |
+
parser.add_argument("--safe-directory", type=Path, required=True)
|
| 302 |
+
parser.add_argument("--output", type=Path, required=True)
|
| 303 |
+
args = parser.parse_args()
|
| 304 |
+
if not Path("/.dockerenv").exists():
|
| 305 |
+
raise RuntimeError("reference fixtures must be generated inside the container")
|
| 306 |
+
for name, expected in SAFE_HASHES.items():
|
| 307 |
+
verify(args.safe_directory / f"{name}.safetensors", expected)
|
| 308 |
+
|
| 309 |
+
sys.path.insert(0, str(args.upstream_root / "motionbricks"))
|
| 310 |
+
from motionbricks.motion_backbone.neural_modules.pose_backbone import pose_backbone_network
|
| 311 |
+
from motionbricks.motion_backbone.neural_modules.root_backbone import root_backbone_network
|
| 312 |
+
from motionbricks.vqvae.neural_modules.encdec_double_cond import DoubleCondDecoder
|
| 313 |
+
|
| 314 |
+
torch.manual_seed(7)
|
| 315 |
+
torch.set_num_threads(1)
|
| 316 |
+
torch.use_deterministic_algorithms(True)
|
| 317 |
+
args.output.mkdir(parents=True, exist_ok=True)
|
| 318 |
+
generated = {
|
| 319 |
+
"pose": write_fixture(args.output / "pose.safetensors", pose_fixture(pose_backbone_network, args.safe_directory)),
|
| 320 |
+
"root": write_fixture(args.output / "root.safetensors", root_fixture(root_backbone_network, args.safe_directory)),
|
| 321 |
+
"vq-decoder": write_fixture(
|
| 322 |
+
args.output / "vq-decoder.safetensors", decoder_fixture(DoubleCondDecoder, args.safe_directory)
|
| 323 |
+
),
|
| 324 |
+
}
|
| 325 |
+
manifest = {
|
| 326 |
+
"format": "motionbricks-reference-fixtures-v1",
|
| 327 |
+
"upstream_revision": UPSTREAM_REVISION,
|
| 328 |
+
"torch": torch.__version__, "numpy": np.__version__, "fixtures": generated,
|
| 329 |
+
}
|
| 330 |
+
temporary = args.output / "manifest.json.tmp"
|
| 331 |
+
temporary.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
| 332 |
+
os.replace(temporary, args.output / "manifest.json")
|
| 333 |
+
for name, record in generated.items():
|
| 334 |
+
print(f"{name}: {record['tensor_count']} tensors, {record['sha256']}")
|
| 335 |
+
|
| 336 |
+
|
| 337 |
+
if __name__ == "__main__":
|
| 338 |
+
main()
|
scripts/convert_fixtures_to_gguf.py
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Package deterministic safetensors reference fixtures as GGUF v3."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
import argparse
|
| 7 |
+
import importlib.util
|
| 8 |
+
import sys
|
| 9 |
+
from pathlib import Path
|
| 10 |
+
|
| 11 |
+
from safetensors import safe_open
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
def load_converter(path: Path):
|
| 15 |
+
spec = importlib.util.spec_from_file_location("motionbricks_gguf_converter", path)
|
| 16 |
+
if spec is None or spec.loader is None:
|
| 17 |
+
raise RuntimeError(f"cannot import {path}")
|
| 18 |
+
module = importlib.util.module_from_spec(spec)
|
| 19 |
+
sys.modules[spec.name] = module
|
| 20 |
+
spec.loader.exec_module(module)
|
| 21 |
+
return module
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
def main() -> None:
|
| 25 |
+
parser = argparse.ArgumentParser()
|
| 26 |
+
parser.add_argument("--fixtures", type=Path, required=True)
|
| 27 |
+
parser.add_argument("--output", type=Path, required=True)
|
| 28 |
+
args = parser.parse_args()
|
| 29 |
+
converter = load_converter(Path(__file__).with_name("convert_to_gguf.py"))
|
| 30 |
+
args.output.mkdir(parents=True, exist_ok=True)
|
| 31 |
+
for fixture in ("pose", "root", "vq-decoder"):
|
| 32 |
+
source_path = args.fixtures / f"{fixture}.safetensors"
|
| 33 |
+
values = []
|
| 34 |
+
with safe_open(source_path, framework="numpy") as source:
|
| 35 |
+
for name in source.keys():
|
| 36 |
+
values.append((name, source.get_tensor(name)))
|
| 37 |
+
converter.write_component(
|
| 38 |
+
args.output / f"{fixture}.gguf", f"fixture-{fixture}", values,
|
| 39 |
+
converter.sha256(source_path),
|
| 40 |
+
)
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
if __name__ == "__main__":
|
| 44 |
+
main()
|
scripts/convert_styles.py
ADDED
|
@@ -0,0 +1,85 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Convert the verified upstream G1 exemplar clips into checked style GGUFs."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
import argparse
|
| 7 |
+
import importlib.util
|
| 8 |
+
import json
|
| 9 |
+
import sys
|
| 10 |
+
from pathlib import Path
|
| 11 |
+
|
| 12 |
+
import numpy as np
|
| 13 |
+
from safetensors import safe_open
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
CLIP_SHA256 = "7ea40c20130c8a9799d83c0438750b51a5be9b1e09357c4928887961fc992d33"
|
| 17 |
+
STYLES = (
|
| 18 |
+
("idle", 0.0, 6), ("slow_walk", 0.3, 6), ("walk", 1.0, 6),
|
| 19 |
+
("hand_crawling", 0.5, 6), ("walk_boxing", 1.0, 11),
|
| 20 |
+
("elbow_crawling", 0.8, 11), ("stealth_walk", 1.0, 6),
|
| 21 |
+
("injured_walk", 0.5, 6), ("walk_stealth", 0.7, 6),
|
| 22 |
+
("walk_happy_dance", 1.0, 6), ("walk_zombie", 0.6, 6),
|
| 23 |
+
("walk_gun", 0.6, 6), ("walk_scared", 0.6, 6),
|
| 24 |
+
("walk_left", 0.2, 4), ("walk_right", 0.2, 4),
|
| 25 |
+
)
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def converter_module(path: Path):
|
| 29 |
+
spec = importlib.util.spec_from_file_location("motionbricks_converter", path)
|
| 30 |
+
if spec is None or spec.loader is None:
|
| 31 |
+
raise RuntimeError(f"cannot import {path}")
|
| 32 |
+
module = importlib.util.module_from_spec(spec)
|
| 33 |
+
sys.modules[spec.name] = module
|
| 34 |
+
spec.loader.exec_module(module)
|
| 35 |
+
return module
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def main() -> None:
|
| 39 |
+
parser = argparse.ArgumentParser()
|
| 40 |
+
parser.add_argument("--safe-directory", type=Path, required=True)
|
| 41 |
+
parser.add_argument("--output", type=Path, required=True)
|
| 42 |
+
args = parser.parse_args()
|
| 43 |
+
converter = converter_module(Path(__file__).with_name("convert_to_gguf.py"))
|
| 44 |
+
source = args.safe_directory / "clips.safetensors"
|
| 45 |
+
if converter.sha256(source) != CLIP_SHA256:
|
| 46 |
+
raise ValueError("upstream clip safetensors hash mismatch")
|
| 47 |
+
with safe_open(source, framework="numpy") as clips:
|
| 48 |
+
positions = clips.get_tensor("global_joint_positions")
|
| 49 |
+
rotations = clips.get_tensor("global_joint_rotations")
|
| 50 |
+
roots = clips.get_tensor("global_root_positions")
|
| 51 |
+
headings = clips.get_tensor("global_headings")
|
| 52 |
+
lengths = clips.get_tensor("num_frames_per_clip")
|
| 53 |
+
headings = headings.copy()
|
| 54 |
+
headings[3, 0] = 0.0
|
| 55 |
+
headings[5] -= 0.95
|
| 56 |
+
args.output.mkdir(parents=True, exist_ok=True)
|
| 57 |
+
records = {}
|
| 58 |
+
for index, (name, speed, allowed_count) in enumerate(STYLES):
|
| 59 |
+
frames = int(lengths[index])
|
| 60 |
+
allowed = np.zeros(11, dtype=np.int32)
|
| 61 |
+
allowed[:allowed_count] = 1
|
| 62 |
+
values = [
|
| 63 |
+
("global_joint_positions", positions[index, :frames]),
|
| 64 |
+
("global_joint_rotations", rotations[index, :frames].reshape(frames, 34, 9)),
|
| 65 |
+
("global_root_positions", roots[index, :frames]),
|
| 66 |
+
("global_headings", headings[index, :frames]),
|
| 67 |
+
("allowed_tokens", allowed),
|
| 68 |
+
]
|
| 69 |
+
records[name] = converter.write_component(
|
| 70 |
+
args.output / f"{name}.mbstyle", "style", values, CLIP_SHA256,
|
| 71 |
+
general_name=name,
|
| 72 |
+
extra_metadata=[
|
| 73 |
+
converter.kv_string("motionbricks.style_name", name),
|
| 74 |
+
converter.kv_f32("motionbricks.style_speed", speed),
|
| 75 |
+
converter.kv_u32("motionbricks.style_frames", frames),
|
| 76 |
+
converter.kv_u32("motionbricks.style_index", index),
|
| 77 |
+
],
|
| 78 |
+
)
|
| 79 |
+
(args.output / "manifest.json").write_text(
|
| 80 |
+
json.dumps({"format": "motionbricks-style-bundle-v1", "styles": records},
|
| 81 |
+
indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
if __name__ == "__main__":
|
| 85 |
+
main()
|
scripts/convert_to_gguf.py
ADDED
|
@@ -0,0 +1,307 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Convert verified MotionBricks safetensors into inference-only GGUF v3.
|
| 3 |
+
|
| 4 |
+
This converter deliberately rejects PyTorch checkpoint files. The pinned
|
| 5 |
+
reference container must first extract the upstream pickle archives into the
|
| 6 |
+
safe, deterministic safetensors lane documented in reference/README.md.
|
| 7 |
+
"""
|
| 8 |
+
|
| 9 |
+
from __future__ import annotations
|
| 10 |
+
|
| 11 |
+
import argparse
|
| 12 |
+
import hashlib
|
| 13 |
+
import json
|
| 14 |
+
import os
|
| 15 |
+
import struct
|
| 16 |
+
from dataclasses import dataclass
|
| 17 |
+
from pathlib import Path
|
| 18 |
+
from typing import Callable
|
| 19 |
+
|
| 20 |
+
import numpy as np
|
| 21 |
+
from safetensors import safe_open
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
GGUF_MAGIC = b"GGUF"
|
| 25 |
+
GGUF_VERSION = 3
|
| 26 |
+
ALIGNMENT = 32
|
| 27 |
+
|
| 28 |
+
GGML_TYPE_F32 = 0
|
| 29 |
+
# ggml v0.20.2 / GGUF v3 enum value (the submodule is pinned with the bundle).
|
| 30 |
+
GGML_TYPE_I32 = 26
|
| 31 |
+
GGUF_TYPE_UINT32 = 4
|
| 32 |
+
GGUF_TYPE_FLOAT32 = 6
|
| 33 |
+
GGUF_TYPE_STRING = 8
|
| 34 |
+
GGUF_TYPE_UINT64 = 10
|
| 35 |
+
|
| 36 |
+
UPSTREAM_REVISION = "a0732b642c0333077e127a2f56ab0014c196bca4"
|
| 37 |
+
SAFE_FORMAT = "motionbricks-safe-manifest-v1"
|
| 38 |
+
BUNDLE_FORMAT = "motionbricks-gguf-bundle-v1"
|
| 39 |
+
SAFE_HASHES = {
|
| 40 |
+
"pose": "01327768be7413111dc927947a95cfb3e9ee7c52acfa35a054e8c2f3b838b888",
|
| 41 |
+
"root": "d1529a8c9da915cb7dd0499272baba61db480660c0aae92b67fbe69828e83c5a",
|
| 42 |
+
"vqvae": "544782e605ed96d60bf999243ef8f44640ea021a5655ef20af2d2853c3e18b5b",
|
| 43 |
+
"support": "229b764411652b2ab0f824481d6daf897f701a97223029759444fc5bc241ea22",
|
| 44 |
+
}
|
| 45 |
+
|
| 46 |
+
JOINT_NAMES = (
|
| 47 |
+
"pelvis_skel",
|
| 48 |
+
"left_hip_pitch_skel", "left_hip_roll_skel", "left_hip_yaw_skel",
|
| 49 |
+
"left_knee_skel", "left_ankle_pitch_skel", "left_ankle_roll_skel", "left_toe_base",
|
| 50 |
+
"right_hip_pitch_skel", "right_hip_roll_skel", "right_hip_yaw_skel",
|
| 51 |
+
"right_knee_skel", "right_ankle_pitch_skel", "right_ankle_roll_skel", "right_toe_base",
|
| 52 |
+
"waist_yaw_skel", "waist_roll_skel", "waist_pitch_skel",
|
| 53 |
+
"left_shoulder_pitch_skel", "left_shoulder_roll_skel", "left_shoulder_yaw_skel",
|
| 54 |
+
"left_elbow_skel", "left_wrist_roll_skel", "left_wrist_pitch_skel",
|
| 55 |
+
"left_wrist_yaw_skel", "left_hand_roll_skel",
|
| 56 |
+
"right_shoulder_pitch_skel", "right_shoulder_roll_skel", "right_shoulder_yaw_skel",
|
| 57 |
+
"right_elbow_skel", "right_wrist_roll_skel", "right_wrist_pitch_skel",
|
| 58 |
+
"right_wrist_yaw_skel", "right_hand_roll_skel",
|
| 59 |
+
)
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
def encoded(value: str) -> bytes:
|
| 63 |
+
data = value.encode("utf-8")
|
| 64 |
+
return struct.pack("<Q", len(data)) + data
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
def kv_string(key: str, value: str) -> bytes:
|
| 68 |
+
return encoded(key) + struct.pack("<I", GGUF_TYPE_STRING) + encoded(value)
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
def kv_u32(key: str, value: int) -> bytes:
|
| 72 |
+
return encoded(key) + struct.pack("<II", GGUF_TYPE_UINT32, value)
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
def kv_f32(key: str, value: float) -> bytes:
|
| 76 |
+
return encoded(key) + struct.pack("<If", GGUF_TYPE_FLOAT32, value)
|
| 77 |
+
|
| 78 |
+
|
| 79 |
+
def kv_u64(key: str, value: int) -> bytes:
|
| 80 |
+
return encoded(key) + struct.pack("<IQ", GGUF_TYPE_UINT64, value)
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
def aligned(value: int) -> int:
|
| 84 |
+
return (value + ALIGNMENT - 1) // ALIGNMENT * ALIGNMENT
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
def sha256(path: Path) -> str:
|
| 88 |
+
digest = hashlib.sha256()
|
| 89 |
+
with path.open("rb") as stream:
|
| 90 |
+
while block := stream.read(8 << 20):
|
| 91 |
+
digest.update(block)
|
| 92 |
+
return digest.hexdigest()
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
def compact_name(name: str) -> str:
|
| 96 |
+
"""Keep upstream names readable while respecting GGML_MAX_NAME (64)."""
|
| 97 |
+
if len(name.encode("utf-8")) < 64:
|
| 98 |
+
return name
|
| 99 |
+
replacements = (
|
| 100 |
+
("_root_token_transformer_encoder.layers.", "root.l."),
|
| 101 |
+
("_shared_transformer_encoder.layers.", "shared.l."),
|
| 102 |
+
("_transformer_encoder.layers.", "transformer.l."),
|
| 103 |
+
("self_attn.", "attn."),
|
| 104 |
+
("in_proj_weight", "qkv.weight"),
|
| 105 |
+
("in_proj_bias", "qkv.bias"),
|
| 106 |
+
("external_cond_blocks.", "xcond."),
|
| 107 |
+
("target_cond_blocks.", "tcond."),
|
| 108 |
+
)
|
| 109 |
+
output = name
|
| 110 |
+
for old, new in replacements:
|
| 111 |
+
output = output.replace(old, new)
|
| 112 |
+
if len(output.encode("utf-8")) >= 64:
|
| 113 |
+
suffix = hashlib.sha256(name.encode("utf-8")).hexdigest()[:12]
|
| 114 |
+
output = output.encode("utf-8")[:48].decode("utf-8", "ignore") + "." + suffix
|
| 115 |
+
return output
|
| 116 |
+
|
| 117 |
+
|
| 118 |
+
@dataclass
|
| 119 |
+
class Tensor:
|
| 120 |
+
name: str
|
| 121 |
+
value: np.ndarray
|
| 122 |
+
kind: int
|
| 123 |
+
dimensions: tuple[int, ...]
|
| 124 |
+
offset: int
|
| 125 |
+
|
| 126 |
+
@property
|
| 127 |
+
def size(self) -> int:
|
| 128 |
+
return int(self.value.size) * 4
|
| 129 |
+
|
| 130 |
+
def bytes(self) -> bytes:
|
| 131 |
+
dtype = "<f4" if self.kind == GGML_TYPE_F32 else "<i4"
|
| 132 |
+
return np.asarray(self.value, dtype=dtype, order="C").tobytes(order="C")
|
| 133 |
+
|
| 134 |
+
|
| 135 |
+
def load_selected(path: Path, select: Callable[[str], bool], strip: str) -> list[tuple[str, np.ndarray]]:
|
| 136 |
+
output: list[tuple[str, np.ndarray]] = []
|
| 137 |
+
with safe_open(path, framework="numpy") as source:
|
| 138 |
+
for original in source.keys():
|
| 139 |
+
if not select(original):
|
| 140 |
+
continue
|
| 141 |
+
name = original.removeprefix(strip)
|
| 142 |
+
value = source.get_tensor(original)
|
| 143 |
+
output.append((name, value))
|
| 144 |
+
if not output:
|
| 145 |
+
raise ValueError(f"selection produced no tensors from {path}")
|
| 146 |
+
return output
|
| 147 |
+
|
| 148 |
+
|
| 149 |
+
def tensor_kind(component: str, name: str, value: np.ndarray) -> int:
|
| 150 |
+
if np.issubdtype(value.dtype, np.floating):
|
| 151 |
+
return GGML_TYPE_F32
|
| 152 |
+
if np.issubdtype(value.dtype, np.integer) or np.issubdtype(value.dtype, np.bool_):
|
| 153 |
+
return GGML_TYPE_I32
|
| 154 |
+
raise ValueError(f"unsupported tensor dtype for {component}:{name}: {value.dtype}")
|
| 155 |
+
|
| 156 |
+
|
| 157 |
+
def write_component(path: Path, component: str, values: list[tuple[str, np.ndarray]],
|
| 158 |
+
source_hash: str, *, general_name: str = "NVIDIA MotionBricks G1",
|
| 159 |
+
extra_metadata: list[bytes] | None = None) -> dict[str, object]:
|
| 160 |
+
tensors: list[Tensor] = []
|
| 161 |
+
names: set[str] = set()
|
| 162 |
+
offset = 0
|
| 163 |
+
parameter_count = 0
|
| 164 |
+
for original_name, value in sorted(values):
|
| 165 |
+
name = compact_name(original_name)
|
| 166 |
+
if name in names:
|
| 167 |
+
raise ValueError(f"compacted tensor name collision: {name}")
|
| 168 |
+
names.add(name)
|
| 169 |
+
if value.ndim > 4:
|
| 170 |
+
raise ValueError(f"GGML supports at most four dimensions: {name} has {value.shape}")
|
| 171 |
+
kind = tensor_kind(component, name, value)
|
| 172 |
+
shape = tuple(int(item) for item in value.shape) or (1,)
|
| 173 |
+
tensor = Tensor(name, value, kind, tuple(reversed(shape)), offset)
|
| 174 |
+
tensors.append(tensor)
|
| 175 |
+
parameter_count += int(value.size)
|
| 176 |
+
offset = aligned(offset + tensor.size)
|
| 177 |
+
|
| 178 |
+
metadata = [
|
| 179 |
+
kv_string("general.architecture", "motionbricks"),
|
| 180 |
+
kv_string("general.name", general_name),
|
| 181 |
+
kv_u32("general.alignment", ALIGNMENT),
|
| 182 |
+
kv_u32("general.file_type", 0),
|
| 183 |
+
kv_u32("motionbricks.format_version", 1),
|
| 184 |
+
kv_string("motionbricks.component", component),
|
| 185 |
+
kv_string("motionbricks.skeleton", "g1skel34"),
|
| 186 |
+
kv_string("motionbricks.upstream_revision", UPSTREAM_REVISION),
|
| 187 |
+
kv_string("motionbricks.source_sha256", source_hash),
|
| 188 |
+
kv_u64("motionbricks.parameter_count", parameter_count),
|
| 189 |
+
]
|
| 190 |
+
if extra_metadata:
|
| 191 |
+
metadata.extend(extra_metadata)
|
| 192 |
+
if component == "support":
|
| 193 |
+
metadata.append(kv_string("motionbricks.joint_names", ",".join(JOINT_NAMES)))
|
| 194 |
+
|
| 195 |
+
header = bytearray(GGUF_MAGIC)
|
| 196 |
+
header += struct.pack("<IQQ", GGUF_VERSION, len(tensors), len(metadata))
|
| 197 |
+
for item in metadata:
|
| 198 |
+
header += item
|
| 199 |
+
for tensor in tensors:
|
| 200 |
+
header += encoded(tensor.name)
|
| 201 |
+
header += struct.pack("<I", len(tensor.dimensions))
|
| 202 |
+
header += struct.pack("<" + "Q" * len(tensor.dimensions), *tensor.dimensions)
|
| 203 |
+
header += struct.pack("<IQ", tensor.kind, tensor.offset)
|
| 204 |
+
header += bytes(aligned(len(header)) - len(header))
|
| 205 |
+
|
| 206 |
+
temporary = path.with_suffix(path.suffix + ".tmp")
|
| 207 |
+
path.parent.mkdir(parents=True, exist_ok=True)
|
| 208 |
+
with temporary.open("wb") as stream:
|
| 209 |
+
stream.write(header)
|
| 210 |
+
position = 0
|
| 211 |
+
for tensor in tensors:
|
| 212 |
+
stream.write(bytes(tensor.offset - position))
|
| 213 |
+
payload = tensor.bytes()
|
| 214 |
+
if len(payload) != tensor.size:
|
| 215 |
+
raise AssertionError(f"incorrect payload size for {tensor.name}")
|
| 216 |
+
stream.write(payload)
|
| 217 |
+
position = tensor.offset + len(payload)
|
| 218 |
+
# GGUF's declared data blob includes alignment after the final tensor.
|
| 219 |
+
stream.write(bytes(offset - position))
|
| 220 |
+
stream.flush()
|
| 221 |
+
os.fsync(stream.fileno())
|
| 222 |
+
os.replace(temporary, path)
|
| 223 |
+
result = {
|
| 224 |
+
"path": path.name,
|
| 225 |
+
"sha256": sha256(path),
|
| 226 |
+
"tensor_count": len(tensors),
|
| 227 |
+
"parameter_count": parameter_count,
|
| 228 |
+
}
|
| 229 |
+
print(f"wrote {path}: {len(tensors)} tensors, {parameter_count:,} values")
|
| 230 |
+
return result
|
| 231 |
+
|
| 232 |
+
|
| 233 |
+
def main() -> None:
|
| 234 |
+
parser = argparse.ArgumentParser()
|
| 235 |
+
parser.add_argument("--safe-directory", type=Path, required=True)
|
| 236 |
+
parser.add_argument("--output", type=Path, required=True)
|
| 237 |
+
args = parser.parse_args()
|
| 238 |
+
|
| 239 |
+
safe_manifest = json.loads((args.safe_directory / "manifest.json").read_text(encoding="utf-8"))
|
| 240 |
+
if safe_manifest.get("format") != SAFE_FORMAT:
|
| 241 |
+
raise ValueError("not a compatible trusted-extraction manifest")
|
| 242 |
+
if safe_manifest.get("upstream_revision") != UPSTREAM_REVISION:
|
| 243 |
+
raise ValueError("trusted extraction came from a different upstream revision")
|
| 244 |
+
for component, expected in SAFE_HASHES.items():
|
| 245 |
+
path = args.safe_directory / f"{component}.safetensors"
|
| 246 |
+
actual = sha256(path)
|
| 247 |
+
recorded = safe_manifest["components"][component]["sha256"]
|
| 248 |
+
if actual != expected or recorded != expected:
|
| 249 |
+
raise ValueError(f"safe input hash mismatch for {component}: {actual}")
|
| 250 |
+
|
| 251 |
+
selections = {
|
| 252 |
+
"pose": load_selected(
|
| 253 |
+
args.safe_directory / "pose.safetensors",
|
| 254 |
+
lambda name: name.startswith("backbone_net.") and name != "backbone_net.initted",
|
| 255 |
+
"backbone_net.",
|
| 256 |
+
),
|
| 257 |
+
"root": load_selected(
|
| 258 |
+
args.safe_directory / "root.safetensors",
|
| 259 |
+
lambda name: name.startswith("backbone_net."),
|
| 260 |
+
"backbone_net.",
|
| 261 |
+
),
|
| 262 |
+
"vq-decoder": load_selected(
|
| 263 |
+
args.safe_directory / "vqvae.safetensors",
|
| 264 |
+
lambda name: name.startswith("pose_net.decoder.")
|
| 265 |
+
or name == "pose_net.quantizer.vq._codebook.embed",
|
| 266 |
+
"pose_net.",
|
| 267 |
+
),
|
| 268 |
+
"support": load_selected(
|
| 269 |
+
args.safe_directory / "support.safetensors", lambda _name: True, ""
|
| 270 |
+
),
|
| 271 |
+
}
|
| 272 |
+
source_for = {"pose": "pose", "root": "root", "vq-decoder": "vqvae", "support": "support"}
|
| 273 |
+
expected = {
|
| 274 |
+
"pose": (209, 136_588_272),
|
| 275 |
+
"root": (150, 34_122_833),
|
| 276 |
+
"vq-decoder": (51, 12_437_277),
|
| 277 |
+
"support": (4, 972),
|
| 278 |
+
}
|
| 279 |
+
components: dict[str, object] = {}
|
| 280 |
+
for component, values in selections.items():
|
| 281 |
+
actual = (len(values), sum(int(value.size) for _, value in values))
|
| 282 |
+
if actual != expected[component]:
|
| 283 |
+
raise ValueError(f"unexpected {component} inventory: {actual}, expected {expected[component]}")
|
| 284 |
+
source = source_for[component]
|
| 285 |
+
components[component] = write_component(
|
| 286 |
+
args.output / f"{component}.gguf", component, values, SAFE_HASHES[source]
|
| 287 |
+
)
|
| 288 |
+
|
| 289 |
+
bundle = {
|
| 290 |
+
"format": BUNDLE_FORMAT,
|
| 291 |
+
"upstream_revision": UPSTREAM_REVISION,
|
| 292 |
+
"skeleton": "g1skel34",
|
| 293 |
+
"joint_count": len(JOINT_NAMES),
|
| 294 |
+
"inference_parameter_count": sum(
|
| 295 |
+
int(components[name]["parameter_count"]) for name in ("pose", "root", "vq-decoder")
|
| 296 |
+
),
|
| 297 |
+
"components": components,
|
| 298 |
+
"safe_manifest_sha256": sha256(args.safe_directory / "manifest.json"),
|
| 299 |
+
}
|
| 300 |
+
temporary = args.output / "manifest.json.tmp"
|
| 301 |
+
temporary.write_text(json.dumps(bundle, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
| 302 |
+
os.replace(temporary, args.output / "manifest.json")
|
| 303 |
+
print(f"inference parameters: {bundle['inference_parameter_count']:,}")
|
| 304 |
+
|
| 305 |
+
|
| 306 |
+
if __name__ == "__main__":
|
| 307 |
+
main()
|
scripts/download_gguf_weights.py
ADDED
|
@@ -0,0 +1,117 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Download and verify the published MotionBricks GGUF/style distribution."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
import argparse
|
| 7 |
+
import hashlib
|
| 8 |
+
import json
|
| 9 |
+
import os
|
| 10 |
+
import shutil
|
| 11 |
+
import sys
|
| 12 |
+
import urllib.parse
|
| 13 |
+
import urllib.request
|
| 14 |
+
from pathlib import Path
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
ROOT = Path(__file__).resolve().parent.parent
|
| 18 |
+
DEFAULT_REPO = "LocalAI-io/MotionBricks-G1-GGML"
|
| 19 |
+
DEFAULT_REVISION = "cc2a47603dbc203a4f18f35dd06ed3611833f506"
|
| 20 |
+
DEFAULT_MANIFEST = ROOT / "scripts/hf/MotionBricks-G1-GGML/MANIFEST.json"
|
| 21 |
+
FORMAT = "motionbricks-gguf-distribution-v1"
|
| 22 |
+
UPSTREAM_REVISION = "a0732b642c0333077e127a2f56ab0014c196bca4"
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def digest(path: Path) -> str:
|
| 26 |
+
value = hashlib.sha256()
|
| 27 |
+
with path.open("rb") as stream:
|
| 28 |
+
for block in iter(lambda: stream.read(8 << 20), b""):
|
| 29 |
+
value.update(block)
|
| 30 |
+
return value.hexdigest()
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def valid(path: Path, entry: dict[str, object]) -> bool:
|
| 34 |
+
return (path.is_file() and path.stat().st_size == entry["bytes"]
|
| 35 |
+
and digest(path) == entry["sha256"])
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def safe_relative(value: object) -> Path:
|
| 39 |
+
path = Path(str(value))
|
| 40 |
+
if path.is_absolute() or ".." in path.parts or not path.parts:
|
| 41 |
+
raise ValueError(f"unsafe distribution path: {value!r}")
|
| 42 |
+
if path.parts[0] not in {"g1-f32", "styles"}:
|
| 43 |
+
raise ValueError(f"unexpected distribution path: {value!r}")
|
| 44 |
+
if path.suffix not in {".gguf", ".mbstyle", ".json"}:
|
| 45 |
+
raise ValueError(f"unexpected distribution file type: {value!r}")
|
| 46 |
+
return path
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
def load_manifest(path: Path, repo: str) -> dict[str, object]:
|
| 50 |
+
value = json.loads(path.read_text(encoding="utf-8"))
|
| 51 |
+
if value.get("format") != FORMAT:
|
| 52 |
+
raise ValueError(f"unsupported manifest format in {path}")
|
| 53 |
+
if value.get("repository") != repo:
|
| 54 |
+
raise ValueError(f"manifest is for {value.get('repository')}, not {repo}")
|
| 55 |
+
if value.get("upstream_revision") != UPSTREAM_REVISION:
|
| 56 |
+
raise ValueError("manifest refers to an unexpected upstream revision")
|
| 57 |
+
files = value.get("files")
|
| 58 |
+
if not isinstance(files, list) or not files:
|
| 59 |
+
raise ValueError("manifest has no files")
|
| 60 |
+
for entry in files:
|
| 61 |
+
if (not isinstance(entry, dict) or not isinstance(entry.get("bytes"), int)
|
| 62 |
+
or entry["bytes"] <= 0 or not isinstance(entry.get("sha256"), str)
|
| 63 |
+
or len(entry["sha256"]) != 64):
|
| 64 |
+
raise ValueError("manifest contains a malformed file record")
|
| 65 |
+
safe_relative(entry.get("path"))
|
| 66 |
+
return value
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
def download(url: str, destination: Path) -> None:
|
| 70 |
+
destination.parent.mkdir(parents=True, exist_ok=True)
|
| 71 |
+
temporary = destination.with_name(destination.name + ".download")
|
| 72 |
+
request = urllib.request.Request(url, headers={"User-Agent": "motion-bricks.cpp/0.1"})
|
| 73 |
+
try:
|
| 74 |
+
with urllib.request.urlopen(request) as response, temporary.open("wb") as output:
|
| 75 |
+
shutil.copyfileobj(response, output, length=8 << 20)
|
| 76 |
+
output.flush()
|
| 77 |
+
os.fsync(output.fileno())
|
| 78 |
+
os.replace(temporary, destination)
|
| 79 |
+
finally:
|
| 80 |
+
temporary.unlink(missing_ok=True)
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
def main() -> int:
|
| 84 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 85 |
+
parser.add_argument("--output", type=Path, default=ROOT / "generated",
|
| 86 |
+
help="destination containing g1-f32/ and styles/")
|
| 87 |
+
parser.add_argument("--repo", default=DEFAULT_REPO, help="Hugging Face model repository")
|
| 88 |
+
parser.add_argument("--revision", default=DEFAULT_REVISION,
|
| 89 |
+
help="pinned Hugging Face commit or revision")
|
| 90 |
+
parser.add_argument("--manifest", type=Path, default=DEFAULT_MANIFEST,
|
| 91 |
+
help="version-controlled distribution manifest")
|
| 92 |
+
args = parser.parse_args()
|
| 93 |
+
|
| 94 |
+
try:
|
| 95 |
+
manifest = load_manifest(args.manifest, args.repo)
|
| 96 |
+
quoted_repo = urllib.parse.quote(args.repo, safe="/")
|
| 97 |
+
quoted_revision = urllib.parse.quote(args.revision, safe="")
|
| 98 |
+
base = f"https://huggingface.co/{quoted_repo}/resolve/{quoted_revision}"
|
| 99 |
+
for entry in manifest["files"]:
|
| 100 |
+
relative = safe_relative(entry["path"])
|
| 101 |
+
destination = args.output / relative
|
| 102 |
+
if valid(destination, entry):
|
| 103 |
+
print(f"verified {destination}")
|
| 104 |
+
continue
|
| 105 |
+
print(f"downloading {relative} from {args.repo}@{args.revision}")
|
| 106 |
+
download(f"{base}/{urllib.parse.quote(relative.as_posix(), safe='/')}", destination)
|
| 107 |
+
if not valid(destination, entry):
|
| 108 |
+
raise ValueError(f"downloaded file failed size/SHA-256 verification: {destination}")
|
| 109 |
+
except (OSError, ValueError, urllib.error.URLError, json.JSONDecodeError) as error:
|
| 110 |
+
print(f"error: {error}", file=sys.stderr)
|
| 111 |
+
return 1
|
| 112 |
+
print("MotionBricks GGUF and style bundles are present and verified")
|
| 113 |
+
return 0
|
| 114 |
+
|
| 115 |
+
|
| 116 |
+
if __name__ == "__main__":
|
| 117 |
+
sys.exit(main())
|
scripts/hf/MotionBricks-G1-GGML/MANIFEST.json
ADDED
|
@@ -0,0 +1,114 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"files": [
|
| 3 |
+
{
|
| 4 |
+
"bytes": 1101,
|
| 5 |
+
"path": "g1-f32/manifest.json",
|
| 6 |
+
"sha256": "35321e8f760316e9a0e73619e23af4089deeedcc8efe7e7266fddbdb8d3c3cfc"
|
| 7 |
+
},
|
| 8 |
+
{
|
| 9 |
+
"bytes": 546369984,
|
| 10 |
+
"path": "g1-f32/pose.gguf",
|
| 11 |
+
"sha256": "6db82e5a67a355052f283bbcab104b71ecfc333dc8d9017c3812eb6f9650756e"
|
| 12 |
+
},
|
| 13 |
+
{
|
| 14 |
+
"bytes": 136504000,
|
| 15 |
+
"path": "g1-f32/root.gguf",
|
| 16 |
+
"sha256": "46268d77e5c15449d68a39ce6ec7145e9eb457c5114d5dca934bd3ab539b29db"
|
| 17 |
+
},
|
| 18 |
+
{
|
| 19 |
+
"bytes": 5472,
|
| 20 |
+
"path": "g1-f32/support.gguf",
|
| 21 |
+
"sha256": "5d41cae4bc494e612ef19e25f599351aafbc9ef1cb46d4ac3f1e42bb7ab07200"
|
| 22 |
+
},
|
| 23 |
+
{
|
| 24 |
+
"bytes": 49753440,
|
| 25 |
+
"path": "g1-f32/vq-decoder.gguf",
|
| 26 |
+
"sha256": "6a1114b6906b07fab9deb18d602b4e731e1b5794318b1643ffd0605692f62d1e"
|
| 27 |
+
},
|
| 28 |
+
{
|
| 29 |
+
"bytes": 9408,
|
| 30 |
+
"path": "styles/elbow_crawling.mbstyle",
|
| 31 |
+
"sha256": "8ce3e04aa92d91628f10824a2f19a0cc0ba5a9c16008cd95e091b05c9528dc54"
|
| 32 |
+
},
|
| 33 |
+
{
|
| 34 |
+
"bytes": 50624,
|
| 35 |
+
"path": "styles/hand_crawling.mbstyle",
|
| 36 |
+
"sha256": "5687f3cd037d7d0942df180727729e04c08d6f0af48b32e9bf793729c5c69fd8"
|
| 37 |
+
},
|
| 38 |
+
{
|
| 39 |
+
"bytes": 50624,
|
| 40 |
+
"path": "styles/idle.mbstyle",
|
| 41 |
+
"sha256": "ae800a700d29a8d1082d32c2c82e893aa292c6c18848ffd2c0c67c6f4d64531e"
|
| 42 |
+
},
|
| 43 |
+
{
|
| 44 |
+
"bytes": 14304,
|
| 45 |
+
"path": "styles/injured_walk.mbstyle",
|
| 46 |
+
"sha256": "b7b62643281b46ceb0a37fc336f201565b595afda21e93a2cbfc3dda3509a779"
|
| 47 |
+
},
|
| 48 |
+
{
|
| 49 |
+
"bytes": 3114,
|
| 50 |
+
"path": "styles/manifest.json",
|
| 51 |
+
"sha256": "762f7eadb5653af53bb9a03f812b724845b3af6967b76c5f52546de97f1edac4"
|
| 52 |
+
},
|
| 53 |
+
{
|
| 54 |
+
"bytes": 50624,
|
| 55 |
+
"path": "styles/slow_walk.mbstyle",
|
| 56 |
+
"sha256": "9e59ec536d22f61c53b430aecabbef92e694d6ab5caef077ac7d0cfbca6d73f0"
|
| 57 |
+
},
|
| 58 |
+
{
|
| 59 |
+
"bytes": 34112,
|
| 60 |
+
"path": "styles/stealth_walk.mbstyle",
|
| 61 |
+
"sha256": "4d2d82e40f9f43c1c15c201b45e30dfb194e4a28cf7fd8a23870394c4589d87a"
|
| 62 |
+
},
|
| 63 |
+
{
|
| 64 |
+
"bytes": 50624,
|
| 65 |
+
"path": "styles/walk.mbstyle",
|
| 66 |
+
"sha256": "c1657f56795f26ac22ab944cc6676dd9a7c12e37fc654b08e744a18963597d59"
|
| 67 |
+
},
|
| 68 |
+
{
|
| 69 |
+
"bytes": 17664,
|
| 70 |
+
"path": "styles/walk_boxing.mbstyle",
|
| 71 |
+
"sha256": "32349902a37d79f049e52a98929680ff18f9300a270ace1709f16d5d87d70892"
|
| 72 |
+
},
|
| 73 |
+
{
|
| 74 |
+
"bytes": 126400,
|
| 75 |
+
"path": "styles/walk_gun.mbstyle",
|
| 76 |
+
"sha256": "6b5f0f6febb92b1fbb090f246e48c4762c23f7d101bc2b371fd93a13a1425245"
|
| 77 |
+
},
|
| 78 |
+
{
|
| 79 |
+
"bytes": 34112,
|
| 80 |
+
"path": "styles/walk_happy_dance.mbstyle",
|
| 81 |
+
"sha256": "0900c03fbdcaf4f5fac8e4e3d4ebde5965a8cf561a4ecab07db256a136f3e2e8"
|
| 82 |
+
},
|
| 83 |
+
{
|
| 84 |
+
"bytes": 9408,
|
| 85 |
+
"path": "styles/walk_left.mbstyle",
|
| 86 |
+
"sha256": "7660e369e64bc711b153caba744c1ccb3d763060f87b368969fd1ec4819694e5"
|
| 87 |
+
},
|
| 88 |
+
{
|
| 89 |
+
"bytes": 9408,
|
| 90 |
+
"path": "styles/walk_right.mbstyle",
|
| 91 |
+
"sha256": "106419603b7844d3f2f804b4debbff47b1ec71969f8dd4c4b4dbaef36d4e5c83"
|
| 92 |
+
},
|
| 93 |
+
{
|
| 94 |
+
"bytes": 149504,
|
| 95 |
+
"path": "styles/walk_scared.mbstyle",
|
| 96 |
+
"sha256": "b88776e3d57c7008e94e2a51c8da19d0715a3a42565b47c2019cabb40b13578d"
|
| 97 |
+
},
|
| 98 |
+
{
|
| 99 |
+
"bytes": 34112,
|
| 100 |
+
"path": "styles/walk_stealth.mbstyle",
|
| 101 |
+
"sha256": "bbb4b69e29cd78cad102da433dc553f6dc16b6710f9a7d4c7df0148f1492f778"
|
| 102 |
+
},
|
| 103 |
+
{
|
| 104 |
+
"bytes": 149504,
|
| 105 |
+
"path": "styles/walk_zombie.mbstyle",
|
| 106 |
+
"sha256": "9ce2769010ecb20c5ad38d36f841ec72e29dd1e4556dff8f0ecb258855a9705b"
|
| 107 |
+
}
|
| 108 |
+
],
|
| 109 |
+
"format": "motionbricks-gguf-distribution-v1",
|
| 110 |
+
"repository": "LocalAI-io/MotionBricks-G1-GGML",
|
| 111 |
+
"upstream_license": "NVIDIA Open Model License",
|
| 112 |
+
"upstream_repository": "NVlabs/GR00T-WholeBodyControl",
|
| 113 |
+
"upstream_revision": "a0732b642c0333077e127a2f56ab0014c196bca4"
|
| 114 |
+
}
|
scripts/hf/MotionBricks-G1-GGML/NOTICE
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Licensed by NVIDIA Corporation under the NVIDIA Open Model License.
|
| 2 |
+
|
| 3 |
+
Upstream model: NVlabs/GR00T-WholeBodyControl, motionbricks/, revision
|
| 4 |
+
a0732b642c0333077e127a2f56ab0014c196bca4. This GGUF conversion grants no
|
| 5 |
+
additional rights.
|
scripts/hf/MotionBricks-G1-GGML/README.md
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
license: other
|
| 3 |
+
library_name: ggml
|
| 4 |
+
tags: [gguf, ggml, motion-generation, unitree-g1, motionbricks]
|
| 5 |
+
---
|
| 6 |
+
|
| 7 |
+
# MotionBricks-G1-GGML
|
| 8 |
+
|
| 9 |
+
Native F32 GGML/GGUF conversion of NVIDIA MotionBricks' released G1 pose,
|
| 10 |
+
root, and VQ-decoder checkpoints, plus the 15 released style primitives. It is
|
| 11 |
+
produced by
|
| 12 |
+
[`motion-bricks.cpp`](https://github.com/localai-org/motion-bricks.cpp) for CPU and
|
| 13 |
+
Vulkan inference.
|
| 14 |
+
|
| 15 |
+
NVIDIA currently publishes MotionBricks through Git LFS rather than a separate
|
| 16 |
+
Hugging Face model repository. This conversion is linked to the pinned
|
| 17 |
+
[`NVlabs/GR00T-WholeBodyControl` revision](https://github.com/NVlabs/GR00T-WholeBodyControl/tree/a0732b642c0333077e127a2f56ab0014c196bca4/motionbricks).
|
| 18 |
+
|
| 19 |
+
Install the complete checked bundle at `generated/g1-f32` and
|
| 20 |
+
`generated/styles` with:
|
| 21 |
+
|
| 22 |
+
```sh
|
| 23 |
+
python scripts/download_gguf_weights.py
|
| 24 |
+
```
|
| 25 |
+
|
| 26 |
+
`MANIFEST.json` and `SHA256SUMS` record the source revision, exact paths,
|
| 27 |
+
sizes, and SHA-256 hashes. The bundle contains 183,148,382 learned F32
|
| 28 |
+
parameters and targets MotionBricks' released 34-joint Unitree G1 skeleton.
|
| 29 |
+
|
| 30 |
+
## License and provenance
|
| 31 |
+
|
| 32 |
+
The original checkpoints and this converted representation remain subject to
|
| 33 |
+
the NVIDIA Open Model License in `UPSTREAM_LICENSE`. Redistribution requires
|
| 34 |
+
retaining that agreement and NVIDIA's attribution in `NOTICE`. This conversion
|
| 35 |
+
grants no additional rights. Use is also subject to NVIDIA's Trustworthy AI
|
| 36 |
+
and trade-compliance terms described by that agreement.
|
scripts/hf/MotionBricks-G1-GGML/UPSTREAM_LICENSE
ADDED
|
@@ -0,0 +1,186 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
================================================================================
|
| 2 |
+
DUAL LICENSE NOTICE
|
| 3 |
+
================================================================================
|
| 4 |
+
|
| 5 |
+
This repository is dual-licensed. Different components are under different terms:
|
| 6 |
+
|
| 7 |
+
1. SOURCE CODE - Apache License 2.0
|
| 8 |
+
All source code, scripts, and software components
|
| 9 |
+
|
| 10 |
+
2. MODEL WEIGHTS - NVIDIA Open Model License
|
| 11 |
+
All trained model checkpoints and weights
|
| 12 |
+
|
| 13 |
+
See below for the full text of each license.
|
| 14 |
+
|
| 15 |
+
================================================================================
|
| 16 |
+
PART 1: SOURCE CODE LICENSE (Apache License 2.0)
|
| 17 |
+
================================================================================
|
| 18 |
+
|
| 19 |
+
Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
| 20 |
+
|
| 21 |
+
Licensed under the Apache License, Version 2.0 (the "License");
|
| 22 |
+
you may not use this file except in compliance with the License.
|
| 23 |
+
You may obtain a copy of the License at
|
| 24 |
+
|
| 25 |
+
http://www.apache.org/licenses/LICENSE-2.0
|
| 26 |
+
|
| 27 |
+
Unless required by applicable law or agreed to in writing, software
|
| 28 |
+
distributed under the License is distributed on an "AS IS" BASIS,
|
| 29 |
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 30 |
+
See the License for the specific language governing permissions and
|
| 31 |
+
limitations under the License.
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
================================================================================
|
| 35 |
+
PART 2: MODEL WEIGHTS LICENSE (NVIDIA Open Model License)
|
| 36 |
+
================================================================================
|
| 37 |
+
|
| 38 |
+
NVIDIA OPEN MODEL LICENSE AGREEMENT
|
| 39 |
+
|
| 40 |
+
Last Modified: October 24, 2025
|
| 41 |
+
|
| 42 |
+
NVIDIA Corporation and its affiliates ("NVIDIA") grants permission to use machine
|
| 43 |
+
learning models under specific conditions. Key permissions include creating
|
| 44 |
+
derivative models and distributing them, with NVIDIA retaining no ownership claims
|
| 45 |
+
over outputs generated by users.
|
| 46 |
+
|
| 47 |
+
SECTION 1: DEFINITIONS
|
| 48 |
+
|
| 49 |
+
1.1 "Derivative Model" means any modification of, or works based on or derived
|
| 50 |
+
from, the Model, excluding outputs.
|
| 51 |
+
|
| 52 |
+
1.2 "Legal Entity" means the union of the acting entity and all other entities
|
| 53 |
+
that control, are controlled by, or are under common control with that entity.
|
| 54 |
+
|
| 55 |
+
1.3 "Model" means the machine learning model, software, and any checkpoints,
|
| 56 |
+
weights, algorithms, parameters, configuration files, and documentation that NVIDIA
|
| 57 |
+
makes available under this Agreement.
|
| 58 |
+
|
| 59 |
+
1.4 "NVIDIA Cosmos Model" means a multimodal Model that is covered by this Agreement.
|
| 60 |
+
|
| 61 |
+
1.5 "Special-Purpose Model" means a Model that is limited to narrow,
|
| 62 |
+
purpose-specific tasks.
|
| 63 |
+
|
| 64 |
+
1.6 "You" or "Your" means an individual or Legal Entity exercising permissions
|
| 65 |
+
granted by this Agreement.
|
| 66 |
+
|
| 67 |
+
SECTION 2: CONDITIONS FOR USE, LICENSE GRANT, AI ETHICS AND IP OWNERSHIP
|
| 68 |
+
|
| 69 |
+
2.1 Conditions for Use. You must comply with all terms and conditions of this
|
| 70 |
+
Agreement. If You initiate copyright or patent litigation against any entity
|
| 71 |
+
(including a cross-claim or counterclaim in a lawsuit) alleging that the Model
|
| 72 |
+
constitutes direct or contributory infringement, then Your licenses under this
|
| 73 |
+
Agreement shall terminate. If You circumvent any safety guardrails or safety
|
| 74 |
+
measures built in to the Model without providing comparable alternatives, Your
|
| 75 |
+
rights under this Agreement shall terminate. NVIDIA may update this Agreement at
|
| 76 |
+
any time to comply with applicable law; Your continued use constitutes Your
|
| 77 |
+
acceptance of the updated terms.
|
| 78 |
+
|
| 79 |
+
2.2 License Grant. Subject to the terms and conditions of this Agreement, NVIDIA
|
| 80 |
+
hereby grants You a perpetual, worldwide, non-exclusive, no-charge, royalty-free,
|
| 81 |
+
revocable license to publicly perform, publicly display, reproduce, use, create
|
| 82 |
+
derivative works of, make, have made, sell, offer for sale, distribute and import
|
| 83 |
+
the Model.
|
| 84 |
+
|
| 85 |
+
2.3 AI Ethics. Your use of the Model must be in accordance with NVIDIA's
|
| 86 |
+
Trustworthy AI terms, which can be found at
|
| 87 |
+
https://www.nvidia.com/en-us/agreements/trustworthy-ai/terms/.
|
| 88 |
+
|
| 89 |
+
2.4 IP Ownership. NVIDIA owns the original Model and NVIDIA's Derivative Models.
|
| 90 |
+
You own Your Derivative Models. NVIDIA makes no claim of ownership to outputs. You
|
| 91 |
+
are responsible for outputs and their subsequent uses.
|
| 92 |
+
|
| 93 |
+
SECTION 3: REDISTRIBUTION
|
| 94 |
+
|
| 95 |
+
You may reproduce and distribute copies of the Model or Derivative Models thereof,
|
| 96 |
+
with or without modifications, provided that You meet the following conditions:
|
| 97 |
+
|
| 98 |
+
a. You must include a copy of this Agreement.
|
| 99 |
+
|
| 100 |
+
b. You must include the following attribution notice, which can appear in the same
|
| 101 |
+
location as other third-party notices or license information: "Licensed by NVIDIA
|
| 102 |
+
Corporation under the NVIDIA Open Model License."
|
| 103 |
+
|
| 104 |
+
c. If You are distributing a NVIDIA Cosmos Model, You must also include the phrase
|
| 105 |
+
"Built on NVIDIA Cosmos" on the applicable website, in the user interface, in a
|
| 106 |
+
blog, in an "about" page, or in product documentation.
|
| 107 |
+
|
| 108 |
+
d. You may add Your own copyright statement to Your modifications and may provide
|
| 109 |
+
additional or different license terms and conditions for use, reproduction, or
|
| 110 |
+
distribution of Your modifications or for any Derivative Models as a whole,
|
| 111 |
+
provided Your use, reproduction, and distribution otherwise complies with this
|
| 112 |
+
Agreement.
|
| 113 |
+
|
| 114 |
+
SECTION 4: SEPARATE COMPONENTS
|
| 115 |
+
|
| 116 |
+
The Model may contain components that are subject to separate legal notices or
|
| 117 |
+
governed by separate licenses (including Open Source Software Licenses), as may be
|
| 118 |
+
described in any files made available with the Model. Your use of those separate
|
| 119 |
+
components is subject to the applicable license. This Agreement shall control over
|
| 120 |
+
the separate licenses for third-party Open Source Software to the extent that the
|
| 121 |
+
separate license imposes additional restrictions. "Open Source Software License"
|
| 122 |
+
means any software license approved by the Open Source Initiative, Free Software
|
| 123 |
+
Foundation, or similar recognized organization, or a license identified by SPDX.
|
| 124 |
+
|
| 125 |
+
SECTION 5: TRADEMARKS
|
| 126 |
+
|
| 127 |
+
This Agreement does not grant permission to use the trade names, trademarks,
|
| 128 |
+
service marks, or product names of NVIDIA, except as required for reasonable and
|
| 129 |
+
customary use in describing the origin of the Model and reproducing the content of
|
| 130 |
+
the notice.
|
| 131 |
+
|
| 132 |
+
SECTION 6: DISCLAIMER OF WARRANTY
|
| 133 |
+
|
| 134 |
+
NVIDIA provides the Model on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
|
| 135 |
+
ANY KIND, either express or implied, including, without limitation, any warranties
|
| 136 |
+
or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
| 137 |
+
PARTICULAR PURPOSE. You are solely responsible for reviewing the documentation
|
| 138 |
+
accompanying the Model and determining the appropriateness of using the Model, and
|
| 139 |
+
You understand that Special-Purpose Models are limited to narrow, purpose-specific
|
| 140 |
+
tasks and must not be deployed for uses that are beyond such tasks.
|
| 141 |
+
|
| 142 |
+
SECTION 7: LIMITATION OF LIABILITY
|
| 143 |
+
|
| 144 |
+
In no event and under no legal theory, whether in tort (including negligence),
|
| 145 |
+
contract, or otherwise, unless required by applicable law (such as deliberate and
|
| 146 |
+
grossly negligent acts) or agreed to in writing, will NVIDIA be liable to You for
|
| 147 |
+
damages, including any direct, indirect, special, incidental, or consequential
|
| 148 |
+
damages of any character arising as a result of this Agreement or out of the use
|
| 149 |
+
or inability to use the Model or Derivative Models or outputs (including but not
|
| 150 |
+
limited to damages for loss of goodwill, work stoppage, computer failure or
|
| 151 |
+
malfunction, or any and all other commercial damages or losses), even if NVIDIA has
|
| 152 |
+
been advised of the possibility of such damages.
|
| 153 |
+
|
| 154 |
+
SECTION 8: INDEMNITY
|
| 155 |
+
|
| 156 |
+
You will defend, indemnify and hold harmless NVIDIA and its affiliates, and their
|
| 157 |
+
respective employees, contractors, directors, officers and agents, from and against
|
| 158 |
+
any and all claims, damages, obligations, losses, liabilities, costs or debt, and
|
| 159 |
+
expenses (including but not limited to attorney's fees) arising from Your use or
|
| 160 |
+
distribution of the Model or Derivative Models or outputs.
|
| 161 |
+
|
| 162 |
+
SECTION 9: FEEDBACK
|
| 163 |
+
|
| 164 |
+
NVIDIA may use feedback You provide without restriction and without any
|
| 165 |
+
compensation to You.
|
| 166 |
+
|
| 167 |
+
SECTION 10: GOVERNING LAW
|
| 168 |
+
|
| 169 |
+
This Agreement will be governed in all respects by the laws of the United States
|
| 170 |
+
and of the State of Delaware, without regard to conflict of laws provisions. The
|
| 171 |
+
federal and state courts residing in Santa Clara County, California shall have
|
| 172 |
+
exclusive jurisdiction over any dispute arising out of this Agreement, and You
|
| 173 |
+
hereby consent to the personal jurisdiction of such courts. However, NVIDIA shall
|
| 174 |
+
have the right to seek injunctive relief in any court of competent jurisdiction.
|
| 175 |
+
|
| 176 |
+
SECTION 11: TRADE AND COMPLIANCE
|
| 177 |
+
|
| 178 |
+
You shall comply with all applicable import, export, trade, and economic sanctions
|
| 179 |
+
laws, including without limitation the Export Administration Regulations and
|
| 180 |
+
economic sanctions laws implemented by the Office of Foreign Assets Control, that
|
| 181 |
+
restrict or govern the destination, end-user and end-use of NVIDIA products,
|
| 182 |
+
technology, software, and services.
|
| 183 |
+
|
| 184 |
+
---
|
| 185 |
+
|
| 186 |
+
Version Release Date: October 24, 2025
|
scripts/publish_gguf.py
ADDED
|
@@ -0,0 +1,151 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Validate and optionally publish the MotionBricks GGUF distribution to HF."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
import argparse
|
| 7 |
+
import hashlib
|
| 8 |
+
import io
|
| 9 |
+
import json
|
| 10 |
+
import sys
|
| 11 |
+
from pathlib import Path
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
ROOT = Path(__file__).resolve().parent.parent
|
| 15 |
+
HF_REPO = "LocalAI-io/MotionBricks-G1-GGML"
|
| 16 |
+
UPSTREAM_REPO = "NVlabs/GR00T-WholeBodyControl"
|
| 17 |
+
UPSTREAM_REVISION = "a0732b642c0333077e127a2f56ab0014c196bca4"
|
| 18 |
+
FORMAT = "motionbricks-gguf-distribution-v1"
|
| 19 |
+
CARD_DIR = ROOT / "scripts/hf/MotionBricks-G1-GGML"
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def digest(path: Path) -> str:
|
| 23 |
+
value = hashlib.sha256()
|
| 24 |
+
with path.open("rb") as stream:
|
| 25 |
+
for block in iter(lambda: stream.read(8 << 20), b""):
|
| 26 |
+
value.update(block)
|
| 27 |
+
return value.hexdigest()
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def checked_json(path: Path, expected_format: str) -> dict[str, object]:
|
| 31 |
+
if not path.is_file():
|
| 32 |
+
raise ValueError(f"missing manifest: {path}")
|
| 33 |
+
value = json.loads(path.read_text(encoding="utf-8"))
|
| 34 |
+
if value.get("format") != expected_format:
|
| 35 |
+
raise ValueError(f"unexpected format in {path}")
|
| 36 |
+
return value
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
def artifacts(model: Path, styles: Path) -> list[tuple[Path, str]]:
|
| 40 |
+
model_manifest = checked_json(model / "manifest.json", "motionbricks-gguf-bundle-v1")
|
| 41 |
+
style_manifest = checked_json(styles / "manifest.json", "motionbricks-style-bundle-v1")
|
| 42 |
+
if model_manifest.get("upstream_revision") != UPSTREAM_REVISION:
|
| 43 |
+
raise ValueError("model bundle came from an unexpected upstream revision")
|
| 44 |
+
result = [(model / "manifest.json", "g1-f32/manifest.json")]
|
| 45 |
+
for record in model_manifest["components"].values():
|
| 46 |
+
source = model / record["path"]
|
| 47 |
+
if digest(source) != record["sha256"]:
|
| 48 |
+
raise ValueError(f"model manifest SHA-256 mismatch: {source}")
|
| 49 |
+
result.append((source, f"g1-f32/{source.name}"))
|
| 50 |
+
result.append((styles / "manifest.json", "styles/manifest.json"))
|
| 51 |
+
for record in style_manifest["styles"].values():
|
| 52 |
+
source = styles / record["path"]
|
| 53 |
+
if digest(source) != record["sha256"]:
|
| 54 |
+
raise ValueError(f"style manifest SHA-256 mismatch: {source}")
|
| 55 |
+
result.append((source, f"styles/{source.name}"))
|
| 56 |
+
for source, destination in result:
|
| 57 |
+
if not source.is_file() or source.stat().st_size == 0:
|
| 58 |
+
raise ValueError(f"missing or empty artifact: {source}")
|
| 59 |
+
if source.suffix in {".gguf", ".mbstyle"}:
|
| 60 |
+
with source.open("rb") as stream:
|
| 61 |
+
if stream.read(4) != b"GGUF":
|
| 62 |
+
raise ValueError(f"invalid GGUF magic: {source}")
|
| 63 |
+
path = Path(destination)
|
| 64 |
+
if path.is_absolute() or ".." in path.parts:
|
| 65 |
+
raise ValueError(f"unsafe destination: {destination}")
|
| 66 |
+
return sorted(result, key=lambda item: item[1])
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
def distribution_manifest(repo: str, files: list[tuple[Path, str]]) -> tuple[dict[str, object], bytes]:
|
| 70 |
+
records = [
|
| 71 |
+
{"path": destination, "bytes": source.stat().st_size, "sha256": digest(source)}
|
| 72 |
+
for source, destination in files
|
| 73 |
+
]
|
| 74 |
+
value = {
|
| 75 |
+
"format": FORMAT,
|
| 76 |
+
"repository": repo,
|
| 77 |
+
"upstream_repository": UPSTREAM_REPO,
|
| 78 |
+
"upstream_revision": UPSTREAM_REVISION,
|
| 79 |
+
"upstream_license": "NVIDIA Open Model License",
|
| 80 |
+
"files": records,
|
| 81 |
+
}
|
| 82 |
+
payload = json.dumps(value, indent=2, sort_keys=True).encode() + b"\n"
|
| 83 |
+
return value, payload
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
def main() -> int:
|
| 87 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 88 |
+
parser.add_argument("--model", type=Path, default=ROOT / "generated/g1-f32")
|
| 89 |
+
parser.add_argument("--styles", type=Path, default=ROOT / "generated/styles")
|
| 90 |
+
parser.add_argument("--repo", default=HF_REPO)
|
| 91 |
+
parser.add_argument("--write-manifest", type=Path,
|
| 92 |
+
help="write the checked distribution manifest to this path")
|
| 93 |
+
parser.add_argument("--upload", action="store_true")
|
| 94 |
+
parser.add_argument("--confirm-upstream-licence", action="store_true",
|
| 95 |
+
help="required with --upload; confirms redistribution conditions were reviewed")
|
| 96 |
+
args = parser.parse_args()
|
| 97 |
+
|
| 98 |
+
try:
|
| 99 |
+
for required in (CARD_DIR / "README.md", CARD_DIR / "NOTICE", CARD_DIR / "UPSTREAM_LICENSE"):
|
| 100 |
+
if not required.is_file():
|
| 101 |
+
raise ValueError(f"missing publication asset: {required}")
|
| 102 |
+
files = artifacts(args.model, args.styles)
|
| 103 |
+
manifest, manifest_payload = distribution_manifest(args.repo, files)
|
| 104 |
+
except (OSError, ValueError, KeyError, json.JSONDecodeError) as error:
|
| 105 |
+
print(f"error: {error}", file=sys.stderr)
|
| 106 |
+
return 1
|
| 107 |
+
|
| 108 |
+
sums = "".join(f"{entry['sha256']} {entry['path']}\n" for entry in manifest["files"])
|
| 109 |
+
total = sum(entry["bytes"] for entry in manifest["files"])
|
| 110 |
+
print(f"repo: https://huggingface.co/{args.repo}")
|
| 111 |
+
print(f"files: {len(files)} model/style artifacts, {total / 1e9:.3f} GB")
|
| 112 |
+
for entry in manifest["files"]:
|
| 113 |
+
print(f" {entry['sha256']} {entry['bytes']:>12} {entry['path']}")
|
| 114 |
+
if args.write_manifest:
|
| 115 |
+
args.write_manifest.parent.mkdir(parents=True, exist_ok=True)
|
| 116 |
+
args.write_manifest.write_bytes(manifest_payload)
|
| 117 |
+
print(f"wrote {args.write_manifest}")
|
| 118 |
+
if not args.upload:
|
| 119 |
+
print("[dry-run] nothing uploaded")
|
| 120 |
+
return 0
|
| 121 |
+
if not args.confirm_upstream_licence:
|
| 122 |
+
print("error: --upload requires --confirm-upstream-licence", file=sys.stderr)
|
| 123 |
+
return 2
|
| 124 |
+
|
| 125 |
+
try:
|
| 126 |
+
from huggingface_hub import CommitOperationAdd, HfApi
|
| 127 |
+
|
| 128 |
+
api = HfApi()
|
| 129 |
+
api.create_repo(args.repo, repo_type="model", exist_ok=True)
|
| 130 |
+
operations = [
|
| 131 |
+
CommitOperationAdd(path_in_repo="README.md", path_or_fileobj=CARD_DIR / "README.md"),
|
| 132 |
+
CommitOperationAdd(path_in_repo="NOTICE", path_or_fileobj=CARD_DIR / "NOTICE"),
|
| 133 |
+
CommitOperationAdd(path_in_repo="UPSTREAM_LICENSE", path_or_fileobj=CARD_DIR / "UPSTREAM_LICENSE"),
|
| 134 |
+
*(CommitOperationAdd(path_in_repo=destination, path_or_fileobj=source)
|
| 135 |
+
for source, destination in files),
|
| 136 |
+
CommitOperationAdd(path_in_repo="MANIFEST.json", path_or_fileobj=io.BytesIO(manifest_payload)),
|
| 137 |
+
CommitOperationAdd(path_in_repo="SHA256SUMS", path_or_fileobj=io.BytesIO(sums.encode())),
|
| 138 |
+
]
|
| 139 |
+
commit = api.create_commit(
|
| 140 |
+
repo_id=args.repo, repo_type="model", operations=operations,
|
| 141 |
+
commit_message=f"Publish MotionBricks G1 GGUF bundle from {UPSTREAM_REVISION[:12]}",
|
| 142 |
+
)
|
| 143 |
+
except Exception as error: # huggingface_hub supplies the useful API detail
|
| 144 |
+
print(f"error: Hugging Face upload failed: {error}", file=sys.stderr)
|
| 145 |
+
return 1
|
| 146 |
+
print(f"published: {commit.commit_url}")
|
| 147 |
+
return 0
|
| 148 |
+
|
| 149 |
+
|
| 150 |
+
if __name__ == "__main__":
|
| 151 |
+
sys.exit(main())
|
src/agent.cpp
ADDED
|
@@ -0,0 +1,266 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#include "agent.hpp"
|
| 2 |
+
|
| 3 |
+
#include "handles.hpp"
|
| 4 |
+
#include "motion_rep.hpp"
|
| 5 |
+
#include "planner.hpp"
|
| 6 |
+
|
| 7 |
+
#include <algorithm>
|
| 8 |
+
#include <array>
|
| 9 |
+
#include <cmath>
|
| 10 |
+
#include <cstddef>
|
| 11 |
+
|
| 12 |
+
namespace motionbricks::detail {
|
| 13 |
+
namespace {
|
| 14 |
+
|
| 15 |
+
using mat3 = std::array<float, 9>;
|
| 16 |
+
|
| 17 |
+
mat3 multiply(const mat3 & a, const mat3 & b) {
|
| 18 |
+
mat3 result{};
|
| 19 |
+
for (unsigned row = 0; row < 3U; ++row)
|
| 20 |
+
for (unsigned column = 0; column < 3U; ++column)
|
| 21 |
+
for (unsigned k = 0; k < 3U; ++k)
|
| 22 |
+
result[row * 3U + column] += a[row * 3U + k] * b[k * 3U + column];
|
| 23 |
+
return result;
|
| 24 |
+
}
|
| 25 |
+
|
| 26 |
+
mat3 transpose(const mat3 & value) {
|
| 27 |
+
return {value[0],value[3],value[6],value[1],value[4],value[7],value[2],value[5],value[8]};
|
| 28 |
+
}
|
| 29 |
+
|
| 30 |
+
mat3 y_rotation(float angle) {
|
| 31 |
+
const float cosine = std::cos(angle), sine = std::sin(angle);
|
| 32 |
+
return {cosine,0.0F,sine, 0.0F,1.0F,0.0F, -sine,0.0F,cosine};
|
| 33 |
+
}
|
| 34 |
+
|
| 35 |
+
std::array<float, 3> transform(const mat3 & matrix, const float * value) {
|
| 36 |
+
return {matrix[0]*value[0] + matrix[1]*value[1] + matrix[2]*value[2],
|
| 37 |
+
matrix[3]*value[0] + matrix[4]*value[1] + matrix[5]*value[2],
|
| 38 |
+
matrix[6]*value[0] + matrix[7]*value[1] + matrix[8]*value[2]};
|
| 39 |
+
}
|
| 40 |
+
|
| 41 |
+
std::array<float, 4> matrix_xyzw(const mat3 & matrix) {
|
| 42 |
+
std::array<float, 4> wxyz{};
|
| 43 |
+
const float trace = matrix[0] + matrix[4] + matrix[8];
|
| 44 |
+
if (trace > 0.0F) {
|
| 45 |
+
const float s = 2.0F * std::sqrt(std::max(0.0F, trace + 1.0F));
|
| 46 |
+
wxyz = {0.25F*s, (matrix[7]-matrix[5])/s, (matrix[2]-matrix[6])/s,
|
| 47 |
+
(matrix[3]-matrix[1])/s};
|
| 48 |
+
} else if (matrix[0] > matrix[4] && matrix[0] > matrix[8]) {
|
| 49 |
+
const float s = 2.0F * std::sqrt(std::max(0.0F, 1.0F+matrix[0]-matrix[4]-matrix[8]));
|
| 50 |
+
wxyz = {(matrix[7]-matrix[5])/s,0.25F*s,(matrix[1]+matrix[3])/s,(matrix[2]+matrix[6])/s};
|
| 51 |
+
} else if (matrix[4] > matrix[8]) {
|
| 52 |
+
const float s = 2.0F * std::sqrt(std::max(0.0F, 1.0F+matrix[4]-matrix[0]-matrix[8]));
|
| 53 |
+
wxyz = {(matrix[2]-matrix[6])/s,(matrix[1]+matrix[3])/s,0.25F*s,(matrix[5]+matrix[7])/s};
|
| 54 |
+
} else {
|
| 55 |
+
const float s = 2.0F * std::sqrt(std::max(0.0F, 1.0F+matrix[8]-matrix[0]-matrix[4]));
|
| 56 |
+
wxyz = {(matrix[3]-matrix[1])/s,(matrix[2]+matrix[6])/s,(matrix[5]+matrix[7])/s,0.25F*s};
|
| 57 |
+
}
|
| 58 |
+
const float norm = std::sqrt(wxyz[0]*wxyz[0]+wxyz[1]*wxyz[1]+wxyz[2]*wxyz[2]+wxyz[3]*wxyz[3]);
|
| 59 |
+
for (float & value : wxyz) value /= norm;
|
| 60 |
+
return {wxyz[1],wxyz[2],wxyz[3],wxyz[0]};
|
| 61 |
+
}
|
| 62 |
+
|
| 63 |
+
float wrap(float value) {
|
| 64 |
+
constexpr float pi = 3.14159265358979323846F;
|
| 65 |
+
return std::remainder(value, 2.0F*pi);
|
| 66 |
+
}
|
| 67 |
+
|
| 68 |
+
float spring(float current, float velocity, float target, float half_life, float time) {
|
| 69 |
+
constexpr float ln2 = 0.69314718056F;
|
| 70 |
+
const float y = (4.0F * ln2) / (half_life + 1.0e-5F) / 2.0F;
|
| 71 |
+
const float x = y * time;
|
| 72 |
+
const float decay = 1.0F / (1.0F + x + 0.48F*x*x + 0.235F*x*x*x);
|
| 73 |
+
const float j0 = current - target;
|
| 74 |
+
const float j1 = velocity + j0*y;
|
| 75 |
+
return (j0 + j1*time)*decay + target;
|
| 76 |
+
}
|
| 77 |
+
|
| 78 |
+
mb_status build_constraints(mb_agent & agent, const mb_command & command,
|
| 79 |
+
const mb_style & style, transition_constraints & result,
|
| 80 |
+
std::string & reason) {
|
| 81 |
+
encoded_frames context;
|
| 82 |
+
auto status = encode_context(*agent.model, agent.context_root_xyz,
|
| 83 |
+
agent.context_local_rotations_xyzw, agent.context_frames, context, reason);
|
| 84 |
+
if (status != MB_OK) return status;
|
| 85 |
+
std::copy(context.global_root.begin(), context.global_root.end(), result.global_root.begin());
|
| 86 |
+
std::copy(context.local_root.begin(), context.local_root.end(), result.local_root.begin());
|
| 87 |
+
std::copy(context.poses.begin(), context.poses.end(), result.poses.begin());
|
| 88 |
+
result.has_local_root[3] = 0U;
|
| 89 |
+
result.allowed_tokens = style.allowed_tokens;
|
| 90 |
+
|
| 91 |
+
const float current_x = context.global_root[0], current_z = context.global_root[2];
|
| 92 |
+
const float velocity_x = (context.global_root[5] - current_x) * 30.0F;
|
| 93 |
+
const float velocity_z = (context.global_root[7] - current_z) * 30.0F;
|
| 94 |
+
float direction_x = command.movement_direction[0];
|
| 95 |
+
float direction_z = command.movement_direction[2];
|
| 96 |
+
const float direction_norm = std::hypot(direction_x, direction_z);
|
| 97 |
+
if (direction_norm > 1.0e-5F) { direction_x /= direction_norm; direction_z /= direction_norm; }
|
| 98 |
+
else { direction_x = 0.0F; direction_z = 0.0F; }
|
| 99 |
+
const float desired_speed = command.target_speed >= 0.0F ? command.target_speed : style.speed;
|
| 100 |
+
const float target_x = command.has_world_target != 0U ? command.world_target[0]
|
| 101 |
+
: current_x + 2.0F * desired_speed * direction_x;
|
| 102 |
+
const float target_z = command.has_world_target != 0U ? command.world_target[2]
|
| 103 |
+
: current_z + 2.0F * desired_speed * direction_z;
|
| 104 |
+
|
| 105 |
+
const float current_heading = std::atan2(context.global_root[4], context.global_root[3]);
|
| 106 |
+
const float next_heading = std::atan2(context.global_root[9], context.global_root[8]);
|
| 107 |
+
const float heading_velocity = wrap(next_heading - current_heading) * 30.0F;
|
| 108 |
+
float desired_heading = command.has_world_target != 0U ? command.world_target_heading
|
| 109 |
+
: std::atan2(command.facing_direction[0], command.facing_direction[2]);
|
| 110 |
+
desired_heading = current_heading + wrap(desired_heading - current_heading);
|
| 111 |
+
|
| 112 |
+
std::array<float, 4> target_root_x{}, target_root_z{}, target_heading{};
|
| 113 |
+
for (unsigned frame = 0; frame < 4U; ++frame) {
|
| 114 |
+
const float time = 1.0F + static_cast<float>(frame) / 30.0F;
|
| 115 |
+
target_root_x[frame] = spring(current_x, velocity_x, target_x, 0.4F, time);
|
| 116 |
+
target_root_z[frame] = spring(current_z, velocity_z, target_z, 0.4F, time);
|
| 117 |
+
target_heading[frame] = spring(current_heading, heading_velocity, desired_heading, 0.17F, time);
|
| 118 |
+
}
|
| 119 |
+
const std::uint32_t first = style.frames == 4U ? 0U
|
| 120 |
+
: static_cast<std::uint32_t>(command.seed % (style.frames - 4U));
|
| 121 |
+
for (std::uint32_t frame = 0; frame < 4U; ++frame) {
|
| 122 |
+
const auto source_frame = first + frame;
|
| 123 |
+
const float correction = wrap(target_heading[frame] - style.global_headings[source_frame]);
|
| 124 |
+
const mat3 correction_matrix = y_rotation(correction);
|
| 125 |
+
auto * global = result.global_root.data() + (frame + 4U) * global_root_width;
|
| 126 |
+
const float * style_root_joint = style.global_joint_positions.data() +
|
| 127 |
+
(static_cast<std::size_t>(source_frame) * g1_joint_count) * 3U;
|
| 128 |
+
global[0] = target_root_x[frame];
|
| 129 |
+
global[1] = style.global_root_positions[source_frame * 3U + 1U] + style_root_joint[1];
|
| 130 |
+
global[2] = target_root_z[frame];
|
| 131 |
+
global[3] = std::cos(target_heading[frame]); global[4] = std::sin(target_heading[frame]);
|
| 132 |
+
std::copy_n(global, 3U, result.target_root_translations.data() + frame * 3U);
|
| 133 |
+
float * pose = result.poses.data() + (frame + 4U) * external_pose_width;
|
| 134 |
+
for (std::uint32_t joint = 1; joint < g1_joint_count; ++joint) {
|
| 135 |
+
const float * source = style.global_joint_positions.data() +
|
| 136 |
+
(static_cast<std::size_t>(source_frame) * g1_joint_count + joint) * 3U;
|
| 137 |
+
const auto position = transform(correction_matrix, source);
|
| 138 |
+
std::copy(position.begin(), position.end(), pose + (joint - 1U) * 3U);
|
| 139 |
+
}
|
| 140 |
+
constexpr std::size_t rotation_offset = 99U;
|
| 141 |
+
std::array<mat3, g1_joint_count> target_global_rotations{};
|
| 142 |
+
for (std::uint32_t joint = 0; joint < g1_joint_count; ++joint) {
|
| 143 |
+
const float * source = style.global_joint_rotations.data() +
|
| 144 |
+
(static_cast<std::size_t>(source_frame) * g1_joint_count + joint) * 9U;
|
| 145 |
+
mat3 source_matrix{};
|
| 146 |
+
std::copy_n(source, 9, source_matrix.begin());
|
| 147 |
+
const auto matrix = multiply(correction_matrix, source_matrix);
|
| 148 |
+
target_global_rotations[joint] = matrix;
|
| 149 |
+
float * six = pose + rotation_offset + joint * 6U;
|
| 150 |
+
six[0]=matrix[0]; six[1]=matrix[3]; six[2]=matrix[6];
|
| 151 |
+
six[3]=matrix[1]; six[4]=matrix[4]; six[5]=matrix[7];
|
| 152 |
+
}
|
| 153 |
+
for (std::uint32_t joint = 0; joint < g1_joint_count; ++joint) {
|
| 154 |
+
mat3 local = target_global_rotations[joint];
|
| 155 |
+
const auto parent = agent.model->joint_parents[joint];
|
| 156 |
+
if (parent >= 0)
|
| 157 |
+
local = multiply(transpose(target_global_rotations[static_cast<std::size_t>(parent)]),
|
| 158 |
+
target_global_rotations[joint]);
|
| 159 |
+
const auto xyzw = matrix_xyzw(local);
|
| 160 |
+
std::copy(xyzw.begin(), xyzw.end(), result.target_local_rotations_xyzw.begin() +
|
| 161 |
+
static_cast<std::ptrdiff_t>((static_cast<std::size_t>(frame) * g1_joint_count + joint) * 4U));
|
| 162 |
+
}
|
| 163 |
+
}
|
| 164 |
+
for (std::uint32_t frame = 0; frame < 3U; ++frame) {
|
| 165 |
+
const float * current = result.global_root.data() + (frame + 4U) * global_root_width;
|
| 166 |
+
const float * next = current + global_root_width;
|
| 167 |
+
float * local = result.local_root.data() + (frame + 4U) * local_root_width;
|
| 168 |
+
local[0] = wrap(target_heading[frame + 1U] - target_heading[frame]) * 30.0F;
|
| 169 |
+
local[1] = (next[0] - current[0]) * 30.0F;
|
| 170 |
+
local[2] = (next[2] - current[2]) * 30.0F;
|
| 171 |
+
local[3] = current[1];
|
| 172 |
+
}
|
| 173 |
+
std::copy_n(result.local_root.data() + 6U * local_root_width, local_root_width,
|
| 174 |
+
result.local_root.data() + 7U * local_root_width);
|
| 175 |
+
result.local_root[7U * local_root_width + 3U] = result.global_root[7U * global_root_width + 1U];
|
| 176 |
+
return MB_OK;
|
| 177 |
+
}
|
| 178 |
+
|
| 179 |
+
} // namespace
|
| 180 |
+
|
| 181 |
+
mb_status seed_agent_from_style(mb_agent & agent, const mb_style & style,
|
| 182 |
+
std::string & reason) {
|
| 183 |
+
if (agent.model == nullptr || style.frames < 4U) {
|
| 184 |
+
reason = "agent or initial style is invalid";
|
| 185 |
+
return MB_INVALID_ARGUMENT;
|
| 186 |
+
}
|
| 187 |
+
agent.context_frames = 4U;
|
| 188 |
+
agent.context_root_xyz.resize(12U);
|
| 189 |
+
agent.context_local_rotations_xyzw.resize(4U * g1_joint_count * 4U);
|
| 190 |
+
for (std::uint32_t frame = 0; frame < 4U; ++frame) {
|
| 191 |
+
const float * root_joint = style.global_joint_positions.data() + frame * g1_joint_count * 3U;
|
| 192 |
+
for (unsigned axis = 0; axis < 3U; ++axis)
|
| 193 |
+
agent.context_root_xyz[frame * 3U + axis] =
|
| 194 |
+
style.global_root_positions[frame * 3U + axis] + root_joint[axis];
|
| 195 |
+
for (std::uint32_t joint = 0; joint < g1_joint_count; ++joint) {
|
| 196 |
+
const float * source = style.global_joint_rotations.data() +
|
| 197 |
+
(static_cast<std::size_t>(frame) * g1_joint_count + joint) * 9U;
|
| 198 |
+
mat3 global{}; std::copy_n(source, 9, global.begin());
|
| 199 |
+
const auto parent = agent.model->joint_parents[joint];
|
| 200 |
+
mat3 local = global;
|
| 201 |
+
if (parent >= 0) {
|
| 202 |
+
const float * parent_source = style.global_joint_rotations.data() +
|
| 203 |
+
(static_cast<std::size_t>(frame) * g1_joint_count +
|
| 204 |
+
static_cast<std::size_t>(parent)) * 9U;
|
| 205 |
+
mat3 parent_global{}; std::copy_n(parent_source, 9, parent_global.begin());
|
| 206 |
+
local = multiply(transpose(parent_global), global);
|
| 207 |
+
}
|
| 208 |
+
const auto xyzw = matrix_xyzw(local);
|
| 209 |
+
std::copy(xyzw.begin(), xyzw.end(), agent.context_local_rotations_xyzw.begin() +
|
| 210 |
+
static_cast<std::ptrdiff_t>((static_cast<std::size_t>(frame) * g1_joint_count + joint) * 4U));
|
| 211 |
+
}
|
| 212 |
+
}
|
| 213 |
+
agent.current_motion.reset();
|
| 214 |
+
agent.current_frame = 0U;
|
| 215 |
+
agent.initial_style = &style;
|
| 216 |
+
return MB_OK;
|
| 217 |
+
}
|
| 218 |
+
|
| 219 |
+
mb_status plan_agent(mb_agent & agent, const mb_command & command,
|
| 220 |
+
mb_motion & output, std::string & reason) {
|
| 221 |
+
const mb_style * style = command.style != nullptr ? command.style : agent.initial_style;
|
| 222 |
+
if (agent.model == nullptr || style == nullptr) {
|
| 223 |
+
reason = "agent plan requires a style";
|
| 224 |
+
return MB_INVALID_ARGUMENT;
|
| 225 |
+
}
|
| 226 |
+
if (agent.context_frames < 4U) {
|
| 227 |
+
const auto status = seed_agent_from_style(agent, *style, reason);
|
| 228 |
+
if (status != MB_OK) return status;
|
| 229 |
+
}
|
| 230 |
+
transition_constraints constraints;
|
| 231 |
+
auto status = build_constraints(agent, command, *style, constraints, reason);
|
| 232 |
+
if (status != MB_OK) return status;
|
| 233 |
+
status = run_transition(*agent.model, constraints, output, nullptr, reason);
|
| 234 |
+
if (status != MB_OK) return status;
|
| 235 |
+
output.target_frames = 4U;
|
| 236 |
+
output.target_root_translations.assign(constraints.target_root_translations.begin(),
|
| 237 |
+
constraints.target_root_translations.end());
|
| 238 |
+
output.target_local_rotations_xyzw.assign(constraints.target_local_rotations_xyzw.begin(),
|
| 239 |
+
constraints.target_local_rotations_xyzw.end());
|
| 240 |
+
return MB_OK;
|
| 241 |
+
}
|
| 242 |
+
|
| 243 |
+
mb_status advance_agent(mb_agent & agent, std::uint32_t frames,
|
| 244 |
+
std::string & reason) {
|
| 245 |
+
if (!agent.current_motion || agent.current_motion->frames == 0U) {
|
| 246 |
+
reason = "agent has no planned motion";
|
| 247 |
+
return MB_INVALID_ARGUMENT;
|
| 248 |
+
}
|
| 249 |
+
agent.current_frame = std::min<std::uint32_t>(
|
| 250 |
+
agent.current_frame + frames, static_cast<std::uint32_t>(agent.current_motion->frames - 1U));
|
| 251 |
+
agent.context_frames = 4U;
|
| 252 |
+
agent.context_root_xyz.resize(12U);
|
| 253 |
+
agent.context_local_rotations_xyzw.resize(4U * g1_joint_count * 4U);
|
| 254 |
+
for (std::uint32_t frame = 0; frame < 4U; ++frame) {
|
| 255 |
+
const auto source = std::min<std::uint64_t>(agent.current_frame + frame,
|
| 256 |
+
agent.current_motion->frames - 1U);
|
| 257 |
+
std::copy_n(agent.current_motion->root_translations.data() + source * 3U, 3,
|
| 258 |
+
agent.context_root_xyz.data() + frame * 3U);
|
| 259 |
+
std::copy_n(agent.current_motion->local_rotations_xyzw.data() + source * g1_joint_count * 4U,
|
| 260 |
+
g1_joint_count * 4U,
|
| 261 |
+
agent.context_local_rotations_xyzw.data() + frame * g1_joint_count * 4U);
|
| 262 |
+
}
|
| 263 |
+
return MB_OK;
|
| 264 |
+
}
|
| 265 |
+
|
| 266 |
+
} // namespace motionbricks::detail
|
src/agent.hpp
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#pragma once
|
| 2 |
+
|
| 3 |
+
#include <motionbricks/motionbricks.h>
|
| 4 |
+
|
| 5 |
+
#include <cstdint>
|
| 6 |
+
#include <string>
|
| 7 |
+
|
| 8 |
+
struct mb_agent;
|
| 9 |
+
struct mb_command;
|
| 10 |
+
struct mb_motion;
|
| 11 |
+
struct mb_style;
|
| 12 |
+
|
| 13 |
+
namespace motionbricks::detail {
|
| 14 |
+
|
| 15 |
+
mb_status seed_agent_from_style(mb_agent & agent, const mb_style & style,
|
| 16 |
+
std::string & reason);
|
| 17 |
+
mb_status plan_agent(mb_agent & agent, const mb_command & command,
|
| 18 |
+
mb_motion & output, std::string & reason);
|
| 19 |
+
mb_status advance_agent(mb_agent & agent, std::uint32_t frames,
|
| 20 |
+
std::string & reason);
|
| 21 |
+
|
| 22 |
+
} // namespace motionbricks::detail
|
src/capi.cpp
ADDED
|
@@ -0,0 +1,552 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#include <motionbricks/motionbricks.h>
|
| 2 |
+
|
| 3 |
+
#include "error.hpp"
|
| 4 |
+
#include "agent.hpp"
|
| 5 |
+
#include "handles.hpp"
|
| 6 |
+
#include "model.hpp"
|
| 7 |
+
#include "style.hpp"
|
| 8 |
+
|
| 9 |
+
#include <algorithm>
|
| 10 |
+
#include <cmath>
|
| 11 |
+
#include <cstdint>
|
| 12 |
+
#include <memory>
|
| 13 |
+
#include <new>
|
| 14 |
+
#include <string>
|
| 15 |
+
#include <string_view>
|
| 16 |
+
|
| 17 |
+
namespace {
|
| 18 |
+
|
| 19 |
+
using motionbricks::detail::fail;
|
| 20 |
+
using motionbricks::detail::guard;
|
| 21 |
+
|
| 22 |
+
bool valid_device(mb_device device) noexcept {
|
| 23 |
+
return device == MB_DEVICE_AUTO || device == MB_DEVICE_CPU || device == MB_DEVICE_VULKAN;
|
| 24 |
+
}
|
| 25 |
+
|
| 26 |
+
bool finite3(float x, float y, float z) noexcept {
|
| 27 |
+
return std::isfinite(x) && std::isfinite(y) && std::isfinite(z);
|
| 28 |
+
}
|
| 29 |
+
|
| 30 |
+
} // namespace
|
| 31 |
+
|
| 32 |
+
extern "C" {
|
| 33 |
+
|
| 34 |
+
uint32_t mb_abi_version(void) { return MB_ABI_VERSION; }
|
| 35 |
+
|
| 36 |
+
const char * mb_status_string(mb_status status) {
|
| 37 |
+
switch (status) {
|
| 38 |
+
case MB_OK: return "ok";
|
| 39 |
+
case MB_INVALID_ARGUMENT: return "invalid argument";
|
| 40 |
+
case MB_OUT_OF_MEMORY: return "out of memory";
|
| 41 |
+
case MB_IO_ERROR: return "I/O error";
|
| 42 |
+
case MB_INVALID_FORMAT: return "invalid format";
|
| 43 |
+
case MB_INCOMPATIBLE_MODEL: return "incompatible model";
|
| 44 |
+
case MB_BACKEND_UNAVAILABLE: return "backend unavailable";
|
| 45 |
+
case MB_COMPUTE_FAILED: return "compute failed";
|
| 46 |
+
case MB_NOT_IMPLEMENTED: return "not implemented";
|
| 47 |
+
case MB_INTERNAL_ERROR: return "internal error";
|
| 48 |
+
default: return "unknown status";
|
| 49 |
+
}
|
| 50 |
+
}
|
| 51 |
+
|
| 52 |
+
mb_status mb_runtime_options_create(mb_runtime_options ** output,
|
| 53 |
+
char * error, uint64_t error_capacity) {
|
| 54 |
+
return guard(error, error_capacity, [&]() -> mb_status {
|
| 55 |
+
if (output == nullptr) return fail(MB_INVALID_ARGUMENT, error, error_capacity, "output is null");
|
| 56 |
+
*output = nullptr;
|
| 57 |
+
*output = new mb_runtime_options();
|
| 58 |
+
return MB_OK;
|
| 59 |
+
});
|
| 60 |
+
}
|
| 61 |
+
|
| 62 |
+
void mb_runtime_options_free(mb_runtime_options * value) { delete value; }
|
| 63 |
+
|
| 64 |
+
mb_status mb_runtime_options_set_device(mb_runtime_options * value, mb_device device,
|
| 65 |
+
char * error, uint64_t error_capacity) {
|
| 66 |
+
return guard(error, error_capacity, [&]() -> mb_status {
|
| 67 |
+
if (value == nullptr) return fail(MB_INVALID_ARGUMENT, error, error_capacity, "options are null");
|
| 68 |
+
if (!valid_device(device)) return fail(MB_INVALID_ARGUMENT, error, error_capacity, "invalid device");
|
| 69 |
+
value->device = device;
|
| 70 |
+
return MB_OK;
|
| 71 |
+
});
|
| 72 |
+
}
|
| 73 |
+
|
| 74 |
+
mb_status mb_runtime_options_get_device(const mb_runtime_options * value, mb_device * output,
|
| 75 |
+
char * error, uint64_t error_capacity) {
|
| 76 |
+
return guard(error, error_capacity, [&]() -> mb_status {
|
| 77 |
+
if (value == nullptr || output == nullptr)
|
| 78 |
+
return fail(MB_INVALID_ARGUMENT, error, error_capacity, "options or output is null");
|
| 79 |
+
*output = value->device;
|
| 80 |
+
return MB_OK;
|
| 81 |
+
});
|
| 82 |
+
}
|
| 83 |
+
|
| 84 |
+
mb_status mb_runtime_options_set_threads(mb_runtime_options * value, uint32_t threads,
|
| 85 |
+
char * error, uint64_t error_capacity) {
|
| 86 |
+
return guard(error, error_capacity, [&]() -> mb_status {
|
| 87 |
+
if (value == nullptr) return fail(MB_INVALID_ARGUMENT, error, error_capacity, "options are null");
|
| 88 |
+
value->threads = threads;
|
| 89 |
+
return MB_OK;
|
| 90 |
+
});
|
| 91 |
+
}
|
| 92 |
+
|
| 93 |
+
mb_status mb_runtime_options_get_threads(const mb_runtime_options * value, uint32_t * output,
|
| 94 |
+
char * error, uint64_t error_capacity) {
|
| 95 |
+
return guard(error, error_capacity, [&]() -> mb_status {
|
| 96 |
+
if (value == nullptr || output == nullptr)
|
| 97 |
+
return fail(MB_INVALID_ARGUMENT, error, error_capacity, "options or output is null");
|
| 98 |
+
*output = value->threads;
|
| 99 |
+
return MB_OK;
|
| 100 |
+
});
|
| 101 |
+
}
|
| 102 |
+
|
| 103 |
+
mb_status mb_runtime_options_set_backend_directory(mb_runtime_options * value,
|
| 104 |
+
const char * directory,
|
| 105 |
+
char * error, uint64_t error_capacity) {
|
| 106 |
+
return guard(error, error_capacity, [&]() -> mb_status {
|
| 107 |
+
if (value == nullptr) return fail(MB_INVALID_ARGUMENT, error, error_capacity, "options are null");
|
| 108 |
+
value->backend_directory = directory != nullptr ? directory : "";
|
| 109 |
+
return MB_OK;
|
| 110 |
+
});
|
| 111 |
+
}
|
| 112 |
+
|
| 113 |
+
mb_status mb_runtime_options_get_backend_directory(const mb_runtime_options * value,
|
| 114 |
+
const char ** output,
|
| 115 |
+
char * error, uint64_t error_capacity) {
|
| 116 |
+
return guard(error, error_capacity, [&]() -> mb_status {
|
| 117 |
+
if (value == nullptr || output == nullptr)
|
| 118 |
+
return fail(MB_INVALID_ARGUMENT, error, error_capacity, "options or output is null");
|
| 119 |
+
*output = value->backend_directory.c_str();
|
| 120 |
+
return MB_OK;
|
| 121 |
+
});
|
| 122 |
+
}
|
| 123 |
+
|
| 124 |
+
mb_status mb_command_create(mb_command ** output, char * error, uint64_t error_capacity) {
|
| 125 |
+
return guard(error, error_capacity, [&]() -> mb_status {
|
| 126 |
+
if (output == nullptr) return fail(MB_INVALID_ARGUMENT, error, error_capacity, "output is null");
|
| 127 |
+
*output = nullptr;
|
| 128 |
+
*output = new mb_command();
|
| 129 |
+
return MB_OK;
|
| 130 |
+
});
|
| 131 |
+
}
|
| 132 |
+
|
| 133 |
+
void mb_command_free(mb_command * value) { delete value; }
|
| 134 |
+
|
| 135 |
+
mb_status mb_command_set_style(mb_command * value, const mb_style * style,
|
| 136 |
+
char * error, uint64_t error_capacity) {
|
| 137 |
+
return guard(error, error_capacity, [&]() -> mb_status {
|
| 138 |
+
if (value == nullptr) return fail(MB_INVALID_ARGUMENT, error, error_capacity, "command is null");
|
| 139 |
+
value->style = style;
|
| 140 |
+
return MB_OK;
|
| 141 |
+
});
|
| 142 |
+
}
|
| 143 |
+
|
| 144 |
+
mb_status mb_command_get_style(const mb_command * value, const mb_style ** output,
|
| 145 |
+
char * error, uint64_t error_capacity) {
|
| 146 |
+
return guard(error, error_capacity, [&]() -> mb_status {
|
| 147 |
+
if (value == nullptr || output == nullptr)
|
| 148 |
+
return fail(MB_INVALID_ARGUMENT, error, error_capacity, "command or output is null");
|
| 149 |
+
*output = value->style;
|
| 150 |
+
return MB_OK;
|
| 151 |
+
});
|
| 152 |
+
}
|
| 153 |
+
|
| 154 |
+
mb_status mb_command_set_movement_direction(mb_command * value, float x, float y, float z,
|
| 155 |
+
char * error, uint64_t error_capacity) {
|
| 156 |
+
return guard(error, error_capacity, [&]() -> mb_status {
|
| 157 |
+
if (value == nullptr) return fail(MB_INVALID_ARGUMENT, error, error_capacity, "command is null");
|
| 158 |
+
if (!finite3(x, y, z)) return fail(MB_INVALID_ARGUMENT, error, error_capacity, "movement direction is not finite");
|
| 159 |
+
value->movement_direction = {x, y, z};
|
| 160 |
+
return MB_OK;
|
| 161 |
+
});
|
| 162 |
+
}
|
| 163 |
+
|
| 164 |
+
mb_status mb_command_get_movement_direction(const mb_command * value,
|
| 165 |
+
float * x, float * y, float * z,
|
| 166 |
+
char * error, uint64_t error_capacity) {
|
| 167 |
+
return guard(error, error_capacity, [&]() -> mb_status {
|
| 168 |
+
if (value == nullptr || x == nullptr || y == nullptr || z == nullptr)
|
| 169 |
+
return fail(MB_INVALID_ARGUMENT, error, error_capacity, "command or output is null");
|
| 170 |
+
*x = value->movement_direction[0];
|
| 171 |
+
*y = value->movement_direction[1];
|
| 172 |
+
*z = value->movement_direction[2];
|
| 173 |
+
return MB_OK;
|
| 174 |
+
});
|
| 175 |
+
}
|
| 176 |
+
|
| 177 |
+
mb_status mb_command_set_facing_direction(mb_command * value, float x, float y, float z,
|
| 178 |
+
char * error, uint64_t error_capacity) {
|
| 179 |
+
return guard(error, error_capacity, [&]() -> mb_status {
|
| 180 |
+
if (value == nullptr) return fail(MB_INVALID_ARGUMENT, error, error_capacity, "command is null");
|
| 181 |
+
if (!finite3(x, y, z)) return fail(MB_INVALID_ARGUMENT, error, error_capacity, "facing direction is not finite");
|
| 182 |
+
if (x * x + y * y + z * z <= 1.0e-12F)
|
| 183 |
+
return fail(MB_INVALID_ARGUMENT, error, error_capacity, "facing direction is zero");
|
| 184 |
+
value->facing_direction = {x, y, z};
|
| 185 |
+
return MB_OK;
|
| 186 |
+
});
|
| 187 |
+
}
|
| 188 |
+
|
| 189 |
+
mb_status mb_command_get_facing_direction(const mb_command * value,
|
| 190 |
+
float * x, float * y, float * z,
|
| 191 |
+
char * error, uint64_t error_capacity) {
|
| 192 |
+
return guard(error, error_capacity, [&]() -> mb_status {
|
| 193 |
+
if (value == nullptr || x == nullptr || y == nullptr || z == nullptr)
|
| 194 |
+
return fail(MB_INVALID_ARGUMENT, error, error_capacity, "command or output is null");
|
| 195 |
+
*x = value->facing_direction[0];
|
| 196 |
+
*y = value->facing_direction[1];
|
| 197 |
+
*z = value->facing_direction[2];
|
| 198 |
+
return MB_OK;
|
| 199 |
+
});
|
| 200 |
+
}
|
| 201 |
+
|
| 202 |
+
mb_status mb_command_set_target_speed(mb_command * value, float speed,
|
| 203 |
+
char * error, uint64_t error_capacity) {
|
| 204 |
+
return guard(error, error_capacity, [&]() -> mb_status {
|
| 205 |
+
if (value == nullptr) return fail(MB_INVALID_ARGUMENT, error, error_capacity, "command is null");
|
| 206 |
+
if (!std::isfinite(speed) || speed < -1.0F)
|
| 207 |
+
return fail(MB_INVALID_ARGUMENT, error, error_capacity, "target speed must be finite and at least -1");
|
| 208 |
+
value->target_speed = speed;
|
| 209 |
+
return MB_OK;
|
| 210 |
+
});
|
| 211 |
+
}
|
| 212 |
+
|
| 213 |
+
mb_status mb_command_get_target_speed(const mb_command * value, float * output,
|
| 214 |
+
char * error, uint64_t error_capacity) {
|
| 215 |
+
return guard(error, error_capacity, [&]() -> mb_status {
|
| 216 |
+
if (value == nullptr || output == nullptr)
|
| 217 |
+
return fail(MB_INVALID_ARGUMENT, error, error_capacity, "command or output is null");
|
| 218 |
+
*output = value->target_speed;
|
| 219 |
+
return MB_OK;
|
| 220 |
+
});
|
| 221 |
+
}
|
| 222 |
+
|
| 223 |
+
mb_status mb_command_set_world_target(mb_command * value, float x, float y, float z,
|
| 224 |
+
float heading, uint32_t enabled,
|
| 225 |
+
char * error, uint64_t error_capacity) {
|
| 226 |
+
return guard(error, error_capacity, [&]() -> mb_status {
|
| 227 |
+
if (value == nullptr) return fail(MB_INVALID_ARGUMENT, error, error_capacity, "command is null");
|
| 228 |
+
if (!finite3(x, y, z) || !std::isfinite(heading))
|
| 229 |
+
return fail(MB_INVALID_ARGUMENT, error, error_capacity, "world target is not finite");
|
| 230 |
+
if (enabled > 1) return fail(MB_INVALID_ARGUMENT, error, error_capacity, "enabled must be 0 or 1");
|
| 231 |
+
value->world_target = {x, y, z};
|
| 232 |
+
value->world_target_heading = heading;
|
| 233 |
+
value->has_world_target = enabled;
|
| 234 |
+
return MB_OK;
|
| 235 |
+
});
|
| 236 |
+
}
|
| 237 |
+
|
| 238 |
+
mb_status mb_command_get_world_target(const mb_command * value,
|
| 239 |
+
float * x, float * y, float * z,
|
| 240 |
+
float * heading, uint32_t * enabled,
|
| 241 |
+
char * error, uint64_t error_capacity) {
|
| 242 |
+
return guard(error, error_capacity, [&]() -> mb_status {
|
| 243 |
+
if (value == nullptr || x == nullptr || y == nullptr || z == nullptr ||
|
| 244 |
+
heading == nullptr || enabled == nullptr)
|
| 245 |
+
return fail(MB_INVALID_ARGUMENT, error, error_capacity, "command or output is null");
|
| 246 |
+
*x = value->world_target[0];
|
| 247 |
+
*y = value->world_target[1];
|
| 248 |
+
*z = value->world_target[2];
|
| 249 |
+
*heading = value->world_target_heading;
|
| 250 |
+
*enabled = value->has_world_target;
|
| 251 |
+
return MB_OK;
|
| 252 |
+
});
|
| 253 |
+
}
|
| 254 |
+
|
| 255 |
+
mb_status mb_command_set_seed(mb_command * value, uint64_t seed,
|
| 256 |
+
char * error, uint64_t error_capacity) {
|
| 257 |
+
return guard(error, error_capacity, [&]() -> mb_status {
|
| 258 |
+
if (value == nullptr) return fail(MB_INVALID_ARGUMENT, error, error_capacity, "command is null");
|
| 259 |
+
value->seed = seed;
|
| 260 |
+
return MB_OK;
|
| 261 |
+
});
|
| 262 |
+
}
|
| 263 |
+
|
| 264 |
+
mb_status mb_command_get_seed(const mb_command * value, uint64_t * output,
|
| 265 |
+
char * error, uint64_t error_capacity) {
|
| 266 |
+
return guard(error, error_capacity, [&]() -> mb_status {
|
| 267 |
+
if (value == nullptr || output == nullptr)
|
| 268 |
+
return fail(MB_INVALID_ARGUMENT, error, error_capacity, "command or output is null");
|
| 269 |
+
*output = value->seed;
|
| 270 |
+
return MB_OK;
|
| 271 |
+
});
|
| 272 |
+
}
|
| 273 |
+
|
| 274 |
+
mb_status mb_model_load(const char * bundle_directory, const mb_runtime_options * options, mb_model ** output,
|
| 275 |
+
char * error, uint64_t error_capacity) {
|
| 276 |
+
return guard(error, error_capacity, [&]() -> mb_status {
|
| 277 |
+
if (output == nullptr) return fail(MB_INVALID_ARGUMENT, error, error_capacity, "output is null");
|
| 278 |
+
*output = nullptr;
|
| 279 |
+
if (bundle_directory == nullptr || *bundle_directory == '\0')
|
| 280 |
+
return fail(MB_INVALID_ARGUMENT, error, error_capacity, "bundle directory is empty");
|
| 281 |
+
if (options != nullptr && !valid_device(options->device))
|
| 282 |
+
return fail(MB_INVALID_ARGUMENT, error, error_capacity, "invalid device in runtime options");
|
| 283 |
+
#if !defined(MOTIONBRICKS_HAVE_VULKAN)
|
| 284 |
+
if (options != nullptr && options->device == MB_DEVICE_VULKAN)
|
| 285 |
+
return fail(MB_BACKEND_UNAVAILABLE, error, error_capacity,
|
| 286 |
+
"this build has no Vulkan backend");
|
| 287 |
+
#endif
|
| 288 |
+
auto model = std::make_unique<mb_model>();
|
| 289 |
+
std::string reason;
|
| 290 |
+
const auto status = motionbricks::detail::load_model_bundle(bundle_directory, options, *model, reason);
|
| 291 |
+
if (status != MB_OK) return fail(status, error, error_capacity, reason);
|
| 292 |
+
*output = model.release();
|
| 293 |
+
return MB_OK;
|
| 294 |
+
});
|
| 295 |
+
}
|
| 296 |
+
|
| 297 |
+
void mb_model_free(mb_model * value) { delete value; }
|
| 298 |
+
|
| 299 |
+
mb_status mb_model_get_parameter_count(const mb_model * value, uint64_t * output,
|
| 300 |
+
char * error, uint64_t error_capacity) {
|
| 301 |
+
return guard(error, error_capacity, [&]() -> mb_status {
|
| 302 |
+
if (value == nullptr || output == nullptr)
|
| 303 |
+
return fail(MB_INVALID_ARGUMENT, error, error_capacity, "model or output is null");
|
| 304 |
+
*output = value->parameter_count;
|
| 305 |
+
return MB_OK;
|
| 306 |
+
});
|
| 307 |
+
}
|
| 308 |
+
|
| 309 |
+
mb_status mb_model_get_joint_count(const mb_model * value, uint32_t * output,
|
| 310 |
+
char * error, uint64_t error_capacity) {
|
| 311 |
+
return guard(error, error_capacity, [&]() -> mb_status {
|
| 312 |
+
if (value == nullptr || output == nullptr)
|
| 313 |
+
return fail(MB_INVALID_ARGUMENT, error, error_capacity, "model or output is null");
|
| 314 |
+
*output = static_cast<std::uint32_t>(value->joint_names.size());
|
| 315 |
+
return MB_OK;
|
| 316 |
+
});
|
| 317 |
+
}
|
| 318 |
+
|
| 319 |
+
mb_status mb_model_get_joint_name(const mb_model * value, uint32_t joint, const char ** output,
|
| 320 |
+
char * error, uint64_t error_capacity) {
|
| 321 |
+
return guard(error, error_capacity, [&]() -> mb_status {
|
| 322 |
+
if (value == nullptr || output == nullptr)
|
| 323 |
+
return fail(MB_INVALID_ARGUMENT, error, error_capacity, "model or output is null");
|
| 324 |
+
if (joint >= value->joint_names.size())
|
| 325 |
+
return fail(MB_INVALID_ARGUMENT, error, error_capacity, "joint index is out of range");
|
| 326 |
+
*output = value->joint_names[joint].c_str();
|
| 327 |
+
return MB_OK;
|
| 328 |
+
});
|
| 329 |
+
}
|
| 330 |
+
|
| 331 |
+
mb_status mb_model_get_joint_parent(const mb_model * value, uint32_t joint, int32_t * output,
|
| 332 |
+
char * error, uint64_t error_capacity) {
|
| 333 |
+
return guard(error, error_capacity, [&]() -> mb_status {
|
| 334 |
+
if (value == nullptr || output == nullptr)
|
| 335 |
+
return fail(MB_INVALID_ARGUMENT, error, error_capacity, "model or output is null");
|
| 336 |
+
if (joint >= value->joint_parents.size())
|
| 337 |
+
return fail(MB_INVALID_ARGUMENT, error, error_capacity, "joint index is out of range");
|
| 338 |
+
*output = value->joint_parents[joint];
|
| 339 |
+
return MB_OK;
|
| 340 |
+
});
|
| 341 |
+
}
|
| 342 |
+
|
| 343 |
+
mb_status mb_model_get_neutral_joint_position(const mb_model * value, uint32_t joint,
|
| 344 |
+
float * x, float * y, float * z,
|
| 345 |
+
char * error, uint64_t error_capacity) {
|
| 346 |
+
return guard(error, error_capacity, [&]() -> mb_status {
|
| 347 |
+
if (value == nullptr || x == nullptr || y == nullptr || z == nullptr)
|
| 348 |
+
return fail(MB_INVALID_ARGUMENT, error, error_capacity, "model or output is null");
|
| 349 |
+
if (joint >= value->joint_names.size() || value->neutral_joints.size() != value->joint_names.size() * 3U)
|
| 350 |
+
return fail(MB_INVALID_ARGUMENT, error, error_capacity, "joint index is out of range");
|
| 351 |
+
*x = value->neutral_joints[static_cast<std::size_t>(joint) * 3U];
|
| 352 |
+
*y = value->neutral_joints[static_cast<std::size_t>(joint) * 3U + 1U];
|
| 353 |
+
*z = value->neutral_joints[static_cast<std::size_t>(joint) * 3U + 2U];
|
| 354 |
+
return MB_OK;
|
| 355 |
+
});
|
| 356 |
+
}
|
| 357 |
+
|
| 358 |
+
mb_status mb_style_load(const mb_model * model, const char * style_path, mb_style ** output,
|
| 359 |
+
char * error, uint64_t error_capacity) {
|
| 360 |
+
return guard(error, error_capacity, [&]() -> mb_status {
|
| 361 |
+
if (output == nullptr) return fail(MB_INVALID_ARGUMENT, error, error_capacity, "output is null");
|
| 362 |
+
*output = nullptr;
|
| 363 |
+
if (model == nullptr || style_path == nullptr || *style_path == '\0')
|
| 364 |
+
return fail(MB_INVALID_ARGUMENT, error, error_capacity, "model or style path is null");
|
| 365 |
+
auto style = std::make_unique<mb_style>();
|
| 366 |
+
std::string reason;
|
| 367 |
+
const auto status = motionbricks::detail::load_style_file(style_path, *style, reason);
|
| 368 |
+
if (status != MB_OK) return fail(status, error, error_capacity, reason);
|
| 369 |
+
*output = style.release();
|
| 370 |
+
return MB_OK;
|
| 371 |
+
});
|
| 372 |
+
}
|
| 373 |
+
|
| 374 |
+
void mb_style_free(mb_style * value) { delete value; }
|
| 375 |
+
|
| 376 |
+
mb_status mb_style_get_name(const mb_style * value, const char ** output, char * error, uint64_t error_capacity) {
|
| 377 |
+
return guard(error, error_capacity, [&]() -> mb_status {
|
| 378 |
+
if (value == nullptr || output == nullptr)
|
| 379 |
+
return fail(MB_INVALID_ARGUMENT, error, error_capacity, "style or output is null");
|
| 380 |
+
*output = value->name.c_str();
|
| 381 |
+
return MB_OK;
|
| 382 |
+
});
|
| 383 |
+
}
|
| 384 |
+
|
| 385 |
+
mb_status mb_style_set_speed(mb_style * value, float speed, char * error, uint64_t error_capacity) {
|
| 386 |
+
return guard(error, error_capacity, [&]() -> mb_status {
|
| 387 |
+
if (value == nullptr) return fail(MB_INVALID_ARGUMENT, error, error_capacity, "style is null");
|
| 388 |
+
if (!std::isfinite(speed) || speed < 0.0F)
|
| 389 |
+
return fail(MB_INVALID_ARGUMENT, error, error_capacity, "style speed must be finite and non-negative");
|
| 390 |
+
value->speed = speed;
|
| 391 |
+
return MB_OK;
|
| 392 |
+
});
|
| 393 |
+
}
|
| 394 |
+
|
| 395 |
+
mb_status mb_style_get_speed(const mb_style * value, float * output, char * error, uint64_t error_capacity) {
|
| 396 |
+
return guard(error, error_capacity, [&]() -> mb_status {
|
| 397 |
+
if (value == nullptr || output == nullptr)
|
| 398 |
+
return fail(MB_INVALID_ARGUMENT, error, error_capacity, "style or output is null");
|
| 399 |
+
*output = value->speed;
|
| 400 |
+
return MB_OK;
|
| 401 |
+
});
|
| 402 |
+
}
|
| 403 |
+
|
| 404 |
+
mb_status mb_agent_create(const mb_model * model, mb_agent ** output,
|
| 405 |
+
char * error, uint64_t error_capacity) {
|
| 406 |
+
return guard(error, error_capacity, [&]() -> mb_status {
|
| 407 |
+
if (output == nullptr) return fail(MB_INVALID_ARGUMENT, error, error_capacity, "output is null");
|
| 408 |
+
*output = nullptr;
|
| 409 |
+
if (model == nullptr) return fail(MB_INVALID_ARGUMENT, error, error_capacity, "model is null");
|
| 410 |
+
auto agent = std::make_unique<mb_agent>();
|
| 411 |
+
agent->model = model;
|
| 412 |
+
*output = agent.release();
|
| 413 |
+
return MB_OK;
|
| 414 |
+
});
|
| 415 |
+
}
|
| 416 |
+
|
| 417 |
+
void mb_agent_free(mb_agent * value) { delete value; }
|
| 418 |
+
|
| 419 |
+
mb_status mb_agent_reset(mb_agent * value, const mb_style * style, char * error, uint64_t error_capacity) {
|
| 420 |
+
return guard(error, error_capacity, [&]() -> mb_status {
|
| 421 |
+
if (value == nullptr || style == nullptr)
|
| 422 |
+
return fail(MB_INVALID_ARGUMENT, error, error_capacity, "agent or style is null");
|
| 423 |
+
std::string reason;
|
| 424 |
+
const auto status = motionbricks::detail::seed_agent_from_style(*value, *style, reason);
|
| 425 |
+
return status == MB_OK ? MB_OK : fail(status, error, error_capacity, reason);
|
| 426 |
+
});
|
| 427 |
+
}
|
| 428 |
+
|
| 429 |
+
mb_status mb_agent_set_context(mb_agent * value, const float * roots, const float * rotations,
|
| 430 |
+
uint64_t frames, uint64_t joints,
|
| 431 |
+
char * error, uint64_t error_capacity) {
|
| 432 |
+
return guard(error, error_capacity, [&]() -> mb_status {
|
| 433 |
+
if (value == nullptr || roots == nullptr || rotations == nullptr)
|
| 434 |
+
return fail(MB_INVALID_ARGUMENT, error, error_capacity, "agent or context data is null");
|
| 435 |
+
if (frames < 4U || joints != 34U || frames > 1000000U)
|
| 436 |
+
return fail(MB_INVALID_ARGUMENT, error, error_capacity, "context must contain at least four frames and 34 joints");
|
| 437 |
+
const auto root_count = static_cast<std::size_t>(frames) * 3U;
|
| 438 |
+
const auto rotation_count = static_cast<std::size_t>(frames) * 34U * 4U;
|
| 439 |
+
if (!std::all_of(roots, roots + root_count, [](float item) { return std::isfinite(item); }) ||
|
| 440 |
+
!std::all_of(rotations, rotations + rotation_count, [](float item) { return std::isfinite(item); }))
|
| 441 |
+
return fail(MB_INVALID_ARGUMENT, error, error_capacity, "context contains non-finite values");
|
| 442 |
+
value->context_root_xyz.assign(roots, roots + root_count);
|
| 443 |
+
value->context_local_rotations_xyzw.assign(rotations, rotations + rotation_count);
|
| 444 |
+
value->context_frames = frames;
|
| 445 |
+
value->current_motion.reset();
|
| 446 |
+
value->current_frame = 0U;
|
| 447 |
+
return MB_OK;
|
| 448 |
+
});
|
| 449 |
+
}
|
| 450 |
+
|
| 451 |
+
mb_status mb_agent_plan(mb_agent * value, const mb_command * command, mb_motion ** output,
|
| 452 |
+
char * error, uint64_t error_capacity) {
|
| 453 |
+
return guard(error, error_capacity, [&]() -> mb_status {
|
| 454 |
+
if (output == nullptr) return fail(MB_INVALID_ARGUMENT, error, error_capacity, "output is null");
|
| 455 |
+
*output = nullptr;
|
| 456 |
+
if (value == nullptr || command == nullptr)
|
| 457 |
+
return fail(MB_INVALID_ARGUMENT, error, error_capacity, "agent or command is null");
|
| 458 |
+
auto motion = std::make_unique<mb_motion>();
|
| 459 |
+
std::string reason;
|
| 460 |
+
const auto status = motionbricks::detail::plan_agent(*value, *command, *motion, reason);
|
| 461 |
+
if (status != MB_OK) return fail(status, error, error_capacity, reason);
|
| 462 |
+
value->current_motion = std::make_unique<mb_motion>(*motion);
|
| 463 |
+
value->current_frame = 0U;
|
| 464 |
+
*output = motion.release();
|
| 465 |
+
return MB_OK;
|
| 466 |
+
});
|
| 467 |
+
}
|
| 468 |
+
|
| 469 |
+
mb_status mb_agent_advance(mb_agent * value, uint32_t frames, char * error, uint64_t error_capacity) {
|
| 470 |
+
return guard(error, error_capacity, [&]() -> mb_status {
|
| 471 |
+
if (value == nullptr) return fail(MB_INVALID_ARGUMENT, error, error_capacity, "agent is null");
|
| 472 |
+
std::string reason;
|
| 473 |
+
const auto status = motionbricks::detail::advance_agent(*value, frames, reason);
|
| 474 |
+
return status == MB_OK ? MB_OK : fail(status, error, error_capacity, reason);
|
| 475 |
+
});
|
| 476 |
+
}
|
| 477 |
+
|
| 478 |
+
void mb_motion_free(mb_motion * value) { delete value; }
|
| 479 |
+
|
| 480 |
+
mb_status mb_motion_get_frame_count(const mb_motion * value, uint64_t * output, char * error, uint64_t error_capacity) {
|
| 481 |
+
return guard(error, error_capacity, [&]() -> mb_status {
|
| 482 |
+
if (value == nullptr || output == nullptr)
|
| 483 |
+
return fail(MB_INVALID_ARGUMENT, error, error_capacity, "motion or output is null");
|
| 484 |
+
*output = value->frames;
|
| 485 |
+
return MB_OK;
|
| 486 |
+
});
|
| 487 |
+
}
|
| 488 |
+
|
| 489 |
+
mb_status mb_motion_get_joint_count(const mb_motion * value, uint64_t * output, char * error, uint64_t error_capacity) {
|
| 490 |
+
return guard(error, error_capacity, [&]() -> mb_status {
|
| 491 |
+
if (value == nullptr || output == nullptr)
|
| 492 |
+
return fail(MB_INVALID_ARGUMENT, error, error_capacity, "motion or output is null");
|
| 493 |
+
*output = value->joints;
|
| 494 |
+
return MB_OK;
|
| 495 |
+
});
|
| 496 |
+
}
|
| 497 |
+
|
| 498 |
+
mb_status mb_motion_get_root_translations(const mb_motion * value, const float ** output, uint64_t * values,
|
| 499 |
+
char * error, uint64_t error_capacity) {
|
| 500 |
+
return guard(error, error_capacity, [&]() -> mb_status {
|
| 501 |
+
if (value == nullptr || output == nullptr || values == nullptr)
|
| 502 |
+
return fail(MB_INVALID_ARGUMENT, error, error_capacity, "motion or output is null");
|
| 503 |
+
*output = value->root_translations.data();
|
| 504 |
+
*values = value->root_translations.size();
|
| 505 |
+
return MB_OK;
|
| 506 |
+
});
|
| 507 |
+
}
|
| 508 |
+
|
| 509 |
+
mb_status mb_motion_get_local_rotations_xyzw(const mb_motion * value, const float ** output, uint64_t * values,
|
| 510 |
+
char * error, uint64_t error_capacity) {
|
| 511 |
+
return guard(error, error_capacity, [&]() -> mb_status {
|
| 512 |
+
if (value == nullptr || output == nullptr || values == nullptr)
|
| 513 |
+
return fail(MB_INVALID_ARGUMENT, error, error_capacity, "motion or output is null");
|
| 514 |
+
*output = value->local_rotations_xyzw.data();
|
| 515 |
+
*values = value->local_rotations_xyzw.size();
|
| 516 |
+
return MB_OK;
|
| 517 |
+
});
|
| 518 |
+
}
|
| 519 |
+
|
| 520 |
+
mb_status mb_motion_get_target_frame_count(const mb_motion * value, uint64_t * output,
|
| 521 |
+
char * error, uint64_t error_capacity) {
|
| 522 |
+
return guard(error, error_capacity, [&]() -> mb_status {
|
| 523 |
+
if (value == nullptr || output == nullptr)
|
| 524 |
+
return fail(MB_INVALID_ARGUMENT, error, error_capacity, "motion or output is null");
|
| 525 |
+
*output = value->target_frames;
|
| 526 |
+
return MB_OK;
|
| 527 |
+
});
|
| 528 |
+
}
|
| 529 |
+
|
| 530 |
+
mb_status mb_motion_get_target_root_translations(const mb_motion * value, const float ** output, uint64_t * values,
|
| 531 |
+
char * error, uint64_t error_capacity) {
|
| 532 |
+
return guard(error, error_capacity, [&]() -> mb_status {
|
| 533 |
+
if (value == nullptr || output == nullptr || values == nullptr)
|
| 534 |
+
return fail(MB_INVALID_ARGUMENT, error, error_capacity, "motion or output is null");
|
| 535 |
+
*output = value->target_root_translations.data();
|
| 536 |
+
*values = value->target_root_translations.size();
|
| 537 |
+
return MB_OK;
|
| 538 |
+
});
|
| 539 |
+
}
|
| 540 |
+
|
| 541 |
+
mb_status mb_motion_get_target_local_rotations_xyzw(const mb_motion * value, const float ** output, uint64_t * values,
|
| 542 |
+
char * error, uint64_t error_capacity) {
|
| 543 |
+
return guard(error, error_capacity, [&]() -> mb_status {
|
| 544 |
+
if (value == nullptr || output == nullptr || values == nullptr)
|
| 545 |
+
return fail(MB_INVALID_ARGUMENT, error, error_capacity, "motion or output is null");
|
| 546 |
+
*output = value->target_local_rotations_xyzw.data();
|
| 547 |
+
*values = value->target_local_rotations_xyzw.size();
|
| 548 |
+
return MB_OK;
|
| 549 |
+
});
|
| 550 |
+
}
|
| 551 |
+
|
| 552 |
+
} // extern "C"
|
src/cli.cpp
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#include <motionbricks/motionbricks.h>
|
| 2 |
+
|
| 3 |
+
#include <array>
|
| 4 |
+
#include <iostream>
|
| 5 |
+
#include <memory>
|
| 6 |
+
#include <string_view>
|
| 7 |
+
|
| 8 |
+
namespace {
|
| 9 |
+
|
| 10 |
+
struct model_deleter {
|
| 11 |
+
void operator()(mb_model * value) const noexcept { mb_model_free(value); }
|
| 12 |
+
};
|
| 13 |
+
|
| 14 |
+
int inspect(const char * directory) {
|
| 15 |
+
std::array<char, 1024> error{};
|
| 16 |
+
mb_model * raw = nullptr;
|
| 17 |
+
const auto status = mb_model_load(directory, nullptr, &raw, error.data(), error.size());
|
| 18 |
+
std::unique_ptr<mb_model, model_deleter> model(raw);
|
| 19 |
+
if (status != MB_OK) {
|
| 20 |
+
std::cerr << "model load failed (" << mb_status_string(status) << "): " << error.data() << '\n';
|
| 21 |
+
return 1;
|
| 22 |
+
}
|
| 23 |
+
std::uint64_t parameters = 0;
|
| 24 |
+
std::uint32_t joints = 0;
|
| 25 |
+
if (mb_model_get_parameter_count(model.get(), ¶meters, error.data(), error.size()) != MB_OK ||
|
| 26 |
+
mb_model_get_joint_count(model.get(), &joints, error.data(), error.size()) != MB_OK) {
|
| 27 |
+
std::cerr << "model inspection failed: " << error.data() << '\n';
|
| 28 |
+
return 1;
|
| 29 |
+
}
|
| 30 |
+
std::cout << "architecture: motionbricks\n"
|
| 31 |
+
<< "skeleton: g1skel34\n"
|
| 32 |
+
<< "parameters: " << parameters << '\n'
|
| 33 |
+
<< "joints: " << joints << '\n';
|
| 34 |
+
for (std::uint32_t joint = 0; joint < joints; ++joint) {
|
| 35 |
+
const char * name = nullptr;
|
| 36 |
+
std::int32_t parent = -1;
|
| 37 |
+
if (mb_model_get_joint_name(model.get(), joint, &name, error.data(), error.size()) != MB_OK ||
|
| 38 |
+
mb_model_get_joint_parent(model.get(), joint, &parent, error.data(), error.size()) != MB_OK) {
|
| 39 |
+
std::cerr << "joint inspection failed: " << error.data() << '\n';
|
| 40 |
+
return 1;
|
| 41 |
+
}
|
| 42 |
+
std::cout << "joint[" << joint << "]: " << name << " parent=" << parent << '\n';
|
| 43 |
+
}
|
| 44 |
+
return 0;
|
| 45 |
+
}
|
| 46 |
+
|
| 47 |
+
} // namespace
|
| 48 |
+
|
| 49 |
+
int main(int argc, char ** argv) {
|
| 50 |
+
if (argc == 2 && std::string_view(argv[1]) == "abi") {
|
| 51 |
+
std::cout << "motion-bricks.cpp ABI " << mb_abi_version() << '\n';
|
| 52 |
+
return 0;
|
| 53 |
+
}
|
| 54 |
+
if (argc == 3 && std::string_view(argv[1]) == "inspect") return inspect(argv[2]);
|
| 55 |
+
|
| 56 |
+
std::cerr << "usage: motionbricks-cli abi | inspect BUNDLE_DIRECTORY\n";
|
| 57 |
+
return 2;
|
| 58 |
+
}
|
src/decoder.cpp
ADDED
|
@@ -0,0 +1,223 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#include "decoder.hpp"
|
| 2 |
+
|
| 3 |
+
#include "neural_runtime.hpp"
|
| 4 |
+
|
| 5 |
+
#include <array>
|
| 6 |
+
#include <cstddef>
|
| 7 |
+
#include <cstdint>
|
| 8 |
+
#include <memory>
|
| 9 |
+
#include <string>
|
| 10 |
+
#include <utility>
|
| 11 |
+
#include <vector>
|
| 12 |
+
|
| 13 |
+
#if defined(MOTIONBRICKS_HAVE_GGML)
|
| 14 |
+
#include <ggml-backend.h>
|
| 15 |
+
#include <ggml.h>
|
| 16 |
+
#endif
|
| 17 |
+
|
| 18 |
+
namespace motionbricks::detail {
|
| 19 |
+
namespace {
|
| 20 |
+
|
| 21 |
+
#if defined(MOTIONBRICKS_HAVE_GGML)
|
| 22 |
+
|
| 23 |
+
struct context_deleter { void operator()(ggml_context * value) const noexcept { ggml_free(value); } };
|
| 24 |
+
struct buffer_deleter {
|
| 25 |
+
void operator()(ggml_backend_buffer * value) const noexcept { ggml_backend_buffer_free(value); }
|
| 26 |
+
};
|
| 27 |
+
using context_ptr = std::unique_ptr<ggml_context, context_deleter>;
|
| 28 |
+
using buffer_ptr = std::unique_ptr<ggml_backend_buffer, buffer_deleter>;
|
| 29 |
+
|
| 30 |
+
ggml_tensor * weight(const neural_runtime & runtime, const std::string & name,
|
| 31 |
+
std::string & reason) {
|
| 32 |
+
auto * result = neural_weight(runtime, "vq-decoder", name);
|
| 33 |
+
if (result == nullptr && reason.empty()) reason = "missing decoder weight: " + name;
|
| 34 |
+
return result;
|
| 35 |
+
}
|
| 36 |
+
|
| 37 |
+
ggml_tensor * linear(ggml_context * context, ggml_tensor * input,
|
| 38 |
+
ggml_tensor * matrix, ggml_tensor * bias) {
|
| 39 |
+
return ggml_add(context, ggml_mul_mat(context, matrix, input), bias);
|
| 40 |
+
}
|
| 41 |
+
|
| 42 |
+
ggml_tensor * conv(ggml_context * context, ggml_tensor * input,
|
| 43 |
+
ggml_tensor * kernel, ggml_tensor * bias,
|
| 44 |
+
int padding, int dilation) {
|
| 45 |
+
// ggml_conv_1d deliberately lowers ordinary F32 convolutions through an
|
| 46 |
+
// F16 im2col buffer. MotionBricks' residual decoder is sensitive enough
|
| 47 |
+
// for that loss to become visible, so retain F32 through the lowering.
|
| 48 |
+
auto * columns = ggml_im2col(context, kernel, input, 1, 0, padding, 0,
|
| 49 |
+
dilation, 0, false, GGML_TYPE_F32);
|
| 50 |
+
auto * output = ggml_mul_mat(context,
|
| 51 |
+
ggml_reshape_2d(context, columns, columns->ne[0], columns->ne[2] * columns->ne[1]),
|
| 52 |
+
ggml_reshape_2d(context, kernel, kernel->ne[0] * kernel->ne[1], kernel->ne[2]));
|
| 53 |
+
output = ggml_reshape_3d(context, output, columns->ne[1], kernel->ne[2], columns->ne[2]);
|
| 54 |
+
auto * shaped_bias = ggml_reshape_2d(context, bias, 1, bias->ne[0]);
|
| 55 |
+
return ggml_add(context, output, shaped_bias);
|
| 56 |
+
}
|
| 57 |
+
|
| 58 |
+
ggml_tensor * channels_first_to_frames(ggml_context * context, ggml_tensor * input,
|
| 59 |
+
std::int64_t channels, std::int64_t frames) {
|
| 60 |
+
return ggml_cont_2d(context, ggml_transpose(context, input), channels, frames);
|
| 61 |
+
}
|
| 62 |
+
|
| 63 |
+
ggml_tensor * frames_to_channels_first(ggml_context * context, ggml_tensor * input,
|
| 64 |
+
std::int64_t frames, std::int64_t channels) {
|
| 65 |
+
return ggml_cont_2d(context, ggml_transpose(context, input), frames, channels);
|
| 66 |
+
}
|
| 67 |
+
|
| 68 |
+
ggml_tensor * residual_stack(ggml_context * context, const neural_runtime & runtime,
|
| 69 |
+
ggml_tensor * input, unsigned stage, std::string & reason,
|
| 70 |
+
std::vector<std::pair<std::string, ggml_tensor *>> & traces) {
|
| 71 |
+
static constexpr std::array dilations{27, 9, 3, 1};
|
| 72 |
+
auto * hidden = input;
|
| 73 |
+
for (unsigned block = 0; block < dilations.size(); ++block) {
|
| 74 |
+
const auto prefix = "decoder.model." + std::to_string(stage) + ".0.model." +
|
| 75 |
+
std::to_string(block) + ".";
|
| 76 |
+
auto * branch = ggml_relu(context, hidden);
|
| 77 |
+
branch = conv(context, branch,
|
| 78 |
+
weight(runtime, prefix + "conv1.weight", reason),
|
| 79 |
+
weight(runtime, prefix + "conv1.bias", reason), dilations[block], dilations[block]);
|
| 80 |
+
traces.emplace_back("model." + std::to_string(stage) + ".0.model." +
|
| 81 |
+
std::to_string(block) + ".conv1", branch);
|
| 82 |
+
branch = ggml_relu(context, branch);
|
| 83 |
+
branch = conv(context, branch,
|
| 84 |
+
weight(runtime, prefix + "conv2.weight", reason),
|
| 85 |
+
weight(runtime, prefix + "conv2.bias", reason), 0, 1);
|
| 86 |
+
traces.emplace_back("model." + std::to_string(stage) + ".0.model." +
|
| 87 |
+
std::to_string(block) + ".conv2", branch);
|
| 88 |
+
hidden = ggml_add(context, hidden, branch);
|
| 89 |
+
}
|
| 90 |
+
return hidden;
|
| 91 |
+
}
|
| 92 |
+
|
| 93 |
+
#endif
|
| 94 |
+
|
| 95 |
+
} // namespace
|
| 96 |
+
|
| 97 |
+
mb_status run_vq_decoder(const neural_runtime & runtime,
|
| 98 |
+
std::span<const float> quantized,
|
| 99 |
+
std::span<const float> external_condition,
|
| 100 |
+
std::span<const float> target_condition,
|
| 101 |
+
std::span<const std::uint8_t> has_target_condition,
|
| 102 |
+
std::uint32_t positions,
|
| 103 |
+
std::vector<float> & motion,
|
| 104 |
+
std::string & reason,
|
| 105 |
+
std::vector<decoder_trace> * output_traces) {
|
| 106 |
+
#if !defined(MOTIONBRICKS_HAVE_GGML)
|
| 107 |
+
(void)runtime; (void)quantized; (void)external_condition; (void)target_condition;
|
| 108 |
+
(void)has_target_condition; (void)positions; (void)motion; (void)output_traces;
|
| 109 |
+
reason = "this build has no GGML support";
|
| 110 |
+
return MB_BACKEND_UNAVAILABLE;
|
| 111 |
+
#else
|
| 112 |
+
constexpr std::uint32_t latent_width = 256;
|
| 113 |
+
constexpr std::uint32_t external_width = 2;
|
| 114 |
+
constexpr std::uint32_t target_width = 304;
|
| 115 |
+
constexpr std::uint32_t output_width = 413;
|
| 116 |
+
const auto frames = positions * 4U;
|
| 117 |
+
if (positions < 1U || positions > 16U ||
|
| 118 |
+
quantized.size() != static_cast<std::size_t>(positions) * latent_width ||
|
| 119 |
+
external_condition.size() != static_cast<std::size_t>(frames) * external_width ||
|
| 120 |
+
target_condition.size() != static_cast<std::size_t>(frames) * target_width ||
|
| 121 |
+
has_target_condition.size() != frames) {
|
| 122 |
+
reason = "VQ decoder input shape mismatch";
|
| 123 |
+
return MB_INVALID_ARGUMENT;
|
| 124 |
+
}
|
| 125 |
+
constexpr std::size_t context_bytes = 32U * 1024U * 1024U;
|
| 126 |
+
context_ptr context(ggml_init({context_bytes, nullptr, true}));
|
| 127 |
+
if (!context) {
|
| 128 |
+
reason = "cannot allocate decoder graph metadata";
|
| 129 |
+
return MB_OUT_OF_MEMORY;
|
| 130 |
+
}
|
| 131 |
+
auto * latent_input = ggml_new_tensor_2d(context.get(), GGML_TYPE_F32, positions, latent_width);
|
| 132 |
+
auto * external_input = ggml_new_tensor_2d(context.get(), GGML_TYPE_F32, external_width, frames);
|
| 133 |
+
auto * target_input = ggml_new_tensor_2d(context.get(), GGML_TYPE_F32, target_width, frames);
|
| 134 |
+
auto * target_mask = ggml_new_tensor_2d(context.get(), GGML_TYPE_F32, 1, frames);
|
| 135 |
+
ggml_set_input(latent_input);
|
| 136 |
+
ggml_set_input(external_input);
|
| 137 |
+
ggml_set_input(target_input);
|
| 138 |
+
ggml_set_input(target_mask);
|
| 139 |
+
|
| 140 |
+
std::vector<std::pair<std::string, ggml_tensor *>> traces;
|
| 141 |
+
auto * initial_conv = conv(context.get(), latent_input,
|
| 142 |
+
weight(runtime, "decoder.model.0.weight", reason),
|
| 143 |
+
weight(runtime, "decoder.model.0.bias", reason), 1, 1);
|
| 144 |
+
traces.emplace_back("model.0", initial_conv);
|
| 145 |
+
auto * hidden = ggml_relu(context.get(), initial_conv);
|
| 146 |
+
for (unsigned stage_index = 0; stage_index < 2U; ++stage_index) {
|
| 147 |
+
const unsigned stage = stage_index + 2U;
|
| 148 |
+
const auto frame_group = 1U << (2U - stage_index);
|
| 149 |
+
const auto stage_positions = positions * (1U << stage_index);
|
| 150 |
+
const auto stage_frames = stage_positions * frame_group;
|
| 151 |
+
|
| 152 |
+
auto * target_embedding = linear(context.get(), target_input,
|
| 153 |
+
weight(runtime, "decoder.target_cond_blocks." + std::to_string(stage_index * 2U) + ".weight", reason),
|
| 154 |
+
weight(runtime, "decoder.target_cond_blocks." + std::to_string(stage_index * 2U) + ".bias", reason));
|
| 155 |
+
traces.emplace_back("target_cond_blocks." + std::to_string(stage_index * 2U), target_embedding);
|
| 156 |
+
auto * hidden_frames = channels_first_to_frames(context.get(), hidden, 512, stage_positions);
|
| 157 |
+
hidden_frames = ggml_reshape_2d(context.get(), hidden_frames, 512 / frame_group, stage_frames);
|
| 158 |
+
hidden_frames = ggml_add(context.get(), hidden_frames,
|
| 159 |
+
ggml_mul(context.get(), ggml_sub(context.get(), ggml_relu(context.get(), target_embedding), hidden_frames),
|
| 160 |
+
target_mask));
|
| 161 |
+
hidden_frames = ggml_reshape_2d(context.get(), hidden_frames, 512, stage_positions);
|
| 162 |
+
|
| 163 |
+
auto * external_grouped = ggml_reshape_2d(
|
| 164 |
+
context.get(), external_input, external_width * frame_group, stage_positions);
|
| 165 |
+
auto * fused = ggml_concat(context.get(), hidden_frames, external_grouped, 0);
|
| 166 |
+
auto * fused_linear = linear(context.get(), fused,
|
| 167 |
+
weight(runtime, "decoder.external_cond_blocks." + std::to_string(stage_index * 2U) + ".weight", reason),
|
| 168 |
+
weight(runtime, "decoder.external_cond_blocks." + std::to_string(stage_index * 2U) + ".bias", reason));
|
| 169 |
+
traces.emplace_back("external_cond_blocks." + std::to_string(stage_index * 2U), fused_linear);
|
| 170 |
+
fused = ggml_relu(context.get(), fused_linear);
|
| 171 |
+
hidden = frames_to_channels_first(context.get(), fused, stage_positions, 512);
|
| 172 |
+
hidden = residual_stack(context.get(), runtime, hidden, stage, reason, traces);
|
| 173 |
+
hidden = ggml_interpolate(context.get(), hidden, hidden->ne[0] * 2, hidden->ne[1],
|
| 174 |
+
hidden->ne[2], hidden->ne[3], GGML_SCALE_MODE_NEAREST);
|
| 175 |
+
hidden = conv(context.get(), hidden,
|
| 176 |
+
weight(runtime, "decoder.model." + std::to_string(stage) + ".2.weight", reason),
|
| 177 |
+
weight(runtime, "decoder.model." + std::to_string(stage) + ".2.bias", reason), 1, 1);
|
| 178 |
+
traces.emplace_back("model." + std::to_string(stage) + ".2", hidden);
|
| 179 |
+
}
|
| 180 |
+
auto * post_conv = conv(context.get(), hidden,
|
| 181 |
+
weight(runtime, "decoder.model.4.weight", reason),
|
| 182 |
+
weight(runtime, "decoder.model.4.bias", reason), 1, 1);
|
| 183 |
+
traces.emplace_back("model.4", post_conv);
|
| 184 |
+
hidden = ggml_relu(context.get(), post_conv);
|
| 185 |
+
hidden = conv(context.get(), hidden,
|
| 186 |
+
weight(runtime, "decoder.model.6.weight", reason),
|
| 187 |
+
weight(runtime, "decoder.model.6.bias", reason), 1, 1);
|
| 188 |
+
traces.emplace_back("model.6", hidden);
|
| 189 |
+
auto * output = channels_first_to_frames(context.get(), hidden, output_width, frames);
|
| 190 |
+
if (!reason.empty()) return MB_INCOMPATIBLE_MODEL;
|
| 191 |
+
auto * graph = ggml_new_graph_custom(context.get(), 2048, false);
|
| 192 |
+
ggml_build_forward_expand(graph, output);
|
| 193 |
+
buffer_ptr buffer(ggml_backend_alloc_ctx_tensors(context.get(), neural_backend(runtime)));
|
| 194 |
+
if (!buffer) {
|
| 195 |
+
reason = "cannot allocate decoder compute buffer";
|
| 196 |
+
return MB_OUT_OF_MEMORY;
|
| 197 |
+
}
|
| 198 |
+
std::vector<float> mask(has_target_condition.begin(), has_target_condition.end());
|
| 199 |
+
ggml_backend_tensor_set(latent_input, quantized.data(), 0, ggml_nbytes(latent_input));
|
| 200 |
+
ggml_backend_tensor_set(external_input, external_condition.data(), 0, ggml_nbytes(external_input));
|
| 201 |
+
ggml_backend_tensor_set(target_input, target_condition.data(), 0, ggml_nbytes(target_input));
|
| 202 |
+
ggml_backend_tensor_set(target_mask, mask.data(), 0, ggml_nbytes(target_mask));
|
| 203 |
+
const auto status = ggml_backend_graph_compute(neural_backend(runtime), graph);
|
| 204 |
+
if (status != GGML_STATUS_SUCCESS) {
|
| 205 |
+
reason = std::string("decoder graph failed: ") + ggml_status_to_string(status);
|
| 206 |
+
return MB_COMPUTE_FAILED;
|
| 207 |
+
}
|
| 208 |
+
motion.resize(static_cast<std::size_t>(frames) * output_width);
|
| 209 |
+
ggml_backend_tensor_get(output, motion.data(), 0, motion.size() * sizeof(float));
|
| 210 |
+
if (output_traces != nullptr) {
|
| 211 |
+
output_traces->clear();
|
| 212 |
+
output_traces->reserve(traces.size());
|
| 213 |
+
for (const auto & [name, tensor] : traces) {
|
| 214 |
+
decoder_trace trace{name, std::vector<float>(static_cast<std::size_t>(ggml_nelements(tensor)))};
|
| 215 |
+
ggml_backend_tensor_get(tensor, trace.values.data(), 0, trace.values.size() * sizeof(float));
|
| 216 |
+
output_traces->push_back(std::move(trace));
|
| 217 |
+
}
|
| 218 |
+
}
|
| 219 |
+
return MB_OK;
|
| 220 |
+
#endif
|
| 221 |
+
}
|
| 222 |
+
|
| 223 |
+
} // namespace motionbricks::detail
|
src/decoder.hpp
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#pragma once
|
| 2 |
+
|
| 3 |
+
#include <motionbricks/motionbricks.h>
|
| 4 |
+
|
| 5 |
+
#include <cstdint>
|
| 6 |
+
#include <span>
|
| 7 |
+
#include <string>
|
| 8 |
+
#include <vector>
|
| 9 |
+
|
| 10 |
+
namespace motionbricks::detail {
|
| 11 |
+
|
| 12 |
+
class neural_runtime;
|
| 13 |
+
|
| 14 |
+
struct decoder_trace {
|
| 15 |
+
std::string name;
|
| 16 |
+
std::vector<float> values;
|
| 17 |
+
};
|
| 18 |
+
|
| 19 |
+
mb_status run_vq_decoder(const neural_runtime & runtime,
|
| 20 |
+
std::span<const float> quantized,
|
| 21 |
+
std::span<const float> external_condition,
|
| 22 |
+
std::span<const float> target_condition,
|
| 23 |
+
std::span<const std::uint8_t> has_target_condition,
|
| 24 |
+
std::uint32_t positions,
|
| 25 |
+
std::vector<float> & motion,
|
| 26 |
+
std::string & reason,
|
| 27 |
+
std::vector<decoder_trace> * traces = nullptr);
|
| 28 |
+
|
| 29 |
+
} // namespace motionbricks::detail
|
src/error.cpp
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#include "error.hpp"
|
| 2 |
+
|
| 3 |
+
#include <algorithm>
|
| 4 |
+
#include <cstring>
|
| 5 |
+
#include <limits>
|
| 6 |
+
|
| 7 |
+
namespace motionbricks::detail {
|
| 8 |
+
|
| 9 |
+
void write_error(char * output, std::uint64_t capacity, std::string_view message) noexcept {
|
| 10 |
+
if (output == nullptr || capacity == 0) return;
|
| 11 |
+
const auto usable = capacity - 1;
|
| 12 |
+
const auto bounded = std::min<std::uint64_t>(usable,
|
| 13 |
+
static_cast<std::uint64_t>(std::numeric_limits<std::size_t>::max()));
|
| 14 |
+
const auto count = std::min<std::size_t>(message.size(), static_cast<std::size_t>(bounded));
|
| 15 |
+
if (count != 0) std::memcpy(output, message.data(), count);
|
| 16 |
+
output[count] = '\0';
|
| 17 |
+
}
|
| 18 |
+
|
| 19 |
+
mb_status fail(mb_status status, char * error, std::uint64_t error_capacity,
|
| 20 |
+
std::string_view message) noexcept {
|
| 21 |
+
write_error(error, error_capacity, message);
|
| 22 |
+
return status;
|
| 23 |
+
}
|
| 24 |
+
|
| 25 |
+
} // namespace motionbricks::detail
|